diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 41cb44ef0b5fa3b..a11061afa38e38b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -220,4 +220,14 @@ default Optional getMetaTableFunction(String dbName, String default Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { return Optional.empty(); } + + // Convert from remote database name to local database name, overridden by subclass if necessary + default String fromRemoteDatabaseName(String remoteDatabaseName) { + return remoteDatabaseName; + } + + // Convert from remote table name to local table name, overridden by subclass if necessary + default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return remoteTableName; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 0ec49052cb83c80..53a3dd47b8aa602 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -48,6 +48,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.OperationType; @@ -655,8 +656,8 @@ public boolean externalTableExistInLocal(String dbName, String tableName, String } public void registerExternalTableFromEvent(String dbName, String tableName, - String catalogName, long updateTime, - boolean ignoreIfExists) throws DdlException { + String catalogName, long updateTime, + boolean ignoreIfExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -686,7 +687,8 @@ public void registerExternalTableFromEvent(String dbName, String tableName, db.writeLock(); try { - HMSExternalTable namedTable = new HMSExternalTable(tblId, tableName, dbName, (HMSExternalCatalog) catalog); + HMSExternalTable namedTable = ((HMSExternalDatabase) db) + .buildTableForInit(tableName, tableName, tblId, hmsCatalog, (HMSExternalDatabase) db, false); namedTable.setUpdateTime(updateTime); db.registerTable(namedTable); } finally { @@ -732,7 +734,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName) } public void addExternalPartitions(String catalogName, String dbName, String tableName, - List partitionNames, long updateTime, boolean ignoreIfNotExists) + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { @@ -767,7 +769,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl } public void dropExternalPartitions(String catalogName, String dbName, String tableName, - List partitionNames, long updateTime, boolean ignoreIfNotExists) + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index f3728498f2a9aba..f918bdca9b95d25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -103,9 +103,13 @@ public abstract class ExternalCatalog public static final String DORIS_VERSION = "doris.version"; public static final String DORIS_VERSION_VALUE = Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH; public static final String USE_META_CACHE = "use_meta_cache"; + public static final String CREATE_TIME = "create_time"; public static final boolean DEFAULT_USE_META_CACHE = true; + public static final String FOUND_CONFLICTING = "Found conflicting"; + public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES = "only_test_lower_case_table_names"; + // Properties that should not be shown in the `show create catalog` result public static final Set HIDDEN_PROPERTIES = Sets.newHashSet( CREATE_TIME, @@ -185,6 +189,7 @@ private Configuration buildConf() { /** * set some default properties when creating catalog + * * @return list of database names in this catalog */ protected List listDatabaseNames() { @@ -266,8 +271,9 @@ public final synchronized void makeSureInitialized() { OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, ignored -> getFilteredDatabaseNames(), - dbName -> Optional.ofNullable( - buildDbForInit(dbName, Util.genIdByName(name, dbName), logType, true)), + localDbName -> Optional.ofNullable( + buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, + true)), (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); } setLastUpdateTime(System.currentTimeMillis()); @@ -364,21 +370,28 @@ private void init() { InitCatalogLog initCatalogLog = new InitCatalogLog(); initCatalogLog.setCatalogId(id); initCatalogLog.setType(logType); - List filteredDatabases = getFilteredDatabaseNames(); - for (String dbName : filteredDatabases) { + List> remoteToLocalPairs = getFilteredDatabaseNames(); + for (Pair pair : remoteToLocalPairs) { + String remoteDbName = pair.key(); + String localDbName = pair.value(); long dbId; - if (dbNameToId != null && dbNameToId.containsKey(dbName)) { - dbId = dbNameToId.get(dbName); - tmpDbNameToId.put(dbName, dbId); + if (dbNameToId != null && dbNameToId.containsKey(localDbName)) { + dbId = dbNameToId.get(localDbName); + tmpDbNameToId.put(localDbName, dbId); ExternalDatabase db = idToDb.get(dbId); + // If the remote name is missing during upgrade, all databases in the Map will be reinitialized. + if (Strings.isNullOrEmpty(db.getRemoteName())) { + db.setRemoteName(remoteDbName); + } tmpIdToDb.put(dbId, db); initCatalogLog.addRefreshDb(dbId); } else { dbId = Env.getCurrentEnv().getNextId(); - tmpDbNameToId.put(dbName, dbId); - ExternalDatabase db = buildDbForInit(dbName, dbId, logType, false); + tmpDbNameToId.put(localDbName, dbId); + ExternalDatabase db = + buildDbForInit(remoteDbName, localDbName, dbId, logType, false); tmpIdToDb.put(dbId, db); - initCatalogLog.addCreateDb(dbId, dbName); + initCatalogLog.addCreateDb(dbId, localDbName, remoteDbName); } } @@ -389,15 +402,41 @@ private void init() { Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); } + /** + * Retrieves a filtered list of database names and their corresponding local database names. + * The method applies to include and exclude filters based on the database properties, ensuring + * only the relevant databases are included for further operations. + *

+ * The method also handles conflicts in database names under case-insensitive conditions + * and throws an exception if such conflicts are detected. + *

+ * Steps: + * 1. Fetch all database names from the remote source. + * 2. Apply to include and exclude database filters: + * - Exclude filters take precedence over include filters. + * - If a database is in the exclude list, it is ignored. + * - If a database is not in the include list and the include list is not empty, it is ignored. + * 3. Map the filtered remote database names to local database names. + * 4. Handle conflicts when `lower_case_meta_names` is enabled: + * - Detect cases where multiple remote database names map to the same lower-cased local name. + * - Throw an exception if conflicts are found. + * + * @return A list of pairs where each pair contains the remote database name and local database name. + * @throws RuntimeException if there are conflicting database names under case-insensitive conditions. + */ @NotNull - private List getFilteredDatabaseNames() { + private List> getFilteredDatabaseNames() { List allDatabases = Lists.newArrayList(listDatabaseNames()); allDatabases.remove(InfoSchemaDb.DATABASE_NAME); allDatabases.add(InfoSchemaDb.DATABASE_NAME); allDatabases.remove(MysqlDb.DATABASE_NAME); allDatabases.add(MysqlDb.DATABASE_NAME); + Map includeDatabaseMap = getIncludeDatabaseMap(); Map excludeDatabaseMap = getExcludeDatabaseMap(); + + List> remoteToLocalPairs = Lists.newArrayList(); + allDatabases = allDatabases.stream().filter(dbName -> { if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) { // Exclude database map take effect with higher priority over include database map @@ -410,7 +449,40 @@ private List getFilteredDatabaseNames() { } return true; }).collect(Collectors.toList()); - return allDatabases; + + for (String remoteDbName : allDatabases) { + String localDbName = fromRemoteDatabaseName(remoteDbName); + remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); + } + + // Check for conflicts when lower_case_meta_names = true + if (Boolean.parseBoolean(getLowerCaseMetaNames())) { + // Map to track lowercase local names and their corresponding remote names + Map> lowerCaseToRemoteNames = Maps.newHashMap(); + + // Collect lowercased local names and their remote counterparts + for (Pair pair : remoteToLocalPairs) { + String lowerCaseLocalName = pair.second.toLowerCase(); + lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.first); + } + + // Identify conflicts: multiple remote names mapping to the same lowercase local name + List conflicts = lowerCaseToRemoteNames.values().stream() + .filter(remoteNames -> remoteNames.size() > 1) // Conflict: more than one remote name + .flatMap(List::stream) // Collect all conflicting remote names + .collect(Collectors.toList()); + + // Throw exception if conflicts are found + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + FOUND_CONFLICTING + " database names under case-insensitive conditions. " + + "Conflicting remote database names: %s in catalog %s. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), name)); + } + } + + return remoteToLocalPairs; } public void onRefresh(boolean invalidCache) { @@ -487,6 +559,7 @@ public void setComment(String comment) { /** * Different from 'listDatabases()', this method will return dbnames from cache. * while 'listDatabases()' will return dbnames from remote datasource. + * * @return names of database in this catalog. */ @Override @@ -643,8 +716,17 @@ private void removeAccessController() { } public void replayInitCatalog(InitCatalogLog log) { + // If the remote name is missing during upgrade, all databases in the Map will be reinitialized. + if (log.getRemoteDbNames() == null || log.getRemoteDbNames().isEmpty()) { + dbNameToId = Maps.newConcurrentMap(); + idToDb = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + Map tmpDbNameToId = Maps.newConcurrentMap(); - Map> tmpIdToDb = Maps.newConcurrentMap(); + Map> tmpIdToDb = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { Optional> db = getDbForReplay(log.getRefreshDbIds().get(i)); // Should not return null. @@ -661,7 +743,8 @@ public void replayInitCatalog(InitCatalogLog log) { } for (int i = 0; i < log.getCreateCount(); i++) { ExternalDatabase db = - buildDbForInit(log.getCreateDbNames().get(i), log.getCreateDbIds().get(i), log.getType(), false); + buildDbForInit(log.getRemoteDbNames().get(i), log.getCreateDbNames().get(i), + log.getCreateDbIds().get(i), log.getType(), false); if (db != null) { tmpDbNameToId.put(db.getFullName(), db.getId()); tmpIdToDb.put(db.getId(), db); @@ -688,54 +771,88 @@ public Optional> getDbForReplay(long d * Build a database instance. * If checkExists is true, it will check if the database exists in the remote system. * - * @param dbName + * @param remoteDbName * @param dbId * @param logType * @param checkExists * @return */ - protected ExternalDatabase buildDbForInit(String dbName, long dbId, - InitCatalogLog.Type logType, boolean checkExists) { + protected ExternalDatabase buildDbForInit(String remoteDbName, String localDbName, + long dbId, InitCatalogLog.Type logType, boolean checkExists) { + // Step 1: Map local database name if not already provided + if (localDbName == null && remoteDbName != null) { + localDbName = fromRemoteDatabaseName(remoteDbName); + } + + // Step 2: // When running ut, disable this check to make ut pass. // Because in ut, the database is not created in remote system. if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalCatalog)) { try { List dbNames = getDbNames(); - if (!dbNames.contains(dbName)) { - dbNames = getFilteredDatabaseNames(); - if (!dbNames.contains(dbName)) { + if (!dbNames.contains(localDbName)) { + dbNames = getFilteredDatabaseNames().stream() + .map(Pair::value) + .collect(Collectors.toList()); + if (!dbNames.contains(localDbName)) { + LOG.warn("Database {} does not exist in the remote system. Skipping initialization.", + localDbName); return null; } } - } catch (Throwable t) { + } catch (RuntimeException e) { + // Handle "Found conflicting" exception explicitly + if (e.getMessage().contains(FOUND_CONFLICTING)) { + LOG.error(e.getMessage()); + throw e; // Rethrow to let the caller handle this critical issue + } else { + // Any errors other than name conflicts, we default to not finding the database + LOG.warn("Failed to check db {} exist in remote system, ignore it.", localDbName, e); + return null; + } + } catch (Exception e) { // If connection failed, it will throw exception. // ignore it and treat it as not exist. - LOG.warn("Failed to check db {} exist in remote system, ignore it.", dbName, t); + LOG.warn("Failed to check db {} exist in remote system, ignore it.", localDbName, e); return null; } } - if (dbName.equals(InfoSchemaDb.DATABASE_NAME)) { + // Step 3: Resolve remote database name if using meta cache + if (remoteDbName == null && useMetaCache.orElse(false)) { + if (Boolean.parseBoolean(getLowerCaseMetaNames()) || !Strings.isNullOrEmpty(getMetaNamesMapping())) { + remoteDbName = metaCache.getRemoteName(localDbName); + if (remoteDbName == null) { + LOG.warn("Could not resolve remote database name for local database: {}", localDbName); + return null; + } + } else { + remoteDbName = localDbName; + } + } + + // Step 4: Instantiate the appropriate ExternalDatabase based on logType + if (localDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { return new ExternalInfoSchemaDatabase(this, dbId); } - if (dbName.equals(MysqlDb.DATABASE_NAME)) { + if (localDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { return new ExternalMysqlDatabase(this, dbId); } switch (logType) { case HMS: - return new HMSExternalDatabase(this, dbId, dbName); + return new HMSExternalDatabase(this, dbId, localDbName, remoteDbName); case ES: - return new EsExternalDatabase(this, dbId, dbName); + return new EsExternalDatabase(this, dbId, localDbName, remoteDbName); case JDBC: - return new JdbcExternalDatabase(this, dbId, dbName); + return new JdbcExternalDatabase(this, dbId, localDbName, remoteDbName); case ICEBERG: - return new IcebergExternalDatabase(this, dbId, dbName); + return new IcebergExternalDatabase(this, dbId, localDbName, remoteDbName); case MAX_COMPUTE: - return new MaxComputeExternalDatabase(this, dbId, dbName); + return new MaxComputeExternalDatabase(this, dbId, localDbName, remoteDbName); case TEST: - return new TestExternalDatabase(this, dbId, dbName); + return new TestExternalDatabase(this, dbId, localDbName, remoteDbName); case PAIMON: - return new PaimonExternalDatabase(this, dbId, dbName); + return new PaimonExternalDatabase(this, dbId, localDbName, remoteDbName); default: break; } @@ -877,6 +994,19 @@ private Map getSpecifiedDatabaseMap(String catalogPropertyKey) return specifiedDatabaseMap; } + + public String getLowerCaseMetaNames() { + return catalogProperty.getOrDefault(Resource.LOWER_CASE_META_NAMES, "false"); + } + + public int getOnlyTestLowerCaseTableNames() { + return Integer.parseInt(catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0")); + } + + public String getMetaNamesMapping() { + return catalogProperty.getOrDefault(Resource.META_NAMES_MAPPING, ""); + } + public String bindBrokerName() { return catalogProperty.getProperties().get(HMSExternalCatalog.BIND_BROKER_NAME); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index eda98efb9b6a038..099d76e98e1b1ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -25,7 +25,9 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; @@ -35,12 +37,14 @@ import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlTable; import org.apache.doris.datasource.metacache.MetaCache; +import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterCatalogExecutor; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -76,6 +80,8 @@ public abstract class ExternalDatabase protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "dbProperties") protected DatabaseProperty dbProperties = new DatabaseProperty(); @SerializedName(value = "initialized") @@ -100,11 +106,14 @@ public abstract class ExternalDatabase * @param extCatalog The catalog this database belongs to. * @param id Database id. * @param name Database name. + * @param remoteName Remote database name. */ - public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, InitDatabaseLog.Type dbLogType) { + public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName, + InitDatabaseLog.Type dbLogType) { this.extCatalog = extCatalog; this.id = id; this.name = name; + this.remoteName = remoteName; this.dbLogType = dbLogType; } @@ -112,6 +121,10 @@ public void setExtCatalog(ExternalCatalog extCatalog) { this.extCatalog = extCatalog; } + public void setRemoteName(String remoteName) { + this.remoteName = remoteName; + } + public void setTableExtCatalog(ExternalCatalog extCatalog) { for (T table : idToTbl.values()) { table.setCatalog(extCatalog); @@ -150,9 +163,10 @@ public final synchronized void makeSureInitialized() { OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, ignored -> listTableNames(), - tableName -> Optional.ofNullable( - buildTableForInit(tableName, - Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)), + localTableName -> Optional.ofNullable( + buildTableForInit(null, localTableName, + Util.genIdByName(extCatalog.getName(), name, localTableName), extCatalog, + this, true)), (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); } setLastUpdateTime(System.currentTimeMillis()); @@ -176,6 +190,15 @@ public final synchronized void makeSureInitialized() { } public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { + // If the remote name is missing during upgrade, all tables in the Map will be reinitialized. + if (log.getRemoteTableNames() == null || log.getRemoteTableNames().isEmpty()) { + tableNameToId = Maps.newConcurrentMap(); + idToTbl = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { @@ -191,7 +214,9 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { } } for (int i = 0; i < log.getCreateCount(); i++) { - T table = buildTableForInit(log.getCreateTableNames().get(i), log.getCreateTableIds().get(i), catalog); + T table = + buildTableForInit(log.getRemoteTableNames().get(i), log.getCreateTableNames().get(i), + log.getCreateTableIds().get(i), catalog, this, false); tmpTableNameToId.put(table.getName(), table.getId()); tmpIdToTbl.put(table.getId(), table); } @@ -206,24 +231,34 @@ private void init() { initDatabaseLog.setType(dbLogType); initDatabaseLog.setCatalogId(extCatalog.getId()); initDatabaseLog.setDbId(id); - List tableNames = listTableNames(); - if (tableNames != null) { + List> tableNamePairs = listTableNames(); + if (tableNamePairs != null) { Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newHashMap(); - for (String tableName : tableNames) { + for (Pair pair : tableNamePairs) { + String remoteTableName = pair.first; + String localTableName = pair.second; long tblId; - if (tableNameToId != null && tableNameToId.containsKey(tableName)) { - tblId = tableNameToId.get(tableName); - tmpTableNameToId.put(tableName, tblId); + if (tableNameToId != null && tableNameToId.containsKey(localTableName)) { + tblId = tableNameToId.get(localTableName); + tmpTableNameToId.put(localTableName, tblId); T table = idToTbl.get(tblId); + // If the remote name is missing during upgrade, all tables in the Map will be reinitialized. + if (Strings.isNullOrEmpty(table.getRemoteName())) { + table.setRemoteName(remoteTableName); + } + // If the db is missing, set it. + if (table.getDb() == null) { + table.setDb(this); + } tmpIdToTbl.put(tblId, table); initDatabaseLog.addRefreshTable(tblId); } else { tblId = Env.getCurrentEnv().getNextId(); - tmpTableNameToId.put(tableName, tblId); - T table = buildTableForInit(tableName, tblId, extCatalog); + tmpTableNameToId.put(localTableName, tblId); + T table = buildTableForInit(remoteTableName, localTableName, tblId, extCatalog, this, false); tmpIdToTbl.put(tblId, table); - initDatabaseLog.addCreateTable(tblId, tableName); + initDatabaseLog.addCreateTable(tblId, localTableName, remoteTableName); } } tableNameToId = tmpTableNameToId; @@ -235,26 +270,119 @@ private void init() { Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } - private List listTableNames() { - List tableNames; + private List> listTableNames() { + List> tableNames; if (name.equals(InfoSchemaDb.DATABASE_NAME)) { - tableNames = ExternalInfoSchemaDatabase.listTableNames(); + tableNames = ExternalInfoSchemaDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else if (name.equals(MysqlDb.DATABASE_NAME)) { - tableNames = ExternalMysqlDatabase.listTableNames(); + tableNames = ExternalMysqlDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else { - tableNames = extCatalog.listTableNames(null, name).stream().map(tableName -> { - lowerCaseToTableName.put(tableName.toLowerCase(), tableName); - if (Env.isStoredTableNamesLowerCase()) { - return tableName.toLowerCase(); - } else { - return tableName; + tableNames = extCatalog.listTableNames(null, remoteName).stream().map(tableName -> { + String localTableName = extCatalog.fromRemoteTableName(remoteName, tableName); + if (this.isStoredTableNamesLowerCase()) { + localTableName = localTableName.toLowerCase(); } + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); + return Pair.of(tableName, localTableName); }).collect(Collectors.toList()); } + // Check for conflicts when stored table names or meta names are case-insensitive + if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) + || this.isStoredTableNamesLowerCase() + || this.isTableNamesCaseInsensitive()) { + // Map to track lowercased local names and their corresponding remote names + Map> lowerCaseToRemoteNames = Maps.newHashMap(); + + // Collect lowercased local names and their remote counterparts + for (Pair pair : tableNames) { + String lowerCaseLocalName = pair.value().toLowerCase(); + lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.key()); + } + + // Identify conflicts: multiple remote names mapping to the same lowercased local name + List conflicts = lowerCaseToRemoteNames.values().stream() + .filter(remoteNames -> remoteNames.size() > 1) // Conflict: more than one remote name + .flatMap(List::stream) // Collect all conflicting remote names + .collect(Collectors.toList()); + + // Throw exception if conflicts are found + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + ExternalCatalog.FOUND_CONFLICTING + " table names under case-insensitive conditions. " + + "Conflicting remote table names: %s in remote database '%s' under catalog '%s'. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), remoteName, extCatalog.getName())); + } + } return tableNames; } - protected abstract T buildTableForInit(String tableName, long tblId, ExternalCatalog catalog); + public T buildTableForInit(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, ExternalDatabase db, boolean checkExists) { + + // Step 1: Resolve local table name if not provided + if (localTableName == null && remoteTableName != null) { + localTableName = extCatalog.fromRemoteTableName(remoteName, remoteTableName); + } + + // Step 2: Check if the table exists in the system, if the `checkExists` flag is enabled + if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalDatabase)) { + try { + List tblNames = Lists.newArrayList(getTableNamesWithLock()); + if (!tblNames.contains(localTableName)) { + tblNames = listTableNames().stream() + .map(Pair::value) + .collect(Collectors.toList()); + if (!tblNames.contains(localTableName)) { + LOG.warn("Table {} does not exist in the remote system. Skipping initialization.", + localTableName); + return null; + } + } + } catch (RuntimeException e) { + // Handle "Found conflicting" exception explicitly + if (e.getMessage().contains(ExternalCatalog.FOUND_CONFLICTING)) { + LOG.error(e.getMessage()); + throw e; // Rethrow to let the caller handle this critical issue + } else { + // Any errors other than name conflicts, we default to not finding the table + LOG.warn("Failed to check existence of table {} in the remote system. Ignoring this table.", + localTableName, e); + return null; + } + } catch (Exception e) { + // If connection fails, treat the table as non-existent + LOG.warn("Failed to check existence of table {} in the remote system. Ignoring this table.", + localTableName, e); + return null; + } + } + + // Step 3: Resolve remote table name if using meta cache and it is not provided + if (remoteTableName == null && extCatalog.useMetaCache.get()) { + if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) + || !Strings.isNullOrEmpty(extCatalog.getMetaNamesMapping()) + || this.isStoredTableNamesLowerCase()) { + remoteTableName = metaCache.getRemoteName(localTableName); + if (remoteTableName == null) { + LOG.warn("Could not resolve remote table name for local table: {}", localTableName); + return null; + } + } else { + remoteTableName = localTableName; + } + } + + // Step 4: Build and return the table instance using the resolved names and other parameters + return buildTableInternal(remoteTableName, localTableName, tblId, catalog, db); + } + + protected abstract T buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, ExternalDatabase db); public Optional getTableForReplay(long tableId) { if (extCatalog.getUseMetaCache().get()) { @@ -328,6 +456,10 @@ public String getFullName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public DatabaseProperty getDbProperties() { return dbProperties; @@ -335,10 +467,18 @@ public DatabaseProperty getDbProperties() { @Override public boolean isTableExist(String tableName) { - if (Env.isTableNamesCaseInsensitive()) { - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { - return false; + if (this.isTableNamesCaseInsensitive()) { + String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (realTableName == null) { + // Here we need to execute listTableNames() once to fill in lowerCaseToTableName + // to prevent lowerCaseToTableName from being empty in some cases + listTableNames(); + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return false; + } + } else { + tableName = realTableName; } } return extCatalog.tableExist(ConnectContext.get().getSessionContext(), name, tableName); @@ -391,15 +531,27 @@ public Set getTableNamesWithLock() { @Override public T getTableNullable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase()) { + if (this.isStoredTableNamesLowerCase()) { tableName = tableName.toLowerCase(); } - if (Env.isTableNamesCaseInsensitive()) { - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { - return null; + if (this.isTableNamesCaseInsensitive()) { + String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (realTableName == null) { + // Here we need to execute listTableNames() once to fill in lowerCaseToTableName + // to prevent lowerCaseToTableName from being empty in some cases + listTableNames(); + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return null; + } + } else { + tableName = realTableName; } } + if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true") + && (this.isTableNamesCaseInsensitive())) { + tableName = tableName.toLowerCase(); + } if (extCatalog.getUseMetaCache().get()) { // must use full qualified name to generate id. // otherwise, if 2 databases have the same table name, the id will be the same. @@ -480,7 +632,7 @@ public void gsonPostProcess() throws IOException { @Override public void unregisterTable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase()) { + if (this.isStoredTableNamesLowerCase()) { tableName = tableName.toLowerCase(); } if (LOG.isDebugEnabled()) { @@ -528,7 +680,9 @@ public boolean registerTable(TableIf tableIf) { } else { if (!tableNameToId.containsKey(tableName)) { tableNameToId.put(tableName, tableId); - idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + idToTbl.put(tableId, + buildTableForInit(tableName, extCatalog.fromRemoteTableName(this.remoteName, tableName), + tableId, extCatalog, this, false)); lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } @@ -539,4 +693,16 @@ public boolean registerTable(TableIf tableIf) { public String getQualifiedName(String tblName) { return String.join(".", extCatalog.getName(), name, tblName); } + + private boolean isStoredTableNamesLowerCase() { + // Because we have added a test configuration item, + // it needs to be judged together with Env.isStoredTableNamesLowerCase() + return Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1; + } + + private boolean isTableNamesCaseInsensitive() { + // Because we have added a test configuration item, + // it needs to be judged together with Env.isTableNamesCaseInsensitive() + return Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 24f55e74266863c..48d170b59166b41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -302,7 +303,7 @@ public void invalidatePartitionsCache(long catalogId, String dbName, String tabl public MetaCache buildMetaCache(String name, OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize, - CacheLoader> namesCacheLoader, + CacheLoader>> namesCacheLoader, CacheLoader> metaObjCacheLoader, RemovalListener> removalListener) { MetaCache metaCache = new MetaCache<>(name, commonRefreshExecutor, expireAfterWriteSec, refreshAfterWriteSec, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 237457b5a997d9c..1924ed1b9fc82fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -71,10 +71,13 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "type") protected TableType type = null; @SerializedName(value = "timestamp") protected long timestamp; + // dbName is temporarily retained and will be deleted later. To use dbName, please use db.getFullName() @SerializedName(value = "dbName") protected String dbName; @SerializedName(value = "ta") @@ -86,6 +89,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long dbId; protected boolean objectCreated; protected ExternalCatalog catalog; + protected ExternalDatabase db; /** * No args constructor for persist. @@ -99,15 +103,19 @@ public ExternalTable() { * * @param id Table id. * @param name Table name. + * @param remoteName Remote table name. * @param catalog ExternalCatalog this table belongs to. - * @param dbName Name of the db the this table belongs to. + * @param db ExternalDatabase this table belongs to. * @param type Table type. */ - public ExternalTable(long id, String name, ExternalCatalog catalog, String dbName, TableType type) { + public ExternalTable(long id, String name, String remoteName, ExternalCatalog catalog, ExternalDatabase db, + TableType type) { this.id = id; this.name = name; + this.remoteName = remoteName; this.catalog = catalog; - this.dbName = dbName; + this.db = db; + this.dbName = db.getFullName(); this.type = type; this.objectCreated = false; } @@ -116,6 +124,14 @@ public void setCatalog(ExternalCatalog catalog) { this.catalog = catalog; } + public void setDb(ExternalDatabase db) { + this.db = db; + } + + public void setRemoteName(String remoteName) { + this.remoteName = remoteName; + } + public boolean isView() { return false; } @@ -141,6 +157,10 @@ public String getName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public TableType getType() { return type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index dd30fbf43c95acc..9a1d4968d8fc213 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -62,6 +62,9 @@ public enum Type { @SerializedName(value = "createDbNames") private List createDbNames; + @SerializedName(value = "remoteDbNames") + private List remoteDbNames; + @SerializedName(value = "type") private Type type; @@ -75,6 +78,7 @@ public InitCatalogLog() { refreshDbIds = Lists.newArrayList(); createDbIds = Lists.newArrayList(); createDbNames = Lists.newArrayList(); + remoteDbNames = Lists.newArrayList(); type = Type.UNKNOWN; } @@ -83,10 +87,11 @@ public void addRefreshDb(long id) { refreshDbIds.add(id); } - public void addCreateDb(long id, String name) { + public void addCreateDb(long id, String name, String remoteName) { createCount += 1; createDbIds.add(id); createDbNames.add(name); + remoteDbNames.add(remoteName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 0731181291c8237..9dff8c820bb154f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -66,6 +66,9 @@ public enum Type { @SerializedName(value = "createTableNames") private List createTableNames; + @SerializedName(value = "remoteTableNames") + private List remoteTableNames; + @SerializedName(value = "type") private Type type; @@ -80,6 +83,7 @@ public InitDatabaseLog() { refreshTableIds = Lists.newArrayList(); createTableIds = Lists.newArrayList(); createTableNames = Lists.newArrayList(); + remoteTableNames = Lists.newArrayList(); type = Type.UNKNOWN; } @@ -88,10 +92,11 @@ public void addRefreshTable(long id) { refreshTableIds.add(id); } - public void addCreateTable(long id, String name) { + public void addCreateTable(long id, String name, String remoteName) { createCount += 1; createTableIds.add(id); createTableNames.add(name); + remoteTableNames.add(remoteName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java index 3c77b112d601602..39db1452833eac9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java @@ -32,14 +32,18 @@ public class EsExternalDatabase extends ExternalDatabase { * @param extCatalog External data source this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ES); + public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ES); } @Override - protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog); + public EsExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new EsExternalTable(tblId, localTableName, remoteTableName, (EsExternalCatalog) extCatalog, + (EsExternalDatabase) db); } public void addTableForTest(EsExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java index 4f05d6d29cd89e6..6e9e5731f415329 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java @@ -42,11 +42,12 @@ public class EsExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog EsExternalDataSource. + * @param db Database. */ - public EsExternalTable(long id, String name, String dbName, EsExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ES_EXTERNAL_TABLE); + public EsExternalTable(long id, String name, String remoteName, EsExternalCatalog catalog, EsExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ES_EXTERNAL_TABLE); } protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 85b999f11110478..7498e6edd93c76d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -263,7 +263,7 @@ public void registerDatabase(long dbId, String dbName) { LOG.debug("create database [{}]", dbName); } - ExternalDatabase db = buildDbForInit(dbName, dbId, logType, false); + ExternalDatabase db = buildDbForInit(dbName, null, dbId, logType, false); if (useMetaCache.get()) { if (isInitialized()) { metaCache.updateCache(dbName, db, Util.genIdByName(getQualifiedName(dbName))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java index 3ae9fbcd6e75a9f..86f99527fe421cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java @@ -32,14 +32,18 @@ public class HMSExternalDatabase extends ExternalDatabase { * @param extCatalog External catalog this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.HMS); + public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.HMS); } @Override - protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog); + public HMSExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new HMSExternalTable(tblId, localTableName, remoteTableName, (HMSExternalCatalog) extCatalog, + (HMSExternalDatabase) db); } public void addTableForTest(HMSExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index c6a571808eed89f..8f1ee9dabee8810 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -169,11 +169,13 @@ public enum DLAType { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalCatalog. + * @param remoteName Remote table name. + * @param catalog HMSExternalDataSource. + * @param db Database. */ - public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE); + public HMSExternalTable(long id, String name, String remoteName, HMSExternalCatalog catalog, + HMSExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.HMS_EXTERNAL_TABLE); } // Will throw NotSupportedException if not supported hms table. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java index f56183972e36d22..7a1a53825a15d33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java @@ -28,13 +28,16 @@ public class IcebergExternalDatabase extends ExternalDatabase { - public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ICEBERG); + public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ICEBERG); } @Override - protected IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new IcebergExternalTable(tblId, tableName, name, (IcebergExternalCatalog) extCatalog); + public IcebergExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new IcebergExternalTable(tblId, localTableName, remoteTableName, (IcebergExternalCatalog) extCatalog, + (IcebergExternalDatabase) db); } public String getLocation() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index feded88ea326f03..a357532f2bf1f81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -36,8 +36,9 @@ public class IcebergExternalTable extends ExternalTable { - public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE); + public IcebergExternalTable(long id, String name, String remoteName, IcebergExternalCatalog catalog, + IcebergExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ICEBERG_EXTERNAL_TABLE); } public String getIcebergCatalogType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java index 837f3691962e919..e8ab0690e17c7f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java @@ -36,7 +36,7 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,10 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalInfoSchemaTable(tblId, tableName, catalog); + public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalInfoSchemaTable(tblId, localTableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java index 9d1336396128fdf..b3e0ead9a118a6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java @@ -18,9 +18,9 @@ package org.apache.doris.datasource.infoschema; import org.apache.doris.analysis.SchemaTableType; -import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -31,8 +31,8 @@ public class ExternalInfoSchemaTable extends ExternalTable { - public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java index 5e0653f52781099..da40dc34c604ee9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java @@ -36,7 +36,7 @@ public class ExternalMysqlDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalMysqlDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, MysqlDb.DATABASE_NAME, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,10 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalMysqlTable(tblId, tableName, catalog); + public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalMysqlTable(tblId, localTableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java index 6f277a5690619bf..1077abf81d8614e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java @@ -19,8 +19,8 @@ import org.apache.doris.analysis.SchemaTableType; import org.apache.doris.catalog.MysqlDBTable; -import org.apache.doris.catalog.MysqlDb; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -30,8 +30,8 @@ import java.util.Optional; public class ExternalMysqlTable extends ExternalTable { - public ExternalMysqlTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, MysqlDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalMysqlTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 80cc0f554f637e9..ba004dfa893d403 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -26,11 +26,15 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.jdbc.client.JdbcClient; import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.datasource.jdbc.client.JdbcClientException; +import org.apache.doris.datasource.mapping.IdentifierMapping; +import org.apache.doris.datasource.mapping.JdbcIdentifierMapping; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest; import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult; @@ -52,6 +56,7 @@ import org.apache.thrift.TException; import org.apache.thrift.TSerializer; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -70,12 +75,17 @@ public class JdbcExternalCatalog extends ExternalCatalog { // Must add "transient" for Gson to ignore this field, // or Gson will throw exception with HikariCP private transient JdbcClient jdbcClient; + private IdentifierMapping identifierMapping; public JdbcExternalCatalog(long catalogId, String name, String resource, Map props, String comment) throws DdlException { super(catalogId, name, InitCatalogLog.Type.JDBC, comment); this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props)); + this.identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); } @Override @@ -118,12 +128,12 @@ public void onRefresh(boolean invalidCache) { super.onRefresh(invalidCache); if (jdbcClient != null) { jdbcClient.closeClient(); + jdbcClient = null; } - } - - @Override - public void onRefreshCache(boolean invalidCache) { - onRefresh(invalidCache); + this.identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); } @Override @@ -131,6 +141,7 @@ public void onClose() { super.onClose(); if (jdbcClient != null) { jdbcClient.closeClient(); + jdbcClient = null; } } @@ -181,16 +192,6 @@ public String getOnlySpecifiedDatabase() { JdbcResource.ONLY_SPECIFIED_DATABASE)); } - public String getLowerCaseMetaNames() { - return catalogProperty.getOrDefault(JdbcResource.LOWER_CASE_META_NAMES, JdbcResource.getDefaultPropertyValue( - JdbcResource.LOWER_CASE_META_NAMES)); - } - - public String getMetaNamesMapping() { - return catalogProperty.getOrDefault(JdbcResource.META_NAMES_MAPPING, JdbcResource.getDefaultPropertyValue( - JdbcResource.META_NAMES_MAPPING)); - } - public int getConnectionPoolMinSize() { return Integer.parseInt(catalogProperty.getOrDefault(JdbcResource.CONNECTION_POOL_MIN_SIZE, JdbcResource .getDefaultPropertyValue(JdbcResource.CONNECTION_POOL_MIN_SIZE))); @@ -231,8 +232,6 @@ protected void initLocalObjectsImpl() { .setDriverUrl(getDriverUrl()) .setDriverClass(getDriverClass()) .setOnlySpecifiedDatabase(getOnlySpecifiedDatabase()) - .setIsLowerCaseMetaNames(getLowerCaseMetaNames()) - .setMetaNamesMapping(getMetaNamesMapping()) .setIncludeDatabaseMap(getIncludeDatabaseMap()) .setExcludeDatabaseMap(getExcludeDatabaseMap()) .setConnectionPoolMinSize(getConnectionPoolMinSize()) @@ -244,20 +243,57 @@ protected void initLocalObjectsImpl() { jdbcClient = JdbcClient.createJdbcClient(jdbcClientConfig); } - protected List listDatabaseNames() { + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + if (this.identifierMapping == null) { + identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); + } + } + + @Override + public List listDatabaseNames() { return jdbcClient.getDatabaseNameList(); } + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + return identifierMapping.fromRemoteDatabaseName(remoteDatabaseName); + } + @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); return jdbcClient.getTablesNameList(dbName); } + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return identifierMapping.fromRemoteTableName(remoteDatabaseName, remoteTableName); + } + @Override public boolean tableExist(SessionContext ctx, String dbName, String tblName) { makeSureInitialized(); - return jdbcClient.isTableExist(dbName, tblName); + ExternalDatabase database = this.getDbNullable(dbName); + if (database == null) { + return false; + } + ExternalTable tbl = database.getTableNullable(tblName); + if (tbl == null) { + return false; + } + String remoteDbName = ((ExternalDatabase) tbl.getDatabase()).getRemoteName(); + String remoteTblName = tbl.getRemoteName(); + return jdbcClient.isTableExist(remoteDbName, remoteTblName); + } + + public List listColumns(String remoteDbName, String remoteTblName) { + makeSureInitialized(); + return jdbcClient.getColumnsFromJdbc(remoteDbName, remoteTblName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java index d078a3e238adbc4..1737ec614a64182 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java @@ -30,13 +30,16 @@ public class JdbcExternalDatabase extends ExternalDatabase { * @param id database id. * @param name database name. */ - public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.JDBC); + public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.JDBC); } @Override - protected JdbcExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new JdbcExternalTable(tblId, tableName, name, (JdbcExternalCatalog) extCatalog); + public JdbcExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new JdbcExternalTable(tblId, localTableName, remoteTableName, (JdbcExternalCatalog) extCatalog, + (JdbcExternalDatabase) db); } public void addTableForTest(JdbcExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java index 20520d7c542833c..4d8ac2f1dac7976 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.JdbcResource; import org.apache.doris.catalog.JdbcTable; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.qe.AutoCloseConnectContext; @@ -32,6 +33,8 @@ import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.thrift.TTableDescriptor; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * Jdbc external table. @@ -57,11 +61,13 @@ public class JdbcExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog JdbcExternalCatalog. + * @param db JdbcExternalDatabase. */ - public JdbcExternalTable(long id, String name, String dbName, JdbcExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.JDBC_EXTERNAL_TABLE); + public JdbcExternalTable(long id, String name, String remoteName, JdbcExternalCatalog catalog, + JdbcExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.JDBC_EXTERNAL_TABLE); } @Override @@ -86,21 +92,82 @@ public TTableDescriptor toThrift() { @Override public Optional initSchema() { - return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).getJdbcClient() - .getColumnsFromJdbc(dbName, name))); + String remoteDbName = ((ExternalDatabase) this.getDatabase()).getRemoteName(); + + // 1. Retrieve remote column information + List columns = ((JdbcExternalCatalog) catalog).listColumns(remoteDbName, remoteName); + if (columns == null || columns.isEmpty()) { + return Optional.empty(); + } + + // 2. Generate local column names from remote names + List remoteColumnNames = columns.stream() + .map(Column::getName) + .collect(Collectors.toList()); + List localColumnNames = Lists.newArrayListWithCapacity(remoteColumnNames.size()); + for (String remoteColName : remoteColumnNames) { + String localName = ((JdbcExternalCatalog) catalog).getIdentifierMapping() + .fromRemoteColumnName(remoteDbName, remoteName, remoteColName); + localColumnNames.add(localName); + } + + // 3. Collect potential conflicts in a case-insensitive scenario + Map> lowerCaseToLocalNames = Maps.newHashMap(); + for (String localColName : localColumnNames) { + String lowerName = localColName.toLowerCase(); + lowerCaseToLocalNames + .computeIfAbsent(lowerName, k -> Lists.newArrayList()) + .add(localColName); + } + + // 4. Check for conflicts + List conflicts = lowerCaseToLocalNames.values().stream() + .filter(names -> names.size() > 1) + .flatMap(List::stream) + .distinct() + .collect(Collectors.toList()); + + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + "Found conflicting column names under case-insensitive conditions. " + + "Conflicting column names: %s in remote table '%s.%s' under catalog '%s'. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), remoteDbName, remoteName, catalog.getName())); + } + + // 5. Update column objects with local names + for (int i = 0; i < columns.size(); i++) { + columns.get(i).setName(localColumnNames.get(i)); + } + + // 6. Build remote->local mapping + Map remoteColumnNamesMap = Maps.newHashMap(); + for (int i = 0; i < columns.size(); i++) { + remoteColumnNamesMap.put(localColumnNames.get(i), remoteColumnNames.get(i)); + } + + // 7. Return the SchemaCacheValue + return Optional.of(new JdbcSchemaCacheValue(columns, remoteColumnNamesMap)); } private JdbcTable toJdbcTable() { List schema = getFullSchema(); JdbcExternalCatalog jdbcCatalog = (JdbcExternalCatalog) catalog; - String fullDbName = this.dbName + "." + this.name; - JdbcTable jdbcTable = new JdbcTable(this.id, fullDbName, schema, TableType.JDBC_EXTERNAL_TABLE); - jdbcCatalog.configureJdbcTable(jdbcTable, fullDbName); + String fullTableName = this.dbName + "." + this.name; + JdbcTable jdbcTable = new JdbcTable(this.id, fullTableName, schema, TableType.JDBC_EXTERNAL_TABLE); + jdbcCatalog.configureJdbcTable(jdbcTable, fullTableName); // Set remote properties - jdbcTable.setRemoteDatabaseName(jdbcCatalog.getJdbcClient().getRemoteDatabaseName(this.dbName)); - jdbcTable.setRemoteTableName(jdbcCatalog.getJdbcClient().getRemoteTableName(this.dbName, this.name)); - jdbcTable.setRemoteColumnNames(jdbcCatalog.getJdbcClient().getRemoteColumnNames(this.dbName, this.name)); + jdbcTable.setRemoteDatabaseName(((ExternalDatabase) this.getDatabase()).getRemoteName()); + jdbcTable.setRemoteTableName(this.getRemoteName()); + Map remoteColumnNames = Maps.newHashMap(); + Optional schemaCacheValue = getSchemaCacheValue(); + for (Column column : schema) { + String remoteColumnName = schemaCacheValue.map(value -> ((JdbcSchemaCacheValue) value) + .getremoteColumnName(column.getName())).orElse(column.getName()); + remoteColumnNames.put(column.getName(), remoteColumnName); + } + jdbcTable.setRemoteColumnNames(remoteColumnNames); return jdbcTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java similarity index 51% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java index 20a74724b3e4965..4e21af4fabd99e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java @@ -17,29 +17,21 @@ package org.apache.doris.datasource.jdbc; -import org.apache.doris.datasource.jdbc.client.JdbcClient; -import org.apache.doris.datasource.mapping.IdentifierMapping; +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.SchemaCacheValue; -public class JdbcIdentifierMapping extends IdentifierMapping { - private final JdbcClient jdbcClient; +import java.util.List; +import java.util.Map; - public JdbcIdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping, JdbcClient jdbcClient) { - super(isLowerCaseMetaNames, metaNamesMapping); - this.jdbcClient = jdbcClient; - } - - @Override - protected void loadDatabaseNames() { - jdbcClient.getDatabaseNameList(); - } +public class JdbcSchemaCacheValue extends SchemaCacheValue { + private Map remoteColumnNamesMap; - @Override - protected void loadTableNames(String localDbName) { - jdbcClient.getTablesNameList(localDbName); + public JdbcSchemaCacheValue(List schema, Map remoteColumnNamesMap) { + super(schema); + this.remoteColumnNamesMap = remoteColumnNamesMap; } - @Override - protected void loadColumnNames(String localDbName, String localTableName) { - jdbcClient.getColumnsFromJdbc(localDbName, localTableName); + public String getremoteColumnName(String columnName) { + return remoteColumnNamesMap.get(columnName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index c8f45e4fc23b7fe..08befe4561e8acf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping; import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.ImmutableSet; @@ -62,11 +61,8 @@ public abstract class JdbcClient { protected ClassLoader classLoader = null; protected HikariDataSource dataSource = null; protected boolean isOnlySpecifiedDatabase; - protected boolean isLowerCaseMetaNames; - protected String metaNamesMapping; protected Map includeDatabaseMap; protected Map excludeDatabaseMap; - protected JdbcIdentifierMapping jdbcLowerCaseMetaMatching; public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) { String dbType = parseDbType(jdbcClientConfig.getJdbcUrl()); @@ -103,8 +99,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.catalogName = jdbcClientConfig.getCatalog(); this.jdbcUser = jdbcClientConfig.getUser(); this.isOnlySpecifiedDatabase = Boolean.parseBoolean(jdbcClientConfig.getOnlySpecifiedDatabase()); - this.isLowerCaseMetaNames = Boolean.parseBoolean(jdbcClientConfig.getIsLowerCaseMetaNames()); - this.metaNamesMapping = jdbcClientConfig.getMetaNamesMapping(); this.includeDatabaseMap = Optional.ofNullable(jdbcClientConfig.getIncludeDatabaseMap()).orElse(Collections.emptyMap()); this.excludeDatabaseMap = @@ -113,7 +107,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.dbType = parseDbType(jdbcUrl); initializeClassLoader(jdbcClientConfig); initializeDataSource(jdbcClientConfig); - this.jdbcLowerCaseMetaMatching = new JdbcIdentifierMapping(isLowerCaseMetaNames, metaNamesMapping, this); } protected void setJdbcDriverSystemProperties() { @@ -173,6 +166,7 @@ public static String parseDbType(String jdbcUrl) { public void closeClient() { dataSource.close(); + dataSource = null; } public Connection getConnection() throws JdbcClientException { @@ -311,10 +305,9 @@ public List getDatabaseNameList() { /** * get all tables of one database */ - public List getTablesNameList(String localDbName) { + public List getTablesNameList(String remoteDbName) { List remoteTablesNames = Lists.newArrayList(); String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); processTable(remoteDbName, null, tableTypes, (rs) -> { try { while (rs.next()) { @@ -324,14 +317,12 @@ public List getTablesNameList(String localDbName) { throw new JdbcClientException("failed to get all tables for remote database: `%s`", e, remoteDbName); } }); - return filterTableNames(remoteDbName, remoteTablesNames); + return remoteTablesNames; } - public boolean isTableExist(String localDbName, String localTableName) { + public boolean isTableExist(String remoteDbName, String remoteTableName) { final boolean[] isExist = {false}; String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); processTable(remoteDbName, remoteTableName, tableTypes, (rs) -> { try { if (rs.next()) { @@ -348,12 +339,10 @@ public boolean isTableExist(String localDbName, String localTableName) { /** * get all columns of one table */ - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); @@ -380,21 +369,7 @@ public List getColumnsFromJdbc(String localDbName, String localTableName field.isAllowNull(), field.getRemarks(), true, -1)); } - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - return filterColumnName(remoteDbName, remoteTableName, dorisTableSchema); - } - - public String getRemoteDatabaseName(String localDbname) { - return jdbcLowerCaseMetaMatching.getRemoteDatabaseName(localDbname); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteTableName(localDbName, localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteColumnNames(localDbName, localTableName); + return dorisTableSchema; } // protected methods, for subclass to override @@ -453,7 +428,7 @@ protected List filterDatabaseNames(List remoteDbNames) { } filteredDatabaseNames.add(databaseName); } - return jdbcLowerCaseMetaMatching.setDatabaseNameMapping(filteredDatabaseNames); + return filteredDatabaseNames; } protected Set getFilterInternalDatabases() { @@ -464,14 +439,6 @@ protected Set getFilterInternalDatabases() { .build(); } - protected List filterTableNames(String remoteDbName, List remoteTableNames) { - return jdbcLowerCaseMetaMatching.setTableNameMapping(remoteDbName, remoteTableNames); - } - - protected List filterColumnName(String remoteDbName, String remoteTableName, List remoteColumns) { - return jdbcLowerCaseMetaMatching.setColumnNameMapping(remoteDbName, remoteTableName, remoteColumns); - } - protected abstract Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema); protected Type createDecimalOrStringType(int precision, int scale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java index 7ba393e0d0aae63..086a8a5f393614b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java @@ -87,12 +87,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index b78589faa77380f..f4c331cf35779c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -134,12 +134,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c * get all columns of one table */ @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 9968de79ab3a7de..adffd06c244e54f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -49,12 +49,10 @@ public String getTestQuery() { } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java index 363ef351152a39d..9199ff985116c06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java @@ -17,314 +17,11 @@ package org.apache.doris.datasource.mapping; -import org.apache.doris.catalog.Column; -import org.apache.doris.qe.GlobalVariable; +public interface IdentifierMapping { -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; + String fromRemoteDatabaseName(String remoteDatabaseName); -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; + String fromRemoteTableName(String remoteDatabaseName, String remoteTableName); -public abstract class IdentifierMapping { - private static final Logger LOG = LogManager.getLogger(IdentifierMapping.class); - - private final ObjectMapper mapper = new ObjectMapper(); - private final ConcurrentHashMap localDBToRemoteDB = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> localTableToRemoteTable - = new ConcurrentHashMap<>(); - private final ConcurrentHashMap>> - localColumnToRemoteColumn = new ConcurrentHashMap<>(); - - private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false); - private final ConcurrentHashMap tableNamesLoadedMap = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> columnNamesLoadedMap - = new ConcurrentHashMap<>(); - - private final boolean isLowerCaseMetaNames; - private final String metaNamesMapping; - - public IdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping) { - this.isLowerCaseMetaNames = isLowerCaseMetaNames; - this.metaNamesMapping = metaNamesMapping; - } - - public List setDatabaseNameMapping(List remoteDatabaseNames) { - JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); - - Map databaseNameMapping = Maps.newTreeMap(); - if (databasesNode.isArray()) { - for (JsonNode node : databasesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String mapping = node.path("mapping").asText(); - databaseNameMapping.put(remoteDatabase, mapping); - } - } - - Map> result = nameListToMapping(remoteDatabaseNames, localDBToRemoteDB, - databaseNameMapping, isLowerCaseMetaNames); - List localDatabaseNames = result.get("localNames"); - List conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict database/schema names found when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the names."); - } - return localDatabaseNames; - } - - public List setTableNameMapping(String remoteDbName, List remoteTableNames) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); - - Map tableNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - if (remoteDbName.equals(remoteDatabase)) { - String remoteTable = node.path("remoteTable").asText(); - String mapping = node.path("mapping").asText(); - tableNameMapping.put(remoteTable, mapping); - } - } - } - - localTableToRemoteTable.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - - List localTableNames; - List conflictNames; - - if (GlobalVariable.lowerCaseTableNames == 1) { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, true); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + " when lower_case_table_names is 1: " + conflictNames - + ". Please use meta_name_mapping to specify the names."); - } - } else { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, isLowerCaseMetaNames); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + "when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the table names."); - } - } - return localTableNames; - } - - public List setColumnNameMapping(String remoteDbName, String remoteTableName, List remoteColumns) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "columns"); - - Map columnNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String remoteTable = node.path("remoteTable").asText(); - if (remoteDbName.equals(remoteDatabase) && remoteTable.equals(remoteTableName)) { - String remoteColumn = node.path("remoteColumn").asText(); - String mapping = node.path("mapping").asText(); - columnNameMapping.put(remoteColumn, mapping); - } - } - } - localColumnToRemoteColumn.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - localColumnToRemoteColumn.get(remoteDbName).putIfAbsent(remoteTableName, new ConcurrentHashMap<>()); - - List localColumnNames; - List conflictNames; - - // Get the name from localColumns and save it to List - List remoteColumnNames = Lists.newArrayList(); - for (Column remoteColumn : remoteColumns) { - remoteColumnNames.add(remoteColumn.getName()); - } - - Map> result = nameListToMapping(remoteColumnNames, - localColumnToRemoteColumn.get(remoteDbName).get(remoteTableName), - columnNameMapping, isLowerCaseMetaNames); - localColumnNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict column names found in remote database/schema: " + remoteDbName - + " in remote table: " + remoteTableName - + " when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the column names."); - } - // Replace the name in remoteColumns with localColumnNames - for (int i = 0; i < remoteColumns.size(); i++) { - remoteColumns.get(i).setName(localColumnNames.get(i)); - } - return remoteColumns; - } - - public String getRemoteDatabaseName(String localDbName) { - return getRequiredMapping(localDBToRemoteDB, localDbName, "database", this::loadDatabaseNamesIfNeeded, - localDbName); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - Map tableMap = localTableToRemoteTable.computeIfAbsent(remoteDbName, - k -> new ConcurrentHashMap<>()); - return getRequiredMapping(tableMap, localTableName, "table", () -> loadTableNamesIfNeeded(localDbName), - localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - ConcurrentHashMap> tableColumnMap - = localColumnToRemoteColumn.computeIfAbsent(remoteDbName, k -> new ConcurrentHashMap<>()); - Map columnMap = tableColumnMap.computeIfAbsent(remoteTableName, k -> new ConcurrentHashMap<>()); - if (columnMap.isEmpty()) { - LOG.info("Column name mapping missing, loading column names for localDbName: {}, localTableName: {}", - localDbName, localTableName); - loadColumnNamesIfNeeded(localDbName, localTableName); - columnMap = tableColumnMap.get(remoteTableName); - } - if (columnMap.isEmpty()) { - LOG.warn("No remote column found for localTableName: {}. Please refresh this catalog.", localTableName); - throw new RuntimeException( - "No remote column found for localTableName: " + localTableName + ". Please refresh this catalog."); - } - return columnMap; - } - - - private void loadDatabaseNamesIfNeeded() { - if (dbNamesLoaded.compareAndSet(false, true)) { - try { - loadDatabaseNames(); - } catch (Exception e) { - dbNamesLoaded.set(false); // Reset on failure - LOG.warn("Error loading database names", e); - } - } - } - - private void loadTableNamesIfNeeded(String localDbName) { - AtomicBoolean isLoaded = tableNamesLoadedMap.computeIfAbsent(localDbName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadTableNames(localDbName); - } catch (Exception e) { - tableNamesLoadedMap.get(localDbName).set(false); // Reset on failure - LOG.warn("Error loading table names for localDbName: {}", localDbName, e); - } - } - } - - private void loadColumnNamesIfNeeded(String localDbName, String localTableName) { - columnNamesLoadedMap.putIfAbsent(localDbName, new ConcurrentHashMap<>()); - AtomicBoolean isLoaded = columnNamesLoadedMap.get(localDbName) - .computeIfAbsent(localTableName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadColumnNames(localDbName, localTableName); - } catch (Exception e) { - columnNamesLoadedMap.get(localDbName).get(localTableName).set(false); // Reset on failure - LOG.warn("Error loading column names for localDbName: {}, localTableName: {}", localDbName, - localTableName, e); - } - } - } - - private V getRequiredMapping(Map map, K key, String typeName, Runnable loadIfNeeded, - String entityName) { - if (map.isEmpty() || !map.containsKey(key) || map.get(key) == null) { - LOG.info("{} mapping missing, loading for {}: {}", typeName, typeName, entityName); - loadIfNeeded.run(); - } - V value = map.get(key); - if (value == null) { - LOG.warn("No remote {} found for {}: {}. Please refresh this catalog.", typeName, typeName, entityName); - throw new RuntimeException("No remote " + typeName + " found for " + typeName + ": " + entityName - + ". Please refresh this catalog."); - } - return value; - } - - // Load the database name from the data source. - // In the corresponding getDatabaseNameList(), setDatabaseNameMapping() must be used to update the mapping. - protected abstract void loadDatabaseNames(); - - // Load the table names for the specified database from the data source. - // In the corresponding getTableNameList(), setTableNameMapping() must be used to update the mapping. - protected abstract void loadTableNames(String localDbName); - - // Load the column names for a specified table in a database from the data source. - // In the corresponding getColumnNameList(), setColumnNameMapping() must be used to update the mapping. - protected abstract void loadColumnNames(String localDbName, String localTableName); - - private JsonNode readAndParseJson(String jsonPath, String nodeName) { - JsonNode rootNode; - try { - rootNode = mapper.readTree(jsonPath); - return rootNode.path(nodeName); - } catch (JsonProcessingException e) { - throw new RuntimeException("parse meta_names_mapping property error", e); - } - } - - private Map> nameListToMapping(List remoteNames, - ConcurrentHashMap localNameToRemoteName, - Map nameMapping, boolean isLowerCaseMetaNames) { - List filteredDatabaseNames = Lists.newArrayList(); - Set lowerCaseNames = Sets.newHashSet(); - Map> nameMap = Maps.newHashMap(); - List conflictNames = Lists.newArrayList(); - - for (String name : remoteNames) { - String mappedName = nameMapping.getOrDefault(name, name); - String localName = isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; - - // Use computeIfAbsent to ensure atomicity - localNameToRemoteName.computeIfAbsent(localName, k -> name); - - if (isLowerCaseMetaNames && !lowerCaseNames.add(localName)) { - if (nameMap.containsKey(localName)) { - nameMap.get(localName).add(mappedName); - } - } else { - nameMap.putIfAbsent(localName, Lists.newArrayList(Collections.singletonList(mappedName))); - } - - filteredDatabaseNames.add(localName); - } - - for (List conflictNameList : nameMap.values()) { - if (conflictNameList.size() > 1) { - conflictNames.addAll(conflictNameList); - } - } - - Map> result = Maps.newConcurrentMap(); - result.put("localNames", filteredDatabaseNames); - result.put("conflictNames", conflictNames); - return result; - } + String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnNames); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java new file mode 100644 index 000000000000000..f584c9acee34e49 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mapping; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Set; + +public class JdbcIdentifierMapping implements IdentifierMapping { + private static final Logger LOG = LogManager.getLogger(JdbcIdentifierMapping.class); + + private final ObjectMapper mapper = new ObjectMapper(); + private final boolean isLowerCaseTableNames; + private final boolean isLowerCaseMetaNames; + private final String metaNamesMapping; + + public JdbcIdentifierMapping(boolean isLowerCaseTableNames, boolean isLowerCaseMetaNames, String metaNamesMapping) { + this.isLowerCaseTableNames = isLowerCaseTableNames; + this.isLowerCaseMetaNames = isLowerCaseMetaNames; + this.metaNamesMapping = metaNamesMapping; + validateMappings(); + } + + private boolean isMappingInvalid() { + return metaNamesMapping == null || metaNamesMapping.isEmpty(); + } + + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteDatabaseName; + } + JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); + + Map databaseNameMapping = Maps.newHashMap(); + if (databasesNode.isArray()) { + for (JsonNode node : databasesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + databaseNameMapping.put(remoteDatabase, mapping); + } + } + return getMappedName(remoteDatabaseName, databaseNameMapping); + } + + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteTableName; + } + JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); + + Map tableNameMapping = Maps.newHashMap(); + if (tablesNode.isArray()) { + for (JsonNode node : tablesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + if (remoteDatabaseName.equals(remoteDatabase)) { + String remoteTable = node.path("remoteTable").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + tableNameMapping.put(remoteTable, mapping); + } + } + } + return getMappedName(remoteTableName, tableNameMapping); + } + + @Override + public String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteColumnName; + } + JsonNode columnsNode = readAndParseJson(metaNamesMapping, "columns"); + + Map columnNameMapping = Maps.newHashMap(); + if (columnsNode.isArray()) { + for (JsonNode node : columnsNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String remoteTable = node.path("remoteTable").asText(); + if (remoteDatabaseName.equals(remoteDatabase) && remoteTableName.equals(remoteTable)) { + String remoteColumn = node.path("remoteColumn").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + columnNameMapping.put(remoteColumn, mapping); + } + } + } + return getMappedName(remoteColumnName, columnNameMapping); + } + + private String getMappedName(String name, Map nameMapping) { + String mappedName = nameMapping.getOrDefault(name, name); + return isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; + } + + private JsonNode readAndParseJson(String jsonPath, String nodeName) { + try { + JsonNode rootNode = mapper.readTree(jsonPath); + return rootNode.path(nodeName); + } catch (JsonProcessingException e) { + throw new RuntimeException("JSON format is incorrect, please check the metaNamesMapping property", e); + } + } + + private void validateMappings() { + Map> duplicateErrors = Maps.newLinkedHashMap(); + try { + JsonNode rootNode = mapper.readTree(metaNamesMapping); + + Map> dbMappingCheck = Maps.newHashMap(); + Map>> tableMappingCheck = Maps.newHashMap(); + Map>>> columnMappingCheck = Maps.newHashMap(); + + Set dbKeySet = Sets.newHashSet(); + Map> tableKeySet = Maps.newHashMap(); + Map>> columnKeySet = Maps.newHashMap(); + + validateNode(rootNode.path("databases"), "databases", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + validateNode(rootNode.path("tables"), "tables", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + validateNode(rootNode.path("columns"), "columns", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + + if (!duplicateErrors.isEmpty()) { + StringBuilder errorBuilder = new StringBuilder("Duplicate mapping found:\n"); + duplicateErrors.forEach((key, value) -> { + errorBuilder.append(key).append(":\n"); + value.forEach(error -> errorBuilder.append(" - ").append(error).append("\n")); + }); + throw new RuntimeException(errorBuilder.toString()); + } + } catch (JsonProcessingException e) { + throw new RuntimeException("The JSON format is incorrect, please check the metaNamesMapping property", e); + } + } + + private void validateNode(JsonNode nodes, + String nodeType, + Map> duplicateErrors, + Map> dbMappingCheck, + Map>> tableMappingCheck, + Map>>> columnMappingCheck, + Set dbKeySet, + Map> tableKeySet, + Map>> columnKeySet) { + Map mappingSet = Maps.newHashMap(); + if (nodes.isArray()) { + for (JsonNode node : nodes) { + String remoteKey; + String remoteDb = null; + String remoteTbl = null; + switch (nodeType) { + case "databases": + remoteKey = node.path("remoteDatabase").asText(); + checkDuplicateRemoteDatabaseKey(remoteKey, dbKeySet, duplicateErrors); + break; + case "tables": + remoteDb = node.path("remoteDatabase").asText(); + remoteKey = node.path("remoteTable").asText(); + checkDuplicateRemoteTableKey(remoteDb, remoteKey, tableKeySet, duplicateErrors); + break; + case "columns": + remoteDb = node.path("remoteDatabase").asText(); + remoteTbl = node.path("remoteTable").asText(); + remoteKey = node.path("remoteColumn").asText(); + checkDuplicateRemoteColumnKey(remoteDb, remoteTbl, remoteKey, columnKeySet, duplicateErrors); + break; + default: + throw new IllegalArgumentException("Unknown type: " + nodeType); + } + + String mapping = node.path("mapping").asText(); + + String existed = mappingSet.get(mapping); + if (existed != null) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s, duplicate mapping: %s (original: %s)", + remoteKey, mapping, existed)); + } else { + mappingSet.put(mapping, remoteKey); + } + + switch (nodeType) { + case "databases": + if (isLowerCaseMetaNames) { + checkCaseConflictForDatabase(mapping, dbMappingCheck, duplicateErrors, nodeType, remoteKey); + } + break; + case "tables": + if (isLowerCaseMetaNames || isLowerCaseTableNames) { + checkCaseConflictForTable(remoteDb, mapping, tableMappingCheck, duplicateErrors, + nodeType, remoteKey); + } + break; + case "columns": + checkCaseConflictForColumn(remoteDb, remoteTbl, mapping, columnMappingCheck, duplicateErrors, + nodeType, remoteKey); + break; + default: + break; + } + } + } + } + + private void checkDuplicateRemoteDatabaseKey(String remoteDatabase, + Set dbKeySet, + Map> duplicateErrors) { + if (dbKeySet == null) { + return; + } + if (!dbKeySet.add(remoteDatabase)) { + duplicateErrors + .computeIfAbsent("databases", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteDatabase found: %s", remoteDatabase)); + } + } + + private void checkDuplicateRemoteTableKey(String remoteDb, + String remoteTable, + Map> tableKeySet, + Map> duplicateErrors) { + if (tableKeySet == null) { + return; + } + Set tables = tableKeySet.computeIfAbsent(remoteDb, k -> Sets.newHashSet()); + if (!tables.add(remoteTable)) { + duplicateErrors + .computeIfAbsent("tables", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteTable found in database %s: %s", remoteDb, remoteTable)); + } + } + + private void checkDuplicateRemoteColumnKey(String remoteDb, + String remoteTbl, + String remoteColumn, + Map>> columnKeySet, + Map> duplicateErrors) { + if (columnKeySet == null) { + return; + } + Map> tblMap = columnKeySet.computeIfAbsent(remoteDb, k -> Maps.newHashMap()); + Set columns = tblMap.computeIfAbsent(remoteTbl, k -> Sets.newHashSet()); + if (!columns.add(remoteColumn)) { + duplicateErrors + .computeIfAbsent("columns", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteColumn found in database %s, table %s: %s", + remoteDb, remoteTbl, remoteColumn)); + } + } + + private void checkCaseConflictForDatabase(String mapping, + Map> dbMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (dbMappingCheck == null) { + return; + } + String lower = mapping.toLowerCase(); + Set variants = dbMappingCheck.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s, case-only different mapping found: %s (existing variants: %s)", + remoteKey, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForTable(String remoteDb, + String mapping, + Map>> tableMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (tableMappingCheck == null || remoteDb == null) { + return; + } + Map> dbMap = tableMappingCheck.computeIfAbsent(remoteDb, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = dbMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s (database: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForColumn(String remoteDb, + String remoteTbl, + String mapping, + Map>>> columnMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (columnMappingCheck == null || remoteDb == null || remoteTbl == null) { + return; + } + Map>> dbMap = columnMappingCheck.computeIfAbsent(remoteDb, + k -> Maps.newHashMap()); + Map> tblMap = dbMap.computeIfAbsent(remoteTbl, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = tblMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format( + "Remote name: %s (database: %s, table: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, remoteTbl, mapping, variants)); + } + variants.add(mapping); + } + + private String applyLowerCaseIfNeeded(String value) { + return isLowerCaseMetaNames ? value.toLowerCase() : value; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java index 0a100e3495ff3d9..7cd38b9d13a0072 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java @@ -32,12 +32,16 @@ public class MaxComputeExternalDatabase extends ExternalDatabase columnNameToOdpsColumn = new HashMap(); + private Map columnNameToOdpsColumn = new HashMap(); @Override protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index 6e4198186e82e3d..fffa0a04e428506 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.common.CacheFactory; +import org.apache.doris.common.Pair; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -27,12 +28,14 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; public class MetaCache { - private LoadingCache> namesCache; + private LoadingCache>> namesCache; private Map idToName = Maps.newConcurrentMap(); private LoadingCache> metaObjCache; @@ -43,7 +46,7 @@ public MetaCache(String name, OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize, - CacheLoader> namesCacheLoader, + CacheLoader>> namesCacheLoader, CacheLoader> metaObjCacheLoader, RemovalListener> removalListener) { this.name = name; @@ -71,7 +74,15 @@ public MetaCache(String name, } public List listNames() { - return namesCache.get(""); + return Objects.requireNonNull(namesCache.get("")).stream().map(Pair::value).collect(Collectors.toList()); + } + + public String getRemoteName(String localName) { + return Objects.requireNonNull(namesCache.getIfPresent("")).stream() + .filter(pair -> pair.value().equals(localName)) + .map(Pair::key) + .findFirst() + .orElse(null); } public Optional getMetaObj(String name, long id) { @@ -94,9 +105,9 @@ public void updateCache(String objName, T obj, long id) { metaObjCache.put(objName, Optional.of(obj)); namesCache.asMap().compute("", (k, v) -> { if (v == null) { - return Lists.newArrayList(objName); + return Lists.newArrayList(Pair.of(objName, objName)); } else { - v.add(objName); + v.add(Pair.of(objName, objName)); return v; } }); @@ -121,5 +132,4 @@ public void invalidateAll() { metaObjCache.invalidateAll(); idToName.clear(); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java index 50265f77463428f..fdbad45c5d0af9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java @@ -23,12 +23,15 @@ public class PaimonExternalDatabase extends ExternalDatabase { - public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.PAIMON); + public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.PAIMON); } @Override - protected PaimonExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new PaimonExternalTable(tblId, tableName, name, (PaimonExternalCatalog) extCatalog); + public PaimonExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new PaimonExternalTable(tblId, localTableName, remoteTableName, (PaimonExternalCatalog) extCatalog, + (PaimonExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index e7d1554d9a7f927..3f22ce4c46b32f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -63,8 +63,9 @@ public class PaimonExternalTable extends ExternalTable implements MvccTable { private final Table paimonTable; - public PaimonExternalTable(long id, String name, String dbName, PaimonExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.PAIMON_EXTERNAL_TABLE); + public PaimonExternalTable(long id, String name, String remoteName, PaimonExternalCatalog catalog, + PaimonExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.PAIMON_EXTERNAL_TABLE); this.paimonTable = catalog.getPaimonTable(dbName, name); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java index 2cf1f57d0e4672c..0c11dc8f9400a3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java @@ -23,12 +23,15 @@ public class TestExternalDatabase extends ExternalDatabase { - public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.TEST); + public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.TEST); } @Override - protected TestExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new TestExternalTable(tblId, tableName, name, (TestExternalCatalog) extCatalog); + public TestExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new TestExternalTable(tblId, localTableName, remoteTableName, (TestExternalCatalog) extCatalog, + (TestExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java index 6da0981b97ef548..6d08b10403bb76f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java @@ -33,8 +33,8 @@ public class TestExternalTable extends ExternalTable { private static final Logger LOG = LogManager.getLogger(TestExternalTable.class); - public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE); + public TestExternalTable(long id, String name, String remoteName, TestExternalCatalog catalog, TestExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TEST_EXTERNAL_TABLE); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java new file mode 100644 index 000000000000000..a664d3889241a31 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.trinoconnector; + +import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.MapType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; +import org.apache.doris.thrift.TTrinoConnectorTable; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.trino.Session; +import io.trino.metadata.QualifiedObjectName; +import io.trino.spi.connector.CatalogHandle; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.transaction.IsolationLevel; +import io.trino.spi.type.BigintType; +import io.trino.spi.type.BooleanType; +import io.trino.spi.type.CharType; +import io.trino.spi.type.DateType; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.DoubleType; +import io.trino.spi.type.IntegerType; +import io.trino.spi.type.RealType; +import io.trino.spi.type.RowType; +import io.trino.spi.type.RowType.Field; +import io.trino.spi.type.SmallintType; +import io.trino.spi.type.TimeType; +import io.trino.spi.type.TimestampType; +import io.trino.spi.type.TimestampWithTimeZoneType; +import io.trino.spi.type.TinyintType; +import io.trino.spi.type.VarbinaryType; +import io.trino.spi.type.VarcharType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; + +public class TrinoConnectorExternalTable extends ExternalTable { + + public TrinoConnectorExternalTable(long id, String name, String remoteName, TrinoConnectorExternalCatalog catalog, + TrinoConnectorExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TRINO_CONNECTOR_EXTERNAL_TABLE); + } + + @Override + protected synchronized void makeSureInitialized() { + super.makeSureInitialized(); + if (!objectCreated) { + objectCreated = true; + } + } + + @Override + public Optional initSchema() { + // 1. Get necessary objects + TrinoConnectorExternalCatalog trinoConnectorCatalog = (TrinoConnectorExternalCatalog) catalog; + CatalogHandle catalogHandle = trinoConnectorCatalog.getTrinoCatalogHandle(); + Connector connector = trinoConnectorCatalog.getConnector(); + Session trinoSession = trinoConnectorCatalog.getTrinoSession(); + ConnectorSession connectorSession = trinoSession.toConnectorSession(catalogHandle); + + // 2. Begin transaction and get ConnectorMetadata + ConnectorTransactionHandle connectorTransactionHandle = connector.beginTransaction( + IsolationLevel.READ_UNCOMMITTED, true, true); + ConnectorMetadata connectorMetadata = connector.getMetadata(connectorSession, connectorTransactionHandle); + + // 3. Get ConnectorTableHandle + Optional connectorTableHandle = Optional.empty(); + QualifiedObjectName qualifiedTable = new QualifiedObjectName(trinoConnectorCatalog.getName(), dbName, + name); + if (!qualifiedTable.getCatalogName().isEmpty() + && !qualifiedTable.getSchemaName().isEmpty() + && !qualifiedTable.getObjectName().isEmpty()) { + connectorTableHandle = Optional.ofNullable(connectorMetadata.getTableHandle(connectorSession, + qualifiedTable.asSchemaTableName(), Optional.empty(), Optional.empty())); + } + if (!connectorTableHandle.isPresent()) { + throw new RuntimeException(String.format("Table does not exist: %s.%s.%s", trinoConnectorCatalog.getName(), + dbName, name)); + } + + // 4. Get ColumnHandle + Map handles = connectorMetadata.getColumnHandles(connectorSession, + connectorTableHandle.get()); + ImmutableMap.Builder columnHandleMapBuilder = ImmutableMap.builder(); + for (Entry mapEntry : handles.entrySet()) { + columnHandleMapBuilder.put(mapEntry.getKey().toLowerCase(Locale.ENGLISH), mapEntry.getValue()); + } + Map columnHandleMap = columnHandleMapBuilder.buildOrThrow(); + + // 5. Get ColumnMetadata + ImmutableMap.Builder columnMetadataMapBuilder = ImmutableMap.builder(); + List columns = Lists.newArrayListWithCapacity(columnHandleMap.size()); + for (ColumnHandle columnHandle : columnHandleMap.values()) { + ColumnMetadata columnMetadata = connectorMetadata.getColumnMetadata(connectorSession, + connectorTableHandle.get(), columnHandle); + if (columnMetadata.isHidden()) { + continue; + } + columnMetadataMapBuilder.put(columnMetadata.getName(), columnMetadata); + + Column column = new Column(columnMetadata.getName(), + trinoConnectorTypeToDorisType(columnMetadata.getType()), + true, + null, + true, + columnMetadata.getComment(), + !columnMetadata.isHidden(), + Column.COLUMN_UNIQUE_ID_INIT_VALUE); + columns.add(column); + } + Map columnMetadataMap = columnMetadataMapBuilder.buildOrThrow(); + return Optional.of( + new TrinoSchemaCacheValue(columns, connectorMetadata, connectorTableHandle, connectorTransactionHandle, + columnHandleMap, columnMetadataMap)); + } + + @Override + public TTableDescriptor toThrift() { + List schema = getFullSchema(); + TTrinoConnectorTable tTrinoConnectorTable = new TTrinoConnectorTable(); + tTrinoConnectorTable.setDbName(dbName); + tTrinoConnectorTable.setTableName(name); + tTrinoConnectorTable.setProperties(new HashMap<>()); + + TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), + TTableType.TRINO_CONNECTOR_TABLE, schema.size(), 0, getName(), dbName); + tTableDescriptor.setTrinoConnectorTable(tTrinoConnectorTable); + return tTableDescriptor; + } + + private Type trinoConnectorTypeToDorisType(io.trino.spi.type.Type type) { + if (type instanceof BooleanType) { + return Type.BOOLEAN; + } else if (type instanceof TinyintType) { + return Type.TINYINT; + } else if (type instanceof SmallintType) { + return Type.SMALLINT; + } else if (type instanceof IntegerType) { + return Type.INT; + } else if (type instanceof BigintType) { + return Type.BIGINT; + } else if (type instanceof RealType) { + return Type.FLOAT; + } else if (type instanceof DoubleType) { + return Type.DOUBLE; + } else if (type instanceof CharType) { + return Type.CHAR; + } else if (type instanceof VarcharType) { + return Type.STRING; + // } else if (type instanceof BinaryType) { + // return Type.STRING; + } else if (type instanceof VarbinaryType) { + return Type.STRING; + } else if (type instanceof DecimalType) { + DecimalType decimal = (DecimalType) type; + return ScalarType.createDecimalV3Type(decimal.getPrecision(), decimal.getScale()); + } else if (type instanceof TimeType) { + return Type.STRING; + } else if (type instanceof DateType) { + return ScalarType.createDateV2Type(); + } else if (type instanceof TimestampType) { + TimestampType timestampType = (TimestampType) type; + return ScalarType.createDatetimeV2Type(timestampType.getPrecision()); + } else if (type instanceof TimestampWithTimeZoneType) { + TimestampWithTimeZoneType timestampWithTimeZoneType = (TimestampWithTimeZoneType) type; + return ScalarType.createDatetimeV2Type(timestampWithTimeZoneType.getPrecision()); + } else if (type instanceof io.trino.spi.type.ArrayType) { + Type elementType = trinoConnectorTypeToDorisType( + ((io.trino.spi.type.ArrayType) type).getElementType()); + return ArrayType.create(elementType, true); + } else if (type instanceof io.trino.spi.type.MapType) { + Type keyType = trinoConnectorTypeToDorisType( + ((io.trino.spi.type.MapType) type).getKeyType()); + Type valueType = trinoConnectorTypeToDorisType( + ((io.trino.spi.type.MapType) type).getValueType()); + return new MapType(keyType, valueType, true, true); + } else if (type instanceof RowType) { + ArrayList dorisFields = Lists.newArrayList(); + for (Field field : ((RowType) type).getFields()) { + Type childType = trinoConnectorTypeToDorisType(field.getType()); + if (field.getName().isPresent()) { + dorisFields.add(new StructField(field.getName().get(), childType)); + } else { + dorisFields.add(new StructField(childType)); + } + } + return new StructType(dorisFields); + } else { + throw new IllegalArgumentException("Cannot transform unknown type: " + type); + } + } + + public ConnectorTableHandle getConnectorTableHandle() { + makeSureInitialized(); + Optional schemaCacheValue = getSchemaCacheValue(); + return schemaCacheValue.map(value -> ((TrinoSchemaCacheValue) value).getConnectorTableHandle().get()) + .orElse(null); + } + + public ConnectorMetadata getConnectorMetadata() { + makeSureInitialized(); + Optional schemaCacheValue = getSchemaCacheValue(); + return schemaCacheValue.map(value -> ((TrinoSchemaCacheValue) value).getConnectorMetadata()).orElse(null); + } + + public ConnectorTransactionHandle getConnectorTransactionHandle() { + makeSureInitialized(); + Optional schemaCacheValue = getSchemaCacheValue(); + return schemaCacheValue.map(value -> ((TrinoSchemaCacheValue) value).getConnectorTransactionHandle()) + .orElse(null); + } + + public Map getColumnHandleMap() { + makeSureInitialized(); + Optional schemaCacheValue = getSchemaCacheValue(); + return schemaCacheValue.map(value -> ((TrinoSchemaCacheValue) value).getColumnHandleMap()).orElse(null); + } + + public Map getColumnMetadataMap() { + makeSureInitialized(); + Optional schemaCacheValue = getSchemaCacheValue(); + return schemaCacheValue.map(value -> ((TrinoSchemaCacheValue) value).getColumnMetadataMap()).orElse(null); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java index 07c7abf7ce0c265..a38b5f49fc070f7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java @@ -239,8 +239,8 @@ void addConstraintLogPersistForExternalTableTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); @@ -302,8 +302,8 @@ void dropConstraintLogPersistForExternalTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es2"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es2"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index aa5fa313be3c91d..9d36d843e9fb7cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -163,15 +163,15 @@ private void createDbAndTableForCatalog(CatalogIf catalog) { schema.add(new Column("k1", PrimitiveType.INT)); if (catalog instanceof HMSExternalCatalog) { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1"); - HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1", "hive_db1"); + HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); hmsCatalog.addDatabaseForTest(db); } else if (catalog instanceof EsExternalCatalog) { EsExternalCatalog esCatalog = (EsExternalCatalog) catalog; - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); esCatalog.addDatabaseForTest(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java index ab4e5fd0fc557c0..af331b708969920 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java @@ -219,7 +219,7 @@ public Table getTable(String dbName, String tblName) { @Mock public ExternalDatabase getDbNullable(String dbName) { if (createdDbs.contains(dbName)) { - return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName); + return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName, dbName); } return null; } @@ -230,7 +230,7 @@ public ExternalDatabase getDbNullable(String dbName) { HMSExternalTable getTableNullable(String tableName) { for (Table table : createdTables) { if (table.getTableName().equals(tableName)) { - return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog); + return new HMSExternalTable(0, tableName, tableName, hmsExternalCatalog, (HMSExternalDatabase) hmsExternalCatalog.getDbNullable(mockedDbName)); } } return null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java index 46d3e1b897d1110..8f268f19426c3ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -68,7 +68,7 @@ public void init() { new MockUp(HMSExternalCatalog.class) { @Mock public ExternalDatabase getDbNullable(String dbName) { - return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb"); + return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb", "mockedDb"); } @Mock diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 439f6f2aa7de217..2300ece6253b5f2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -84,7 +84,7 @@ public static void beforeClass() throws Throwable { } else { icebergCatalog.setInitialized(true); } - IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName); + IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName); icebergCatalog.addDatabaseForTest(db); // context diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java new file mode 100644 index 000000000000000..c5f53a3cb8ba101 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import mockit.Verifications; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.transforms.Days; +import org.apache.iceberg.transforms.Hours; +import org.apache.iceberg.transforms.Months; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class IcebergExternalTableTest { + + @Test + public void testIsSupportedPartitionTable(@Mocked org.apache.iceberg.Table icebergTable, + @Mocked PartitionSpec spec, + @Mocked PartitionField field, + @Mocked Schema schema) { + IcebergExternalDatabase database = new IcebergExternalDatabase(null, 1L, "2", "2"); + IcebergExternalTable table = new IcebergExternalTable(1, "1", "1", null, database); + Map specs = Maps.newHashMap(); + new MockUp() { + @Mock + private void makeSureInitialized() { + } + + @Mock + public Table getIcebergTable() { + return icebergTable; + } + }; + // Test null + specs.put(0, null); + new Expectations() {{ + icebergTable.specs(); + result = specs; + }}; + table.setTable(icebergTable); + Assertions.assertFalse(table.isValidRelatedTableCached()); + Assertions.assertFalse(table.isValidRelatedTable()); + new Verifications() {{ + icebergTable.specs(); + times = 1; + }}; + Assertions.assertTrue(table.isValidRelatedTableCached()); + Assertions.assertFalse(table.validRelatedTableCache()); + + // Test spec fields are empty. + specs.put(0, spec); + table.setIsValidRelatedTableCached(false); + Assertions.assertFalse(table.isValidRelatedTableCached()); + new Expectations() {{ + icebergTable.specs(); + result = specs; + }}; + List fields = Lists.newArrayList(); + new Expectations() {{ + spec.fields(); + result = fields; + }}; + Assertions.assertFalse(table.isValidRelatedTable()); + new Verifications() {{ + spec.fields(); + times = 1; + }}; + Assertions.assertTrue(table.isValidRelatedTableCached()); + Assertions.assertFalse(table.validRelatedTableCache()); + + // Test true + fields.add(field); + table.setIsValidRelatedTableCached(false); + Assertions.assertFalse(table.isValidRelatedTableCached()); + new Expectations() { + { + icebergTable.schema(); + result = schema; + + schema.findColumnName(anyInt); + result = "col1"; + } + }; + new Expectations() {{ + field.transform(); + result = new Hours(); + }}; + Assertions.assertTrue(table.isValidRelatedTable()); + Assertions.assertTrue(table.isValidRelatedTableCached()); + Assertions.assertTrue(table.validRelatedTableCache()); + new Verifications() {{ + schema.findColumnName(anyInt); + times = 1; + }}; + new Expectations() {{ + field.transform(); + result = new Days(); + }}; + table.setIsValidRelatedTableCached(false); + Assertions.assertFalse(table.isValidRelatedTableCached()); + Assertions.assertTrue(table.isValidRelatedTable()); + new Expectations() {{ + field.transform(); + result = new Months(); + }}; + table.setIsValidRelatedTableCached(false); + Assertions.assertFalse(table.isValidRelatedTableCached()); + Assertions.assertTrue(table.isValidRelatedTable()); + Assertions.assertTrue(table.isValidRelatedTableCached()); + Assertions.assertTrue(table.validRelatedTableCache()); + } + + @Test + public void testGetPartitionRange() throws AnalysisException { + IcebergExternalDatabase database = new IcebergExternalDatabase(null, 1L, "2", "2"); + IcebergExternalTable table = new IcebergExternalTable(1, "1", "1", null, database); + Column c = new Column("ts", PrimitiveType.DATETIMEV2); + List partitionColumns = Lists.newArrayList(c); + table.setPartitionColumns(partitionColumns); + + // Test null partition value + Range nullRange = table.getPartitionRange(null, "hour", partitionColumns); + Assertions.assertEquals("0000-01-01 00:00:00", + nullRange.lowerEndpoint().getPartitionValuesAsStringList().get(0)); + Assertions.assertEquals("0000-01-01 00:00:01", + nullRange.upperEndpoint().getPartitionValuesAsStringList().get(0)); + + // Test hour transform. + Range hour = table.getPartitionRange("100", "hour", partitionColumns); + PartitionKey lowKey = hour.lowerEndpoint(); + PartitionKey upKey = hour.upperEndpoint(); + Assertions.assertEquals("1970-01-05 04:00:00", lowKey.getPartitionValuesAsStringList().get(0)); + Assertions.assertEquals("1970-01-05 05:00:00", upKey.getPartitionValuesAsStringList().get(0)); + + // Test day transform. + Range day = table.getPartitionRange("100", "day", partitionColumns); + lowKey = day.lowerEndpoint(); + upKey = day.upperEndpoint(); + Assertions.assertEquals("1970-04-11 00:00:00", lowKey.getPartitionValuesAsStringList().get(0)); + Assertions.assertEquals("1970-04-12 00:00:00", upKey.getPartitionValuesAsStringList().get(0)); + + // Test month transform. + Range month = table.getPartitionRange("100", "month", partitionColumns); + lowKey = month.lowerEndpoint(); + upKey = month.upperEndpoint(); + Assertions.assertEquals("1978-05-01 00:00:00", lowKey.getPartitionValuesAsStringList().get(0)); + Assertions.assertEquals("1978-06-01 00:00:00", upKey.getPartitionValuesAsStringList().get(0)); + + // Test year transform. + Range year = table.getPartitionRange("100", "year", partitionColumns); + lowKey = year.lowerEndpoint(); + upKey = year.upperEndpoint(); + Assertions.assertEquals("2070-01-01 00:00:00", lowKey.getPartitionValuesAsStringList().get(0)); + Assertions.assertEquals("2071-01-01 00:00:00", upKey.getPartitionValuesAsStringList().get(0)); + + // Test unsupported transform + Exception exception = Assertions.assertThrows(RuntimeException.class, () -> { + table.getPartitionRange("100", "bucket", partitionColumns); + }); + Assertions.assertEquals("Unsupported transform bucket", exception.getMessage()); + } + + @Test + public void testSortRange() throws AnalysisException { + IcebergExternalDatabase database = new IcebergExternalDatabase(null, 1L, "2", "2"); + IcebergExternalTable table = new IcebergExternalTable(1, "1", "1", null, database); + Column c = new Column("c", PrimitiveType.DATETIMEV2); + ArrayList columns = Lists.newArrayList(c); + table.setPartitionColumns(Lists.newArrayList(c)); + PartitionItem nullRange = new RangePartitionItem(table.getPartitionRange(null, "hour", columns)); + PartitionItem year1970 = new RangePartitionItem(table.getPartitionRange("0", "year", columns)); + PartitionItem year1971 = new RangePartitionItem(table.getPartitionRange("1", "year", columns)); + PartitionItem month197002 = new RangePartitionItem(table.getPartitionRange("1", "month", columns)); + PartitionItem month197103 = new RangePartitionItem(table.getPartitionRange("14", "month", columns)); + PartitionItem month197204 = new RangePartitionItem(table.getPartitionRange("27", "month", columns)); + PartitionItem day19700202 = new RangePartitionItem(table.getPartitionRange("32", "day", columns)); + PartitionItem day19730101 = new RangePartitionItem(table.getPartitionRange("1096", "day", columns)); + Map map = Maps.newHashMap(); + map.put("nullRange", nullRange); + map.put("year1970", year1970); + map.put("year1971", year1971); + map.put("month197002", month197002); + map.put("month197103", month197103); + map.put("month197204", month197204); + map.put("day19700202", day19700202); + map.put("day19730101", day19730101); + List> entries = table.sortPartitionMap(map); + Assertions.assertEquals(8, entries.size()); + Assertions.assertEquals("nullRange", entries.get(0).getKey()); + Assertions.assertEquals("year1970", entries.get(1).getKey()); + Assertions.assertEquals("month197002", entries.get(2).getKey()); + Assertions.assertEquals("day19700202", entries.get(3).getKey()); + Assertions.assertEquals("year1971", entries.get(4).getKey()); + Assertions.assertEquals("month197103", entries.get(5).getKey()); + Assertions.assertEquals("month197204", entries.get(6).getKey()); + Assertions.assertEquals("day19730101", entries.get(7).getKey()); + + Map> stringSetMap = table.mergeOverlapPartitions(map); + Assertions.assertEquals(2, stringSetMap.size()); + Assertions.assertTrue(stringSetMap.containsKey("year1970")); + Assertions.assertTrue(stringSetMap.containsKey("year1971")); + + Set names1970 = stringSetMap.get("year1970"); + Assertions.assertEquals(3, names1970.size()); + Assertions.assertTrue(names1970.contains("year1970")); + Assertions.assertTrue(names1970.contains("month197002")); + Assertions.assertTrue(names1970.contains("day19700202")); + + Set names1971 = stringSetMap.get("year1971"); + Assertions.assertEquals(2, names1971.size()); + Assertions.assertTrue(names1971.contains("year1971")); + Assertions.assertTrue(names1971.contains("month197103")); + + Assertions.assertEquals(5, map.size()); + Assertions.assertTrue(map.containsKey("nullRange")); + Assertions.assertTrue(map.containsKey("year1970")); + Assertions.assertTrue(map.containsKey("year1971")); + Assertions.assertTrue(map.containsKey("month197204")); + Assertions.assertTrue(map.containsKey("day19730101")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java new file mode 100644 index 000000000000000..acc357d85d684bf --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameComparedLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseMetaCacheFalseTest$ExternalTableNameComparedLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 2; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table1").getName(); + Assertions.assertEquals("TABLE1", tblName); + String tblName2 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table2").getName(); + Assertions.assertEquals("TABLE2", tblName2); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("TABLE1")); + Assertions.assertTrue(tableNames.contains("TABLE2")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("TABLE1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("TABLE2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java similarity index 82% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java index c48e18cb2711dfd..ade73f3fef21755 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; import org.apache.doris.analysis.SwitchStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -27,6 +28,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.utframe.TestWithFeService; @@ -39,7 +41,7 @@ import java.util.Map; import java.util.Set; -public class ExternalTableNameComparedLowercaseTest extends TestWithFeService { +public class ExternalTableNameComparedLowercaseMetaCacheTrueTest extends TestWithFeService { private static Env env; private ConnectContext rootCtx; @@ -50,8 +52,9 @@ protected void runBeforeAll() throws Exception { // 1. create test catalog CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"true\",\n" + " \"catalog_provider.class\" " - + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseTest$ExternalTableNameComparedLowercaseProvider\"\n" + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseMetaCacheTrueTest$ExternalTableNameComparedLowercaseProvider\"\n" + ");", rootCtx); env.getCatalogMgr().createCatalog(testCatalog); @@ -77,6 +80,20 @@ public void testGlobalVariable() { Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); } + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table1").getName(); + Assertions.assertEquals("TABLE1", tblName); + String tblName2 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table2").getName(); + Assertions.assertEquals("TABLE2", tblName2); + } + @Test public void testTableNameLowerCase() { Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java new file mode 100644 index 000000000000000..c4e1fee1768cb87 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameStoredLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseMetaCacheFalseTest$ExternalTableNameStoredLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 1; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(1, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("TABLE1").getName(); + Assertions.assertEquals("table1", tblName); + String tblName3 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table3").getName(); + Assertions.assertEquals("table3", tblName3); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(3, tableNames.size()); + Assertions.assertTrue(tableNames.contains("table1")); + Assertions.assertTrue(tableNames.contains("table2")); + Assertions.assertTrue(tableNames.contains("table3")); + Assertions.assertFalse(tableNames.contains("TABLE1")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("table1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("table2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + tblSchemaMap1.put("TABLE3", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java similarity index 83% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java index d55e209fa63d9fd..d3eca35debb0b1b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; import org.apache.doris.analysis.SwitchStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -27,6 +28,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.utframe.TestWithFeService; @@ -39,7 +41,7 @@ import java.util.Map; import java.util.Set; -public class ExternalTableNameStoredLowercaseTest extends TestWithFeService { +public class ExternalTableNameStoredLowercaseMetaCacheTrueTest extends TestWithFeService { private static Env env; private ConnectContext rootCtx; @@ -50,8 +52,9 @@ protected void runBeforeAll() throws Exception { // 1. create test catalog CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"true\",\n" + " \"catalog_provider.class\" " - + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseTest$ExternalTableNameStoredLowercaseProvider\"\n" + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseMetaCacheTrueTest$ExternalTableNameStoredLowercaseProvider\"\n" + ");", rootCtx); env.getCatalogMgr().createCatalog(testCatalog); @@ -77,6 +80,20 @@ public void testGlobalVariable() { Assertions.assertEquals(1, GlobalVariable.lowerCaseTableNames); } + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("TABLE1").getName(); + Assertions.assertEquals("table1", tblName); + String tblName3 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table3").getName(); + Assertions.assertEquals("table3", tblName3); + } + @Test public void testTableNameLowerCase() { Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java new file mode 100644 index 000000000000000..edf7ab4e29b12a8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java @@ -0,0 +1,277 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mapping; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class JdbcIdentifierMappingTest { + private String validJson; + private String invalidJson; + private String duplicateMappingJson; + private String columnConflictJson; + private String tableConflictJson; + + @Before + public void setUp() { + validJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"doris_local\"}\n" + + " ],\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"mapping\": \"table_a_local\"}\n" + + " ],\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"remoteColumn\": \"COLUMN_X\", " + + "\"mapping\": \"column_x_local\"}\n" + + " ]\n" + + "}"; + + invalidJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"doris_local\"}\n" + + " ],\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"mapping\": \"table_a_local\"}\n" + + " ], // Invalid JSON due to trailing comma\n" + + "}"; + + duplicateMappingJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"conflict\"},\n" + + " {\"remoteDatabase\": \"DORIS_DUP\", \"mapping\": \"CONFLICT\"}\n" + + " ]\n" + + "}"; + + columnConflictJson = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C1\", \"mapping\": \"custom_col\"},\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C2\", \"mapping\": \"CUSTOM_COL\"}\n" + + " ]\n" + + "}"; + + tableConflictJson = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T1\", \"mapping\": \"custom_table\"},\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T2\", \"mapping\": \"CUSTOM_TABLE\"}\n" + + " ]\n" + + "}"; + } + + @Test + public void testIsLowerCaseMetaNamesTrue() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, validJson); + + String databaseName = mapping.fromRemoteDatabaseName("DORIS"); + String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); + String columnName = mapping.fromRemoteColumnName("DORIS", "TABLE_A", "COLUMN_X"); + + Assert.assertEquals("doris_local", databaseName); + Assert.assertEquals("table_a_local", tableName); + Assert.assertEquals("column_x_local", columnName); + } + + @Test + public void testIsLowerCaseMetaNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, validJson); + + String databaseName = mapping.fromRemoteDatabaseName("DORIS"); + String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); + String columnName = mapping.fromRemoteColumnName("DORIS", "TABLE_A", "COLUMN_X"); + + Assert.assertEquals("doris_local", databaseName); + Assert.assertEquals("table_a_local", tableName); + Assert.assertEquals("column_x_local", columnName); + } + + @Test(expected = RuntimeException.class) + public void testInvalidJson() { + new JdbcIdentifierMapping(true, false, invalidJson); + } + + @Test + public void testDuplicateMappingWhenLowerCaseMetaNamesTrue() { + try { + new JdbcIdentifierMapping(false, true, duplicateMappingJson); + Assert.fail("Expected RuntimeException due to duplicate mappings"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("Duplicate mapping found")); + } + } + + @Test + public void testDuplicateMappingWhenLowerCaseMetaNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, duplicateMappingJson); + + String databaseName1 = mapping.fromRemoteDatabaseName("DORIS"); + String databaseName2 = mapping.fromRemoteDatabaseName("DORIS_DUP"); + + Assert.assertEquals("conflict", databaseName1); + Assert.assertEquals("CONFLICT", databaseName2); + } + + @Test + public void testColumnCaseConflictAlwaysChecked() { + try { + new JdbcIdentifierMapping(false, false, columnConflictJson); + Assert.fail("Expected RuntimeException due to column case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseAndLowerCaseTableNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, tableConflictJson); + String tableName1 = mapping.fromRemoteTableName("D2", "T1"); + String tableName2 = mapping.fromRemoteTableName("D2", "T2"); + Assert.assertEquals("custom_table", tableName1); + Assert.assertEquals("CUSTOM_TABLE", tableName2); + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesTrue() { + try { + new JdbcIdentifierMapping(true, false, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseButLowerCaseTableNamesTrue() { + try { + new JdbcIdentifierMapping(false, true, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_local", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_LOCAL", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForColumnWithoutLowerCaseMetaNames() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_CLOCAL", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } + + @Test + public void testUppercaseMappingForColumnWithLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_clocal", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index d15c11e55cc0a34..991e2432ae9532f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -98,7 +98,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index 358254cd2bb95ed..11df79402bf3f1a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -115,7 +115,7 @@ private void init(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/regression-test/data/external_table_p0/lower_case/test_conflict_name.out b/regression-test/data/external_table_p0/lower_case/test_conflict_name.out new file mode 100644 index 000000000000000..be86d4d3b3ed311 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_conflict_name.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_external_conflict_name_db -- + +-- !select_external_conflict_name_tbl -- + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out new file mode 100644 index 000000000000000..1332a98aac4f9d4 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out @@ -0,0 +1,49 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_cache_false_lower_false1 -- +1 lower + +-- !sql_test_cache_false_lower_false2 -- +1 UPPER + +-- !sql_test_cache_false_lower_false1_insert -- +1 + +-- !sql_test_cache_false_lower_false2_insert -- +1 + +-- !sql_test_cache_false_lower_false1 -- +1 lower + +-- !sql_test_cache_false_lower_false2 -- +1 UPPER + +-- !sql_test_cache_false_lower_false1_insert -- +1 + +-- !sql_test_cache_false_lower_false2_insert -- +1 + +-- !sql_test_cache_false_lower_true1 -- +1 lower + +-- !sql_test_cache_false_lower_true2 -- +1 UPPER + +-- !sql_test_cache_false_lower_true1_insert -- +1 + +-- !sql_test_cache_false_lower_true2_insert -- +1 + +-- !sql_test_cache_true_lower_true1 -- +1 lower + +-- !sql_test_cache_true_lower_true2 -- +1 UPPER + +-- !sql_test_cache_true_lower_true1_insert -- +1 + +-- !sql_test_cache_true_lower_true2_insert -- +1 + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out new file mode 100644 index 000000000000000..6be380ed0d0161d --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out @@ -0,0 +1,241 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_cache_false_lower_false_with_conf1_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf1_2 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf1_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_2_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_3_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_4_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf2_2 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf2_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_2_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_3_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_4_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf1_2 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf1_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_3_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_4_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf2_2 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf2_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_3_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_4_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf1_2 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf1_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_3_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_4_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf2_2 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf2_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_3_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_4_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf1_2 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf1_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_3_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_4_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf2_2 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf2_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_3_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_4_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf0_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf0_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf0_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf0_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf0_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf0_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf0_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf0_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf0_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf0_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf0_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf0_2_insert -- +1 + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out new file mode 100644 index 000000000000000..f958424e65c6268 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- + diff --git a/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out new file mode 100644 index 000000000000000..7669a50d30cd211 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_meta_cache_lower_true_select_without_refresh_select_table1 -- +1 table1 + +-- !test_meta_cache_lower_false_select_without_refresh_select_table1 -- +1 table1 + +-- !test_meta_cache_lower_false_select_without_refresh_select_table2 -- +1 TABLE2 + diff --git a/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out b/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out new file mode 100644 index 000000000000000..71551d168912964 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_meta_mapping_select_lower -- +1 lowercase 100 + +-- !sql_meta_mapping_select_upper -- +2 UPPERCASE 200 + +-- !sql_meta_mapping_select_lower_lower_case_true -- +1 lowercase 100 + +-- !sql_meta_mapping_select_upper_lower_case_true -- +2 UPPERCASE 200 + diff --git a/regression-test/data/external_table_p0/lower_case/upgrade/load.out b/regression-test/data/external_table_p0/lower_case/upgrade/load.out new file mode 100644 index 000000000000000..bdd9b17d0c6e3dd --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/upgrade/load.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_upgrade_lower_case_catalog_1 -- +1 lower + +-- !sql_test_upgrade_lower_case_catalog_2 -- +1 UPPER + diff --git a/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out b/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out new file mode 100644 index 000000000000000..bdd9b17d0c6e3dd --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_upgrade_lower_case_catalog_1 -- +1 lower + +-- !sql_test_upgrade_lower_case_catalog_2 -- +1 UPPER + diff --git a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy new file mode 100644 index 000000000000000..6187d47b6455644 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_conflict_name", "p0,external,doris,meta_names_mapping") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_conflict_name; """ + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ + sql """create database if not exists internal.external_conflict_name; """ + sql """create database if not exists internal.EXTERNAL_CONFLICT_NAME; """ + sql """create table if not exists internal.external_conflict_name.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """drop catalog if exists test_conflict_name """ + sql """ CREATE CATALOG `test_conflict_name` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_conflict_name,EXTERNAL_CONFLICT_NAME" + )""" + + test { + sql """show databases from test_conflict_name""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_CONFLICT_NAME, external_conflict_name in catalog test_conflict_name. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_name""" + + test { + sql """select * from test_conflict_name.external_conflict_name.table_test""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_CONFLICT_NAME, external_conflict_name in catalog test_conflict_name. Please use meta_names_mapping to handle name mapping.""" + } + + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ + + sql """refresh catalog test_conflict_name""" + + qt_select_external_conflict_name_db "select * from test_conflict_name.external_conflict_name.table_test" + + sql """create table if not exists internal.external_conflict_name.TABLE_TEST + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """refresh catalog test_conflict_name""" + + test { + sql """show tables from test_conflict_name.external_conflict_name""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_conflict_name' under catalog 'test_conflict_name'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_name""" + + test { + sql """select * from test_conflict_name.external_conflict_name.TABLE_TEST""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_conflict_name' under catalog 'test_conflict_name'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """drop table if exists internal.external_conflict_name.TABLE_TEST; """ + + qt_select_external_conflict_name_tbl "select * from test_conflict_name.external_conflict_name.table_test" + + sql """drop database if exists internal.external_conflict_name; """ + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy new file mode 100644 index 000000000000000..854eb06a8e27556 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_include", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String mapping_db = """ + { + "databases": [ + {"remoteDatabase": "external_INCLUDE", "mapping": "external_include_test"}, + {"remoteDatabase": "external_EXCLUDE", "mapping": "external_exclude_test"} + ] + } + """ + + sql """drop database if exists internal.external_INCLUDE; """ + sql """drop database if exists internal.external_EXCLUDE; """ + sql """create database if not exists internal.external_INCLUDE; """ + sql """create database if not exists internal.external_EXCLUDE; """ + + // Test include + sql """drop catalog if exists test_lower_case_include """ + + sql """ CREATE CATALOG `test_lower_case_include` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_INCLUDE" + )""" + + test { + sql """show databases from test_lower_case_include""" + + rowNum 3 + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_include"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_include """ + + // Test exclude + sql """drop catalog if exists test_lower_case_exclude """ + + sql """ CREATE CATALOG `test_lower_case_exclude` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "exclude_database_list" = "external_EXCLUDE" + )""" + + test { + sql """show databases from test_lower_case_exclude""" + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_exclude"] + expectedDatabases.each { dbName -> + assertFalse(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + // Test include with mapping + sql """drop catalog if exists test_lower_case_include """ + + sql """ CREATE CATALOG `test_lower_case_include` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + 'meta_names_mapping' = '${mapping_db}', + "include_database_list" = "external_INCLUDE" + )""" + + test { + sql """show databases from test_lower_case_include""" + + rowNum 3 + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_include_test"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_include """ + + // Test exclude with mapping + sql """drop catalog if exists test_lower_case_exclude """ + + sql """ CREATE CATALOG `test_lower_case_exclude` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + 'meta_names_mapping' = '${mapping_db}', + "exclude_database_list" = "external_EXCLUDE" + )""" + + test { + sql """show databases from test_lower_case_exclude""" + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_exclude_test"] + expectedDatabases.each { dbName -> + assertFalse(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_exclude """ + sql """drop database if exists internal.external_INCLUDE; """ + sql """drop database if exists internal.external_EXCLUDE; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy new file mode 100644 index 000000000000000..d4efd141c2d4686 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_test_lower; """ + sql """drop database if exists internal.external_test_UPPER; """ + sql """create database if not exists internal.external_test_lower; """ + sql """create database if not exists internal.external_test_UPPER; """ + sql """create table if not exists internal.external_test_lower.lower + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower.UPPER + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_test_lower.lower values(1, 'lower')""" + sql """insert into internal.external_test_lower.UPPER values(1, 'UPPER')""" + + sql """create table if not exists internal.external_test_lower.lower_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower.UPPER_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + // Test for cache false and lower false + sql """drop catalog if exists test_cache_false_lower_false """ + + sql """ CREATE CATALOG `test_cache_false_lower_false` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_false_lower_false""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_UPPER"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_false_lower_false.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "UPPER"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_false1 "select * from test_cache_false_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2 "select * from test_cache_false_lower_false.external_test_lower.UPPER" + + qt_sql_test_cache_false_lower_false1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_false_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_false_lower_false.external_test_lower.UPPER" + + sql """drop catalog if exists test_cache_false_lower_false """ + + // Test for cache true and lower false + sql """drop catalog if exists test_cache_true_lower_false """ + + sql """ CREATE CATALOG `test_cache_true_lower_false` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_true_lower_false""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_UPPER"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_true_lower_false.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "UPPER"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_false1 "select * from test_cache_true_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2 "select * from test_cache_true_lower_false.external_test_lower.UPPER" + + qt_sql_test_cache_false_lower_false1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_true_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_true_lower_false.external_test_lower.UPPER" + + + sql """drop catalog if exists test_cache_true_lower_false """ + + // Test for cache false and lower true + sql """drop catalog if exists test_cache_false_lower_true """ + + sql """ CREATE CATALOG `test_cache_false_lower_true` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_false_lower_true""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_false_lower_true.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_true1 "select * from test_cache_false_lower_true.external_test_lower.lower" + qt_sql_test_cache_false_lower_true2 "select * from test_cache_false_lower_true.external_test_lower.upper" + + qt_sql_test_cache_false_lower_true1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_false_lower_true.external_test_lower.lower" + qt_sql_test_cache_false_lower_true2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_false_lower_true.external_test_lower.upper" + + sql """drop catalog if exists test_cache_false_lower_true """ + + // Test for cache true and lower true + sql """drop catalog if exists test_cache_true_lower_true """ + + sql """ CREATE CATALOG `test_cache_true_lower_true` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_true_lower_true""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_true_lower_true.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_true_lower_true1 "select * from test_cache_true_lower_true.external_test_lower.lower" + qt_sql_test_cache_true_lower_true2 "select * from test_cache_true_lower_true.external_test_lower.upper" + + qt_sql_test_cache_true_lower_true1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_true_lower_true.external_test_lower.lower" + qt_sql_test_cache_true_lower_true2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_true_lower_true.external_test_lower.upper" + + sql """drop catalog if exists test_cache_true_lower_true """ + + + sql """drop database if exists internal.external_test_lower; """ + sql """drop database if exists internal.external_test_UPPER; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy new file mode 100644 index 000000000000000..c3da24e1f55c1d5 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -0,0 +1,702 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_test_lower_with_conf; """ + sql """drop database if exists internal.external_test_UPPER_with_conf; """ + sql """create database if not exists internal.external_test_lower_with_conf; """ + sql """create table if not exists internal.external_test_lower_with_conf.lower_with_conf + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower_with_conf.UPPER_with_conf + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_test_lower_with_conf.lower_with_conf values(1, 'lower')""" + sql """insert into internal.external_test_lower_with_conf.UPPER_with_conf values(1, 'UPPER')""" + + sql """create table if not exists internal.external_test_lower_with_conf.with_conf_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + // Test for cache false and lower false and lower conf 1 + sql """drop catalog if exists test_cache_false_lower_false_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_false_lower_false_with_conf1_1 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_2 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_3 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_4 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + + test { + sql """show tables from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf1 """ + + // Test for cache false and lower false and lower conf 2 + sql """drop catalog if exists test_cache_false_lower_false_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_false_lower_false_with_conf2_1 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_2 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_3 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_4 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf2 """ + + + + // Test for cache true and lower false and lower conf 1 + sql """drop catalog if exists test_cache_true_lower_false_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_true_lower_false_with_conf1_1 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_2 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_3 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_4 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf1 """ + + // Test for cache true and lower false and lower conf 2 + sql """drop catalog if exists test_cache_true_lower_false_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_true_lower_false_with_conf2_1 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_2 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_3 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_4 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf2 """ + + // Test for cache false and lower true and lower conf 1 + sql """drop catalog if exists test_cache_false_lower_true_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_false_lower_true_with_conf1_1 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_2 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_3 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_4 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf1 """ + + // Test for cache false and lower true and lower conf 2 + sql """drop catalog if exists test_cache_false_lower_true_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_false_lower_true_with_conf2_1 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_2 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_3 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_4 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf2 """ + + + // Test for cache true and lower true and lower conf 1 + sql """drop catalog if exists test_cache_true_lower_true_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_true_lower_true_with_conf1_1 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_2 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_3 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_4 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf1 """ + + // Test for cache true and lower true and lower conf 2 + sql """drop catalog if exists test_cache_true_lower_true_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "truee", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_true_lower_true_with_conf2_1 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_2 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_3 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_4 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf2 """ + + + // Test for cache false and lower false and lower conf 0 + sql """drop catalog if exists test_cache_false_lower_false_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_false_lower_false_with_conf0_1 "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_2 "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf0 """ + + + // Test for cache true and lower false and lower conf 0 + sql """drop catalog if exists test_cache_true_lower_false_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_true_lower_false_with_conf0_1 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_2 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf0 """ + + + // Test for cache false and lower true and lower conf 0 + sql """drop catalog if exists test_cache_false_lower_true_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_false_lower_true_with_conf0_1 "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_2 "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf0 """ + + + // Test for cache true and lower true and lower conf 0 + sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_true_lower_true_with_conf0_1 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_2 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """ + + sql """drop database if exists internal.external_test_lower_with_conf; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy new file mode 100644 index 000000000000000..40322a22afe737c --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_mtmv", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.EXTERNAL_LOWER_MTMV; """ + sql """create database if not exists internal.EXTERNAL_LOWER_MTMV;""" + sql """create table if not exists internal.EXTERNAL_LOWER_MTMV.TABLE_TEST + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.EXTERNAL_LOWER_MTMV.TABLE_TEST values(1, 'lower')""" + + sql """drop catalog if exists test_lower_case_mtmv """ + + sql """ CREATE CATALOG `test_lower_case_mtmv` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "EXTERNAL_LOWER_MTMV" + )""" + + + sql """CREATE MATERIALIZED VIEW internal.EXTERNAL_LOWER_MTMV.MTMV_TEST + REFRESH COMPLETE ON SCHEDULE EVERY 1 minute + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + 'replication_num' = '1' + ) + AS SELECT * FROM test_lower_case_mtmv.external_lower_mtmv.table_test;""" + + qt_select """select * from internal.EXTERNAL_LOWER_MTMV.MTMV_TEST""" + + sql """drop catalog if exists test_lower_case_mtmv """ + sql """drop database if exists internal.EXTERNAL_LOWER_MTMV """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy new file mode 100644 index 000000000000000..20e6b0f0032e961 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_meta_cache_select_without_refresh", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """ drop database if exists internal.external_lower_select_without_refresh; """ + sql """create database if not exists internal.external_lower_select_without_refresh;""" + + // Test include + sql """drop catalog if exists test_meta_cache_lower_true_select_without_refresh """ + + sql """ CREATE CATALOG `test_meta_cache_lower_true_select_without_refresh` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_lower_select_without_refresh" + )""" + + sql """drop catalog if exists test_meta_cache_lower_false_select_without_refresh """ + + sql """ CREATE CATALOG `test_meta_cache_lower_false_select_without_refresh` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_lower_select_without_refresh" + )""" + + sql """create table if not exists internal.external_lower_select_without_refresh.table1 + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_lower_select_without_refresh.table1 values(1, 'table1')""" + + qt_test_meta_cache_lower_true_select_without_refresh_select_table1 "select * from test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table1;" + + qt_test_meta_cache_lower_false_select_without_refresh_select_table1 "select * from test_meta_cache_lower_false_select_without_refresh.external_lower_select_without_refresh.table1;" + + sql """create table if not exists internal.external_lower_select_without_refresh.TABLE2 + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_lower_select_without_refresh.TABLE2 values(1, 'TABLE2')""" + + test { + sql """select * from test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table2;""" + + exception "Table [table2] does not exist in database [external_lower_select_without_refresh]." + } + + qt_test_meta_cache_lower_false_select_without_refresh_select_table2 "select * from test_meta_cache_lower_false_select_without_refresh.external_lower_select_without_refresh.TABLE2;" + + sql """drop catalog if exists test_meta_cache_lower_true_select_without_refresh """ + sql """drop catalog if exists test_meta_cache_lower_false_select_without_refresh """ + sql """drop database if exists internal.external_lower_select_without_refresh; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy new file mode 100644 index 000000000000000..1cf48b17c87ab98 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String validMetaNamesMapping = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ], + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "table_test", "mapping": "table_test_lower"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"} + ], + "columns": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "remoteColumn": "column_test", "mapping": "column_test_local"} + ] + } + """ + + sql """drop database if exists internal.external_meta_names_mapping; """ + sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """ + sql """create database if not exists internal.external_meta_names_mapping; """ + sql """create database if not exists internal.EXTERNAL_META_NAMES_MAPPING; """ + + sql """create table if not exists internal.external_meta_names_mapping.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + sql """create table if not exists internal.external_meta_names_mapping.TABLE_TEST + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + sql """create table if not exists internal.EXTERNAL_META_NAMES_MAPPING.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_meta_names_mapping.table_test values(1, 'lowercase', 100);""" + sql """insert into internal.external_meta_names_mapping.TABLE_TEST values(2, 'UPPERCASE', 200);""" + sql """insert into internal.EXTERNAL_META_NAMES_MAPPING.table_test values(3, 'MIXEDCASE', 300);""" + + sql """drop catalog if exists test_valid_meta_names_mapping """ + sql """ CREATE CATALOG `test_valid_meta_names_mapping` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${validMetaNamesMapping}' + )""" + + test { + sql """show databases from test_valid_meta_names_mapping""" + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_meta_names_mapping_upper", "external_meta_names_mapping_lower"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_valid_meta_names_mapping.external_meta_names_mapping_lower""" + check { result, ex, startTime, endTime -> + def expectedTables = ["table_test_lower", "table_test_upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + test { + sql """describe test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_upper""" + check { result, ex, startTime, endTime -> + def expectedColumns = ["column_test_local"] + expectedColumns.each { columnName -> + assertTrue(result.collect { it[0] }.contains(columnName), "Expected column '${columnName}' not found in result") + } + } + } + + qt_sql_meta_mapping_select_lower "select * from test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_lower" + qt_sql_meta_mapping_select_upper "select * from test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_upper" + + sql """drop catalog if exists test_valid_meta_names_mapping """ + + sql """ drop catalog if exists test_conflict_meta_names """ + sql """ CREATE CATALOG `test_conflict_meta_names` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + )""" + + test { + sql """show databases from test_conflict_meta_names""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_META_NAMES_MAPPING, external_meta_names_mapping in catalog test_conflict_meta_names. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_meta_names""" + + test { + sql """select * from test_conflict_meta_names.external_meta_names_mapping.table_test""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_META_NAMES_MAPPING, external_meta_names_mapping in catalog test_conflict_meta_names. Please use meta_names_mapping to handle name mapping.""" + } + + String validMetaNamesMapping2 = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ] + } + """ + + sql """alter catalog test_conflict_meta_names set properties('meta_names_mapping' = '${validMetaNamesMapping2}')""" + + test { + sql """show tables from test_conflict_meta_names.external_meta_names_mapping_lower""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_meta_names_mapping' under catalog 'test_conflict_meta_names'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_meta_names""" + + test { + sql """select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_meta_names_mapping' under catalog 'test_conflict_meta_names'. Please use meta_names_mapping to handle name mapping.""" + } + + String validMetaNamesMapping3 = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ], + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "table_test", "mapping": "table_test_lower"} + ] + } + """ + + sql """alter catalog test_conflict_meta_names set properties('meta_names_mapping' = '${validMetaNamesMapping3}')""" + + test { + sql """show databases from test_conflict_meta_names""" + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_meta_names_mapping_upper", "external_meta_names_mapping_lower"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_conflict_meta_names.external_meta_names_mapping_lower""" + check { result, ex, startTime, endTime -> + def expectedTables = ["table_test_lower", "table_test_upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_meta_mapping_select_lower_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_lower" + qt_sql_meta_mapping_select_upper_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_upper" + + sql """drop catalog if exists test_conflict_meta_names """ + + String error_mapping_db = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_lower"} + ] + } + """ + + sql """drop catalog if exists test_error_mapping_db """ + + test { + sql """ CREATE CATALOG `test_error_mapping_db` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${error_mapping_db}' + )""" + + exception "Duplicate remoteDatabase found: EXTERNAL_META_NAMES_MAPPING" + } + + sql """drop catalog if exists test_error_mapping_db """ + + String error_mapping_tbl = """ + { + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_lower"} + ] + } + """ + + sql """drop catalog if exists test_error_mapping_tbl """ + + test { + sql """ CREATE CATALOG `test_error_mapping_tbl` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${error_mapping_tbl}' + )""" + + exception "Duplicate remoteTable found in database external_meta_names_mapping: TABLE_TEST" + } + + sql """drop catalog if exists test_error_mapping_tbl """ + + sql """drop catalog if exists test_alter_error_mapping """ + + sql """ CREATE CATALOG `test_alter_error_mapping` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + )""" + + test { + sql """alter catalog test_alter_error_mapping set properties('meta_names_mapping' = '${error_mapping_db}')""" + exception "Duplicate remoteDatabase found: EXTERNAL_META_NAMES_MAPPING" + } + + test { + sql """alter catalog test_alter_error_mapping set properties('meta_names_mapping' = '${error_mapping_tbl}')""" + exception "Duplicate remoteTable found in database external_meta_names_mapping: TABLE_TEST" + } + + sql """drop catalog if exists test_alter_error_mapping """ + + sql """drop database if exists internal.external_meta_names_mapping; """ + sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy new file mode 100644 index 000000000000000..19d7e2db18089ab --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String mapping = """ + { + "databases": [ + {"remoteDatabase": "external_timing_refresh_catalog", "mapping": "db"} + ], + "tables": [ + {"remoteDatabase": "external_timing_refresh_catalog", "remoteTable": "tbl", "mapping": "table_t"} + ], + "columns": [ + {"remoteDatabase": "external_timing_refresh_catalog", "remoteTable": "tbl", "remoteColumn": "id", "mapping": "id_c"} + ] + } + """ + + sql """drop database if exists internal.external_timing_refresh_catalog; """ + sql """create database if not exists internal.external_timing_refresh_catalog;""" + sql """create table if not exists internal.external_timing_refresh_catalog.tbl + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_timing_refresh_catalog.tbl values(1, 'lower')""" + + sql """drop catalog if exists test_timing_refresh_catalog1 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog" + )""" + + sql """drop catalog if exists test_timing_refresh_catalog2 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog" + )""" + + test { + def catalogName = "test_timing_refresh_catalog1" + for (int i = 0; i < 10; i++) { + sql """ + select * from ${catalogName}.external_timing_refresh_catalog.tbl + """ + Thread.sleep(1000) + } + } + + test { + def catalogName = "test_timing_refresh_catalog2" + for (int i = 0; i < 10; i++) { + sql """ + select * from ${catalogName}.external_timing_refresh_catalog.tbl + """ + Thread.sleep(1000) + } + } + + // with mapping + sql """drop catalog if exists test_timing_refresh_catalog1 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog", + 'meta_names_mapping' = '${mapping}' + )""" + + sql """drop catalog if exists test_timing_refresh_catalog2 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog", + 'meta_names_mapping' = '${mapping}' + )""" + + test { + def catalogName = "test_timing_refresh_catalog1" + for (int i = 0; i < 10; i++) { + sql """ + select id_c from ${catalogName}.db.table_t + """ + Thread.sleep(1000) + } + } + + test { + def catalogName = "test_timing_refresh_catalog2" + for (int i = 0; i < 10; i++) { + sql """ + select id_c from ${catalogName}.db.table_t + """ + Thread.sleep(1000) + } + } + + sql """drop catalog if exists test_timing_refresh_catalog1 """ + sql """drop catalog if exists test_timing_refresh_catalog2 """ + sql """drop database if exists internal.external_timing_refresh_catalog """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy new file mode 100644 index 000000000000000..0ea89d4012d1cb7 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_lower_case_catalog_prepare", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.upgrade_lower_case_catalog_lower; """ + sql """drop database if exists internal.upgrade_lower_case_catalog_UPPER; """ + sql """create database if not exists internal.upgrade_lower_case_catalog_lower; """ + sql """create database if not exists internal.upgrade_lower_case_catalog_UPPER; """ + sql """create table if not exists internal.upgrade_lower_case_catalog_lower.lower + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.upgrade_lower_case_catalog_lower.UPPER + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.upgrade_lower_case_catalog_lower.lower values(1, 'lower')""" + sql """insert into internal.upgrade_lower_case_catalog_lower.UPPER values(1, 'UPPER')""" + + // Test for cache false and lower false + sql """drop catalog if exists test_upgrade_lower_case_catalog """ + + sql """ CREATE CATALOG `test_upgrade_lower_case_catalog` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "upgrade_lower_case_catalog_lower,upgrade_lower_case_catalog_UPPER" + )""" + + test { + sql """show databases from test_upgrade_lower_case_catalog""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["upgrade_lower_case_catalog_lower", "upgrade_lower_case_catalog_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_upgrade_lower_case_catalog_1 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.lower" + qt_sql_test_upgrade_lower_case_catalog_2 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.upper" + +} diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy new file mode 100644 index 000000000000000..634e5e5737ef6de --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_lower_case_catalog", "p0,external,doris,external_docker,external_docker_doris") { + + test { + sql """show databases from test_upgrade_lower_case_catalog""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["upgrade_lower_case_catalog_lower", "upgrade_lower_case_catalog_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_upgrade_lower_case_catalog_1 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.lower" + qt_sql_test_upgrade_lower_case_catalog_2 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.upper" + +} \ No newline at end of file