[improvement](jdbc catalog) Optimize JdbcCatalog case mapping stabili…

…ty (#41510)

1. Cancel the preservation of name mapping by IdentifierMapping, and only use it as the logic of case processing
2. Save the remote name in ExternalDatabase/ExternalTable object for get
3. Use fromRemoteXXName to process the remote name and map it to the local. Currently, only JdbcExternalCatalog uses IdentifierMapping for processing in this method, and other Catalogs are not processed. If processing is required, it should be extended by yourself
4. When lower_case_meta_name is true, the database, table, and column names mapped remotely to local will be checked to see if they are only case-sensitive.
5. Since Doris's column names are case-insensitive, column names will be checked at any time
6. When Fe config's lower_case_table_names is 1 or 2, the table names mapped from remote to local will be checked for only case-sensitive.
7. include/exclude_database_lists The remote name must be used
8. meta_names_mapping is used to resolve the conflicts mentioned above
9. If the same remoteName is in include_database_lists and meta_names_mapping, the filtered remoteName will be displayed according to the name mapped by meta_names_mapping
10. Since IdentifierMapping is no longer used to store the uppercase and lowercase name mapping, ExternalObject itself is used instead, which solves the problem of frequent refresh and small mapping loss
11. Remove the logic of rebuilding JdbcClient every time refreshCatalog

fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java

@@ -220,4 +220,14 @@ default Optional<TableValuedFunction> getMetaTableFunction(String dbName, String
     default Optional<TableValuedFunctionRef> getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) {
         return Optional.empty();
     }
+
+    // Convert from remote database name to local database name, overridden by subclass if necessary
+    default String fromRemoteDatabaseName(String remoteDatabaseName) {
+        return remoteDatabaseName;
+    }
+
+    // Convert from remote table name to local table name, overridden by subclass if necessary
+    default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) {
+        return remoteTableName;
+    }
 }

fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java

@@ -48,6 +48,7 @@
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalDatabase;
 import org.apache.doris.datasource.hive.HMSExternalTable;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.persist.OperationType;
@@ -655,8 +656,8 @@ public boolean externalTableExistInLocal(String dbName, String tableName, String
     }
 
     public void registerExternalTableFromEvent(String dbName, String tableName,
-                                               String catalogName, long updateTime,
-                                               boolean ignoreIfExists) throws DdlException {
+            String catalogName, long updateTime,
+            boolean ignoreIfExists) throws DdlException {
         CatalogIf catalog = nameToCatalog.get(catalogName);
         if (catalog == null) {
             throw new DdlException("No catalog found with name: " + catalogName);
@@ -686,7 +687,8 @@ public void registerExternalTableFromEvent(String dbName, String tableName,
 
         db.writeLock();
         try {
-            HMSExternalTable namedTable = new HMSExternalTable(tblId, tableName, dbName, (HMSExternalCatalog) catalog);
+            HMSExternalTable namedTable = ((HMSExternalDatabase) db)
+                    .buildTableForInit(tableName, tableName, tblId, hmsCatalog, (HMSExternalDatabase) db, false);
             namedTable.setUpdateTime(updateTime);
             db.registerTable(namedTable);
         } finally {
@@ -732,7 +734,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName)
     }
 
     public void addExternalPartitions(String catalogName, String dbName, String tableName,
-                                      List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
+            List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
             throws DdlException {
         CatalogIf catalog = nameToCatalog.get(catalogName);
         if (catalog == null) {
@@ -767,7 +769,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl
     }
 
     public void dropExternalPartitions(String catalogName, String dbName, String tableName,
-                                       List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
+            List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
             throws DdlException {
         CatalogIf catalog = nameToCatalog.get(catalogName);
         if (catalog == null) {

fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java

@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.Type;
 import org.apache.doris.cluster.ClusterNamespace;
 import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
 import org.apache.doris.common.ThreadPoolManager;
 import org.apache.doris.datasource.hive.HMSExternalCatalog;
 import org.apache.doris.datasource.hive.HMSExternalTable;
@@ -302,7 +303,7 @@ public void invalidatePartitionsCache(long catalogId, String dbName, String tabl
 
     public <T> MetaCache<T> buildMetaCache(String name,
             OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize,
-            CacheLoader<String, List<String>> namesCacheLoader,
+            CacheLoader<String, List<Pair<String, String>>> namesCacheLoader,
             CacheLoader<String, Optional<T>> metaObjCacheLoader,
             RemovalListener<String, Optional<T>> removalListener) {
         MetaCache<T> metaCache = new MetaCache<>(name, commonRefreshExecutor, expireAfterWriteSec, refreshAfterWriteSec,

fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java

@@ -71,10 +71,13 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
     protected long id;
     @SerializedName(value = "name")
     protected String name;
+    @SerializedName(value = "remoteName")
+    protected String remoteName;
     @SerializedName(value = "type")
     protected TableType type = null;
     @SerializedName(value = "timestamp")
     protected long timestamp;
+    // dbName is temporarily retained and will be deleted later. To use dbName, please use db.getFullName()
     @SerializedName(value = "dbName")
     protected String dbName;
     @SerializedName(value = "ta")
@@ -86,6 +89,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
     protected long dbId;
     protected boolean objectCreated;
     protected ExternalCatalog catalog;
+    protected ExternalDatabase db;
 
     /**
      * No args constructor for persist.
@@ -99,15 +103,19 @@ public ExternalTable() {
      *
      * @param id Table id.
      * @param name Table name.
+     * @param remoteName Remote table name.
      * @param catalog ExternalCatalog this table belongs to.
-     * @param dbName Name of the db the this table belongs to.
+     * @param db ExternalDatabase this table belongs to.
      * @param type Table type.
      */
-    public ExternalTable(long id, String name, ExternalCatalog catalog, String dbName, TableType type) {
+    public ExternalTable(long id, String name, String remoteName, ExternalCatalog catalog, ExternalDatabase db,
+            TableType type) {
         this.id = id;
         this.name = name;
+        this.remoteName = remoteName;
         this.catalog = catalog;
-        this.dbName = dbName;
+        this.db = db;
+        this.dbName = db.getFullName();
         this.type = type;
         this.objectCreated = false;
     }
@@ -116,6 +124,14 @@ public void setCatalog(ExternalCatalog catalog) {
         this.catalog = catalog;
     }
 
+    public void setDb(ExternalDatabase db) {
+        this.db = db;
+    }
+
+    public void setRemoteName(String remoteName) {
+        this.remoteName = remoteName;
+    }
+
     public boolean isView() {
         return false;
     }
@@ -141,6 +157,10 @@ public String getName() {
         return name;
     }
 
+    public String getRemoteName() {
+        return remoteName;
+    }
+
     @Override
     public TableType getType() {
         return type;

fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java

@@ -62,6 +62,9 @@ public enum Type {
     @SerializedName(value = "createDbNames")
     private List<String> createDbNames;
 
+    @SerializedName(value = "remoteDbNames")
+    private List<String> remoteDbNames;
+
     @SerializedName(value = "type")
     private Type type;
 
@@ -75,6 +78,7 @@ public InitCatalogLog() {
         refreshDbIds = Lists.newArrayList();
         createDbIds = Lists.newArrayList();
         createDbNames = Lists.newArrayList();
+        remoteDbNames = Lists.newArrayList();
         type = Type.UNKNOWN;
     }
 
@@ -83,10 +87,11 @@ public void addRefreshDb(long id) {
         refreshDbIds.add(id);
     }
 
-    public void addCreateDb(long id, String name) {
+    public void addCreateDb(long id, String name, String remoteName) {
         createCount += 1;
         createDbIds.add(id);
         createDbNames.add(name);
+        remoteDbNames.add(remoteName);
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java

@@ -66,6 +66,9 @@ public enum Type {
     @SerializedName(value = "createTableNames")
     private List<String> createTableNames;
 
+    @SerializedName(value = "remoteTableNames")
+    private List<String> remoteTableNames;
+
     @SerializedName(value = "type")
     private Type type;
 
@@ -80,6 +83,7 @@ public InitDatabaseLog() {
         refreshTableIds = Lists.newArrayList();
         createTableIds = Lists.newArrayList();
         createTableNames = Lists.newArrayList();
+        remoteTableNames = Lists.newArrayList();
         type = Type.UNKNOWN;
     }
 
@@ -88,10 +92,11 @@ public void addRefreshTable(long id) {
         refreshTableIds.add(id);
     }
 
-    public void addCreateTable(long id, String name) {
+    public void addCreateTable(long id, String name, String remoteName) {
         createCount += 1;
         createTableIds.add(id);
         createTableNames.add(name);
+        remoteTableNames.add(remoteName);
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java

@@ -32,14 +32,18 @@ public class EsExternalDatabase extends ExternalDatabase<EsExternalTable> {
      * @param extCatalog External data source this database belongs to.
      * @param id database id.
      * @param name database name.
+     * @param remoteName remote database name.
      */
-    public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
-        super(extCatalog, id, name, InitDatabaseLog.Type.ES);
+    public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) {
+        super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ES);
     }
 
     @Override
-    protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog);
+    public EsExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new EsExternalTable(tblId, localTableName, remoteTableName, (EsExternalCatalog) extCatalog,
+                (EsExternalDatabase) db);
     }
 
     public void addTableForTest(EsExternalTable tbl) {

fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java

@@ -42,11 +42,12 @@ public class EsExternalTable extends ExternalTable {
      *
      * @param id Table id.
      * @param name Table name.
-     * @param dbName Database name.
-     * @param catalog HMSExternalDataSource.
+     * @param remoteName Remote table name.
+     * @param catalog EsExternalDataSource.
+     * @param db Database.
      */
-    public EsExternalTable(long id, String name, String dbName, EsExternalCatalog catalog) {
-        super(id, name, catalog, dbName, TableType.ES_EXTERNAL_TABLE);
+    public EsExternalTable(long id, String name, String remoteName, EsExternalCatalog catalog, EsExternalDatabase db) {
+        super(id, name, remoteName, catalog, db, TableType.ES_EXTERNAL_TABLE);
     }
 
     protected synchronized void makeSureInitialized() {

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java

@@ -263,7 +263,7 @@ public void registerDatabase(long dbId, String dbName) {
             LOG.debug("create database [{}]", dbName);
         }
 
-        ExternalDatabase<? extends ExternalTable> db = buildDbForInit(dbName, dbId, logType, false);
+        ExternalDatabase<? extends ExternalTable> db = buildDbForInit(dbName, null,  dbId, logType, false);
         if (useMetaCache.get()) {
             if (isInitialized()) {
                 metaCache.updateCache(dbName, db, Util.genIdByName(getQualifiedName(dbName)));

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java

@@ -32,14 +32,18 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
      * @param extCatalog External catalog this database belongs to.
      * @param id database id.
      * @param name database name.
+     * @param remoteName remote database name.
      */
-    public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
-        super(extCatalog, id, name, InitDatabaseLog.Type.HMS);
+    public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) {
+        super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.HMS);
     }
 
     @Override
-    protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog);
+    public HMSExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new HMSExternalTable(tblId, localTableName, remoteTableName, (HMSExternalCatalog) extCatalog,
+                (HMSExternalDatabase) db);
     }
 
     public void addTableForTest(HMSExternalTable tbl) {

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java

@@ -169,11 +169,13 @@ public enum DLAType {
      *
      * @param id Table id.
      * @param name Table name.
-     * @param dbName Database name.
-     * @param catalog HMSExternalCatalog.
+     * @param remoteName Remote table name.
+     * @param catalog HMSExternalDataSource.
+     * @param db Database.
      */
-    public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog) {
-        super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE);
+    public HMSExternalTable(long id, String name, String remoteName, HMSExternalCatalog catalog,
+            HMSExternalDatabase db) {
+        super(id, name, remoteName, catalog, db, TableType.HMS_EXTERNAL_TABLE);
     }
 
     // Will throw NotSupportedException if not supported hms table.

fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java

@@ -28,13 +28,16 @@
 
 public class IcebergExternalDatabase extends ExternalDatabase<IcebergExternalTable> {
 
-    public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name) {
-        super(extCatalog, id, name, InitDatabaseLog.Type.ICEBERG);
+    public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) {
+        super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ICEBERG);
     }
 
     @Override
-    protected IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new IcebergExternalTable(tblId, tableName, name, (IcebergExternalCatalog) extCatalog);
+    public IcebergExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new IcebergExternalTable(tblId, localTableName, remoteTableName, (IcebergExternalCatalog) extCatalog,
+                (IcebergExternalDatabase) db);
     }
 
     public String getLocation() {

fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java

@@ -36,8 +36,9 @@
 
 public class IcebergExternalTable extends ExternalTable {
 
-    public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) {
-        super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);
+    public IcebergExternalTable(long id, String name, String remoteName, IcebergExternalCatalog catalog,
+            IcebergExternalDatabase db) {
+        super(id, name, remoteName, catalog, db, TableType.ICEBERG_EXTERNAL_TABLE);
     }
 
     public String getIcebergCatalogType() {

fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java

@@ -36,16 +36,18 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase {
      * @param dbId The id of this database.
      */
     public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) {
-        super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB);
+        super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB);
     }
 
     public static List<String> listTableNames() {
         return Lists.newArrayList(SchemaTable.TABLE_MAP.keySet());
     }
 
     @Override
-    protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
-        return new ExternalInfoSchemaTable(tblId, tableName, catalog);
+    public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
+            ExternalCatalog catalog,
+            ExternalDatabase db) {
+        return new ExternalInfoSchemaTable(tblId, localTableName, catalog, db);
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java

@@ -18,9 +18,9 @@
 package org.apache.doris.datasource.infoschema;
 
 import org.apache.doris.analysis.SchemaTableType;
-import org.apache.doris.catalog.InfoSchemaDb;
 import org.apache.doris.catalog.SchemaTable;
 import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
 import org.apache.doris.datasource.ExternalTable;
 import org.apache.doris.datasource.SchemaCacheValue;
 import org.apache.doris.thrift.TSchemaTable;
@@ -31,8 +31,8 @@
 
 public class ExternalInfoSchemaTable extends ExternalTable {
 
-    public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) {
-        super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA);
+    public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) {
+        super(id, name, name, catalog, db, TableType.SCHEMA);
     }
 
     @Override