From a2ad5edbe3fa79b519c88348da3bc9a827ba7ccb Mon Sep 17 00:00:00 2001 From: RaigorJiang Date: Thu, 22 Aug 2024 17:58:51 +0800 Subject: [PATCH 1/3] Collect table type in PostgreSQLMetaDataLoader --- .../data/loader/MySQLMetaDataLoader.java | 6 +-- .../data/loader/PostgreSQLMetaDataLoader.java | 37 ++++++++++++++++--- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/infra/database/type/mysql/src/main/java/org/apache/shardingsphere/infra/database/mysql/metadata/data/loader/MySQLMetaDataLoader.java b/infra/database/type/mysql/src/main/java/org/apache/shardingsphere/infra/database/mysql/metadata/data/loader/MySQLMetaDataLoader.java index 1484baac74177..0b3499d007a95 100644 --- a/infra/database/type/mysql/src/main/java/org/apache/shardingsphere/infra/database/mysql/metadata/data/loader/MySQLMetaDataLoader.java +++ b/infra/database/type/mysql/src/main/java/org/apache/shardingsphere/infra/database/mysql/metadata/data/loader/MySQLMetaDataLoader.java @@ -103,7 +103,7 @@ private String getViewMetaDataSQL(final Collection tableNames) { } private Map> loadConstraintMetaDataMap(final DataSource dataSource, final Collection tables) throws SQLException { - Map> result = new LinkedHashMap<>(); + Map> result = new LinkedHashMap<>(tables.size(), 1F); try ( Connection connection = dataSource.getConnection(); PreparedStatement preparedStatement = connection.prepareStatement(getConstraintMetaDataSQL(tables))) { @@ -127,7 +127,7 @@ private String getConstraintMetaDataSQL(final Collection tableNames) { } private Map> loadColumnMetaDataMap(final DataSource dataSource, final Collection tables) throws SQLException { - Map> result = new HashMap<>(); + Map> result = new HashMap<>(tables.size(), 1F); try ( Connection connection = dataSource.getConnection(); PreparedStatement preparedStatement = connection.prepareStatement(getTableMetaDataSQL(tables))) { @@ -167,7 +167,7 @@ private String getTableMetaDataSQL(final Collection tables) { } private Map> loadIndexMetaData(final DataSource dataSource, final Collection tableNames) throws SQLException { - Map> tableToIndex = new HashMap<>(); + Map> tableToIndex = new HashMap<>(tableNames.size(), 1F); try ( Connection connection = dataSource.getConnection(); PreparedStatement preparedStatement = connection.prepareStatement(getIndexMetaDataSQL(tableNames))) { diff --git a/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java b/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java index c2d292a0f60cc..6879dc544f56a 100644 --- a/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java +++ b/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java @@ -28,6 +28,7 @@ import org.apache.shardingsphere.infra.database.core.metadata.data.model.SchemaMetaData; import org.apache.shardingsphere.infra.database.core.metadata.data.model.TableMetaData; import org.apache.shardingsphere.infra.database.core.metadata.database.datatype.DataTypeLoader; +import org.apache.shardingsphere.infra.database.core.metadata.database.enums.TableType; import org.apache.shardingsphere.infra.database.core.type.DatabaseType; import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader; @@ -36,6 +37,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -76,6 +78,8 @@ public final class PostgreSQLMetaDataLoader implements DialectMetaDataLoader { private static final String LOAD_FILTERED_ROLE_TABLE_GRANTS_SQL = LOAD_ALL_ROLE_TABLE_GRANTS_SQL + " WHERE table_name IN (%s)"; + private static final String VIEW_META_DATA_SQL = "SELECT table_schema, table_name FROM information_schema.views WHERE table_schema IN (%s) and table_name IN (%s)"; + @Override public Collection load(final MetaDataLoaderMaterial material) throws SQLException { try (Connection connection = material.getDataSource().getConnection()) { @@ -83,19 +87,21 @@ public Collection load(final MetaDataLoaderMaterial material) th Map> schemaIndexMetaDataMap = loadIndexMetaDataMap(connection, schemaNames); Map> schemaColumnMetaDataMap = loadColumnMetaDataMap(connection, material.getActualTableNames(), schemaNames); Map> schemaConstraintMetaDataMap = loadConstraintMetaDataMap(connection, schemaNames); + Map> schemaViewNames = loadViewNames(connection, schemaNames, material.getActualTableNames()); Collection result = new LinkedList<>(); for (String each : schemaNames) { Multimap tableIndexMetaDataMap = schemaIndexMetaDataMap.getOrDefault(each, LinkedHashMultimap.create()); Multimap tableColumnMetaDataMap = schemaColumnMetaDataMap.getOrDefault(each, LinkedHashMultimap.create()); Multimap tableConstraintMetaDataMap = schemaConstraintMetaDataMap.getOrDefault(each, LinkedHashMultimap.create()); - result.add(new SchemaMetaData(each, createTableMetaDataList(tableIndexMetaDataMap, tableColumnMetaDataMap, tableConstraintMetaDataMap))); + Collection viewNames = schemaViewNames.getOrDefault(each, Collections.emptySet()); + result.add(new SchemaMetaData(each, createTableMetaDataList(tableIndexMetaDataMap, tableColumnMetaDataMap, tableConstraintMetaDataMap, viewNames))); } return result; } } private Map> loadIndexMetaDataMap(final Connection connection, final Collection schemaNames) throws SQLException { - Map> result = new LinkedHashMap<>(); + Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); try (PreparedStatement preparedStatement = connection.prepareStatement(getIndexMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("schemaname"); @@ -136,7 +142,7 @@ private String getAdvanceIndexMetaDataSQL(final Collection schemaNames) private Map> loadColumnMetaDataMap(final Connection connection, final Collection tables, final Collection schemaNames) throws SQLException { - Map> result = new LinkedHashMap<>(); + Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); Collection roleTableGrants = loadRoleTableGrants(connection, tables); try (PreparedStatement preparedStatement = connection.prepareStatement(getColumnMetaDataSQL(schemaNames, tables)); ResultSet resultSet = preparedStatement.executeQuery()) { Map dataTypes = new DataTypeLoader().load(connection.getMetaData(), getType()); @@ -207,7 +213,7 @@ private ColumnMetaData loadColumnMetaData(final Map dataTypeMap } private Map> loadConstraintMetaDataMap(final Connection connection, final Collection schemaNames) throws SQLException { - Map> result = new LinkedHashMap<>(); + Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); try (PreparedStatement preparedStatement = connection.prepareStatement(getConstraintKeyMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); @@ -225,14 +231,33 @@ private String getConstraintKeyMetaDataSQL(final Collection schemaNames) return String.format(FOREIGN_KEY_META_DATA_SQL, schemaNames.stream().map(each -> String.format("'%s'", each)).collect(Collectors.joining(","))); } + private Map> loadViewNames(final Connection connection, final Collection schemaNames, final Collection tables) throws SQLException { + Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); + try (PreparedStatement preparedStatement = connection.prepareStatement(getViewMetaDataSQL(schemaNames, tables)); + ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + String schemaName = resultSet.getString("table_schema"); + Collection viewMetaData = result.computeIfAbsent(schemaName, key -> new LinkedList<>()); + String tableName = resultSet.getString("table_name"); + viewMetaData.add(tableName); + } + } + return result; + } + + private String getViewMetaDataSQL(final Collection schemaNames, final Collection tables) { + return String.format(VIEW_META_DATA_SQL, schemaNames.stream().map(each -> String.format("'%s'", each)).collect(Collectors.joining(",")), + tables.stream().map(each -> String.format("'%s'", each)).collect(Collectors.joining(","))); + } + private Collection createTableMetaDataList(final Multimap tableIndexMetaDataMap, final Multimap tableColumnMetaDataMap, - final Multimap tableConstraintMetaDataMap) { + final Multimap tableConstraintMetaDataMap, final Collection viewNames) { Collection result = new LinkedList<>(); for (String each : tableColumnMetaDataMap.keySet()) { Collection columnMetaDataList = tableColumnMetaDataMap.get(each); Collection indexMetaDataList = tableIndexMetaDataMap.get(each); Collection constraintMetaDataList = tableConstraintMetaDataMap.get(each); - result.add(new TableMetaData(each, columnMetaDataList, indexMetaDataList, constraintMetaDataList)); + result.add(new TableMetaData(each, columnMetaDataList, indexMetaDataList, constraintMetaDataList, viewNames.contains(each) ? TableType.VIEW : TableType.TABLE)); } return result; } From 57a363382e4fc6bbffe631d6b962ff9c35646c00 Mon Sep 17 00:00:00 2001 From: RaigorJiang Date: Thu, 22 Aug 2024 18:08:48 +0800 Subject: [PATCH 2/3] Fix checkstyle in try block --- .../loader/type/ColumnMetaDataLoader.java | 4 ++- .../data/loader/OpenGaussMetaDataLoader.java | 16 +++++++--- .../data/loader/PostgreSQLMetaDataLoader.java | 29 ++++++++++++++----- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/infra/database/core/src/main/java/org/apache/shardingsphere/infra/database/core/metadata/data/loader/type/ColumnMetaDataLoader.java b/infra/database/core/src/main/java/org/apache/shardingsphere/infra/database/core/metadata/data/loader/type/ColumnMetaDataLoader.java index 33c1d66aadfbd..440c335849964 100644 --- a/infra/database/core/src/main/java/org/apache/shardingsphere/infra/database/core/metadata/data/loader/type/ColumnMetaDataLoader.java +++ b/infra/database/core/src/main/java/org/apache/shardingsphere/infra/database/core/metadata/data/loader/type/ColumnMetaDataLoader.java @@ -79,7 +79,9 @@ public static Collection load(final Connection connection, final } } } - try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(generateEmptyResultSQL(tableNamePattern, columnNames, databaseType))) { + try ( + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(generateEmptyResultSQL(tableNamePattern, columnNames, databaseType))) { for (int i = 0; i < columnNames.size(); i++) { boolean generated = resultSet.getMetaData().isAutoIncrement(i + 1); caseSensitiveFlags.add(resultSet.getMetaData().isCaseSensitive(resultSet.findColumn(columnNames.get(i)))); diff --git a/infra/database/type/opengauss/src/main/java/org/apache/shardingsphere/infra/database/opengauss/metadata/data/loader/OpenGaussMetaDataLoader.java b/infra/database/type/opengauss/src/main/java/org/apache/shardingsphere/infra/database/opengauss/metadata/data/loader/OpenGaussMetaDataLoader.java index ac2c19488946d..baa86be01fa69 100644 --- a/infra/database/type/opengauss/src/main/java/org/apache/shardingsphere/infra/database/opengauss/metadata/data/loader/OpenGaussMetaDataLoader.java +++ b/infra/database/type/opengauss/src/main/java/org/apache/shardingsphere/infra/database/opengauss/metadata/data/loader/OpenGaussMetaDataLoader.java @@ -84,7 +84,9 @@ public Collection load(final MetaDataLoaderMaterial material) th private Map> loadIndexMetaDataMap(final Connection connection, final Collection schemaNames) throws SQLException { Map> result = new LinkedHashMap<>(); - try (PreparedStatement preparedStatement = connection.prepareStatement(getIndexMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getIndexMetaDataSQL(schemaNames)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("schemaname"); String tableName = resultSet.getString("tablename"); @@ -93,7 +95,9 @@ private Map> loadIndexMetaDataMap(final indexMetaDataMap.put(tableName, new IndexMetaData(indexName)); } } - try (PreparedStatement preparedStatement = connection.prepareStatement(getAdvanceIndexMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getAdvanceIndexMetaDataSQL(schemaNames)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("index_schema"); String tableName = resultSet.getString("table_name"); @@ -125,7 +129,9 @@ private String getAdvanceIndexMetaDataSQL(final Collection schemaNames) private Map> loadColumnMetaDataMap(final Connection connection, final Collection tables, final Collection schemaNames) throws SQLException { Map> result = new LinkedHashMap<>(); - try (PreparedStatement preparedStatement = connection.prepareStatement(getColumnMetaDataSQL(schemaNames, tables)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getColumnMetaDataSQL(schemaNames, tables)); + ResultSet resultSet = preparedStatement.executeQuery()) { Map dataTypes = new DataTypeLoader().load(connection.getMetaData(), getType()); Collection primaryKeys = loadPrimaryKeys(connection, schemaNames); while (resultSet.next()) { @@ -146,7 +152,9 @@ private String getColumnMetaDataSQL(final Collection schemaNames, final private Collection loadPrimaryKeys(final Connection connection, final Collection schemaNames) throws SQLException { Collection result = new HashSet<>(); - try (PreparedStatement preparedStatement = connection.prepareStatement(getPrimaryKeyMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getPrimaryKeyMetaDataSQL(schemaNames)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); String tableName = resultSet.getString("table_name"); diff --git a/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java b/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java index 6879dc544f56a..16f340c1148f0 100644 --- a/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java +++ b/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java @@ -102,7 +102,9 @@ public Collection load(final MetaDataLoaderMaterial material) th private Map> loadIndexMetaDataMap(final Connection connection, final Collection schemaNames) throws SQLException { Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); - try (PreparedStatement preparedStatement = connection.prepareStatement(getIndexMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getIndexMetaDataSQL(schemaNames)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("schemaname"); String tableName = resultSet.getString("tablename"); @@ -111,7 +113,9 @@ private Map> loadIndexMetaDataMap(final indexMetaDataMap.put(tableName, new IndexMetaData(indexName)); } } - try (PreparedStatement preparedStatement = connection.prepareStatement(getAdvanceIndexMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getAdvanceIndexMetaDataSQL(schemaNames)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("index_schema"); String tableName = resultSet.getString("table_name"); @@ -144,7 +148,9 @@ private Map> loadColumnMetaDataMap(fina final Collection schemaNames) throws SQLException { Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); Collection roleTableGrants = loadRoleTableGrants(connection, tables); - try (PreparedStatement preparedStatement = connection.prepareStatement(getColumnMetaDataSQL(schemaNames, tables)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getColumnMetaDataSQL(schemaNames, tables)); + ResultSet resultSet = preparedStatement.executeQuery()) { Map dataTypes = new DataTypeLoader().load(connection.getMetaData(), getType()); Collection primaryKeys = loadPrimaryKeys(connection, schemaNames); while (resultSet.next()) { @@ -162,7 +168,9 @@ private Map> loadColumnMetaDataMap(fina private Collection loadRoleTableGrants(final Connection connection, final Collection tables) throws SQLException { Collection result = new HashSet<>(tables.size(), 1F); - try (PreparedStatement preparedStatement = connection.prepareStatement(getLoadRoleTableGrantsSQL(tables)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getLoadRoleTableGrantsSQL(tables)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { result.add(resultSet.getString("table_name")); } @@ -183,7 +191,9 @@ private String getColumnMetaDataSQL(final Collection schemaNames, final private Set loadPrimaryKeys(final Connection connection, final Collection schemaNames) throws SQLException { Set result = new HashSet<>(); - try (PreparedStatement preparedStatement = connection.prepareStatement(getPrimaryKeyMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getPrimaryKeyMetaDataSQL(schemaNames)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); String tableName = resultSet.getString("table_name"); @@ -214,7 +224,9 @@ private ColumnMetaData loadColumnMetaData(final Map dataTypeMap private Map> loadConstraintMetaDataMap(final Connection connection, final Collection schemaNames) throws SQLException { Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); - try (PreparedStatement preparedStatement = connection.prepareStatement(getConstraintKeyMetaDataSQL(schemaNames)); ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getConstraintKeyMetaDataSQL(schemaNames)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); Multimap constraintMetaData = result.computeIfAbsent(schemaName, key -> LinkedHashMultimap.create()); @@ -233,8 +245,9 @@ private String getConstraintKeyMetaDataSQL(final Collection schemaNames) private Map> loadViewNames(final Connection connection, final Collection schemaNames, final Collection tables) throws SQLException { Map> result = new LinkedHashMap<>(schemaNames.size(), 1F); - try (PreparedStatement preparedStatement = connection.prepareStatement(getViewMetaDataSQL(schemaNames, tables)); - ResultSet resultSet = preparedStatement.executeQuery()) { + try ( + PreparedStatement preparedStatement = connection.prepareStatement(getViewMetaDataSQL(schemaNames, tables)); + ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); Collection viewMetaData = result.computeIfAbsent(schemaName, key -> new LinkedList<>()); From 9231db0d7bd96b28e80e51b18020dc5764bdc385 Mon Sep 17 00:00:00 2001 From: RaigorJiang Date: Thu, 22 Aug 2024 18:27:53 +0800 Subject: [PATCH 3/3] Change LinkedList to HashSet --- .../metadata/data/loader/PostgreSQLMetaDataLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java b/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java index 16f340c1148f0..253c1cf2f01f5 100644 --- a/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java +++ b/infra/database/type/postgresql/src/main/java/org/apache/shardingsphere/infra/database/postgresql/metadata/data/loader/PostgreSQLMetaDataLoader.java @@ -250,7 +250,7 @@ private Map> loadViewNames(final Connection connectio ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String schemaName = resultSet.getString("table_schema"); - Collection viewMetaData = result.computeIfAbsent(schemaName, key -> new LinkedList<>()); + Collection viewMetaData = result.computeIfAbsent(schemaName, key -> new HashSet<>()); String tableName = resultSet.getString("table_name"); viewMetaData.add(tableName); }