From fb38c41e71e011b0b40882a31afbbb54a81bacec Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 6 Aug 2024 11:23:51 +0800 Subject: [PATCH] [chore](sql) Forbid show hidden columns and create table with hidden column (#38796) Forbid show hidden columns and create table with hidden column (cherry picked from commit 9eae4ba773b52970265aa03c7405a06285021bb1) --- .../doris/analysis/CreateTableStmt.java | 5 +++++ .../java/org/apache/doris/catalog/Column.java | 3 +++ .../java/org/apache/doris/catalog/Env.java | 8 +------ .../plans/commands/info/CreateTableInfo.java | 5 +++++ .../doris/regression/action/TestAction.groovy | 2 +- .../test_show_create_table_and_views.groovy | 22 +++++++++++++++++++ ...show_create_table_and_views_nereids.groovy | 20 +++++++++++++++++ 7 files changed, 57 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 315c56a2864cdc..70a62f1c2c61c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -311,6 +311,11 @@ public void analyze(Analyzer analyzer) throws UserException { if (Objects.equals(columnDef.getType(), Type.ALL)) { throw new AnalysisException("Disable to create table with `ALL` type columns."); } + String columnNameUpperCase = columnDef.getName().toUpperCase(); + if (columnNameUpperCase.startsWith("__DORIS_")) { + throw new AnalysisException( + "Disable to create table column with name start with __DORIS_: " + columnNameUpperCase); + } if (Objects.equals(columnDef.getType(), Type.DATE) && Config.disable_datev1) { throw new AnalysisException("Disable to create table with `DATE` type columns, please use `DATEV2`."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index b30b387865a6c0..6aefc66eacdd11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -55,6 +55,7 @@ */ public class Column implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(Column.class); + // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__"; public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__"; public static final String SEQUENCE_COL = "__DORIS_SEQUENCE_COL__"; @@ -62,6 +63,8 @@ public class Column implements Writable, GsonPostProcessable { public static final String ROW_STORE_COL = "__DORIS_ROW_STORE_COL__"; public static final String DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__"; public static final String VERSION_COL = "__DORIS_VERSION_COL__"; + // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + private static final String COLUMN_ARRAY_CHILDREN = "item"; private static final String COLUMN_STRUCT_CHILDREN = "field"; private static final String COLUMN_AGG_ARGUMENT_CHILDREN = "argument"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index f3a3b22d7dd10f..dae7e732497461 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3520,13 +3520,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append(" (\n"); int idx = 0; - List columns; - // when 'create table B like A', always return schema of A without hidden columns - if (getDdlForLike) { - columns = table.getBaseSchema(false); - } else { - columns = table.getBaseSchema(); - } + List columns = table.getBaseSchema(false); for (Column column : columns) { if (idx++ != 0) { sb.append(",\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index ed573fd98663ed..8136b464f5839c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -262,6 +262,11 @@ public void validate(ConnectContext ctx) { //check datev1 and decimalv2 for (ColumnDefinition columnDef : columns) { + String columnNameUpperCase = columnDef.getName().toUpperCase(); + if (columnNameUpperCase.startsWith("__DORIS_")) { + throw new AnalysisException( + "Disable to create table column with name start with __DORIS_: " + columnNameUpperCase); + } if (columnDef.getType().isDateType() && Config.disable_datev1) { throw new AnalysisException( "Disable to create table with `DATE` type columns, please use `DATEV2`."); diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy index 684cb9d5beb86c..3febe6bbad57cf 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy @@ -256,7 +256,7 @@ class TestAction implements SuiteAction { this.exception = exceptionMsgSupplier.call() } - void check(@ClosureParams(value = FromString, options = ["String,Throwable,Long,Long"]) Closure check) { + void check(@ClosureParams(value = FromString, options = ["List>,Throwable,Long,Long"]) Closure check) { this.check = check } diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy index f7497f9ee517c1..6dffc47d14ea68 100644 --- a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy +++ b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy @@ -15,8 +15,30 @@ // specific language governing permissions and limitations // under the License. +import org.apache.doris.regression.util.JdbcUtils + suite("test_show_create_table_and_views", "show") { sql "SET enable_nereids_planner=false;" + + def shouldNotShowHiddenColumnsAndCreateWithHiddenColumns = { + connect { + multi_sql """ + SET enable_nereids_planner=false; + drop table if exists test_show_create_table_no_hidden_column; + create table test_show_create_table_no_hidden_column(id int, name varchar(50)) unique key(id) distributed by hash(id) properties('replication_num'='1'); + set show_hidden_columns=true; + """ + + def result = JdbcUtils.executeToMapArray(context.getConnection(), "show create table test_show_create_table_no_hidden_column") + assertTrue(!result[0].get("Create Table").toString().contains("__DORIS_DELETE_SIGN__")) + + test { + sql "create table table_with_hidden_sign(id int, __DORIS_DELETE_SIGN__ int) distributed by hash(id) properties('replication_num'='1')" + exception "Disable to create table column with name start with __DORIS_: " + } + } + }() + def ret = sql "SHOW FRONTEND CONFIG like '%enable_feature_binlog%';" logger.info("${ret}") if (ret.size() != 0 && ret[0].size() > 1 && ret[0][1] == 'false') { diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy index 81102d90c3a56d..07bdfbfd821e04 100644 --- a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy +++ b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy @@ -15,7 +15,27 @@ // specific language governing permissions and limitations // under the License. +import org.apache.doris.regression.util.JdbcUtils + suite("test_show_create_table_and_views_nereids", "show") { + def shouldNotShowHiddenColumnsAndCreateWithHiddenColumns = { + connect { + multi_sql """ + drop table if exists test_show_create_table_no_hidden_column; + create table test_show_create_table_no_hidden_column(id int, name varchar(50)) unique key(id) distributed by hash(id) properties('replication_num'='1'); + set show_hidden_columns=true; + """ + + def result = JdbcUtils.executeToMapArray(context.getConnection(), "show create table test_show_create_table_no_hidden_column") + assertTrue(!result[0].get("Create Table").toString().contains("__DORIS_DELETE_SIGN__")) + + test { + sql "create table table_with_hidden_sign(id int, __DORIS_DELETE_SIGN__ int) distributed by hash(id) properties('replication_num'='1')" + exception "Disable to create table column with name start with __DORIS_: " + } + } + }() + def ret = sql "SHOW FRONTEND CONFIG like '%enable_feature_binlog%';" logger.info("${ret}") if (ret.size() != 0 && ret[0].size() > 1 && ret[0][1] == 'false') {