diff --git a/be/src/olap/partial_update_info.cpp b/be/src/olap/partial_update_info.cpp index bff3f4196369db2..898a4a0fdbba337 100644 --- a/be/src/olap/partial_update_info.cpp +++ b/be/src/olap/partial_update_info.cpp @@ -25,6 +25,7 @@ #include "olap/rowset/rowset_writer_context.h" #include "olap/tablet_schema.h" #include "olap/utils.h" +#include "util/bitmap_value.h" #include "vec/common/assert_cast.h" #include "vec/core/block.h" @@ -144,6 +145,11 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids( DateV2Value dv; dv.from_unixtime(timestamp_ms / 1000, timezone); default_value = dv.debug_string(); + } else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT && + to_lower(column.default_value()).find(to_lower("BITMAP_EMPTY")) != + std::string::npos)) { + BitmapValue v = BitmapValue {}; + default_value = v.to_string(); } else { default_value = column.default_value(); } diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 42092b896822c2d..2b3d0bc2a3a9053 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -114,6 +114,7 @@ BINARY: 'BINARY'; BINLOG: 'BINLOG'; BITAND: 'BITAND'; BITMAP: 'BITMAP'; +BITMAP_EMPTY: 'BITMAP_EMPTY'; BITMAP_UNION: 'BITMAP_UNION'; BITOR: 'BITOR'; BITXOR: 'BITXOR'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index b46b55a96e0da8f..ffd20f8e5ff647c 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1194,7 +1194,7 @@ columnDef ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)? ((NOT)? nullable=NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? - (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | stringValue=STRING_LITERAL + (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | PI | BITMAP_EMPTY | stringValue=STRING_LITERAL | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? (COMMENT comment=STRING_LITERAL)? @@ -1656,6 +1656,7 @@ nonReserved | BIN | BITAND | BITMAP + | BITMAP_EMPTY | BITMAP_UNION | BITOR | BITXOR diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index e01a4f117934557..dd5faeb779149f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -422,8 +422,10 @@ public void analyze(boolean isOlap) throws AnalysisException { } if (type.getPrimitiveType() == PrimitiveType.BITMAP) { - if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Bitmap type column can not set default value"); + if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE + && !defaultValue.value.equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value)) { + throw new AnalysisException("Bitmap type column default value only support null or " + + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value); } defaultValue = DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 7369b714f7439c1..1a6d0bef8149b50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2780,6 +2780,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { defaultValue = Optional.of(DefaultValue.CURRENT_DATE_DEFAULT_VALUE); } else if (ctx.PI() != null) { defaultValue = Optional.of(DefaultValue.PI_DEFAULT_VALUE); + } else if (ctx.BITMAP_EMPTY() != null) { + defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } } if (ctx.UPDATE() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 0b2694cc311b4c0..184b6d2fa555f61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -318,13 +318,15 @@ public void validate(boolean isOlap, Set keysSet, Set clusterKey } defaultValue = Optional.of(DefaultValue.HLL_EMPTY_DEFAULT_VALUE); } else if (type.isBitmapType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Bitmap type column can not set default value"); + if (defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE + && !defaultValue.get().getValue().equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.getValue())) { + throw new AnalysisException("Bitmap type column default value only support " + + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } else if (type.isArrayType() && defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get() - .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { + .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { throw new AnalysisException("Array type column default value only support null or " + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE); } else if (type.isMapType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java index 6df1d3d7cf1451f..366082fa9b431e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java @@ -29,6 +29,7 @@ public class DefaultValue { public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP"; public static String NOW = "now"; public static String HLL_EMPTY = "HLL_EMPTY"; + public static String BITMAP_EMPTY = "BITMAP_EMPTY"; public static DefaultValue CURRENT_DATE_DEFAULT_VALUE = new DefaultValue(CURRENT_DATE, CURRENT_DATE.toLowerCase()); public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new DefaultValue(CURRENT_TIMESTAMP, NOW); // default null @@ -38,7 +39,7 @@ public class DefaultValue { // default "value", "0" means empty hll public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, HLL_EMPTY); // default "value", "0" means empty bitmap - public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO); + public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, BITMAP_EMPTY); // default "value", "[]" means empty array public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue("[]"); // default "value", "3.14159265358979323846" means pi, refer to M_PI in . diff --git a/regression-test/data/correctness_p0/test_default_bitmap_empty.out b/regression-test/data/correctness_p0/test_default_bitmap_empty.out new file mode 100644 index 000000000000000..33cc825e767f557 --- /dev/null +++ b/regression-test/data/correctness_p0/test_default_bitmap_empty.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !insert_into1 -- +0 +0 +0 +0 + +-- !stream_load_csv1 -- +0 +0 +0 +0 +0 +0 + +-- !select_2 -- +0 +0 +0 +0 + +-- !stream_load_csv2 -- +0 +0 +0 +0 +0 +0 + +-- !insert_into3 -- +0 +0 +0 +0 + +-- !stream_load_csv3 -- +0 +0 +0 +0 +0 +0 + diff --git a/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv b/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv new file mode 100644 index 000000000000000..f4ec2d7748a0a2c --- /dev/null +++ b/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv @@ -0,0 +1,2 @@ +5,5 +6,6 \ No newline at end of file diff --git a/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy b/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy new file mode 100644 index 000000000000000..60b6ff73cb37e87 --- /dev/null +++ b/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_default_bitmap_empty") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + def tableName = "test_default_bitmap_empty" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 bitmap NOT NULL DEFAULT bitmap_empty, + v2 INT + ) + UNIQUE KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + // test insert into. + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + qt_insert_into1 """ select bitmap_count(v1) from ${tableName}; """ + + // test csv stream load. + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'columns', 'k, v1=bitmap_empty(), v2' + + file 'test_default_bitmap_empty_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv1 """ select bitmap_count(v1) from ${tableName}; """ + + // test partial update + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 bitmap NOT NULL DEFAULT bitmap_empty, + v2 INT + ) + UNIQUE KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + + qt_select_2 "select bitmap_count(v1) from ${tableName};" + + streamLoad { + table "${tableName}" + + set 'partial_columns', 'true' + set 'column_separator', ',' + set 'columns', 'k, v2' + + file 'test_default_bitmap_empty_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv2 """ select bitmap_count(v1) from ${tableName}; """ + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 bitmap BITMAP_UNION default BITMAP_EMPTY, + v2 INT replace_if_not_null + ) + aggregate KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + // test insert into. + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + qt_insert_into3 """ select bitmap_count(v1) from ${tableName}; """ + + // test csv stream load. + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'columns', 'k, v1=bitmap_empty(), v2' + + file 'test_default_bitmap_empty_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv3 """ select bitmap_count(v1) from ${tableName}; """ +} \ No newline at end of file