Merge branch 'StarRocks:main' into show_catalog_recycle_bin

StarRocks · Sep 20, 2024 · 6db6550 · 6db6550
2 parents c29ecac + ddd1a33
commit 6db6550
Show file tree

Hide file tree

Showing 512 changed files with 14,904 additions and 6,371 deletions.
diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml
@@ -691,7 +691,7 @@ jobs:
         run: |
           echo "package=${package}" >> "$GITHUB_ENV"
           if [[ "${be_change}" != "true" && "${fe_change}" != "true" ]]; then
-            oss_path=oss://${bucket_prefix}-ci-release/${BRANCH}/${build_type}/inspection/pr/ubuntu/StarRocks-
+            oss_path=oss://${bucket_prefix}-us-ci-release/${BRANCH}/${build_type}/inspection/pr/ubuntu/StarRocks-
             package=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "tar.gz" | sort -n -r | head -n 1 | awk '{print $NF}')
             echo "package=${package}" >> $GITHUB_ENV
           fi
@@ -799,7 +799,7 @@ jobs:
           
           if [[ "${{ needs.build.outputs.is_self_build }}" == 'true' ]]; then 
             bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
-            tar_path="oss://${bucket_prefix}-ci-release/${{github.base_ref}}/Release/pr/${LINUX_DISTRO}/${{needs.build.outputs.build_output_tar}}"
+            tar_path="oss://${bucket_prefix}-us-ci-release/${{github.base_ref}}/Release/pr/${LINUX_DISTRO}/${{needs.build.outputs.build_output_tar}}"
           else
             tar_path="${{ needs.build.outputs.build_output_tar }}"
           fi
@@ -872,7 +872,7 @@ jobs:
         run: |
           branch=${{ steps.branch.outputs.branch }}
           bucket_prefix=${{ steps.branch.outputs.bucket_prefix }}
-          xml_oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
+          xml_oss_path=oss://${bucket_prefix}-us-ci-release/$branch/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
           ossutil64 --config-file ~/.ossutilconfig rm ${xml_oss_path} -rf
           ossutil64 --config-file ~/.ossutilconfig cp test/ ${xml_oss_path} --include "*.xml" --recursive --force --tagging="type=ci"
 
@@ -989,7 +989,7 @@ jobs:
         run: |
           branch=${{ steps.branch.outputs.branch }}
           bucket_prefix=${{ steps.branch.outputs.bucket_prefix }}
-          xml_oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/Admit-XML/${PR_NUMBER}/
+          xml_oss_path=oss://${bucket_prefix}-us-ci-release/$branch/Release/pr/Admit-XML/${PR_NUMBER}/
           ossutil64 --config-file ~/.ossutilconfig rm ${xml_oss_path} -rf
           ossutil64 --config-file ~/.ossutilconfig cp StarRocksTest/result ${xml_oss_path} --include "*.xml" --recursive --force --tagging="type=ci"
 

diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml
@@ -96,7 +96,7 @@ jobs:
       - name: Download FE UT XML
         id: download-ut-xml
         run: |
-          oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
+          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
           size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/fe_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
           echo "size=${size}" >> $GITHUB_OUTPUT
           if [[ "$size" != "0" ]]; then
@@ -159,7 +159,7 @@ jobs:
           java -jar $package_path/jacococli.jar report ${fe_path}/merge.exec --classfiles ${fe_core_path}/target/classes/ \
             --html ./result --xml ${{ github.workspace }}/coverage.xml \
             --sourcefiles ${fe_core_path}/src/main/java/ --encoding utf-8 --name fe-coverage
-          oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}/
+          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}/
           ossutil64 --config-file ~/.ossutilconfig cp coverage.xml ${oss_path} -f
 
       # Incremental Total Coverage
@@ -227,7 +227,7 @@ jobs:
       - name: Download BE UT XML
         id: download-ut-xml
         run: |
-          oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
+          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
           be_ut_res_path=${oss_path}/flag/be_ut_res
           size=$(ossutil64 --config-file ~/.ossutilconfig ls ${be_ut_res_path} | grep "Object Number is" | awk '{print $NF}')
           echo "size=${size}" >> $GITHUB_OUTPUT
@@ -313,7 +313,7 @@ jobs:
       - name: Download Java Extension UT XML
         id: download-ut-xml
         run: |
-          oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
+          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
           size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/java_extension_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
           echo "size=${size}" >> $GITHUB_OUTPUT
           if [[ "$size" != "0" ]]; then
@@ -390,7 +390,7 @@ jobs:
           bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
         run: |
           mkdir sql-tester-result && cd sql-tester-result
-          oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
+          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
           size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
           echo "size=${size}" >> $GITHUB_OUTPUT
           if [[ "$size" != "0" ]]; then
@@ -435,7 +435,7 @@ jobs:
 
         run: |
           mkdir admit-result && cd admit-result
-          oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/Admit-XML/${PR_NUMBER}/
+          oss_path=oss://${bucket_prefix}-us-ci-release/$BASE_REF/Release/pr/Admit-XML/${PR_NUMBER}/
           size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
           echo "size=${size}" >> $GITHUB_OUTPUT
           if [[ "$size" != "0" ]]; then

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -2,7 +2,7 @@
 
 StarRocks is built by an open and friendly community. We are dedicated to building a collaborative, inspiring, and exuberant open-source community for our members. Everyone is more than welcome to join our community to get help and to contribute to StarRocks.
 
-## Table of content
+## Table of contents
 
 - [How to contribute](#How-to-contribute)
 - [Contributing guideline](#Contributing-guideline)

diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
@@ -245,6 +245,10 @@ endforeach()
 find_package(AWSSDK REQUIRED COMPONENTS core s3 s3-crt transfer identity-management sts)
 include_directories(${AWSSDK_INCLUDE_DIRS})
 
+set(Poco_ROOT_DIR ${THIRDPARTY_DIR})
+find_package(Poco REQUIRED COMPONENTS Net NetSSL)
+include_directories(&{Poco_INCLUDE_DIRS})
+
 add_library(libevent STATIC IMPORTED)
 set_target_properties(libevent PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libevent.a)
 
@@ -493,7 +497,6 @@ if ("${USE_STAROS}" STREQUAL "ON")
 endif()
 
 # LLVM
-# TODO(yueyang): Adjust libraries during the implementation of JIT.
 set (LLVM_LIBRARIES
     LLVMRuntimeDyld
     LLVMBitstreamReader
@@ -554,6 +557,7 @@ foreach(lib IN ITEMS ${LLVM_LIBRARIES})
     set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/llvm/lib/lib${lib}.a)
 endforeach()
 
+
 # Check Clang-Tidy
 include(cmake_modules/FindClangTidy.cmake)
 
@@ -673,6 +677,11 @@ if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new")
 endif()
 
+if (STARROCKS_JIT_ENABLE)
+    message(STATUS "Enable JIT")
+    set(CXX_GCC_FLAGS "${CXX_GCC_FLAGS} -DSTARROCKS_JIT_ENABLE")
+endif()
+
 # For any gcc builds:
 #   -g: Enable symbols for profiler tools
 #   -Wno-unused-local-typedefs: Do not warn for local typedefs that are unused.
@@ -833,14 +842,20 @@ set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
 )
 set_target_properties(aws-cpp-sdk-core PROPERTIES INTERFACE_LINK_LIBRARIES AWS::aws-crt-cpp)
 
+if (STARROCKS_JIT_ENABLE)
+    set(STARROCKS_DEPENDENCIES 
+        ${STARROCKS_DEPENDENCIES}
+        ${WL_START_GROUP}
+        ${LLVM_LIBRARIES}
+        ${WL_END_GROUP}
+    )
+endif()
+
 # Set thirdparty libraries
 set(STARROCKS_DEPENDENCIES
     ${STARROCKS_DEPENDENCIES}
     build_version
     ${WL_START_GROUP}
-    ${LLVM_LIBRARIES}
-    ${WL_END_GROUP}
-    ${WL_START_GROUP}
     clucene-core
     clucene-shared
     clucene-contribs-lib
@@ -858,6 +873,7 @@ set(STARROCKS_DEPENDENCIES
     lz4
     libevent
     curl
+    ${Poco_LIBRARIES}
     libz
     libbz2
     gflags

diff --git a/be/src/agent/publish_version.cpp b/be/src/agent/publish_version.cpp
@@ -249,12 +249,16 @@ void run_publish_version_task(ThreadPoolToken* token, const TPublishVersionReque
                                             tablet_info.tablet_id, partition_version.version, transaction_id);
             } else {
                 const int64_t max_continuous_version =
-                        enable_sync_publish ? tablet->max_continuous_version() : tablet->max_readable_version();
+                        enable_sync_publish ? tablet->max_readable_version() : tablet->max_continuous_version();
                 if (max_continuous_version > 0) {
                     auto& pair = tablet_versions.emplace_back();
                     pair.__set_tablet_id(tablet_info.tablet_id);
                     pair.__set_version(max_continuous_version);
                 }
+
+                if (enable_sync_publish && tablet_tasks.empty() && max_continuous_version < partition_version.version) {
+                    error_tablet_ids.push_back(tablet_info.tablet_id);
+                }
             }
         }
     }

diff --git a/be/src/column/column_hash.h b/be/src/column/column_hash.h
@@ -29,6 +29,8 @@
 #include <xmmintrin.h>
 #endif
 
+#include "column/type_traits.h"
+#include "types/logical_type.h"
 #include "util/hash_util.hpp"
 #include "util/slice.h"
 #include "util/unaligned_access.h"
@@ -343,4 +345,26 @@ struct HashTypeTraits<int128_t> {
     using HashFunc = Hash128WithSeed<PhmapSeed2>;
 };
 
+template <LogicalType LT, PhmapSeed seed>
+struct PhmapDefaultHashFunc {
+    std::size_t operator()(const RunTimeCppType<LT>& value) const {
+        static_assert(is_supported(), "unsupported logical type");
+
+        if constexpr (lt_is_largeint<LT> || lt_is_decimal128<LT>) {
+            return Hash128WithSeed<seed>()(value);
+        } else if constexpr (lt_is_fixedlength<LT>) {
+            return StdHashWithSeed<RunTimeCppType<LT>, seed>()(value);
+        } else if constexpr (lt_is_string<LT> || lt_is_binary<LT>) {
+            return SliceHashWithSeed<seed>()(value);
+        } else {
+            assert(false);
+        }
+    }
+
+    constexpr static bool is_supported() {
+        return lt_is_largeint<LT> || lt_is_decimal128<LT> || lt_is_fixedlength<LT> || lt_is_string<LT> ||
+               lt_is_binary<LT>;
+    }
+};
+
 } // namespace starrocks
diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -703,6 +703,12 @@ CONF_Int32(metric_late_materialization_ratio, "1000");
 
 // Max batched bytes for each transmit request. (256KB)
 CONF_Int64(max_transmit_batched_bytes, "262144");
+// max chunk size for each tablet write request. (512MB)
+// see: https://github.com/StarRocks/starrocks/pull/50302
+// NOTE: If there are a large number of columns when loading,
+// a too small max_tablet_write_chunk_bytes may cause more frequent RPCs, which may affect performance.
+// In this case, we can try to increase the value to avoid the problem.
+CONF_mInt64(max_tablet_write_chunk_bytes, "536870912");
 
 CONF_Int16(bitmap_max_filter_items, "30");
 
@@ -922,6 +928,9 @@ CONF_String(aws_sdk_logging_trace_level, "trace");
 // Enabling RFC-3986 encoding will make sure these characters are properly encoded.
 CONF_Bool(aws_sdk_enable_compliant_rfc3986_encoding, "false");
 
+// use poco client to replace default curl client
+CONF_Bool(enable_poco_client_for_aws_sdk, "true");
+
 // default: 16MB
 CONF_mInt64(experimental_s3_max_single_part_size, "16777216");
 // default: 16MB

diff --git a/be/src/connector/connector.cpp b/be/src/connector/connector.cpp
@@ -114,14 +114,9 @@ StatusOr<pipeline::MorselQueuePtr> DataSourceProvider::convert_scan_range_to_mor
     peek_scan_ranges(scan_ranges);
 
     pipeline::Morsels morsels;
-    // If this scan node does not accept non-empty scan ranges, create a placeholder one.
-    if (!accept_empty_scan_ranges() && scan_ranges.empty()) {
-        morsels.emplace_back(std::make_unique<pipeline::ScanMorsel>(node_id, TScanRangeParams()));
-    } else {
-        for (const auto& scan_range : scan_ranges) {
-            morsels.emplace_back(std::make_unique<pipeline::ScanMorsel>(node_id, scan_range));
-        }
-    }
+    bool has_more_morsel = false;
+    pipeline::ScanMorsel::build_scan_morsels(node_id, scan_ranges, accept_empty_scan_ranges(), &morsels,
+                                             &has_more_morsel);
 
     if (partition_order_hint().has_value()) {
         bool asc = partition_order_hint().value();
@@ -143,7 +138,7 @@ StatusOr<pipeline::MorselQueuePtr> DataSourceProvider::convert_scan_range_to_mor
         });
     }
 
-    auto morsel_queue = std::make_unique<pipeline::DynamicMorselQueue>(std::move(morsels));
+    auto morsel_queue = std::make_unique<pipeline::DynamicMorselQueue>(std::move(morsels), has_more_morsel);
     if (scan_parallelism > 0) {
         morsel_queue->set_max_degree_of_parallelism(scan_parallelism);
     }