Merge branch 'master' into fix-lock

apache · Feb 19, 2025 · 8fd364c · 8fd364c
2 parents 56a5a98 + 3477d51
commit 8fd364c
Show file tree

Hide file tree

Showing 21 changed files with 1,604 additions and 84 deletions.
diff --git a/be/src/vec/core/sort_cursor.h b/be/src/vec/core/sort_cursor.h
@@ -206,7 +206,7 @@ struct BlockSupplierSortCursorImpl : public MergeSortCursorImpl {
         }
         block->clear();
         THROW_IF_ERROR(_block_supplier(block.get(), &_is_eof));
-        DCHECK(!block->empty() || _is_eof);
+        DCHECK(!block->empty() xor _is_eof);
         if (!block->empty()) {
             DCHECK_EQ(_ordering_expr.size(), desc.size());
             for (int i = 0; i < desc.size(); ++i) {

diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
@@ -272,6 +272,7 @@ struct WhichDataType {
     bool is_struct() const { return idx == TypeIndex::Struct; }
     bool is_map() const { return idx == TypeIndex::Map; }
     bool is_set() const { return idx == TypeIndex::Set; }
+    bool is_fixed_length_object() const { return idx == TypeIndex::FixedLengthObject; }
 
     bool is_nothing() const { return idx == TypeIndex::Nothing; }
     bool is_nullable() const { return idx == TypeIndex::Nullable; }
@@ -371,6 +372,11 @@ bool is_string_or_fixed_string(const T& data_type) {
     return WhichDataType(data_type).is_string_or_fixed_string();
 }
 
+template <typename T>
+bool is_fixed_length_object(const T& data_type) {
+    return WhichDataType(data_type).is_fixed_length_object();
+}
+
 inline bool is_not_decimal_but_comparable_to_decimal(const DataTypePtr& data_type) {
     WhichDataType which(data_type);
     return which.is_int() || which.is_uint();

diff --git a/be/test/testutil/column_helper.h b/be/test/testutil/column_helper.h
@@ -21,27 +21,40 @@
 #include <type_traits>
 #include <vector>
 
+#include "vec/columns/column_nullable.h"
 #include "vec/core/block.h"
 #include "vec/data_types/data_type_string.h"
 
 namespace doris::vectorized {
 struct ColumnHelper {
 public:
     template <typename DataType>
-    static ColumnPtr create_column(const std::vector<typename DataType::FieldType>& datas) {
+    static ColumnPtr create_column(const std::vector<typename DataType::FieldType>& data) {
         auto column = DataType::ColumnType::create();
         if constexpr (std::is_same_v<DataTypeString, DataType>) {
-            for (const auto& data : datas) {
-                column->insert_data(data.data(), data.size());
+            for (const auto& datum : data) {
+                column->insert_data(datum.data(), datum.size());
             }
         } else {
-            for (const auto& data : datas) {
-                column->insert_value(data);
+            for (const auto& datum : data) {
+                column->insert_value(datum);
             }
         }
         return std::move(column);
     }
 
+    template <typename DataType>
+    static ColumnPtr create_nullable_column(
+            const std::vector<typename DataType::FieldType>& data,
+            const std::vector<typename NullMap::value_type>& null_map) {
+        auto null_col = ColumnUInt8::create();
+        for (const auto& datum : null_map) {
+            null_col->insert_value(datum);
+        }
+        auto ptr = create_column<DataType>(data);
+        return ColumnNullable::create(std::move(ptr), std::move(null_col));
+    }
+
     static bool column_equal(const ColumnPtr& column1, const ColumnPtr& column2) {
         if (column1->size() != column2->size()) {
             return false;
@@ -67,13 +80,22 @@ struct ColumnHelper {
     }
 
     template <typename DataType>
-    static Block create_block(const std::vector<typename DataType::FieldType>& datas) {
-        auto column = create_column<DataType>(datas);
+    static Block create_block(const std::vector<typename DataType::FieldType>& data) {
+        auto column = create_column<DataType>(data);
         auto data_type = std::make_shared<DataType>();
         Block block({ColumnWithTypeAndName(column, data_type, "column")});
         return block;
     }
 
+    template <typename DataType>
+    static Block create_nullable_block(const std::vector<typename DataType::FieldType>& data,
+                                       const std::vector<typename NullMap::value_type>& null_map) {
+        auto column = create_nullable_column<DataType>(data, null_map);
+        auto data_type = std::make_shared<DataTypeNullable>(std::make_shared<DataType>());
+        Block block({ColumnWithTypeAndName(column, data_type, "column")});
+        return block;
+    }
+
     template <typename DataType>
     static ColumnWithTypeAndName create_column_with_name(
             const std::vector<typename DataType::FieldType>& datas) {

diff --git a/be/test/vec/data_types/common_data_type_test.h b/be/test/vec/data_types/common_data_type_test.h
@@ -22,6 +22,8 @@
 #include <fstream>
 #include <iostream>
 
+#include "agent/be_exec_version_manager.h"
+#include "olap/schema.h"
 #include "vec/columns/column.h"
 #include "vec/core/field.h"
 #include "vec/core/types.h"
@@ -52,6 +54,10 @@ namespace doris::vectorized {
 static bool gen_check_data_in_assert = true;
 
 class CommonDataTypeTest : public ::testing::Test {
+public:
+    CommonDataTypeTest() = default;
+    void TestBody() override {}
+
 protected:
     // Helper function to load data from CSV, with index which splited by spliter and load to columns
     void load_data_from_csv(const DataTypeSerDeSPtrs serders, MutableColumns& columns,
@@ -164,7 +170,8 @@ class CommonDataTypeTest : public ::testing::Test {
             ASSERT_EQ(const_col->operator[](i), default_const_col->operator[](i));
         }
         // get_uncompressed_serialized_bytes
-        ASSERT_EQ(data_type->get_uncompressed_serialized_bytes(*column, 0),
+        ASSERT_EQ(data_type->get_uncompressed_serialized_bytes(
+                          *column, BeExecVersionManager::get_newest_version()),
                   uncompressed_serialized_bytes);
     }
 
@@ -249,7 +256,7 @@ class CommonDataTypeTest : public ::testing::Test {
         }
         // nt be_exec_version, PBlock* pblock, size_t* uncompressed_bytes,
         //                     size_t* compressed_bytes, segment_v2::CompressionTypePB compression_type,
-        size_t be_exec_version = 2;
+        size_t be_exec_version = BeExecVersionManager::get_newest_version();
         auto pblock = std::make_unique<PBlock>();
         size_t uncompressed_bytes = 0;
         size_t compressed_bytes = 0;
@@ -262,9 +269,9 @@ class CommonDataTypeTest : public ::testing::Test {
         st = block_1->deserialize(*pblock);
         ASSERT_EQ(st.ok(), true);
         // check block_1 and block is same
-        for (int i = 0; i < block->rows(); ++i) {
-            auto& col = block->get_by_position(i);
-            auto& col_1 = block_1->get_by_position(i);
+        for (auto col_idx = 0; col_idx < block->columns(); ++col_idx) {
+            auto& col = block->get_by_position(col_idx);
+            auto& col_1 = block_1->get_by_position(col_idx);
             ASSERT_EQ(col.column->size(), col_1.column->size());
             for (int j = 0; j < col.column->size(); ++j) {
                 ASSERT_EQ(col.column->operator[](j), col_1.column->operator[](j));