From e08b7c0c6ca782dfd6d4d88e8c33f87a762d7d8e Mon Sep 17 00:00:00 2001 From: Mryange Date: Wed, 26 Aug 2026 10:55:23 +0800 Subject: [PATCH] [refine](column) support const sources in column insertion ### What problem does this PR solve? Issue Number: N/A Problem Summary: MutableBlock and sort paths could receive top-level ColumnConst sources, but the virtual range and index insertion interfaces required concrete non-const columns. Callers therefore had to detect and expand constant columns manually, and some paths could fail with a bad cast. This change applies a non-virtual interface around the insertion methods, routes ColumnConst sources through insert_many_from(), and moves concrete implementations behind insert_range_from_impl() and insert_indices_from_impl(). ### Release note None ### Check List (For Author) - Test: BE build with sh build.sh --be -j48; ASAN BE UT with ColumnConstTest.* and BlockTest.MutableBlockAddRowsSupportsConstNullableSource (19 tests passed) - Behavior changed: No - Does this need documentation: No --- be/src/core/column/column.cpp | 21 ++++++++++ be/src/core/column/column.h | 18 ++++++--- be/src/core/column/column_array.cpp | 6 +-- be/src/core/column/column_array.h | 6 +-- be/src/core/column/column_complex.h | 6 +-- be/src/core/column/column_const.h | 6 +-- be/src/core/column/column_decimal.cpp | 2 +- be/src/core/column/column_decimal.h | 6 +-- be/src/core/column/column_dictionary.h | 6 +-- be/src/core/column/column_dummy.h | 6 +-- .../core/column/column_fixed_length_object.h | 6 +-- be/src/core/column/column_map.cpp | 6 +-- be/src/core/column/column_map.h | 6 +-- be/src/core/column/column_nullable.cpp | 6 +-- be/src/core/column/column_nullable.h | 6 +-- be/src/core/column/column_string.cpp | 8 ++-- be/src/core/column/column_string.h | 6 +-- be/src/core/column/column_struct.cpp | 6 +-- be/src/core/column/column_struct.h | 6 +-- be/src/core/column/column_varbinary.cpp | 6 +-- be/src/core/column/column_varbinary.h | 6 +-- be/src/core/column/column_variant.cpp | 8 ++-- be/src/core/column/column_variant.h | 6 +-- be/src/core/column/column_vector.cpp | 6 +-- be/src/core/column/column_vector.h | 6 +-- .../column/variant_v2/column_variant_v2.cpp | 4 +- .../column/variant_v2/column_variant_v2.h | 6 +-- be/src/exec/sort/sorter.cpp | 9 +---- be/test/core/block/block_test.cpp | 26 +++++++++++++ be/test/core/column/column_const_test.cpp | 39 +++++++++++++++++++ 30 files changed, 173 insertions(+), 88 deletions(-) diff --git a/be/src/core/column/column.cpp b/be/src/core/column/column.cpp index ef0d61076b1f9b..a01560f1e77fa2 100644 --- a/be/src/core/column/column.cpp +++ b/be/src/core/column/column.cpp @@ -184,6 +184,27 @@ void IColumn::insert_from(const IColumn& src, size_t n) { insert(src[n]); } +void IColumn::insert_range_from(const IColumn& src, size_t start, size_t length) { + if (const auto* const_src = check_and_get_column(src)) { + if (!is_column_const(*this)) { + insert_many_from(const_src->get_data_column(), 0, length); + return; + } + } + insert_range_from_impl(src, start, length); +} + +void IColumn::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { + if (const auto* const_src = check_and_get_column(src)) { + if (!is_column_const(*this)) { + insert_many_from(const_src->get_data_column(), 0, indices_end - indices_begin); + return; + } + } + insert_indices_from_impl(src, indices_begin, indices_end); +} + void IColumn::sort_column(const ColumnSorter* sorter, EqualFlags& flags, IColumn::Permutation& perms, EqualRange& range, bool last_column) const { sorter->sort_column(static_cast(*this), flags, perms, range, last_column); diff --git a/be/src/core/column/column.h b/be/src/core/column/column.h index 513383e2855df9..242f58af12dcda 100644 --- a/be/src/core/column/column.h +++ b/be/src/core/column/column.h @@ -198,9 +198,9 @@ class IColumn : public COW { /// Is used in merge-sort and merges. It could be implemented in inherited classes more optimally than default implementation. virtual void insert_from(const IColumn& src, size_t n); - /// Appends range of elements from other column with the same type. - /// Could be used to concatenate columns. - virtual void insert_range_from(const IColumn& src, size_t start, size_t length) = 0; + /// Appends range of elements from another logical column with the same type. + /// A top-level ColumnConst source is appended without materializing the whole source column. + void insert_range_from(const IColumn& src, size_t start, size_t length); /// Appends range of elements from other column with the same type. /// Do not need throw execption in ColumnString overflow uint32, only @@ -223,11 +223,12 @@ class IColumn : public COW { virtual void insert_from_multi_column(const std::vector& srcs, const std::vector& positions) = 0; - /// Appends a batch elements from other column with the same type + /// Appends a batch of elements from another logical column with the same type. + /// A top-level ColumnConst source is appended without materializing the whole source column. /// Also here should make sure indices_end is bigger than indices_begin /// indices_begin + indices_end represent the row indices of column src - virtual void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) = 0; + void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end); /// Appends data located in specified memory chunk if it is possible (throws an exception if it cannot be implemented). /// used in ColumnString, ColumnFixedString, ColumnVector, not support in ColumnArray|ColumnMap... @@ -779,6 +780,11 @@ class IColumn : public COW { virtual void replace_float_special_values() {} protected: + virtual void insert_range_from_impl(const IColumn& src, size_t start, size_t length) = 0; + + virtual void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) = 0; + template void append_data_by_selector_impl(MutablePtr& res, const Selector& selector) const { append_data_by_selector_impl(res, selector, 0, selector.size()); diff --git a/be/src/core/column/column_array.cpp b/be/src/core/column/column_array.cpp index 604c46948539b1..b4ad5cc0a013fe 100644 --- a/be/src/core/column/column_array.cpp +++ b/be/src/core/column/column_array.cpp @@ -590,7 +590,7 @@ bool ColumnArray::has_equal_offsets(const ColumnArray& other) const { 0 == memcmp(offsets1.data(), offsets2.data(), sizeof(offsets1[0]) * offsets1.size())); } -void ColumnArray::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnArray::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { if (length == 0) return; const ColumnArray& src_concrete = assert_cast(src); @@ -970,8 +970,8 @@ size_t ColumnArray::filter(const Filter& filter) { } } -void ColumnArray::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnArray::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { for (const auto* x = indices_begin; x != indices_end; ++x) { ColumnArray::insert_from(src, *x); } diff --git a/be/src/core/column/column_array.h b/be/src/core/column/column_array.h index 645b1662d7b238..43b9d2932b98b0 100644 --- a/be/src/core/column/column_array.h +++ b/be/src/core/column/column_array.h @@ -166,7 +166,7 @@ class ColumnArray final : public COWHelper { void update_crc32c_single(size_t start, size_t end, uint32_t& hash, const uint8_t* __restrict null_map) const override; - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; void insert_range_from_ignore_overflow(const IColumn& src, size_t start, size_t length) override; void insert(const Field& x) override; @@ -230,8 +230,8 @@ class ColumnArray final : public COWHelper { return IColumn::convert_column_if_overflow(); } - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, diff --git a/be/src/core/column/column_complex.h b/be/src/core/column/column_complex.h index 9f0d7b45e72cf4..fef810b7746464 100644 --- a/be/src/core/column/column_complex.h +++ b/be/src/core/column/column_complex.h @@ -150,7 +150,7 @@ class ColumnComplexType final : public COWHelper> throw doris::Exception(ErrorCode::INTERNAL_ERROR, "get field not implemented"); } - void insert_range_from(const IColumn& src, size_t start, size_t length) override { + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override { auto& col = assert_cast(src); auto& src_data = col.get_data(); auto st = src_data.begin() + start; @@ -158,8 +158,8 @@ class ColumnComplexType final : public COWHelper> data.insert(data.end(), st, ed); } - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override { + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override { const Self& src_vec = assert_cast(src); auto new_size = indices_end - indices_begin; diff --git a/be/src/core/column/column_const.h b/be/src/core/column/column_const.h index 44b1f9d477a966..e6b124c36d3f36 100644 --- a/be/src/core/column/column_const.h +++ b/be/src/core/column/column_const.h @@ -145,7 +145,7 @@ class ColumnConst final : public COWHelper { bool is_null_at(size_t) const override { return data->is_null_at(0); } - void insert_range_from(const IColumn& src, size_t /*start*/, size_t length) override { + void insert_range_from_impl(const IColumn& src, size_t /*start*/, size_t length) override { if (!is_column_const(src) || compare_at(0, 0, src, 0) != 0) { throw Exception( ErrorCode::INTERNAL_ERROR, @@ -163,8 +163,8 @@ class ColumnConst final : public COWHelper { s += length; } - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override { + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override { if (this->s != 0 /*not empty*/ && (!is_column_const(src) || compare_at(0, 0, src, 0) != 0)) { throw Exception( diff --git a/be/src/core/column/column_decimal.cpp b/be/src/core/column/column_decimal.cpp index 314bccd5d54a54..b61113d5cd3ca9 100644 --- a/be/src/core/column/column_decimal.cpp +++ b/be/src/core/column/column_decimal.cpp @@ -417,7 +417,7 @@ void ColumnDecimal::insert_many_from(const IColumn& src, size_t position, siz } template -void ColumnDecimal::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnDecimal::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { const ColumnDecimal& src_vec = assert_cast(src); if (start + length > src_vec.data.size()) { diff --git a/be/src/core/column/column_decimal.h b/be/src/core/column/column_decimal.h index 8cd7be991a783f..c8837717a44f82 100644 --- a/be/src/core/column/column_decimal.h +++ b/be/src/core/column/column_decimal.h @@ -111,8 +111,8 @@ class ColumnDecimal final : public COWHelper> { data.push_back(assert_cast(src).get_data()[n]); } - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override { + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override { auto origin_size = size(); auto new_size = indices_end - indices_begin; data.resize(origin_size + new_size); @@ -140,7 +140,7 @@ class ColumnDecimal final : public COWHelper> { void insert_data(const char* pos, size_t /*length*/) override; void insert_default() override { data.push_back(value_type()); } void insert(const Field& x) override { data.push_back(x.template get()); } - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; void insert_many_defaults(size_t length) override { size_t old_size = data.size(); diff --git a/be/src/core/column/column_dictionary.h b/be/src/core/column/column_dictionary.h index 7c60c7613e4a7c..ca2d62b0f077cb 100644 --- a/be/src/core/column/column_dictionary.h +++ b/be/src/core/column/column_dictionary.h @@ -66,13 +66,13 @@ class ColumnDictI32 final : public COWHelper { "insert_from not supported in ColumnDictionary"); } - void insert_range_from(const IColumn& src, size_t start, size_t length) override { + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override { throw doris::Exception(ErrorCode::INTERNAL_ERROR, "insert_range_from not supported in ColumnDictionary"); } - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override { + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override { throw doris::Exception(ErrorCode::INTERNAL_ERROR, "insert_indices_from not supported in ColumnDictionary"); } diff --git a/be/src/core/column/column_dummy.h b/be/src/core/column/column_dummy.h index 30264f672cc5ac..42f9b852ead5ce 100644 --- a/be/src/core/column/column_dummy.h +++ b/be/src/core/column/column_dummy.h @@ -80,12 +80,12 @@ class IColumnDummy : public IColumn { void insert_from(const IColumn&, size_t) override { ++s; } - void insert_range_from(const IColumn& /*src*/, size_t /*start*/, size_t length) override { + void insert_range_from_impl(const IColumn& /*src*/, size_t /*start*/, size_t length) override { s += length; } - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override { + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override { s += (indices_end - indices_begin); } diff --git a/be/src/core/column/column_fixed_length_object.h b/be/src/core/column/column_fixed_length_object.h index 3789eeb868d150..c1c0a2213b1b2c 100644 --- a/be/src/core/column/column_fixed_length_object.h +++ b/be/src/core/column/column_fixed_length_object.h @@ -83,8 +83,8 @@ class ColumnFixedLengthObject final : public COWHelper(src); auto origin_size = size(); auto new_size = indices_end - indices_begin; @@ -128,7 +128,7 @@ class ColumnFixedLengthObject final : public COWHelper().data(), _item_size); } - void insert_range_from(const IColumn& src, size_t start, size_t length) override { + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override { const auto& src_col = assert_cast(src); CHECK_EQ(src_col._item_size, _item_size); diff --git a/be/src/core/column/column_map.cpp b/be/src/core/column/column_map.cpp index 2d9c266e73a688..b6ef8a9b5f5644 100644 --- a/be/src/core/column/column_map.cpp +++ b/be/src/core/column/column_map.cpp @@ -229,8 +229,8 @@ void ColumnMap::insert_from(const IColumn& src_, size_t n) { get_offsets().push_back(get_offsets().back() + size); } -void ColumnMap::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnMap::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { for (const auto* x = indices_begin; x != indices_end; ++x) { ColumnMap::insert_from(src, *x); } @@ -483,7 +483,7 @@ void ColumnMap::update_crc32c_single(size_t start, size_t end, uint32_t& hash, } } -void ColumnMap::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnMap::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { if (length == 0) { return; } diff --git a/be/src/core/column/column_map.h b/be/src/core/column/column_map.h index 0781a2cb503569..e4333b95bbd5b8 100644 --- a/be/src/core/column/column_map.h +++ b/be/src/core/column/column_map.h @@ -109,7 +109,7 @@ class ColumnMap final : public COWHelper { Field operator[](size_t n) const override; void get(size_t n, Field& res) const override; - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; void insert_range_from_ignore_overflow(const IColumn& src, size_t start, size_t length) override; void insert_from(const IColumn& src_, size_t n) override; @@ -127,8 +127,8 @@ class ColumnMap final : public COWHelper { int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override; - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; void insert_many_from(const IColumn& src, size_t position, size_t length) override; diff --git a/be/src/core/column/column_nullable.cpp b/be/src/core/column/column_nullable.cpp index a3857c2af047f6..59b2d001f8e826 100644 --- a/be/src/core/column/column_nullable.cpp +++ b/be/src/core/column/column_nullable.cpp @@ -364,14 +364,14 @@ void ColumnNullable::insert_range_from_ignore_overflow(const doris::IColumn& src length); } -void ColumnNullable::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnNullable::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { const auto& nullable_col = assert_cast(src); get_null_map_column().insert_range_from(nullable_col.get_null_map_column(), start, length); get_nested_column().insert_range_from(*nullable_col._nested_column, start, length); } -void ColumnNullable::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnNullable::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { const auto& src_concrete = assert_cast(src); get_nested_column().insert_indices_from(src_concrete.get_nested_column(), indices_begin, indices_end); diff --git a/be/src/core/column/column_nullable.h b/be/src/core/column/column_nullable.h index bc4bc9358a1302..b0c3fd8de22339 100644 --- a/be/src/core/column/column_nullable.h +++ b/be/src/core/column/column_nullable.h @@ -130,13 +130,13 @@ class ColumnNullable final : public COWHelper { void deserialize(StringRef* keys, size_t num_rows) override; - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; void insert_range_from_ignore_overflow(const IColumn& src, size_t start, size_t length) override; - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; void insert_indices_from_not_has_null(const IColumn& src, const uint32_t* indices_begin, const uint32_t* indices_end); diff --git a/be/src/core/column/column_string.cpp b/be/src/core/column/column_string.cpp index 1ea2f24c1e7695..a658bca1efa3f8 100644 --- a/be/src/core/column/column_string.cpp +++ b/be/src/core/column/column_string.cpp @@ -158,7 +158,7 @@ bool ColumnStr::has_enough_capacity(const IColumn& src) const { } template -void ColumnStr::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnStr::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { if (length == 0) { return; } @@ -193,7 +193,7 @@ void ColumnStr::insert_range_from(const IColumn& src, size_t start, size_t le } } }; - // insert_range_from maybe called by ColumnArray::insert_indices_from(which is used by hash join operator), + // insert_range_from maybe called by ColumnArray::insert_indices_from (which is used by hash join operator), // so we need to support both ColumnStr and ColumnStr if (src.is_column_string64()) { do_insert(assert_cast&>(src)); @@ -227,8 +227,8 @@ void ColumnStr::insert_many_from(const IColumn& src, size_t position, size_t } template -void ColumnStr::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnStr::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { auto do_insert = [&](const auto& src_str) { const auto* __restrict src_offset_data = src_str.get_offsets().data(); diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index 54d224c0c22c6b..a5b3465917e01c 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -571,13 +571,13 @@ class ColumnStr final : public COWHelper> { } } - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; void insert_range_from_ignore_overflow(const IColumn& src, size_t start, size_t length) override; - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; ColumnPtr filter(const IColumn::Filter& filt, ssize_t result_size_hint) const override; size_t filter(const IColumn::Filter& filter) override; diff --git a/be/src/core/column/column_struct.cpp b/be/src/core/column/column_struct.cpp index fb15785df8bb2d..8ceac64f30520f 100644 --- a/be/src/core/column/column_struct.cpp +++ b/be/src/core/column/column_struct.cpp @@ -271,8 +271,8 @@ void ColumnStruct::update_crc32c_single(size_t start, size_t end, uint32_t& hash } } -void ColumnStruct::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnStruct::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { const auto& src_concrete = assert_cast(src); for (size_t i = 0; i < columns.size(); ++i) { columns[i]->insert_indices_from(src_concrete.get_column(i), indices_begin, indices_end); @@ -286,7 +286,7 @@ void ColumnStruct::insert_many_from(const IColumn& src, size_t position, size_t } } -void ColumnStruct::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnStruct::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { const size_t tuple_size = columns.size(); for (size_t i = 0; i < tuple_size; ++i) { columns[i]->insert_range_from( diff --git a/be/src/core/column/column_struct.h b/be/src/core/column/column_struct.h index 83affe7296558f..586a96a2740fa4 100644 --- a/be/src/core/column/column_struct.h +++ b/be/src/core/column/column_struct.h @@ -131,8 +131,8 @@ class ColumnStruct final : public COWHelper { void update_crc32c_single(size_t start, size_t end, uint32_t& hash, const uint8_t* __restrict null_map) const override; - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; void insert_many_from(const IColumn& src, size_t position, size_t length) override; @@ -141,7 +141,7 @@ class ColumnStruct final : public COWHelper { "Method replace_column_data is not supported for " + get_name()); } - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; void insert_range_from_ignore_overflow(const IColumn& src, size_t start, size_t length) override; ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override; diff --git a/be/src/core/column/column_varbinary.cpp b/be/src/core/column/column_varbinary.cpp index 4a54ca421d0a65..1106f565919da7 100644 --- a/be/src/core/column/column_varbinary.cpp +++ b/be/src/core/column/column_varbinary.cpp @@ -47,7 +47,7 @@ MutableColumnPtr ColumnVarbinary::clone_resized(size_t size) const { return res; } -void ColumnVarbinary::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnVarbinary::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { if (length == 0) { return; } @@ -66,8 +66,8 @@ void ColumnVarbinary::insert_range_from(const IColumn& src, size_t start, size_t } } -void ColumnVarbinary::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnVarbinary::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { const Self& src_vec = assert_cast(src); auto new_size = indices_end - indices_begin; diff --git a/be/src/core/column/column_varbinary.h b/be/src/core/column/column_varbinary.h index caad77e28ad44f..91182bc6701097 100644 --- a/be/src/core/column/column_varbinary.h +++ b/be/src/core/column/column_varbinary.h @@ -138,12 +138,12 @@ class ColumnVarbinary final : public COWHelper { return pos + deserialize_impl(pos); } - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; MutableColumnPtr clone_resized(size_t size) const override; - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; size_t allocated_bytes() const override { return _data.allocated_bytes() + _arena.size(); } diff --git a/be/src/core/column/column_variant.cpp b/be/src/core/column/column_variant.cpp index e00de8cd75bac1..8ca2780f279378 100644 --- a/be/src/core/column/column_variant.cpp +++ b/be/src/core/column/column_variant.cpp @@ -1212,7 +1212,7 @@ bool ColumnVariant::try_add_new_subcolumn(const PathInData& path) { return false; } -void ColumnVariant::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnVariant::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { const auto& src_object = assert_cast(src); ENABLE_CHECK_CONSISTENCY(&src_object); ENABLE_CHECK_CONSISTENCY(this); @@ -2477,8 +2477,8 @@ DataTypePtr ColumnVariant::get_root_type() const { return subcolumns.get_root()->data.get_least_common_type(); } -void ColumnVariant::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnVariant::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { // optimize when src and this column are scalar variant, since try_insert is inefficiency const auto* src_v = check_and_get_column(src); @@ -2512,7 +2512,7 @@ void ColumnVariant::insert_indices_from(const IColumn& src, const uint32_t* indi finalize(); } -// void ColumnVariant::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, +// void ColumnVariant::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, // const uint32_t* indices_end) { // for (const auto* x = indices_begin; x != indices_end; ++x) { // ColumnVariant::insert_from(src, *x); diff --git a/be/src/core/column/column_variant.h b/be/src/core/column/column_variant.h index c834ebec1038da..735fb42607c340 100644 --- a/be/src/core/column/column_variant.h +++ b/be/src/core/column/column_variant.h @@ -481,12 +481,12 @@ class ColumnVariant final : public COWHelper { // Do nothing, call try_insert instead void insert(const Field& field) override { try_insert(field); } - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; void insert_from(const IColumn& src, size_t n) override; - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; void insert_default() override; diff --git a/be/src/core/column/column_vector.cpp b/be/src/core/column/column_vector.cpp index 00de7a17ba78a5..30279e556f6656 100644 --- a/be/src/core/column/column_vector.cpp +++ b/be/src/core/column/column_vector.cpp @@ -358,7 +358,7 @@ MutableColumnPtr ColumnVector::clone_resized(size_t size) const { } template -void ColumnVector::insert_range_from(const IColumn& src, size_t start, size_t length) { +void ColumnVector::insert_range_from_impl(const IColumn& src, size_t start, size_t length) { const ColumnVector& src_vec = assert_cast(src); // size_t(start) start > src_vec.data.size() || length > src_vec.data.size() should not be negative which cause overflow if (start + length > src_vec.data.size()) { @@ -374,8 +374,8 @@ void ColumnVector::insert_range_from(const IColumn& src, size_t start, size_t } template -void ColumnVector::insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) { +void ColumnVector::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) { auto origin_size = size(); auto new_size = indices_end - indices_begin; data.resize(origin_size + new_size); diff --git a/be/src/core/column/column_vector.h b/be/src/core/column/column_vector.h index 5c6b6f294f5bcf..ec10e05cc8487e 100644 --- a/be/src/core/column/column_vector.h +++ b/be/src/core/column/column_vector.h @@ -339,10 +339,10 @@ class ColumnVector final : public COWHelper> { // uint8 value and then insert into column. void insert(const Field& x) override { data.push_back(x.get()); } - void insert_range_from(const IColumn& src, size_t start, size_t length) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; ColumnPtr filter(const IColumn::Filter& filt, ssize_t result_size_hint) const override; size_t filter(const IColumn::Filter& filter) override; diff --git a/be/src/core/column/variant_v2/column_variant_v2.cpp b/be/src/core/column/variant_v2/column_variant_v2.cpp index 70d3a4adf1ad94..318ba75db7512f 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.cpp +++ b/be/src/core/column/variant_v2/column_variant_v2.cpp @@ -1153,7 +1153,7 @@ void ColumnVariantV2::insert_from(const IColumn& src, size_t row) { } // Range insertion handles typed and encoded state pairs. -void ColumnVariantV2::insert_range_from( // NOLINT(readability-function-size) +void ColumnVariantV2::insert_range_from_impl( // NOLINT(readability-function-size) const IColumn& src, size_t start, size_t length) { const auto& source = assert_cast(src); DORIS_CHECK_LE(start, source.size()) << "source range starts past source size"; @@ -1281,7 +1281,7 @@ void ColumnVariantV2::insert_range_from( // NOLINT(readability-function-size) } // Indexed insertion handles typed and encoded state pairs. -void ColumnVariantV2::insert_indices_from( // NOLINT(readability-function-size) +void ColumnVariantV2::insert_indices_from_impl( // NOLINT(readability-function-size) const IColumn& src, const uint32_t* indices_begin, const uint32_t* indices_end) { const auto& source = assert_cast(src); const size_t rows = validate_selected_indices(indices_begin, indices_end, source.size()); diff --git a/be/src/core/column/variant_v2/column_variant_v2.h b/be/src/core/column/variant_v2/column_variant_v2.h index 91addbab20f03a..c8a1b17666eaaa 100644 --- a/be/src/core/column/variant_v2/column_variant_v2.h +++ b/be/src/core/column/variant_v2/column_variant_v2.h @@ -205,9 +205,9 @@ class ColumnVariantV2 final : public COWHelper { void insert_many_defaults(size_t length) override; void insert_from(const IColumn& src, size_t row) override; - void insert_range_from(const IColumn& src, size_t start, size_t length) override; - void insert_indices_from(const IColumn& src, const uint32_t* indices_begin, - const uint32_t* indices_end) override; + void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override; + void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin, + const uint32_t* indices_end) override; void pop_back(size_t length) override; StringRef get_data_at(size_t row) const override; diff --git a/be/src/exec/sort/sorter.cpp b/be/src/exec/sort/sorter.cpp index 5f878ecd7279ea..1e2b01e08011a4 100644 --- a/be/src/exec/sort/sorter.cpp +++ b/be/src/exec/sort/sorter.cpp @@ -292,14 +292,7 @@ Status FullSorter::append_block(Block* block) { DCHECK(data[i].type->equals(*(arrival_data[i].type))) << " type1: " << data[i].type->get_name() << " type2: " << arrival_data[i].type->get_name() << " i: " << i; - if (is_column_const(*arrival_data[i].column)) { - data[i].column->assert_mutable()->insert_many_from( - assert_cast(arrival_data[i].column.get()) - ->get_data_column(), - 0, sz); - } else { - data[i].column->assert_mutable()->insert_range_from(*arrival_data[i].column, 0, sz); - } + data[i].column->assert_mutable()->insert_range_from(*arrival_data[i].column, 0, sz); } block->clear_column_data(); } diff --git a/be/test/core/block/block_test.cpp b/be/test/core/block/block_test.cpp index 566eaccc9826d5..cbe250ada6f832 100644 --- a/be/test/core/block/block_test.cpp +++ b/be/test/core/block/block_test.cpp @@ -1186,6 +1186,32 @@ TEST(BlockTest, MergeMaterializesConstNullableDestination) { } } +TEST(BlockTest, MutableBlockAddRowsSupportsConstNullableSource) { + auto type = make_nullable(std::make_shared()); + auto source_column = type->create_column_const_with_default_value(4); + Block source({ColumnWithTypeAndName(std::move(source_column), type, "const_nullable")}); + + { + Block destination({ColumnWithTypeAndName(type->create_column(), type, "const_nullable")}); + MutableBlock mutable_block(std::move(destination)); + ASSERT_TRUE(mutable_block.add_rows(&source, 1, 2).ok()); + ASSERT_EQ(mutable_block.rows(), 2); + EXPECT_FALSE(is_column_const(*mutable_block.get_column_by_position(0))); + EXPECT_TRUE(mutable_block.get_column_by_position(0)->only_null()); + } + + { + Block destination({ColumnWithTypeAndName(type->create_column(), type, "const_nullable")}); + MutableBlock mutable_block(std::move(destination)); + std::vector indices = {3, 0, 2}; + ASSERT_TRUE(mutable_block.add_rows(&source, indices.data(), indices.data() + indices.size()) + .ok()); + ASSERT_EQ(mutable_block.rows(), indices.size()); + EXPECT_FALSE(is_column_const(*mutable_block.get_column_by_position(0))); + EXPECT_TRUE(mutable_block.get_column_by_position(0)->only_null()); + } +} + TEST(BlockTest, ctor) { TDescriptorTableBuilder builder; TTupleDescriptorBuilder tuple_builder; diff --git a/be/test/core/column/column_const_test.cpp b/be/test/core/column/column_const_test.cpp index 3056d5276a8263..6826e54d2f2cd2 100644 --- a/be/test/core/column/column_const_test.cpp +++ b/be/test/core/column/column_const_test.cpp @@ -27,7 +27,9 @@ #include "core/column/column_nullable.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" #include "core/data_type/primitive_type.h" #include "testutil/column_helper.h" @@ -83,6 +85,43 @@ TEST(ColumnConstTest, ConstNullableNullValueKeepsNullSemantics) { EXPECT_TRUE(const_null->only_null()); } +TEST(ColumnConstTest, InsertRangeAndIndicesFromConstSource) { + std::vector> test_cases; + test_cases.emplace_back(std::make_shared(), Field::create_field(7)); + test_cases.emplace_back(std::make_shared(), + Field::create_field("const string")); + test_cases.emplace_back(make_nullable(std::make_shared()), Field()); + test_cases.emplace_back( + std::make_shared(std::make_shared()), + Field::create_field(Array {Field::create_field(1), + Field::create_field(2)})); + + for (const auto& [type, value] : test_cases) { + SCOPED_TRACE(type->get_name()); + auto source = type->create_column_const(4, value); + + auto range_destination = type->create_column(); + range_destination->insert_range_from(*source, 1, 2); + ASSERT_EQ(range_destination->size(), 2); + EXPECT_EQ((*range_destination)[0], value); + EXPECT_EQ((*range_destination)[1], value); + + auto indices_destination = type->create_column(); + std::vector indices = {3, 1, 0}; + indices_destination->insert_indices_from(*source, indices.data(), + indices.data() + indices.size()); + ASSERT_EQ(indices_destination->size(), indices.size()); + for (size_t i = 0; i < indices.size(); ++i) { + EXPECT_EQ((*indices_destination)[i], value); + } + + range_destination->insert_range_from(*source, source->size(), 0); + indices_destination->insert_indices_from(*source, indices.data(), indices.data()); + EXPECT_EQ(range_destination->size(), 2); + EXPECT_EQ(indices_destination->size(), indices.size()); + } +} + TEST(ColumnConstTest, clone_resized_clones_nested_data) { auto column_data = ColumnHelper::create_column({7}); auto column_const = ColumnConst::create(column_data, 3);