Skip to content

[refine](column) support const sources in column insertion #67145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Mryange wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[refine](column) support const sources in column insertion #67145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions be/src/core/column/column.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,27 @@ void IColumn::insert_from(const IColumn& src, size_t n) {
insert(src[n]);
}

void IColumn::insert_range_from(const IColumn& src, size_t start, size_t length) {
if (const auto* const_src = check_and_get_column<ColumnConst>(src)) {
if (!is_column_const(*this)) {
insert_many_from(const_src->get_data_column(), 0, length);
return;
}
}
insert_range_from_impl(src, start, length);
}

void IColumn::insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
if (const auto* const_src = check_and_get_column<ColumnConst>(src)) {
if (!is_column_const(*this)) {
insert_many_from(const_src->get_data_column(), 0, indices_end - indices_begin);
return;
}
}
insert_indices_from_impl(src, indices_begin, indices_end);
}

void IColumn::sort_column(const ColumnSorter* sorter, EqualFlags& flags,
IColumn::Permutation& perms, EqualRange& range, bool last_column) const {
sorter->sort_column(static_cast<const IColumn&>(*this), flags, perms, range, last_column);
Expand Down
18 changes: 12 additions & 6 deletions be/src/core/column/column.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,9 @@ class IColumn : public COW<IColumn> {
/// Is used in merge-sort and merges. It could be implemented in inherited classes more optimally than default implementation.
virtual void insert_from(const IColumn& src, size_t n);

/// Appends range of elements from other column with the same type.
/// Could be used to concatenate columns.
virtual void insert_range_from(const IColumn& src, size_t start, size_t length) = 0;
/// Appends range of elements from another logical column with the same type.
/// A top-level ColumnConst source is appended without materializing the whole source column.
void insert_range_from(const IColumn& src, size_t start, size_t length);

/// Appends range of elements from other column with the same type.
/// Do not need throw execption in ColumnString overflow uint32, only
Expand All @@ -223,11 +223,12 @@ class IColumn : public COW<IColumn> {
virtual void insert_from_multi_column(const std::vector<const IColumn*>& srcs,
const std::vector<size_t>& positions) = 0;

/// Appends a batch elements from other column with the same type
/// Appends a batch of elements from another logical column with the same type.
/// A top-level ColumnConst source is appended without materializing the whole source column.
/// Also here should make sure indices_end is bigger than indices_begin
/// indices_begin + indices_end represent the row indices of column src
virtual void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) = 0;
void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end);

/// Appends data located in specified memory chunk if it is possible (throws an exception if it cannot be implemented).
/// used in ColumnString, ColumnFixedString, ColumnVector, not support in ColumnArray|ColumnMap...
Expand Down Expand Up @@ -779,6 +780,11 @@ class IColumn : public COW<IColumn> {
virtual void replace_float_special_values() {}

protected:
virtual void insert_range_from_impl(const IColumn& src, size_t start, size_t length) = 0;

virtual void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) = 0;

template <typename Derived>
void append_data_by_selector_impl(MutablePtr& res, const Selector& selector) const {
append_data_by_selector_impl<Derived>(res, selector, 0, selector.size());
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_array.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,7 @@ bool ColumnArray::has_equal_offsets(const ColumnArray& other) const {
0 == memcmp(offsets1.data(), offsets2.data(), sizeof(offsets1[0]) * offsets1.size()));
}

void ColumnArray::insert_range_from(const IColumn& src, size_t start, size_t length) {
void ColumnArray::insert_range_from_impl(const IColumn& src, size_t start, size_t length) {
if (length == 0) return;

const ColumnArray& src_concrete = assert_cast<const ColumnArray&>(src);
Expand Down Expand Up @@ -970,8 +970,8 @@ size_t ColumnArray::filter(const Filter& filter) {
}
}

void ColumnArray::insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
void ColumnArray::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
for (const auto* x = indices_begin; x != indices_end; ++x) {
ColumnArray::insert_from(src, *x);
}
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_array.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray> {
void update_crc32c_single(size_t start, size_t end, uint32_t& hash,
const uint8_t* __restrict null_map) const override;

void insert_range_from(const IColumn& src, size_t start, size_t length) override;
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override;
void insert_range_from_ignore_overflow(const IColumn& src, size_t start,
size_t length) override;
void insert(const Field& x) override;
Expand Down Expand Up @@ -230,8 +230,8 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray> {
return IColumn::convert_column_if_overflow();
}

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;

void replace_column_data(const IColumn& rhs, size_t row, size_t self_row = 0) override {
throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_complex.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,16 +150,16 @@ class ColumnComplexType final : public COWHelper<IColumn, ColumnComplexType<T>>
throw doris::Exception(ErrorCode::INTERNAL_ERROR, "get field not implemented");
}

void insert_range_from(const IColumn& src, size_t start, size_t length) override {
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override {
auto& col = assert_cast<const Self&>(src);
auto& src_data = col.get_data();
auto st = src_data.begin() + start;
auto ed = st + length;
data.insert(data.end(), st, ed);
}

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
const Self& src_vec = assert_cast<const Self&>(src);
auto new_size = indices_end - indices_begin;

Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_const.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst> {

bool is_null_at(size_t) const override { return data->is_null_at(0); }

void insert_range_from(const IColumn& src, size_t /*start*/, size_t length) override {
void insert_range_from_impl(const IColumn& src, size_t /*start*/, size_t length) override {
if (!is_column_const(src) || compare_at(0, 0, src, 0) != 0) {
throw Exception(
ErrorCode::INTERNAL_ERROR,
Expand All @@ -163,8 +163,8 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst> {
s += length;
}

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
if (this->s != 0 /*not empty*/ &&
(!is_column_const(src) || compare_at(0, 0, src, 0) != 0)) {
throw Exception(
Expand Down
2 changes: 1 addition & 1 deletion be/src/core/column/column_decimal.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,7 @@ void ColumnDecimal<T>::insert_many_from(const IColumn& src, size_t position, siz
}

template <PrimitiveType T>
void ColumnDecimal<T>::insert_range_from(const IColumn& src, size_t start, size_t length) {
void ColumnDecimal<T>::insert_range_from_impl(const IColumn& src, size_t start, size_t length) {
const ColumnDecimal& src_vec = assert_cast<const ColumnDecimal&>(src);

if (start + length > src_vec.data.size()) {
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_decimal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ class ColumnDecimal final : public COWHelper<IColumn, ColumnDecimal<T>> {
data.push_back(assert_cast<const Self&, TypeCheckOnRelease::DISABLE>(src).get_data()[n]);
}

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
auto origin_size = size();
auto new_size = indices_end - indices_begin;
data.resize(origin_size + new_size);
Expand Down Expand Up @@ -140,7 +140,7 @@ class ColumnDecimal final : public COWHelper<IColumn, ColumnDecimal<T>> {
void insert_data(const char* pos, size_t /*length*/) override;
void insert_default() override { data.push_back(value_type()); }
void insert(const Field& x) override { data.push_back(x.template get<T>()); }
void insert_range_from(const IColumn& src, size_t start, size_t length) override;
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override;

void insert_many_defaults(size_t length) override {
size_t old_size = data.size();
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_dictionary.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,13 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {
"insert_from not supported in ColumnDictionary");
}

void insert_range_from(const IColumn& src, size_t start, size_t length) override {
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"insert_range_from not supported in ColumnDictionary");
}

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
throw doris::Exception(ErrorCode::INTERNAL_ERROR,
"insert_indices_from not supported in ColumnDictionary");
}
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_dummy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,12 @@ class IColumnDummy : public IColumn {

void insert_from(const IColumn&, size_t) override { ++s; }

void insert_range_from(const IColumn& /*src*/, size_t /*start*/, size_t length) override {
void insert_range_from_impl(const IColumn& /*src*/, size_t /*start*/, size_t length) override {
s += length;
}

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
s += (indices_end - indices_begin);
}

Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_fixed_length_object.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ class ColumnFixedLengthObject final : public COWHelper<IColumn, ColumnFixedLengt
return res;
}

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override {
const Self& src_vec = assert_cast<const Self&>(src);
auto origin_size = size();
auto new_size = indices_end - indices_begin;
Expand Down Expand Up @@ -128,7 +128,7 @@ class ColumnFixedLengthObject final : public COWHelper<IColumn, ColumnFixedLengt
insert_data(x.get<TYPE_STRING>().data(), _item_size);
}

void insert_range_from(const IColumn& src, size_t start, size_t length) override {
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override {
const auto& src_col = assert_cast<const ColumnFixedLengthObject&>(src);
CHECK_EQ(src_col._item_size, _item_size);

Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_map.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,8 @@ void ColumnMap::insert_from(const IColumn& src_, size_t n) {
get_offsets().push_back(get_offsets().back() + size);
}

void ColumnMap::insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
void ColumnMap::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
for (const auto* x = indices_begin; x != indices_end; ++x) {
ColumnMap::insert_from(src, *x);
}
Expand Down Expand Up @@ -483,7 +483,7 @@ void ColumnMap::update_crc32c_single(size_t start, size_t end, uint32_t& hash,
}
}

void ColumnMap::insert_range_from(const IColumn& src, size_t start, size_t length) {
void ColumnMap::insert_range_from_impl(const IColumn& src, size_t start, size_t length) {
if (length == 0) {
return;
}
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_map.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {

Field operator[](size_t n) const override;
void get(size_t n, Field& res) const override;
void insert_range_from(const IColumn& src, size_t start, size_t length) override;
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override;
void insert_range_from_ignore_overflow(const IColumn& src, size_t start,
size_t length) override;
void insert_from(const IColumn& src_, size_t n) override;
Expand All @@ -127,8 +127,8 @@ class ColumnMap final : public COWHelper<IColumn, ColumnMap> {

int compare_at(size_t n, size_t m, const IColumn& rhs_, int nan_direction_hint) const override;

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;

void insert_many_from(const IColumn& src, size_t position, size_t length) override;

Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_nullable.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,14 +364,14 @@ void ColumnNullable::insert_range_from_ignore_overflow(const doris::IColumn& src
length);
}

void ColumnNullable::insert_range_from(const IColumn& src, size_t start, size_t length) {
void ColumnNullable::insert_range_from_impl(const IColumn& src, size_t start, size_t length) {
const auto& nullable_col = assert_cast<const ColumnNullable&>(src);
get_null_map_column().insert_range_from(nullable_col.get_null_map_column(), start, length);
get_nested_column().insert_range_from(*nullable_col._nested_column, start, length);
}

void ColumnNullable::insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
void ColumnNullable::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
const auto& src_concrete = assert_cast<const ColumnNullable&>(src);
get_nested_column().insert_indices_from(src_concrete.get_nested_column(), indices_begin,
indices_end);
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_nullable.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,13 +130,13 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {

void deserialize(StringRef* keys, size_t num_rows) override;

void insert_range_from(const IColumn& src, size_t start, size_t length) override;
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override;

void insert_range_from_ignore_overflow(const IColumn& src, size_t start,
size_t length) override;

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;
void insert_indices_from_not_has_null(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end);

Expand Down
8 changes: 4 additions & 4 deletions be/src/core/column/column_string.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ bool ColumnStr<T>::has_enough_capacity(const IColumn& src) const {
}

template <typename T>
void ColumnStr<T>::insert_range_from(const IColumn& src, size_t start, size_t length) {
void ColumnStr<T>::insert_range_from_impl(const IColumn& src, size_t start, size_t length) {
if (length == 0) {
return;
}
Expand Down Expand Up @@ -193,7 +193,7 @@ void ColumnStr<T>::insert_range_from(const IColumn& src, size_t start, size_t le
}
}
};
// insert_range_from maybe called by ColumnArray::insert_indices_from(which is used by hash join operator),
// insert_range_from maybe called by ColumnArray::insert_indices_from (which is used by hash join operator),
// so we need to support both ColumnStr<uint32_t> and ColumnStr<uint64_t>
if (src.is_column_string64()) {
do_insert(assert_cast<const ColumnStr<uint64_t>&>(src));
Expand Down Expand Up @@ -227,8 +227,8 @@ void ColumnStr<T>::insert_many_from(const IColumn& src, size_t position, size_t
}

template <typename T>
void ColumnStr<T>::insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
void ColumnStr<T>::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
auto do_insert = [&](const auto& src_str) {
const auto* __restrict src_offset_data = src_str.get_offsets().data();

Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_string.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,13 +571,13 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
}
}

void insert_range_from(const IColumn& src, size_t start, size_t length) override;
void insert_range_from_impl(const IColumn& src, size_t start, size_t length) override;

void insert_range_from_ignore_overflow(const IColumn& src, size_t start,
size_t length) override;

void insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;
void insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) override;

ColumnPtr filter(const IColumn::Filter& filt, ssize_t result_size_hint) const override;
size_t filter(const IColumn::Filter& filter) override;
Expand Down
6 changes: 3 additions & 3 deletions be/src/core/column/column_struct.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,8 +271,8 @@ void ColumnStruct::update_crc32c_single(size_t start, size_t end, uint32_t& hash
}
}

void ColumnStruct::insert_indices_from(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
void ColumnStruct::insert_indices_from_impl(const IColumn& src, const uint32_t* indices_begin,
const uint32_t* indices_end) {
const auto& src_concrete = assert_cast<const ColumnStruct&>(src);
for (size_t i = 0; i < columns.size(); ++i) {
columns[i]->insert_indices_from(src_concrete.get_column(i), indices_begin, indices_end);
Expand All @@ -286,7 +286,7 @@ void ColumnStruct::insert_many_from(const IColumn& src, size_t position, size_t
}
}

void ColumnStruct::insert_range_from(const IColumn& src, size_t start, size_t length) {
void ColumnStruct::insert_range_from_impl(const IColumn& src, size_t start, size_t length) {
const size_t tuple_size = columns.size();
for (size_t i = 0; i < tuple_size; ++i) {
columns[i]->insert_range_from(
Expand Down
Loading
Loading