Skip to content

Commit 50fd013

Browse files
xylaaaaaxylaaaaa
committed
refactor(core): align columnar context naming and row ref usage
1 parent 763d1aa commit 50fd013

5 files changed

Lines changed: 38 additions & 37 deletions

File tree

‎src/paimon/common/data/columnar/columnar_array.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@
2626
#include "arrow/util/checked_cast.h"
2727
#include "arrow/util/decimal.h"
2828
#include "fmt/format.h"
29+
#include "paimon/common/data/columnar/columnar_batch_context.h"
2930
#include "paimon/common/data/columnar/columnar_map.h"
30-
#include "paimon/common/data/columnar/columnar_row.h"
31+
#include "paimon/common/data/columnar/columnar_row_ref.h"
3132
#include "paimon/common/utils/date_time_utils.h"
3233

3334
namespace paimon {
@@ -84,7 +85,8 @@ std::shared_ptr<InternalMap> ColumnarArray::GetMap(int32_t pos) const {
8485
std::shared_ptr<InternalRow> ColumnarArray::GetRow(int32_t pos, int32_t num_fields) const {
8586
auto struct_array = arrow::internal::checked_cast<const arrow::StructArray*>(array_);
8687
assert(struct_array);
87-
return std::make_shared<ColumnarRow>(struct_array->fields(), pool_, offset_ + pos);
88+
auto row_ctx = std::make_shared<ColumnarBatchContext>(nullptr, struct_array->fields(), pool_);
89+
return std::make_shared<ColumnarRowRef>(std::move(row_ctx), offset_ + pos);
8890
}
8991

9092
Result<std::vector<char>> ColumnarArray::ToBooleanArray() const {

‎src/paimon/common/data/columnar/columnar_batch_context.h‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2024-present Alibaba Inc.
2+
* Copyright 2026-present Alibaba Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -29,21 +29,19 @@ namespace paimon {
2929
class MemoryPool;
3030

3131
struct ColumnarBatchContext {
32-
ColumnarBatchContext(std::shared_ptr<arrow::StructArray> struct_array_in,
33-
const arrow::ArrayVector& array_vec_in,
34-
std::shared_ptr<MemoryPool> pool_in)
35-
: struct_array(std::move(struct_array_in)),
36-
pool(std::move(pool_in)),
37-
array_vec_holder(array_vec_in) {
38-
array_vec.reserve(array_vec_holder.size());
39-
for (const auto& array : array_vec_holder) {
40-
array_vec.push_back(array.get());
32+
ColumnarBatchContext(const std::shared_ptr<arrow::StructArray>& struct_array_in,
33+
const arrow::ArrayVector& field_arrays_in,
34+
const std::shared_ptr<MemoryPool>& pool_in)
35+
: struct_array(struct_array_in), pool(pool_in), field_arrays(field_arrays_in) {
36+
array_ptrs.reserve(field_arrays.size());
37+
for (const auto& array : field_arrays) {
38+
array_ptrs.push_back(array.get());
4139
}
4240
}
4341

4442
std::shared_ptr<arrow::StructArray> struct_array;
4543
std::shared_ptr<MemoryPool> pool;
46-
arrow::ArrayVector array_vec_holder;
47-
std::vector<const arrow::Array*> array_vec;
44+
arrow::ArrayVector field_arrays;
45+
std::vector<const arrow::Array*> array_ptrs;
4846
};
4947
} // namespace paimon

‎src/paimon/common/data/columnar/columnar_row_ref.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2024-present Alibaba Inc.
2+
* Copyright 2026-present Alibaba Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -31,7 +31,7 @@
3131
namespace paimon {
3232
Decimal ColumnarRowRef::GetDecimal(int32_t pos, int32_t precision, int32_t scale) const {
3333
using ArrayType = typename arrow::TypeTraits<arrow::Decimal128Type>::ArrayType;
34-
auto array = arrow::internal::checked_cast<const ArrayType*>(ctx_->array_vec[pos]);
34+
auto array = arrow::internal::checked_cast<const ArrayType*>(ctx_->array_ptrs[pos]);
3535
assert(array);
3636
arrow::Decimal128 decimal(array->GetValue(row_id_));
3737
return Decimal(precision, scale,
@@ -40,7 +40,7 @@ Decimal ColumnarRowRef::GetDecimal(int32_t pos, int32_t precision, int32_t scale
4040

4141
Timestamp ColumnarRowRef::GetTimestamp(int32_t pos, int32_t precision) const {
4242
using ArrayType = typename arrow::TypeTraits<arrow::TimestampType>::ArrayType;
43-
auto array = arrow::internal::checked_cast<const ArrayType*>(ctx_->array_vec[pos]);
43+
auto array = arrow::internal::checked_cast<const ArrayType*>(ctx_->array_ptrs[pos]);
4444
assert(array);
4545
int64_t data = array->Value(row_id_);
4646
auto timestamp_type =
@@ -55,23 +55,23 @@ Timestamp ColumnarRowRef::GetTimestamp(int32_t pos, int32_t precision) const {
5555

5656
std::shared_ptr<InternalRow> ColumnarRowRef::GetRow(int32_t pos, int32_t num_fields) const {
5757
auto struct_array =
58-
arrow::internal::checked_pointer_cast<arrow::StructArray>(ctx_->array_vec_holder[pos]);
58+
arrow::internal::checked_pointer_cast<arrow::StructArray>(ctx_->field_arrays[pos]);
5959
assert(struct_array);
6060
auto nested_ctx =
6161
std::make_shared<ColumnarBatchContext>(struct_array, struct_array->fields(), ctx_->pool);
6262
return std::make_shared<ColumnarRowRef>(std::move(nested_ctx), row_id_);
6363
}
6464

6565
std::shared_ptr<InternalArray> ColumnarRowRef::GetArray(int32_t pos) const {
66-
auto list_array = arrow::internal::checked_cast<const arrow::ListArray*>(ctx_->array_vec[pos]);
66+
auto list_array = arrow::internal::checked_cast<const arrow::ListArray*>(ctx_->array_ptrs[pos]);
6767
assert(list_array);
6868
int32_t offset = list_array->value_offset(row_id_);
6969
int32_t length = list_array->value_length(row_id_);
7070
return std::make_shared<ColumnarArray>(list_array->values(), ctx_->pool, offset, length);
7171
}
7272

7373
std::shared_ptr<InternalMap> ColumnarRowRef::GetMap(int32_t pos) const {
74-
auto map_array = arrow::internal::checked_cast<const arrow::MapArray*>(ctx_->array_vec[pos]);
74+
auto map_array = arrow::internal::checked_cast<const arrow::MapArray*>(ctx_->array_ptrs[pos]);
7575
assert(map_array);
7676
int32_t offset = map_array->value_offset(row_id_);
7777
int32_t length = map_array->value_length(row_id_);

‎src/paimon/common/data/columnar/columnar_row_ref.h‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2024-present Alibaba Inc.
2+
* Copyright 2026-present Alibaba Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -51,68 +51,69 @@ class ColumnarRowRef : public InternalRow {
5151
}
5252

5353
int32_t GetFieldCount() const override {
54-
return static_cast<int32_t>(ctx_->array_vec.size());
54+
return static_cast<int32_t>(ctx_->array_ptrs.size());
5555
}
5656

5757
bool IsNullAt(int32_t pos) const override {
58-
return ctx_->array_vec[pos]->IsNull(row_id_);
58+
return ctx_->array_ptrs[pos]->IsNull(row_id_);
5959
}
6060

6161
bool GetBoolean(int32_t pos) const override {
62-
return ColumnarUtils::GetGenericValue<arrow::BooleanType, bool>(ctx_->array_vec[pos],
62+
return ColumnarUtils::GetGenericValue<arrow::BooleanType, bool>(ctx_->array_ptrs[pos],
6363
row_id_);
6464
}
6565

6666
char GetByte(int32_t pos) const override {
67-
return ColumnarUtils::GetGenericValue<arrow::Int8Type, char>(ctx_->array_vec[pos], row_id_);
67+
return ColumnarUtils::GetGenericValue<arrow::Int8Type, char>(ctx_->array_ptrs[pos],
68+
row_id_);
6869
}
6970

7071
int16_t GetShort(int32_t pos) const override {
71-
return ColumnarUtils::GetGenericValue<arrow::Int16Type, int16_t>(ctx_->array_vec[pos],
72+
return ColumnarUtils::GetGenericValue<arrow::Int16Type, int16_t>(ctx_->array_ptrs[pos],
7273
row_id_);
7374
}
7475

7576
int32_t GetInt(int32_t pos) const override {
76-
return ColumnarUtils::GetGenericValue<arrow::Int32Type, int32_t>(ctx_->array_vec[pos],
77+
return ColumnarUtils::GetGenericValue<arrow::Int32Type, int32_t>(ctx_->array_ptrs[pos],
7778
row_id_);
7879
}
7980

8081
int32_t GetDate(int32_t pos) const override {
81-
return ColumnarUtils::GetGenericValue<arrow::Date32Type, int32_t>(ctx_->array_vec[pos],
82+
return ColumnarUtils::GetGenericValue<arrow::Date32Type, int32_t>(ctx_->array_ptrs[pos],
8283
row_id_);
8384
}
8485

8586
int64_t GetLong(int32_t pos) const override {
86-
return ColumnarUtils::GetGenericValue<arrow::Int64Type, int64_t>(ctx_->array_vec[pos],
87+
return ColumnarUtils::GetGenericValue<arrow::Int64Type, int64_t>(ctx_->array_ptrs[pos],
8788
row_id_);
8889
}
8990

9091
float GetFloat(int32_t pos) const override {
91-
return ColumnarUtils::GetGenericValue<arrow::FloatType, float>(ctx_->array_vec[pos],
92+
return ColumnarUtils::GetGenericValue<arrow::FloatType, float>(ctx_->array_ptrs[pos],
9293
row_id_);
9394
}
9495

9596
double GetDouble(int32_t pos) const override {
96-
return ColumnarUtils::GetGenericValue<arrow::DoubleType, double>(ctx_->array_vec[pos],
97+
return ColumnarUtils::GetGenericValue<arrow::DoubleType, double>(ctx_->array_ptrs[pos],
9798
row_id_);
9899
}
99100

100101
BinaryString GetString(int32_t pos) const override {
101-
auto bytes = ColumnarUtils::GetBytes<arrow::StringType>(ctx_->array_vec[pos], row_id_,
102+
auto bytes = ColumnarUtils::GetBytes<arrow::StringType>(ctx_->array_ptrs[pos], row_id_,
102103
ctx_->pool.get());
103104
return BinaryString::FromBytes(bytes);
104105
}
105106

106107
std::string_view GetStringView(int32_t pos) const override {
107-
return ColumnarUtils::GetView(ctx_->array_vec[pos], row_id_);
108+
return ColumnarUtils::GetView(ctx_->array_ptrs[pos], row_id_);
108109
}
109110

110111
Decimal GetDecimal(int32_t pos, int32_t precision, int32_t scale) const override;
111112

112113
Timestamp GetTimestamp(int32_t pos, int32_t precision) const override;
113114

114115
std::shared_ptr<Bytes> GetBinary(int32_t pos) const override {
115-
return ColumnarUtils::GetBytes<arrow::BinaryType>(ctx_->array_vec[pos], row_id_,
116+
return ColumnarUtils::GetBytes<arrow::BinaryType>(ctx_->array_ptrs[pos], row_id_,
116117
ctx_->pool.get());
117118
}
118119

‎src/paimon/core/io/key_value_data_file_record_reader.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,10 @@ bool KeyValueDataFileRecordReader::Iterator::HasNext() const {
6868

6969
Result<KeyValue> KeyValueDataFileRecordReader::Iterator::Next() {
7070
assert(HasNext());
71-
// as key is only used in merge sort, do not hold the data in ColumnarRowRef
71+
// key is only used in merge sort; key context does not hold parent struct array
7272
auto key = std::make_unique<ColumnarRowRef>(reader_->key_ctx_, cursor_);
73-
// as value is used in merge sort and projection (maybe async and multi-thread), hold the data
74-
// in ColumnarRowRef
73+
// value is used in merge sort and projection (maybe async and multi-thread), so value context
74+
// holds parent struct array to ensure data remains valid
7575
auto value = std::make_unique<ColumnarRowRef>(reader_->value_ctx_, cursor_);
7676
PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
7777
RowKind::FromByteValue(reader_->row_kind_array_->Value(cursor_)));

0 commit comments

Comments
 (0)