Skip to content

Commit 84f5653

Browse files
lxy-9602lxy-9602
committed
fix little
1 parent 46a21e9 commit 84f5653

13 files changed

Lines changed: 52 additions & 30 deletions

‎include/paimon/global_index/global_indexer.h‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,7 @@ class PAIMON_EXPORT GlobalIndexer {
3838
virtual ~GlobalIndexer() = default;
3939

4040
/// Returns additional table fields required during index construction.
41-
virtual Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const {
42-
return std::optional<std::vector<std::string>>(std::nullopt);
43-
}
41+
virtual Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const = 0;
4442

4543
/// Creates a writer for building a global index on a specific field.
4644
///

‎src/paimon/common/global_index/bitmap/bitmap_global_index.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919
#include "paimon/common/global_index/wrap/file_index_writer_wrapper.h"
2020

2121
namespace paimon {
22+
Result<std::optional<std::vector<std::string>>> BitmapGlobalIndex::GetExtraFieldNames() const {
23+
return std::optional<std::vector<std::string>>(std::nullopt);
24+
}
25+
2226
Result<std::shared_ptr<GlobalIndexWriter>> BitmapGlobalIndex::CreateWriter(
2327
const std::string& field_name, ::ArrowSchema* arrow_schema,
2428
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/common/global_index/bitmap/bitmap_global_index.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#pragma once
1818

1919
#include <memory>
20+
#include <optional>
2021
#include <string>
2122
#include <vector>
2223

@@ -29,6 +30,8 @@ class BitmapGlobalIndex : public GlobalIndexer {
2930
public:
3031
explicit BitmapGlobalIndex(const std::shared_ptr<BitmapFileIndex>& index) : index_(index) {}
3132

33+
Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
34+
3235
Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
3336
const std::string& field_name, ::ArrowSchema* arrow_schema,
3437
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/common/global_index/btree/btree_global_indexer.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ Result<std::unique_ptr<BTreeGlobalIndexer>> BTreeGlobalIndexer::Create(
5454
return std::unique_ptr<BTreeGlobalIndexer>(new BTreeGlobalIndexer(cache_manager, options));
5555
}
5656

57+
Result<std::optional<std::vector<std::string>>> BTreeGlobalIndexer::GetExtraFieldNames() const {
58+
return std::optional<std::vector<std::string>>(std::nullopt);
59+
}
60+
5761
Result<std::shared_ptr<GlobalIndexWriter>> BTreeGlobalIndexer::CreateWriter(
5862
const std::string& field_name, ::ArrowSchema* arrow_schema,
5963
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/common/global_index/btree/btree_global_indexer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ class BTreeGlobalIndexer : public GlobalIndexer {
5353
static Result<std::unique_ptr<BTreeGlobalIndexer>> Create(
5454
const std::map<std::string, std::string>& options);
5555

56+
Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
57+
5658
Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
5759
const std::string& field_name, ::ArrowSchema* arrow_schema,
5860
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919
#include "paimon/common/global_index/wrap/file_index_writer_wrapper.h"
2020

2121
namespace paimon {
22+
Result<std::optional<std::vector<std::string>>> RangeBitmapGlobalIndex::GetExtraFieldNames() const {
23+
return std::optional<std::vector<std::string>>(std::nullopt);
24+
}
25+
2226
Result<std::shared_ptr<GlobalIndexWriter>> RangeBitmapGlobalIndex::CreateWriter(
2327
const std::string& field_name, ::ArrowSchema* arrow_schema,
2428
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#pragma once
1818

1919
#include <memory>
20+
#include <optional>
2021
#include <string>
2122
#include <vector>
2223

@@ -30,6 +31,8 @@ class RangeBitmapGlobalIndex : public GlobalIndexer {
3031
explicit RangeBitmapGlobalIndex(const std::shared_ptr<RangeBitmapFileIndex>& index)
3132
: index_(index) {}
3233

34+
Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
35+
3336
Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
3437
const std::string& field_name, ::ArrowSchema* arrow_schema,
3538
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/global_index/lucene/lucene_global_index.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ namespace paimon::lucene {
3535
LuceneGlobalIndex::LuceneGlobalIndex(const std::map<std::string, std::string>& options)
3636
: options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {}
3737

38+
Result<std::optional<std::vector<std::string>>> LuceneGlobalIndex::GetExtraFieldNames() const {
39+
return std::optional<std::vector<std::string>>(std::nullopt);
40+
}
41+
3842
Result<std::shared_ptr<GlobalIndexWriter>> LuceneGlobalIndex::CreateWriter(
3943
const std::string& field_name, ::ArrowSchema* arrow_schema,
4044
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/global_index/lucene/lucene_global_index.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#pragma once
1818

1919
#include <memory>
20+
#include <optional>
2021
#include <string>
2122
#include <vector>
2223

@@ -31,6 +32,8 @@ class LuceneGlobalIndex : public GlobalIndexer {
3132
public:
3233
explicit LuceneGlobalIndex(const std::map<std::string, std::string>& options);
3334

35+
Result<std::optional<std::vector<std::string>>> GetExtraFieldNames() const override;
36+
3437
Result<std::shared_ptr<GlobalIndexWriter>> CreateWriter(
3538
const std::string& field_name, ::ArrowSchema* arrow_schema,
3639
const std::shared_ptr<GlobalIndexFileWriter>& file_writer,

‎src/paimon/global_index/lumina/lumina_global_index.cpp‎

Lines changed: 13 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,6 @@ Result<LuminaTagField> ParseTagField(const rapidjson::Value& obj, const std::str
9696
return Status::Invalid(
9797
fmt::format("lumina tag_schema {} key_name must not be empty", tag_label));
9898
}
99-
if (key_name.size() > ::lumina::core::kMaxTagKNameLength) {
100-
return Status::Invalid(fmt::format("lumina tag_schema {} key_name exceeds max length {}",
101-
tag_label, ::lumina::core::kMaxTagKNameLength));
102-
}
103-
10499
LuminaTagField::Type parsed_type;
105100
if (type == std::string(::lumina::core::kExtensionTagTypeEnum)) {
106101
parsed_type = LuminaTagField::Type::ENUM;
@@ -178,6 +173,15 @@ Status AppendTagValue(const std::shared_ptr<arrow::Array>& array, int64_t index,
178173
return Status::OK();
179174
}
180175

176+
auto validate_array_type = [&](arrow::Type::type expected_type,
177+
const char* value_type_name) -> Status {
178+
if (array->type_id() != expected_type) {
179+
return Status::Invalid(fmt::format("lumina {} tag field has unsupported arrow type {}",
180+
value_type_name, array->type()->ToString()));
181+
}
182+
return Status::OK();
183+
};
184+
181185
if constexpr (std::is_same_v<ValueType, int32_t>) {
182186
switch (array->type_id()) {
183187
case arrow::Type::INT8:
@@ -195,30 +199,16 @@ Status AppendTagValue(const std::shared_ptr<arrow::Array>& array, int64_t index,
195199
array->type()->ToString()));
196200
}
197201
} else if constexpr (std::is_same_v<ValueType, int64_t>) {
198-
if (array->type_id() != arrow::Type::INT64) {
199-
return Status::Invalid(fmt::format(
200-
"lumina int64 tag field has unsupported arrow type {}", array->type()->ToString()));
201-
}
202+
PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::INT64, "int64"));
202203
AppendPrimitiveTagValue<ValueType, arrow::Int64Array>(array, index, values);
203204
} else if constexpr (std::is_same_v<ValueType, float>) {
204-
if (array->type_id() != arrow::Type::FLOAT) {
205-
return Status::Invalid(fmt::format(
206-
"lumina float tag field has unsupported arrow type {}", array->type()->ToString()));
207-
}
205+
PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::FLOAT, "float"));
208206
AppendPrimitiveTagValue<ValueType, arrow::FloatArray>(array, index, values);
209207
} else if constexpr (std::is_same_v<ValueType, double>) {
210-
if (array->type_id() != arrow::Type::DOUBLE) {
211-
return Status::Invalid(
212-
fmt::format("lumina double tag field has unsupported arrow type {}",
213-
array->type()->ToString()));
214-
}
208+
PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::DOUBLE, "double"));
215209
AppendPrimitiveTagValue<ValueType, arrow::DoubleArray>(array, index, values);
216210
} else if constexpr (std::is_same_v<ValueType, std::string>) {
217-
if (array->type_id() != arrow::Type::STRING) {
218-
return Status::Invalid(
219-
fmt::format("lumina string tag field has unsupported arrow type {}",
220-
array->type()->ToString()));
221-
}
211+
PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::STRING, "string"));
222212
auto string_array = static_cast<const arrow::StringArray*>(array.get());
223213
auto view = string_array->GetView(index);
224214
values->emplace_back(view.data(), view.size());

0 commit comments

Comments
 (0)