Skip to content

Commit 8c93f60

Browse files
zjw1111zjw1111
committed
fix
1 parent 799cb74 commit 8c93f60

7 files changed

Lines changed: 63 additions & 69 deletions

File tree

‎include/paimon/read_context.h‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
#include "paimon/result.h"
2828
#include "paimon/type_fwd.h"
2929
#include "paimon/utils/read_ahead_cache.h"
30-
#include "paimon/utils/special_field_ids.h"
3130
#include "paimon/visibility.h"
3231

3332
namespace paimon {
@@ -179,9 +178,9 @@ class PAIMON_EXPORT ReadContextBuilder {
179178
/// @param read_field_ids Vector of field ids to read from the table.
180179
/// @return Reference to this builder for method chaining.
181180
/// @note Currently supports top-level field selection. Future versions may support
182-
/// nested field selection using ArrowSchema for more granular projection,
183-
/// If SetReadFieldIds() call and SetReadSchema() are natually are mutually
184-
/// exclusive. Calling both will ignore the read schema set by SetReadSchema().
181+
/// nested field selection using ArrowSchema for more granular projection.
182+
/// @note SetReadFieldIds() and SetReadSchema() are mutually exclusive.
183+
/// Calling both will ignore the read schema set by SetReadSchema().
185184
ReadContextBuilder& SetReadFieldIds(const std::vector<int32_t>& read_field_ids);
186185

187186
/// Set a configuration options map to set some option entries which are not defined in the

‎src/paimon/core/postpone/postpone_bucket_writer_test.cpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,10 @@ class PostponeBucketWriterTest : public ::testing::Test,
9898
ASSERT_OK_AND_ASSIGN(auto reader_builder,
9999
file_format->CreateReaderBuilder(/*batch_size=*/10));
100100
ASSERT_OK_AND_ASSIGN(auto batch_reader, reader_builder->Build(input_stream));
101-
// if (file_format_str == "avro") {
102101
auto c_schema = std::make_unique<::ArrowSchema>();
103102
ASSERT_TRUE(arrow::ExportType(*file_schema, c_schema.get()).ok());
104103
ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr,
105104
/*selection_bitmap=*/std::nullopt));
106-
// }
107105
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> result_array,
108106
ReadResultCollector::CollectResult(batch_reader.get()));
109107
ASSERT_TRUE(expected_array->Equals(result_array)) << result_array->ToString() << "\n != \n"

‎src/paimon/format/avro/avro_direct_encoder.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@
2828
#include "arrow/api.h"
2929
#include "arrow/type.h"
3030
#include "arrow/util/checked_cast.h"
31+
#include "fmt/format.h"
3132
#include "paimon/common/utils/date_time_utils.h"
33+
#include "paimon/format/avro/avro_utils.h"
3234
#include "paimon/result.h"
3335

3436
namespace paimon::avro {
@@ -174,12 +176,11 @@ Status AvroDirectEncoder::EncodeArrowToAvro(const ::avro::NodePtr& avro_node,
174176
timestamp *
175177
DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::MILLISECOND]);
176178
} else {
177-
std::stringstream logical_type_str;
178-
avro_node->logicalType().printJson(logical_type_str);
179-
return Status::Invalid(fmt::format(
180-
"Unsupported timestamp type with avro logical type {} and arrow time "
181-
"unit {}.",
182-
logical_type_str.str(), DateTimeUtils::GetArrowTimeUnitStr(unit)));
179+
return Status::Invalid(
180+
fmt::format("Unsupported timestamp type with avro logical type {} and "
181+
"arrow time unit {}.",
182+
AvroUtils::ToString(avro_node->logicalType()),
183+
DateTimeUtils::GetArrowTimeUnitStr(unit)));
183184
}
184185
return Status::OK();
185186
}

‎src/paimon/format/avro/avro_file_format_test.cpp‎

Lines changed: 34 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@
2121
#include <vector>
2222

2323
#include "arrow/api.h"
24-
#include "arrow/array/array_base.h"
25-
#include "arrow/array/array_nested.h"
2624
#include "arrow/c/abi.h"
2725
#include "arrow/c/bridge.h"
2826
#include "arrow/ipc/json_simple.h"
@@ -135,11 +133,11 @@ TEST_P(AvroFileFormatTest, TestComplexTypes) {
135133
arrow::field("f11", arrow::list(arrow::int32())),
136134
arrow::field("f12", arrow::map(arrow::utf8(), arrow::utf8())),
137135
arrow::field("f13", arrow::map(arrow::int32(), arrow::utf8())),
138-
arrow::field("f14", arrow::map(arrow::struct_({field("f0", arrow::int32())}),
136+
arrow::field("f14", arrow::map(arrow::struct_({arrow::field("f0", arrow::int32())}),
139137
arrow::map(arrow::int32(), arrow::utf8()))),
140-
arrow::field("f15",
141-
arrow::struct_({field("sub1", arrow::int64()), field("sub2", arrow::float64()),
142-
field("sub3", arrow::boolean())})),
138+
arrow::field("f15", arrow::struct_({arrow::field("sub1", arrow::int64()),
139+
arrow::field("sub2", arrow::float64()),
140+
arrow::field("sub3", arrow::boolean())})),
143141
};
144142
auto schema = arrow::schema(fields);
145143
auto data_type = arrow::struct_(fields);
@@ -213,25 +211,28 @@ TEST_P(AvroFileFormatTest, TestNestedMap) {
213211
arrow::map(
214212
arrow::int32(),
215213
arrow::struct_(
216-
{field("f5.a",
217-
arrow::struct_(
218-
{field("f5.a.0", arrow::utf8()), field("f5.sub2", arrow::int32()),
219-
field("f5.a.1", arrow::timestamp(arrow::TimeUnit::MICRO))})),
220-
field("f5.b", arrow::list(arrow::utf8())),
221-
field("f5.c", arrow::map(arrow::utf8(), arrow::int32()))}))),
214+
{arrow::field(
215+
"f5.a",
216+
arrow::struct_(
217+
{arrow::field("f5.a.0", arrow::utf8()),
218+
arrow::field("f5.sub2", arrow::int32()),
219+
arrow::field("f5.a.1", arrow::timestamp(arrow::TimeUnit::MICRO))})),
220+
arrow::field("f5.b", arrow::list(arrow::utf8())),
221+
arrow::field("f5.c", arrow::map(arrow::utf8(), arrow::int32()))}))),
222222
arrow::field(
223223
"f6", arrow::map(arrow::utf8(), arrow::map(arrow::utf8(), arrow::list(arrow::utf8())))),
224224
arrow::field("f7", arrow::map(arrow::int32(), arrow::boolean())),
225-
arrow::field("f8", arrow::map(arrow::int64(), arrow::decimal(2, 2))),
225+
arrow::field("f8", arrow::map(arrow::int64(), arrow::decimal128(2, 2))),
226226
arrow::field("f9", arrow::map(arrow::date32(), arrow::float32())),
227227
arrow::field("f10", arrow::map(arrow::binary(), arrow::float64())),
228228
arrow::field("f11", arrow::map(arrow::int32(), arrow::list(arrow::int64()))),
229229
arrow::field(
230-
"f12", arrow::map(arrow::utf8(),
231-
arrow::list(arrow::struct_(
232-
{field("name", arrow::utf8()),
233-
field("scores", arrow::list(arrow::float32())),
234-
field("info", arrow::map(arrow::float32(), arrow::utf8()))}))))};
230+
"f12",
231+
arrow::map(arrow::utf8(),
232+
arrow::list(arrow::struct_(
233+
{arrow::field("name", arrow::utf8()),
234+
arrow::field("scores", arrow::list(arrow::float32())),
235+
arrow::field("info", arrow::map(arrow::float32(), arrow::utf8()))}))))};
235236

236237
auto schema = arrow::schema(fields);
237238
auto data_type = arrow::struct_(fields);
@@ -319,25 +320,28 @@ TEST_P(AvroFileFormatTest, TestReadRow) {
319320
arrow::map(
320321
arrow::int32(),
321322
arrow::struct_(
322-
{field("f5.a",
323-
arrow::struct_(
324-
{field("f5.a.0", arrow::utf8()), field("f5.sub2", arrow::int32()),
325-
field("f5.a.1", arrow::timestamp(arrow::TimeUnit::MICRO))})),
326-
field("f5.b", arrow::list(arrow::utf8())),
327-
field("f5.c", arrow::map(arrow::utf8(), arrow::int32()))}))),
323+
{arrow::field(
324+
"f5.a",
325+
arrow::struct_(
326+
{arrow::field("f5.a.0", arrow::utf8()),
327+
arrow::field("f5.sub2", arrow::int32()),
328+
arrow::field("f5.a.1", arrow::timestamp(arrow::TimeUnit::MICRO))})),
329+
arrow::field("f5.b", arrow::list(arrow::utf8())),
330+
arrow::field("f5.c", arrow::map(arrow::utf8(), arrow::int32()))}))),
328331
arrow::field(
329332
"f6", arrow::map(arrow::utf8(), arrow::map(arrow::utf8(), arrow::list(arrow::utf8())))),
330333
arrow::field("f7", arrow::map(arrow::int32(), arrow::boolean())),
331-
arrow::field("f8", arrow::map(arrow::int64(), arrow::decimal(2, 2))),
334+
arrow::field("f8", arrow::map(arrow::int64(), arrow::decimal128(2, 2))),
332335
arrow::field("f9", arrow::map(arrow::date32(), arrow::float32())),
333336
arrow::field("f10", arrow::map(arrow::binary(), arrow::float64())),
334337
arrow::field("f11", arrow::map(arrow::int32(), arrow::list(arrow::int64()))),
335338
arrow::field(
336-
"f12", arrow::map(arrow::utf8(),
337-
arrow::list(arrow::struct_(
338-
{field("name", arrow::utf8()),
339-
field("scores", arrow::list(arrow::float32())),
340-
field("info", arrow::map(arrow::float32(), arrow::utf8()))}))))};
339+
"f12",
340+
arrow::map(arrow::utf8(),
341+
arrow::list(arrow::struct_(
342+
{arrow::field("name", arrow::utf8()),
343+
arrow::field("scores", arrow::list(arrow::float32())),
344+
arrow::field("info", arrow::map(arrow::float32(), arrow::utf8()))}))))};
341345

342346
auto schema = arrow::schema(fields);
343347
auto data_type = arrow::struct_(fields);

‎src/paimon/format/avro/avro_stats_extractor.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ class AvroStatsExtractor : public FormatStatsExtractor {
5050
const std::shared_ptr<FileSystem>& file_system, const std::string& path,
5151
const std::shared_ptr<MemoryPool>& pool) override {
5252
assert(false);
53-
return paimon::Status::Invalid("avro format do not support ExtractWithFileInfo");
53+
return Status::Invalid("Avro format does not support ExtractWithFileInfo.");
5454
}
5555

5656
private:

‎src/paimon/format/avro/avro_stats_extractor_test.cpp‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,8 @@
1919
#include <vector>
2020

2121
#include "arrow/api.h"
22-
#include "arrow/array/array_base.h"
23-
#include "arrow/array/array_nested.h"
2422
#include "arrow/c/abi.h"
2523
#include "arrow/c/bridge.h"
26-
#include "arrow/c/helpers.h"
2724
#include "arrow/ipc/json_simple.h"
2825
#include "gtest/gtest.h"
2926
#include "paimon/common/data/binary_row.h"
@@ -32,11 +29,13 @@
3229
#include "paimon/core/stats/simple_stats_converter.h"
3330
#include "paimon/format/avro/avro_file_format.h"
3431
#include "paimon/format/avro/avro_format_writer.h"
32+
#include "paimon/format/file_format_factory.h"
3533
#include "paimon/fs/file_system.h"
3634
#include "paimon/fs/local/local_file_system.h"
3735
#include "paimon/memory/memory_pool.h"
3836
#include "paimon/status.h"
3937
#include "paimon/testing/utils/testharness.h"
38+
4039
namespace paimon::avro::test {
4140

4241
class AvroStatsExtractorTest : public ::testing::Test {
@@ -50,9 +49,10 @@ class AvroStatsExtractorTest : public ::testing::Test {
5049
::ArrowSchema c_schema;
5150
ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
5251

53-
AvroFileFormat format({{"file.format", "avro"}, {"manifest.format", "avro"}});
52+
ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileFormat> file_format,
53+
FileFormatFactory::Get("avro", {}));
5454
ASSERT_OK_AND_ASSIGN(auto writer_builder,
55-
format.CreateWriterBuilder(&c_schema, /*batch_size=*/1024));
55+
file_format->CreateWriterBuilder(&c_schema, /*batch_size=*/1024));
5656

5757
auto fs = std::make_shared<LocalFileSystem>();
5858
ASSERT_OK_AND_ASSIGN(std::unique_ptr<OutputStream> output_stream,
@@ -72,13 +72,6 @@ class AvroStatsExtractorTest : public ::testing::Test {
7272
}
7373
};
7474

75-
static AvroStatsExtractor MakeExtractor() {
76-
std::map<std::string, std::string> opts;
77-
opts["file.format"] = "avro";
78-
opts["manifest.format"] = "avro";
79-
return AvroStatsExtractor(opts);
80-
}
81-
8275
TEST_F(AvroStatsExtractorTest, TestPrimitiveStatsExtractor) {
8376
auto timezone = DateTimeUtils::GetLocalTimezoneName();
8477
arrow::FieldVector fields = {
@@ -106,10 +99,10 @@ TEST_F(AvroStatsExtractorTest, TestPrimitiveStatsExtractor) {
10699
auto schema = std::make_shared<arrow::Schema>(fields);
107100
auto array = std::dynamic_pointer_cast<arrow::StructArray>(
108101
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields}), R"([
109-
[1, 11, 111, 1111, 1.1, 1.11, "Hello", "你好", 1234, "2033-05-18 03:33:20.0", "1.22", true, "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0"],
110-
[2, 22, 222, 2222, 2.2, 2.22, "World", "世界", -1234, "1899-01-01 00:59:20.001001001", "2.22", false, "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001","1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001"],
111-
[null, null, 0, null, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null]
112-
])")
102+
[1, 11, 111, 1111, 1.1, 1.11, "Hello", "你好", 1234, "2033-05-18 03:33:20.0", "1.22", true, "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0", "2033-05-18 03:33:20.0"],
103+
[2, 22, 222, 2222, 2.2, 2.22, "World", "世界", -1234, "1899-01-01 00:59:20.001001001", "2.22", false, "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001","1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20", "1899-01-01 00:59:20.001001001"],
104+
[null, null, 0, null, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null]
105+
])")
113106
.ValueOrDie());
114107
auto src_chunk_array = std::make_shared<arrow::ChunkedArray>(arrow::ArrayVector({array}));
115108

@@ -139,10 +132,10 @@ TEST_F(AvroStatsExtractorTest, TestNestedType) {
139132
auto schema = arrow::schema(fields);
140133
auto array = std::dynamic_pointer_cast<arrow::StructArray>(
141134
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields}), R"([
142-
[null, [true, 2]],
143-
[[0.1, 0.3], [true, 1]],
144-
[[1.1, 1.2], null]
145-
])")
135+
[null, [true, 2]],
136+
[[0.1, 0.3], [true, 1]],
137+
[[1.1, 1.2], null]
138+
])")
146139
.ValueOrDie());
147140
auto src_chunk_array = std::make_shared<arrow::ChunkedArray>(arrow::ArrayVector({array}));
148141

@@ -151,7 +144,7 @@ TEST_F(AvroStatsExtractorTest, TestNestedType) {
151144
std::string file_path = dir->Str() + "/test.avro";
152145
WriteAvroFile(file_path, src_chunk_array, schema);
153146

154-
AvroStatsExtractor extractor = MakeExtractor();
147+
AvroStatsExtractor extractor({});
155148
auto fs = std::make_shared<LocalFileSystem>();
156149
ASSERT_OK_AND_ASSIGN(auto results, extractor.Extract(fs, file_path, GetDefaultPool()));
157150

@@ -194,8 +187,8 @@ TEST_F(AvroStatsExtractorTest, TestNullForAllType) {
194187
auto schema = std::make_shared<arrow::Schema>(fields);
195188
auto src_array = std::dynamic_pointer_cast<arrow::StructArray>(
196189
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields}), R"([
197-
[null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null]
198-
])")
190+
[null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null]
191+
])")
199192
.ValueOrDie());
200193
auto src_chunk_array = std::make_shared<arrow::ChunkedArray>(arrow::ArrayVector({src_array}));
201194

@@ -204,7 +197,7 @@ TEST_F(AvroStatsExtractorTest, TestNullForAllType) {
204197
std::string file_path = dir->Str() + "/test.avro";
205198
WriteAvroFile(file_path, src_chunk_array, schema);
206199

207-
AvroStatsExtractor extractor = MakeExtractor();
200+
AvroStatsExtractor extractor({});
208201
auto fs = std::make_shared<LocalFileSystem>();
209202
ASSERT_OK_AND_ASSIGN(auto column_stats, extractor.Extract(fs, file_path, GetDefaultPool()));
210203

‎test/inte/scan_and_read_inte_test.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2619,9 +2619,8 @@ TEST_F(ScanAndReadInteTest, TestAvroWithAppendTable) {
26192619
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), result_json)
26202620
.ValueOrDie());
26212621
ASSERT_TRUE(expected);
2622-
auto copied_array = arrow::Concatenate(read_result->chunks()).ValueOrDie();
26232622
ASSERT_TRUE(expected->Equals(read_result))
2624-
<< "read_result: " << copied_array->ToString() << "expected: " << expected->ToString();
2623+
<< "read_result: " << read_result->ToString() << "expected: " << expected->ToString();
26252624
};
26262625

26272626
read_data(1, R"([

0 commit comments

Comments
 (0)