Skip to content

Commit 1addb37

Browse files
duanyyyyyyyduanyyyyyyy
authored
Merge branch 'main' into duanyan/remap_predicate_field_index
2 parents a849a59 + 83c3b04 commit 1addb37

12 files changed

Lines changed: 202 additions & 375 deletions

‎src/paimon/common/utils/binary_row_partition_computer_test.cpp‎

Lines changed: 36 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -43,16 +43,14 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
4343
arrow::field("f6", arrow::int32()),
4444
arrow::field("f7", arrow::int64()),
4545
arrow::field("f8", arrow::int64()),
46-
arrow::field("f9", arrow::float32()),
47-
arrow::field("f10", arrow::float64()),
48-
arrow::field("f11", arrow::utf8()),
49-
arrow::field("f12", arrow::utf8()),
50-
arrow::field("f13", arrow::date32()),
46+
arrow::field("f9", arrow::utf8()),
47+
arrow::field("f10", arrow::utf8()),
48+
arrow::field("f11", arrow::date32()),
5149
arrow::field("non-partition-field", arrow::int32())};
5250

5351
auto schema = arrow::schema(fields);
54-
std::vector<std::string> partition_keys = {"f0", "f2", "f1", "f3", "f4", "f5", "f6",
55-
"f7", "f8", "f9", "f10", "f11", "f12", "f13"};
52+
std::vector<std::string> partition_keys = {"f0", "f2", "f1", "f3", "f4", "f5",
53+
"f6", "f7", "f8", "f9", "f10", "f11"};
5654
{
5755
// simple case with legacy_partition_name_enabled = true
5856
ASSERT_OK_AND_ASSIGN(
@@ -69,14 +67,12 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
6967
{"f6", "-448489"},
7068
{"f7", "-9223372036854775808"},
7169
{"f8", "182737474"},
72-
{"f9", "0.334"},
73-
{"f10", "467.66472"},
74-
{"f11", "abcde"},
75-
{"f12", "这是一个很长很长的中文"},
76-
{"f13", "5"},
70+
{"f9", "abcde"},
71+
{"f10", "这是一个很长很长的中文"},
72+
{"f11", "5"},
7773
};
7874
ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map));
79-
ASSERT_EQ(14, row.GetFieldCount());
75+
ASSERT_EQ(12, row.GetFieldCount());
8076
ASSERT_EQ(true, row.GetBoolean(0));
8177
ASSERT_EQ(-20, row.GetByte(1));
8278
ASSERT_EQ(10, row.GetByte(2));
@@ -86,15 +82,13 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
8682
ASSERT_EQ(-448489, row.GetInt(6));
8783
ASSERT_EQ(std::numeric_limits<int64_t>::min(), row.GetLong(7));
8884
ASSERT_EQ(182737474l, row.GetLong(8));
89-
ASSERT_NEAR(0.334, row.GetFloat(9), 0.0000001);
90-
ASSERT_NEAR(467.66472, row.GetDouble(10), 0.0000001);
91-
ASSERT_EQ("abcde", row.GetString(11).ToString());
92-
ASSERT_EQ("这是一个很长很长的中文", row.GetString(12).ToString());
93-
ASSERT_EQ(5, row.GetDate(13));
85+
ASSERT_EQ("abcde", row.GetString(9).ToString());
86+
ASSERT_EQ("这是一个很长很长的中文", row.GetString(10).ToString());
87+
ASSERT_EQ(5, row.GetDate(11));
9488

9589
std::vector<std::pair<std::string, std::string>> part_values;
9690
ASSERT_OK_AND_ASSIGN(part_values, computer->GeneratePartitionVector(row));
97-
ASSERT_EQ(14, part_values.size());
91+
ASSERT_EQ(12, part_values.size());
9892
std::map<std::string, std::string> actual_part_values_map;
9993
for (const auto& [key, value] : part_values) {
10094
actual_part_values_map[key] = value;
@@ -117,14 +111,12 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
117111
{"f6", "-448489"},
118112
{"f7", "-9223372036854775808"},
119113
{"f8", "182737474"},
120-
{"f9", "0.334"},
121-
{"f10", "467.66472"},
122-
{"f11", "abcde"},
123-
{"f12", "这是一个很长很长的中文"},
124-
{"f13", "1970-01-06"},
114+
{"f9", "abcde"},
115+
{"f10", "这是一个很长很长的中文"},
116+
{"f11", "1970-01-06"},
125117
};
126118
ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map));
127-
ASSERT_EQ(14, row.GetFieldCount());
119+
ASSERT_EQ(12, row.GetFieldCount());
128120
ASSERT_EQ(true, row.GetBoolean(0));
129121
ASSERT_EQ(-20, row.GetByte(1));
130122
ASSERT_EQ(10, row.GetByte(2));
@@ -134,15 +126,13 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
134126
ASSERT_EQ(-448489, row.GetInt(6));
135127
ASSERT_EQ(std::numeric_limits<int64_t>::min(), row.GetLong(7));
136128
ASSERT_EQ(182737474l, row.GetLong(8));
137-
ASSERT_NEAR(0.334, row.GetFloat(9), 0.0000001);
138-
ASSERT_NEAR(467.66472, row.GetDouble(10), 0.0000001);
139-
ASSERT_EQ("abcde", row.GetString(11).ToString());
140-
ASSERT_EQ("这是一个很长很长的中文", row.GetString(12).ToString());
141-
ASSERT_EQ(5, row.GetDate(13));
129+
ASSERT_EQ("abcde", row.GetString(9).ToString());
130+
ASSERT_EQ("这是一个很长很长的中文", row.GetString(10).ToString());
131+
ASSERT_EQ(5, row.GetDate(11));
142132

143133
std::vector<std::pair<std::string, std::string>> part_values;
144134
ASSERT_OK_AND_ASSIGN(part_values, computer->GeneratePartitionVector(row));
145-
ASSERT_EQ(14, part_values.size());
135+
ASSERT_EQ(12, part_values.size());
146136
std::map<std::string, std::string> actual_part_values_map;
147137
for (const auto& [key, value] : part_values) {
148138
actual_part_values_map[key] = value;
@@ -165,14 +155,12 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
165155
{"f6", "-448489"},
166156
{"f7", "-9223372036854775808"},
167157
{"f8", "182737474"},
168-
{"f9", "0.334"},
169-
{"f10", "467.66472"},
170-
{"f11", " "},
171-
{"f12", "__DEFAULT_PARTITION__"},
172-
{"f13", "5"},
158+
{"f9", " "},
159+
{"f10", "__DEFAULT_PARTITION__"},
160+
{"f11", "5"},
173161
};
174162
ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map));
175-
ASSERT_EQ(14, row.GetFieldCount());
163+
ASSERT_EQ(12, row.GetFieldCount());
176164
ASSERT_EQ(true, row.GetBoolean(0));
177165
ASSERT_EQ(-20, row.GetByte(1));
178166
ASSERT_EQ(10, row.GetByte(2));
@@ -182,15 +170,13 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
182170
ASSERT_EQ(-448489, row.GetInt(6));
183171
ASSERT_EQ(std::numeric_limits<int64_t>::min(), row.GetLong(7));
184172
ASSERT_EQ(182737474l, row.GetLong(8));
185-
ASSERT_NEAR(0.334, row.GetFloat(9), 0.0000001);
186-
ASSERT_NEAR(467.66472, row.GetDouble(10), 0.0000001);
187-
ASSERT_EQ(" ", row.GetString(11).ToString());
188-
ASSERT_TRUE(row.IsNullAt(12));
189-
ASSERT_EQ(5, row.GetInt(13));
173+
ASSERT_EQ(" ", row.GetString(9).ToString());
174+
ASSERT_TRUE(row.IsNullAt(10));
175+
ASSERT_EQ(5, row.GetInt(11));
190176

191177
std::vector<std::pair<std::string, std::string>> part_values;
192178
ASSERT_OK_AND_ASSIGN(part_values, computer->GeneratePartitionVector(row));
193-
ASSERT_EQ(14, part_values.size());
179+
ASSERT_EQ(12, part_values.size());
194180
std::map<std::string, std::string> actual_part_values_map;
195181
for (const auto& [key, value] : part_values) {
196182
actual_part_values_map[key] = value;
@@ -205,11 +191,9 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
205191
{"f6", "-448489"},
206192
{"f7", "-9223372036854775808"},
207193
{"f8", "182737474"},
208-
{"f9", "0.334"},
209-
{"f10", "467.66472"},
210-
{"f11", "__DEFAULT_PARTITION__"},
211-
{"f12", "__DEFAULT_PARTITION__"},
212-
{"f13", "5"},
194+
{"f9", "__DEFAULT_PARTITION__"},
195+
{"f10", "__DEFAULT_PARTITION__"},
196+
{"f11", "5"},
213197
};
214198
ASSERT_EQ(actual_part_values_map, expected_map);
215199
}
@@ -227,10 +211,8 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
227211
{"f6", "-448489"},
228212
{"f7", "-9223372036854775808"},
229213
{"f8", "182737474"},
230-
{"f9", "0.334"},
231-
{"f10", "467.66472"},
232-
{"f11", "abcde"},
233-
{"f12", "这是一个很长很长的中文"}};
214+
{"f9", "abcde"},
215+
{"f10", "这是一个很长很长的中文"}};
234216

235217
ASSERT_NOK_WITH_MSG(computer->ToBinaryRow(partition_map),
236218
"can not find partition key 'f4' in input partition");
@@ -251,10 +233,8 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) {
251233
{"f6", "abcd"},
252234
{"f7", "-9223372036854775808"},
253235
{"f8", "182737474"},
254-
{"f9", "0.334"},
255-
{"f10", "467.66472"},
256-
{"f11", "abcde"},
257-
{"f12", "这是一个很长很长的中文"}};
236+
{"f9", "abcde"},
237+
{"f10", "这是一个很长很长的中文"}};
258238
ASSERT_NOK_WITH_MSG(computer->ToBinaryRow(partition_map),
259239
"cannot convert field idx 6, field value abcd to type INT32");
260240
}

‎src/paimon/common/utils/data_converter_utils.h‎

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -113,26 +113,6 @@ class DataConverterUtils {
113113
return Status::OK();
114114
};
115115
break;
116-
case arrow::Type::FLOAT:
117-
converter = [](const std::string& value_str, int32_t field_idx,
118-
BinaryRowWriter* writer) {
119-
auto value = StringUtils::StringToValue<float>(value_str);
120-
RETURN_INVALID_WITH_FIELD_INFO(value, field_idx, value_str,
121-
arrow::internal::ToString(arrow::Type::FLOAT));
122-
writer->WriteFloat(field_idx, value.value());
123-
return Status::OK();
124-
};
125-
break;
126-
case arrow::Type::DOUBLE:
127-
converter = [](const std::string& value_str, int32_t field_idx,
128-
BinaryRowWriter* writer) {
129-
auto value = StringUtils::StringToValue<double>(value_str);
130-
RETURN_INVALID_WITH_FIELD_INFO(value, field_idx, value_str,
131-
arrow::internal::ToString(arrow::Type::DOUBLE));
132-
writer->WriteDouble(field_idx, value.value());
133-
return Status::OK();
134-
};
135-
break;
136116
case arrow::Type::STRING:
137117
converter = [pool](const std::string& value_str, int32_t field_idx,
138118
BinaryRowWriter* writer) {
@@ -158,39 +138,6 @@ class DataConverterUtils {
158138
return converter;
159139
}
160140

161-
// support float and double
162-
template <typename T>
163-
static std::string FloatValueToString(const T& value, int32_t precision) {
164-
std::stringstream oss;
165-
if (value >= 1e-3 && value <= 1e7) {
166-
oss << std::fixed << std::setprecision(sizeof(T)) << value;
167-
std::string result = oss.str();
168-
auto pos = result.find_last_not_of('0');
169-
result.erase(pos + (result[pos] == '.') + 1, std::string::npos);
170-
return result;
171-
}
172-
oss << std::uppercase << std::scientific << std::setprecision(precision) << value;
173-
std::string result = oss.str();
174-
auto e_pos = result.find('E');
175-
if (e_pos != std::string::npos) {
176-
if (result[e_pos + 1] == '+') {
177-
result.erase(e_pos + 1, 1 + (result[e_pos + 2] == '0'));
178-
} else {
179-
if (result[e_pos + 1] == '-' && result[e_pos + 2] == '0') {
180-
result.erase(e_pos + 2, 1);
181-
}
182-
}
183-
auto zero_pos = e_pos - 1;
184-
while (zero_pos >= 1 && result[zero_pos] == '0' && result[zero_pos - 1] != '.') {
185-
zero_pos--;
186-
}
187-
if (e_pos - zero_pos - 1 > 0) {
188-
result.erase(zero_pos + 1, e_pos - zero_pos - 1);
189-
}
190-
}
191-
return result;
192-
}
193-
194141
static Result<BinaryRowFieldToStrConverter> CreateBinaryRowFieldToStringConverter(
195142
arrow::Type::type type, bool legacy_partition_name_enabled) {
196143
BinaryRowFieldToStrConverter converter;
@@ -226,18 +173,6 @@ class DataConverterUtils {
226173
return std::to_string(data);
227174
};
228175
break;
229-
case arrow::Type::FLOAT:
230-
converter = [](const BinaryRow& row, int32_t field_idx) {
231-
float data = row.GetFloat(field_idx);
232-
return FloatValueToString<float>(data, 6);
233-
};
234-
break;
235-
case arrow::Type::DOUBLE:
236-
converter = [](const BinaryRow& row, int32_t field_idx) {
237-
double data = row.GetDouble(field_idx);
238-
return FloatValueToString<double>(data, 15);
239-
};
240-
break;
241176
case arrow::Type::STRING:
242177
converter = [](const BinaryRow& row, int32_t field_idx) {
243178
BinaryString data = row.GetString(field_idx);

‎src/paimon/common/utils/data_converter_utils_test.cpp‎

Lines changed: 6 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,6 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithLegacyPartitionName
3838
{"-448489", arrow::Type::INT32},
3939
{"279039", arrow::Type::INT64},
4040
{"1234567", arrow::Type::INT64},
41-
{"0.334", arrow::Type::FLOAT},
42-
{"467.66472", arrow::Type::DOUBLE},
4341
{"abcde", arrow::Type::STRING},
4442
{"这是一个很长很长的中文", arrow::Type::STRING},
4543
{"10440", arrow::Type::DATE32}};
@@ -77,11 +75,9 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithLegacyPartitionName
7775
ASSERT_EQ(-448489, row.GetInt(6));
7876
ASSERT_EQ(279039, row.GetLong(7));
7977
ASSERT_EQ(1234567, row.GetLong(8));
80-
ASSERT_NEAR(0.334, row.GetFloat(9), 0.0000001);
81-
ASSERT_NEAR(467.66472, row.GetDouble(10), 0.0000001);
82-
ASSERT_EQ("abcde", row.GetString(11).ToString());
83-
ASSERT_EQ("这是一个很长很长的中文", row.GetString(12).ToString());
84-
ASSERT_EQ(10440, row.GetDate(13));
78+
ASSERT_EQ("abcde", row.GetString(9).ToString());
79+
ASSERT_EQ("这是一个很长很长的中文", row.GetString(10).ToString());
80+
ASSERT_EQ(10440, row.GetDate(11));
8581

8682
for (size_t idx = 0; idx < data.size(); idx++) {
8783
ASSERT_OK_AND_ASSIGN(auto partition_field_str, reconverters[idx](row, idx));
@@ -101,8 +97,6 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithNoLegacyPartitionNa
10197
{"-448489", arrow::Type::INT32},
10298
{"279039", arrow::Type::INT64},
10399
{"1234567", arrow::Type::INT64},
104-
{"0.334", arrow::Type::FLOAT},
105-
{"467.66472", arrow::Type::DOUBLE},
106100
{"abcde", arrow::Type::STRING},
107101
{"这是一个很长很长的中文", arrow::Type::STRING},
108102
{"1998-08-02", arrow::Type::DATE32}};
@@ -135,43 +129,14 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithNoLegacyPartitionNa
135129
ASSERT_EQ(-448489, row.GetInt(6));
136130
ASSERT_EQ(279039, row.GetLong(7));
137131
ASSERT_EQ(1234567, row.GetLong(8));
138-
ASSERT_NEAR(0.334, row.GetFloat(9), 0.0000001);
139-
ASSERT_NEAR(467.66472, row.GetDouble(10), 0.0000001);
140-
ASSERT_EQ("abcde", row.GetString(11).ToString());
141-
ASSERT_EQ("这是一个很长很长的中文", row.GetString(12).ToString());
142-
ASSERT_EQ(10440, row.GetDate(13));
132+
ASSERT_EQ("abcde", row.GetString(9).ToString());
133+
ASSERT_EQ("这是一个很长很长的中文", row.GetString(10).ToString());
134+
ASSERT_EQ(10440, row.GetDate(11));
143135

144136
for (size_t idx = 0; idx < data.size(); idx++) {
145137
ASSERT_OK_AND_ASSIGN(auto partition_field_str, reconverters[idx](row, idx));
146138
ASSERT_EQ(data[idx].first, partition_field_str);
147139
}
148140
}
149141

150-
TEST(DataConverterUtilsTest, TestValueToStringSimple) {
151-
ASSERT_EQ("233.0", DataConverterUtils::FloatValueToString<float>(static_cast<float>(233), 6));
152-
ASSERT_EQ("3.0E-4",
153-
DataConverterUtils::FloatValueToString<float>(static_cast<float>(0.0003), 6));
154-
ASSERT_EQ("3.478589E10",
155-
DataConverterUtils::FloatValueToString<float>(static_cast<float>(34785895352), 6));
156-
ASSERT_EQ("1.0E9",
157-
DataConverterUtils::FloatValueToString<float>(static_cast<float>(1000000000), 6));
158-
ASSERT_EQ("1000000.0",
159-
DataConverterUtils::FloatValueToString<float>(static_cast<float>(1000000), 6));
160-
ASSERT_EQ("467.6647",
161-
DataConverterUtils::FloatValueToString<float>(static_cast<float>(467.6647), 6));
162-
163-
ASSERT_EQ("233.0",
164-
DataConverterUtils::FloatValueToString<double>(static_cast<double>(233), 15));
165-
ASSERT_EQ("3.4785895352E10",
166-
DataConverterUtils::FloatValueToString<double>(static_cast<double>(34785895352), 15));
167-
ASSERT_EQ("1.0E9",
168-
DataConverterUtils::FloatValueToString<double>(static_cast<double>(1000000000), 15));
169-
ASSERT_EQ("1000000.0",
170-
DataConverterUtils::FloatValueToString<double>(static_cast<double>(1000000), 15));
171-
ASSERT_EQ("467.66472",
172-
DataConverterUtils::FloatValueToString<double>(static_cast<double>(467.66472), 6));
173-
ASSERT_EQ("123456.123456", DataConverterUtils::FloatValueToString<double>(
174-
static_cast<double>(123456.123456), 6));
175-
}
176-
177142
} // namespace paimon::test

‎src/paimon/core/io/field_mapping_reader.cpp‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -218,16 +218,6 @@ Result<std::shared_ptr<arrow::Array>> FieldMappingReader::GenerateSinglePartitio
218218
scalar = std::make_shared<arrow::Int64Scalar>(value);
219219
break;
220220
}
221-
case arrow::Type::type::FLOAT: {
222-
float value = partition_.GetFloat(partition_info_.value().idx_in_partition[idx]);
223-
scalar = std::make_shared<arrow::FloatScalar>(value);
224-
break;
225-
}
226-
case arrow::Type::type::DOUBLE: {
227-
double value = partition_.GetDouble(partition_info_.value().idx_in_partition[idx]);
228-
scalar = std::make_shared<arrow::DoubleScalar>(value);
229-
break;
230-
}
231221
case arrow::Type::type::STRING: {
232222
BinaryString value =
233223
partition_.GetString(partition_info_.value().idx_in_partition[idx]);

0 commit comments

Comments
 (0)