Skip to content

Commit 575a3d8

Browse files
SteNicholasSteNicholas
authored
fix(core): support fallback keys for ignore-delete option (alibaba#462)
1 parent f449a9f commit 575a3d8

5 files changed

Lines changed: 127 additions & 2 deletions

File tree

‎include/paimon/defs.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,15 @@ struct PAIMON_EXPORT Options {
320320
/// "ignore-delete" - Whether to ignore delete records. Default value is "false".
321321
static const char IGNORE_DELETE[];
322322

323+
/// "first-row.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`.
324+
static const char FALLBACK_FIRST_ROW_IGNORE_DELETE[];
325+
326+
/// "deduplicate.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`.
327+
static const char FALLBACK_DEDUPLICATE_IGNORE_DELETE[];
328+
329+
/// "partial-update.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`.
330+
static const char FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE[];
331+
323332
/// "fields.default-aggregate-function" - Default aggregate function of all fields for
324333
/// partial-update and aggregate merge function.
325334
static const char FIELDS_DEFAULT_AGG_FUNC[];

‎src/paimon/common/defs.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ const char Options::SEQUENCE_FIELD_SORT_ORDER[] = "sequence.field.sort-order";
7474
const char Options::MERGE_ENGINE[] = "merge-engine";
7575
const char Options::SORT_ENGINE[] = "sort-engine";
7676
const char Options::IGNORE_DELETE[] = "ignore-delete";
77+
const char Options::FALLBACK_FIRST_ROW_IGNORE_DELETE[] = "first-row.ignore-delete";
78+
const char Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE[] = "deduplicate.ignore-delete";
79+
const char Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE[] = "partial-update.ignore-delete";
7780
const char Options::FIELDS_DEFAULT_AGG_FUNC[] = "fields.default-aggregate-function";
7881
const char Options::DELETION_VECTORS_ENABLED[] = "deletion-vectors.enabled";
7982
const char Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE[] =

‎src/paimon/core/core_options.cpp‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -708,8 +708,21 @@ struct CoreOptions::Impl {
708708
PAIMON_RETURN_NOT_OK(parser.ParseSortEngine(&sort_engine));
709709
// Parse merge-engine - merge engine for primary key table, default "deduplicate"
710710
PAIMON_RETURN_NOT_OK(parser.ParseMergeEngine(&merge_engine));
711-
// Parse ignore-delete - whether to ignore delete records, default false
712-
PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::IGNORE_DELETE, &ignore_delete));
711+
// Parse ignore-delete - whether to ignore delete records, default false.
712+
// Java CoreOptions declares first-row.ignore-delete, deduplicate.ignore-delete
713+
// and partial-update.ignore-delete as fallback keys, checked in that order only
714+
// when ignore-delete itself is absent.
715+
std::optional<bool> ignore_delete_value;
716+
PAIMON_RETURN_NOT_OK(parser.Parse<bool>(Options::IGNORE_DELETE, &ignore_delete_value));
717+
for (const char* fallback_key : {Options::FALLBACK_FIRST_ROW_IGNORE_DELETE,
718+
Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE,
719+
Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE}) {
720+
if (ignore_delete_value.has_value()) {
721+
break;
722+
}
723+
PAIMON_RETURN_NOT_OK(parser.Parse<bool>(fallback_key, &ignore_delete_value));
724+
}
725+
ignore_delete = ignore_delete_value.value_or(false);
713726
// Parse fields.default-aggregate-function - default agg function for partial-update
714727
PAIMON_RETURN_NOT_OK(parser.Parse(Options::FIELDS_DEFAULT_AGG_FUNC, &field_default_func));
715728
// Parse changelog-producer - whether to double write to a changelog file, default "none"

‎src/paimon/core/core_options_test.cpp‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -981,6 +981,50 @@ TEST(CoreOptionsTest, TestFallback) {
981981
}
982982
}
983983

984+
TEST(CoreOptionsTest, TestIgnoreDeleteFallbackKeys) {
985+
{
986+
// Tables written by Java may carry first-row.ignore-delete instead of ignore-delete.
987+
ASSERT_OK_AND_ASSIGN(
988+
CoreOptions options,
989+
CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"},
990+
{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "true"}}));
991+
ASSERT_TRUE(options.IgnoreDelete());
992+
}
993+
{
994+
ASSERT_OK_AND_ASSIGN(
995+
CoreOptions options,
996+
CoreOptions::FromMap({{Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE, "true"}}));
997+
ASSERT_TRUE(options.IgnoreDelete());
998+
}
999+
{
1000+
ASSERT_OK_AND_ASSIGN(
1001+
CoreOptions options,
1002+
CoreOptions::FromMap({{Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE, "true"}}));
1003+
ASSERT_TRUE(options.IgnoreDelete());
1004+
}
1005+
{
1006+
// The primary key takes precedence over fallback keys, matching Java CoreOptions.
1007+
ASSERT_OK_AND_ASSIGN(
1008+
CoreOptions options,
1009+
CoreOptions::FromMap({{Options::IGNORE_DELETE, "false"},
1010+
{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "true"}}));
1011+
ASSERT_FALSE(options.IgnoreDelete());
1012+
}
1013+
{
1014+
// Fallback keys are checked in declaration order.
1015+
ASSERT_OK_AND_ASSIGN(
1016+
CoreOptions options,
1017+
CoreOptions::FromMap({{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "false"},
1018+
{Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE, "true"}}));
1019+
ASSERT_FALSE(options.IgnoreDelete());
1020+
}
1021+
{
1022+
ASSERT_NOK_WITH_MSG(
1023+
CoreOptions::FromMap({{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "invalid"}}),
1024+
"Invalid Config [first-row.ignore-delete: invalid]");
1025+
}
1026+
}
1027+
9841028
TEST(CoreOptionsTest, TestMapStorageLayout) {
9851029
// Test shared-shredding layout configured for a specific column
9861030
{

‎src/paimon/core/utils/primary_key_table_utils_test.cpp‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,23 @@
1818

1919
#include <cstdint>
2020
#include <map>
21+
#include <memory>
22+
#include <optional>
2123
#include <utility>
2224

2325
#include "arrow/type.h"
2426
#include "gtest/gtest.h"
2527
#include "paimon/common/types/data_field.h"
28+
#include "paimon/common/types/row_kind.h"
2629
#include "paimon/common/utils/fields_comparator.h"
2730
#include "paimon/core/core_options.h"
31+
#include "paimon/core/key_value.h"
32+
#include "paimon/core/mergetree/compact/merge_function.h"
2833
#include "paimon/defs.h"
34+
#include "paimon/memory/memory_pool.h"
2935
#include "paimon/status.h"
36+
#include "paimon/testing/utils/binary_row_generator.h"
37+
#include "paimon/testing/utils/key_value_checker.h"
3038
#include "paimon/testing/utils/testharness.h"
3139

3240
namespace paimon::test {
@@ -54,4 +62,52 @@ TEST(PrimaryKeyTableUtilsTest, TestCreateSequenceFieldsComparator) {
5462
}
5563
}
5664

65+
TEST(PrimaryKeyTableUtilsTest, TestCreateFirstRowMergeFunctionWithIgnoreDelete) {
66+
auto pool = GetDefaultPool();
67+
auto value_schema = arrow::schema({arrow::field("v0", arrow::int32())});
68+
69+
// ignore-delete can also be configured through the merge-engine-specific
70+
// first-row.ignore-delete key, which must reach FirstRowMergeFunction the same way.
71+
for (const char* ignore_delete_key :
72+
{Options::IGNORE_DELETE, Options::FALLBACK_FIRST_ROW_IGNORE_DELETE}) {
73+
ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
74+
CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"},
75+
{ignore_delete_key, "true"}}));
76+
ASSERT_OK_AND_ASSIGN(
77+
std::unique_ptr<MergeFunction> merge_function,
78+
PrimaryKeyTableUtils::CreateMergeFunction(value_schema, {"k0"}, core_options));
79+
merge_function->Reset();
80+
81+
KeyValue insert_kv(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/
82+
BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
83+
/*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get()));
84+
KeyValue delete_kv(RowKind::Delete(), /*sequence_number=*/1, /*level=*/0, /*key=*/
85+
BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
86+
/*value=*/BinaryRowGenerator::GenerateRowPtr({200}, pool.get()));
87+
ASSERT_OK(merge_function->Add(std::move(insert_kv)));
88+
ASSERT_OK(merge_function->Add(std::move(delete_kv)));
89+
90+
ASSERT_OK_AND_ASSIGN(std::optional<KeyValue> result_kv, merge_function->GetResult());
91+
ASSERT_TRUE(result_kv.has_value());
92+
KeyValue expected(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/
93+
BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
94+
/*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get()));
95+
KeyValueChecker::CheckResult(expected, result_kv.value(), /*key_arity=*/1,
96+
/*value_arity=*/1);
97+
}
98+
99+
// Without the option, the first-row merge engine still rejects retract records.
100+
ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
101+
CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"}}));
102+
ASSERT_OK_AND_ASSIGN(
103+
std::unique_ptr<MergeFunction> merge_function,
104+
PrimaryKeyTableUtils::CreateMergeFunction(value_schema, {"k0"}, core_options));
105+
merge_function->Reset();
106+
KeyValue delete_kv(RowKind::Delete(), /*sequence_number=*/0, /*level=*/0, /*key=*/
107+
BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
108+
/*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get()));
109+
ASSERT_NOK_WITH_MSG(merge_function->Add(std::move(delete_kv)),
110+
"First row merge engine can not accept DELETE/UPDATE_BEFORE records");
111+
}
112+
57113
} // namespace paimon::test

0 commit comments

Comments
 (0)