Skip to content

Commit 3e96684

Browse files
authored
feat(append): add append-only writer, compact task and coordinator (#112)
1 parent e5aa957 commit 3e96684

7 files changed

Lines changed: 1725 additions & 0 deletions

File tree

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#pragma once
20+
21+
#include <map>
22+
#include <memory>
23+
#include <string>
24+
#include <vector>
25+
26+
#include "paimon/result.h"
27+
28+
namespace paimon {
29+
30+
class CommitMessage;
31+
class FileSystem;
32+
class MemoryPool;
33+
34+
/// Compact coordinator for append-only unaware-bucket tables.
35+
///
36+
/// This coordinator scans the latest snapshot for small files, groups them by partition,
37+
/// and generates compaction tasks using a bin-packing algorithm. It then synchronously
38+
/// executes all tasks and returns the resulting commit messages.
39+
///
40+
/// @note This implementation does not support deletion vectors or streaming mode.
41+
/// It only scans the current latest snapshot (batch mode).
42+
class PAIMON_EXPORT AppendCompactCoordinator {
43+
public:
44+
AppendCompactCoordinator() = delete;
45+
~AppendCompactCoordinator() = delete;
46+
/// Run the compaction coordinator.
47+
///
48+
/// Scans the latest snapshot for small files across the specified partitions,
49+
/// generates compact tasks via bin-packing, executes them synchronously,
50+
/// and returns the resulting commit messages.
51+
///
52+
/// @param table_path The root path of the table.
53+
/// @param options User-defined options (will be merged with schema options).
54+
/// @param partitions Partition filters; each element is a partition spec as key-value pairs.
55+
/// Empty vector means all partitions.
56+
/// @param file_system The file system to use. If nullptr, will be created from options.
57+
/// @param pool The memory pool to use. If nullptr, will use default pool.
58+
/// @return Result containing a vector of commit messages from compaction tasks.
59+
static Result<std::vector<std::shared_ptr<CommitMessage>>> Run(
60+
const std::string& table_path, const std::map<std::string, std::string>& options,
61+
const std::vector<std::map<std::string, std::string>>& partitions,
62+
const std::shared_ptr<FileSystem>& file_system, const std::shared_ptr<MemoryPool>& pool);
63+
};
64+
65+
} // namespace paimon

0 commit comments

Comments
 (0)