Skip to content

Commit 914bb06

Browse files
lxy-9602lxy-9602
authored
Merge branch 'main' into sync/apache-pr-12
2 parents f23b387 + a72bb22 commit 914bb06

146 files changed

Lines changed: 4366 additions & 705 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ if(NOT CMAKE_BUILD_TYPE)
3232
endif()
3333

3434
project(paimon
35-
VERSION 0.1.0
35+
VERSION 0.2.0
3636
DESCRIPTION "Paimon C++ Project")
3737

3838
string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE)

‎README.md‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,20 @@ Paimon C++ is a high-performance C++ implementation of [Apache Paimon](https://p
2323

2424
## What's in the Paimon C++ library
2525

26-
* Write: Paimon append table and primary key table write (without compaction).
27-
* Commit: Paimon append table commit. (Note: Limited support — only works for simple append-only tables; table with compaction, index, changelog, and stats are not supported.)
28-
* Scan: Paimon append and primary key table batch and stream scan (without changelog).
29-
* Read: Paimon append table read and primary key table with deletion vector read (raw read) and primary key table with merge on read (merge read).
30-
* Batch read and write interface using the [Arrow Columnar In-Memory Format](https://arrow.apache.org) to increase throughput.
31-
* IO interfaces to file system and built-in local and jindo file system implementation.
32-
* File format interfaces to customize different format and built-in orc, parquet and lance format implementation.
33-
* Memory pool interfaces and a default implementation.
34-
* Thread pool executor interfaces and a default implementation.
35-
* Compatible with Java Paimon format and communication protocol (e.g., commit message, data splits, manifests).
36-
* Note: The current implementation only supports the x86_64 architecture.
26+
- **Write**: append table and primary key table write support with compaction.
27+
- **Commit**: append table commit support for simple append-only tables.
28+
- **Scan**: batch and stream scan for append tables and primary key tables without changelog.
29+
- **Read**: append table read, primary key table read with deletion vector, and primary key table
30+
merge-on-read.
31+
- **Arrow integration**: batch read and write interfaces based on the [Arrow Columnar In-Memory Format](https://arrow.apache.org).
32+
- **File systems**: file system abstraction with built-in local and Jindo file system support.
33+
- **File formats**: file format abstraction with built-in ORC, Parquet, and Avro support.
34+
- **Runtime utilities**: memory pool and thread pool abstractions with default implementations.
35+
- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index capabilities including bitmap index, B-tree index, DiskANN-based vector search with Lumina, and Lucene-based full-text search.
36+
- **Compatibility**: compatibility with Apache Paimon Java format and communication protocols,
37+
including commit messages, data splits, and manifests.
38+
39+
Note: The current implementation only supports the x86_64 architecture.
3740

3841
## Write And Commit Example
3942

@@ -58,8 +61,8 @@ The writing is divided into two stages:
5861
// ...
5962
RecordBatchBuilder batch_builder(&arrow_array);
6063
batch_builder.SetPartition({{"col1", "20240813"}, {"col2", "23"}}).SetBucket(1);
61-
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> batch, batch_builder.Finish());
62-
PAIMON_RETURN_NOT_OK(file_store_write->Write(batch));
64+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RecordBatch> batch, batch_builder.Finish());
65+
PAIMON_RETURN_NOT_OK(file_store_write->Write(std::move(batch)));
6366
PAIMON_ASSIGN_OR_RAISE(std::vector<std::shared_ptr<CommitMessage>> commit_messages,
6467
file_store_write->PrepareCommit());
6568

@@ -96,7 +99,7 @@ The reading is divided into two stages:
9699
TableScan::Create(std::move(scan_context)));
97100
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan, table_scan->CreatePlan());
98101
99-
ReadContextBuilder read_context_builder(table_path, /*schema_id=*/0);
102+
ReadContextBuilder read_context_builder(table_path);
100103
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<ReadContext> read_context,
101104
read_context_builder.SetReadSchema({"f0", "f1"})
102105
.SetPredicate(predicate)
@@ -160,13 +163,15 @@ The supported dependency source values are:
160163

161164
You can also override individual dependencies. The supported dependency set
162165
includes Arrow/Parquet, ORC, Protobuf, Avro, RE2, fmt, RapidJSON, TBB, glog,
163-
GoogleTest, and compression libraries.
166+
GoogleTest, and compression libraries. Arrow and ORC require project-specific
167+
patches, so their supported source values are `AUTO` and `BUNDLED`; `AUTO`
168+
resolves to bundled sources for them.
164169

165170
```
166171
$ cmake -B build \
167172
-DPAIMON_DEPENDENCY_SOURCE=AUTO \
168-
-DArrow_SOURCE=SYSTEM \
169-
-DArrow_ROOT=/opt/arrow \
173+
-Dfmt_SOURCE=SYSTEM \
174+
-Dfmt_ROOT=/opt/fmt \
170175
-Dzstd_SOURCE=BUNDLED
171176
```
172177

@@ -184,16 +189,14 @@ dependency source interface. They can still be used through standard CMake
184189
mechanisms such as `CMAKE_PREFIX_PATH` or `CMAKE_TOOLCHAIN_FILE`, while Paimon
185190
keeps the dependency source values limited to `AUTO`, `BUNDLED`, and `SYSTEM`.
186191

187-
When `Arrow_SOURCE` is explicitly set to `SYSTEM` or `BUNDLED`, the compression
188-
dependencies default to the same source unless individually overridden. Mixing
189-
system and bundled copies of transitive dependencies can cause ABI conflicts,
190-
so prefer keeping Arrow and its compression dependencies from the same source
191-
unless you have a specific reason to override them.
192+
When `Arrow_SOURCE` is explicitly set to `BUNDLED` or left as `AUTO`, the
193+
compression dependencies default to bundled sources unless individually
194+
overridden. Mixing system and bundled copies of transitive dependencies can
195+
cause ABI conflicts, so prefer keeping Arrow and its compression dependencies
196+
from the same source unless you have a specific reason to override them.
192197

193-
When `ORC_SOURCE` is explicitly set, `Protobuf_SOURCE` defaults to the same
194-
source unless individually overridden. In `AUTO` mode, Paimon prechecks for a
195-
system ORC installation and defaults Protobuf to `SYSTEM` only when system ORC
196-
is found; otherwise Protobuf stays bundled with bundled ORC.
198+
When `ORC_SOURCE` is explicitly set to `BUNDLED` or left as `AUTO`,
199+
`Protobuf_SOURCE` defaults to bundled sources unless individually overridden.
197200

198201
CMake prints a dependency resolution summary during configuration showing the
199202
requested source, actual source, compatibility target, and search root for each

‎cmake_modules/DefineOptions.cmake‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -156,11 +156,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
156156
"Prefer shared libraries for system third-party packages" OFF)
157157

158158
define_option_string(Arrow_SOURCE
159-
"Dependency source for Apache Arrow"
159+
"Dependency source for Apache Arrow; SYSTEM is unsupported"
160160
""
161161
AUTO
162-
BUNDLED
163-
SYSTEM)
162+
BUNDLED)
164163
define_option_string(zstd_SOURCE
165164
"Dependency source for zstd"
166165
""
@@ -198,11 +197,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
198197
BUNDLED
199198
SYSTEM)
200199
define_option_string(ORC_SOURCE
201-
"Dependency source for Apache ORC"
200+
"Dependency source for Apache ORC; SYSTEM is unsupported"
202201
""
203202
AUTO
204-
BUNDLED
205-
SYSTEM)
203+
BUNDLED)
206204
define_option_string(fmt_SOURCE
207205
"Dependency source for fmt"
208206
""

‎cmake_modules/FindGTestAlt.cmake‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ if(_PAIMON_GTEST_ROOTS)
1818
set(_PAIMON_GTEST_FIND_ARGS HINTS ${_PAIMON_GTEST_ROOTS} NO_DEFAULT_PATH)
1919
endif()
2020

21+
include(FindPackageUtils)
2122
find_package(GTest CONFIG QUIET ${_PAIMON_GTEST_FIND_ARGS})
2223

2324
if(NOT TARGET GTest::gtest
@@ -65,8 +66,20 @@ if(NOT TARGET GTest::gtest
6566
endif()
6667
endif()
6768
else()
68-
get_target_property(GTEST_INCLUDE_DIR GTest::gtest INTERFACE_INCLUDE_DIRECTORIES)
69-
set(GTestAlt_FOUND TRUE)
69+
paimon_find_target_headers(GTEST_INCLUDE_DIR
70+
GTest::gtest
71+
NAMES
72+
gtest/gtest.h
73+
${_PAIMON_GTEST_FIND_ARGS})
74+
paimon_find_target_headers(GMOCK_INCLUDE_DIR
75+
GTest::gmock
76+
NAMES
77+
gmock/gmock.h
78+
${_PAIMON_GTEST_FIND_ARGS})
79+
80+
include(FindPackageHandleStandardArgs)
81+
find_package_handle_standard_args(GTestAlt REQUIRED_VARS GTEST_INCLUDE_DIR
82+
GMOCK_INCLUDE_DIR)
7083
endif()
7184

7285
if(GTestAlt_FOUND)

‎cmake_modules/FindLZ4Alt.cmake‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ if(_PAIMON_LZ4_ROOTS)
1818
set(_PAIMON_LZ4_FIND_ARGS HINTS ${_PAIMON_LZ4_ROOTS} NO_DEFAULT_PATH)
1919
endif()
2020

21+
include(FindPackageUtils)
2122
find_package(lz4 CONFIG QUIET ${_PAIMON_LZ4_FIND_ARGS})
2223
find_package(LZ4 CONFIG QUIET ${_PAIMON_LZ4_FIND_ARGS})
2324

@@ -30,18 +31,24 @@ foreach(_target IN LISTS _PAIMON_LZ4_TARGETS)
3031
endforeach()
3132

3233
if(_PAIMON_LZ4_TARGET)
33-
get_target_property(LZ4_INCLUDE_DIR ${_PAIMON_LZ4_TARGET}
34-
INTERFACE_INCLUDE_DIRECTORIES)
35-
if(NOT TARGET lz4)
34+
paimon_find_target_headers(LZ4_INCLUDE_DIR
35+
${_PAIMON_LZ4_TARGET}
36+
NAMES
37+
lz4.h
38+
${_PAIMON_LZ4_FIND_ARGS})
39+
40+
include(FindPackageHandleStandardArgs)
41+
find_package_handle_standard_args(LZ4Alt REQUIRED_VARS LZ4_INCLUDE_DIR)
42+
43+
if(LZ4Alt_FOUND AND NOT TARGET lz4)
3644
add_library(lz4 INTERFACE IMPORTED)
37-
if(LZ4_INCLUDE_DIR)
38-
set_target_properties(lz4 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
39-
"${LZ4_INCLUDE_DIR}")
40-
endif()
45+
set_target_properties(lz4 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
46+
"${LZ4_INCLUDE_DIR}")
4147
target_link_libraries(lz4 INTERFACE ${_PAIMON_LZ4_TARGET})
4248
endif()
43-
set(LZ4_LIBRARIES ${_PAIMON_LZ4_TARGET})
44-
set(LZ4Alt_FOUND TRUE)
49+
if(LZ4Alt_FOUND)
50+
set(LZ4_LIBRARIES ${_PAIMON_LZ4_TARGET})
51+
endif()
4552
else()
4653
find_package(PkgConfig QUIET)
4754
if(PkgConfig_FOUND)

‎cmake_modules/FindPackageUtils.cmake‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# Copyright 2026-present Alibaba Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
function(paimon_find_target_headers OUT_VAR TARGET_NAME)
16+
set(options NO_DEFAULT_PATH)
17+
set(one_value_args)
18+
set(multi_value_args NAMES HINTS PATH_SUFFIXES)
19+
cmake_parse_arguments(ARG
20+
"${options}"
21+
"${one_value_args}"
22+
"${multi_value_args}"
23+
${ARGN})
24+
25+
if(NOT TARGET ${TARGET_NAME} OR NOT ARG_NAMES)
26+
set(${OUT_VAR}
27+
"${OUT_VAR}-NOTFOUND"
28+
PARENT_SCOPE)
29+
return()
30+
endif()
31+
32+
set(_target_include_dirs)
33+
foreach(_property INTERFACE_INCLUDE_DIRECTORIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
34+
INCLUDE_DIRECTORIES)
35+
get_target_property(_property_value ${TARGET_NAME} ${_property})
36+
if(_property_value AND NOT _property_value MATCHES "-NOTFOUND$")
37+
list(APPEND _target_include_dirs ${_property_value})
38+
endif()
39+
endforeach()
40+
41+
set(_search_dirs)
42+
foreach(_dir IN LISTS _target_include_dirs)
43+
if(_dir MATCHES "^\\$<BUILD_INTERFACE:(.*)>$")
44+
list(APPEND _search_dirs "${CMAKE_MATCH_1}")
45+
elseif(_dir MATCHES "^\\$<INSTALL_INTERFACE:(.*)>$")
46+
if(IS_ABSOLUTE "${CMAKE_MATCH_1}")
47+
list(APPEND _search_dirs "${CMAKE_MATCH_1}")
48+
endif()
49+
elseif(NOT _dir MATCHES "^\\$<")
50+
list(APPEND _search_dirs "${_dir}")
51+
endif()
52+
endforeach()
53+
54+
list(APPEND _search_dirs ${ARG_HINTS})
55+
if(_search_dirs)
56+
list(REMOVE_DUPLICATES _search_dirs)
57+
endif()
58+
59+
string(MAKE_C_IDENTIFIER "${TARGET_NAME}_${ARG_NAMES}" _header_var_suffix)
60+
set(_header_dir_var "PAIMON_${_header_var_suffix}_HEADER_DIR")
61+
set(_find_args NAMES ${ARG_NAMES})
62+
if(_search_dirs)
63+
list(APPEND _find_args HINTS ${_search_dirs})
64+
endif()
65+
if(ARG_PATH_SUFFIXES)
66+
list(APPEND _find_args PATH_SUFFIXES ${ARG_PATH_SUFFIXES})
67+
endif()
68+
list(APPEND _find_args NO_DEFAULT_PATH)
69+
70+
unset(${_header_dir_var} CACHE)
71+
find_path(${_header_dir_var} ${_find_args})
72+
73+
if(NOT ${_header_dir_var})
74+
get_property(_partial_targets GLOBAL PROPERTY PAIMON_PARTIAL_SYSTEM_TARGETS)
75+
list(APPEND _partial_targets "${TARGET_NAME}: ${ARG_NAMES}")
76+
set_property(GLOBAL PROPERTY PAIMON_PARTIAL_SYSTEM_TARGETS "${_partial_targets}")
77+
endif()
78+
79+
set(${OUT_VAR}
80+
"${${_header_dir_var}}"
81+
PARENT_SCOPE)
82+
83+
unset(${_header_dir_var} CACHE)
84+
unset(_find_args)
85+
unset(_header_dir_var)
86+
unset(_header_var_suffix)
87+
unset(_partial_targets)
88+
unset(_property_value)
89+
unset(_search_dirs)
90+
unset(_target_include_dirs)
91+
endfunction()

‎cmake_modules/FindProtobufAlt.cmake‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ if(_PAIMON_PROTOBUF_ROOTS)
1818
set(_PAIMON_PROTOBUF_FIND_ARGS HINTS ${_PAIMON_PROTOBUF_ROOTS} NO_DEFAULT_PATH)
1919
endif()
2020

21+
include(FindPackageUtils)
2122
find_package(Protobuf CONFIG QUIET ${_PAIMON_PROTOBUF_FIND_ARGS})
2223

2324
set(_PAIMON_PROTOBUF_LIBRARY_TARGETS protobuf::libprotobuf Protobuf::libprotobuf)
@@ -52,29 +53,29 @@ if(_PAIMON_PROTOBUF_LIBRARY_TARGET AND NOT PROTOBUF_COMPILER)
5253
endif()
5354

5455
if(_PAIMON_PROTOBUF_LIBRARY_TARGET)
56+
paimon_find_target_headers(PROTOBUF_INCLUDE_DIR
57+
${_PAIMON_PROTOBUF_LIBRARY_TARGET}
58+
NAMES
59+
google/protobuf/message.h
60+
${_PAIMON_PROTOBUF_FIND_ARGS})
61+
5562
include(FindPackageHandleStandardArgs)
5663
find_package_handle_standard_args(
57-
ProtobufAlt REQUIRED_VARS _PAIMON_PROTOBUF_LIBRARY_TARGET PROTOBUF_COMPILER)
64+
ProtobufAlt REQUIRED_VARS _PAIMON_PROTOBUF_LIBRARY_TARGET PROTOBUF_INCLUDE_DIR
65+
PROTOBUF_COMPILER)
5866
if(ProtobufAlt_FOUND)
59-
get_target_property(PROTOBUF_INCLUDE_DIR ${_PAIMON_PROTOBUF_LIBRARY_TARGET}
60-
INTERFACE_INCLUDE_DIRECTORIES)
61-
6267
if(NOT TARGET libprotobuf)
6368
add_library(libprotobuf INTERFACE IMPORTED)
64-
if(PROTOBUF_INCLUDE_DIR)
65-
set_target_properties(libprotobuf PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
66-
"${PROTOBUF_INCLUDE_DIR}")
67-
endif()
69+
set_target_properties(libprotobuf PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
70+
"${PROTOBUF_INCLUDE_DIR}")
6871
target_link_libraries(libprotobuf
6972
INTERFACE ${_PAIMON_PROTOBUF_LIBRARY_TARGET})
7073
endif()
7174

7275
if(_PAIMON_PROTOC_LIBRARY_TARGET AND NOT TARGET libprotoc)
7376
add_library(libprotoc INTERFACE IMPORTED)
74-
if(PROTOBUF_INCLUDE_DIR)
75-
set_target_properties(libprotoc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
76-
"${PROTOBUF_INCLUDE_DIR}")
77-
endif()
77+
set_target_properties(libprotoc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
78+
"${PROTOBUF_INCLUDE_DIR}")
7879
target_link_libraries(libprotoc INTERFACE ${_PAIMON_PROTOC_LIBRARY_TARGET})
7980
endif()
8081

‎cmake_modules/FindRE2Alt.cmake‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,22 @@ if(_PAIMON_RE2_ROOTS)
1818
set(_PAIMON_RE2_FIND_ARGS HINTS ${_PAIMON_RE2_ROOTS} NO_DEFAULT_PATH)
1919
endif()
2020

21+
include(FindPackageUtils)
2122
find_package(re2 CONFIG QUIET ${_PAIMON_RE2_FIND_ARGS})
2223

2324
if(TARGET re2::re2)
24-
get_target_property(RE2_INCLUDE_DIR re2::re2 INTERFACE_INCLUDE_DIRECTORIES)
25-
set(RE2_LIBRARIES re2::re2)
26-
set(RE2Alt_FOUND TRUE)
25+
paimon_find_target_headers(RE2_INCLUDE_DIR
26+
re2::re2
27+
NAMES
28+
re2/re2.h
29+
${_PAIMON_RE2_FIND_ARGS})
30+
31+
include(FindPackageHandleStandardArgs)
32+
find_package_handle_standard_args(RE2Alt REQUIRED_VARS RE2_INCLUDE_DIR)
33+
34+
if(RE2Alt_FOUND)
35+
set(RE2_LIBRARIES re2::re2)
36+
endif()
2737
else()
2838
find_package(PkgConfig QUIET)
2939
if(PkgConfig_FOUND)

0 commit comments

Comments
 (0)