Skip to content

Commit 39851a5

Browse files
authored
Merge branch 'main' into codex/fix-filesystem-issues
2 parents 6448046 + aac6240 commit 39851a5

146 files changed

Lines changed: 10862 additions & 942 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

LICENSE

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,54 @@ License: https://www.apache.org/licenses/LICENSE-2.0
506506

507507
--------------------------------------------------------------------------------
508508

509+
This product includes code derived from PyTorch TH simd.h.
510+
511+
* SIMD detection code in third_party/roaring_bitmap/roaring.cpp
512+
513+
Copyright (c) 2016- Facebook, Inc (Adam Paszke)
514+
Copyright (c) 2014- Facebook, Inc (Soumith Chintala)
515+
Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
516+
Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu)
517+
Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
518+
Copyright (c) 2011-2013 NYU (Clement Farabet)
519+
Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou,
520+
Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute
521+
(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert,
522+
Samy Bengio, Johnny Mariethoz)
523+
524+
All rights reserved.
525+
526+
License: BSD-3-Clause
527+
528+
Redistribution and use in source and binary forms, with or without
529+
modification, are permitted provided that the following conditions are met:
530+
531+
1. Redistributions of source code must retain the above copyright
532+
notice, this list of conditions and the following disclaimer.
533+
534+
2. Redistributions in binary form must reproduce the above copyright
535+
notice, this list of conditions and the following disclaimer in the
536+
documentation and/or other materials provided with the distribution.
537+
538+
3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories
539+
America and IDIAP Research Institute nor the names of its contributors may be
540+
used to endorse or promote products derived from this software without
541+
specific prior written permission.
542+
543+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
544+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
545+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
546+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
547+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
548+
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
549+
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
550+
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
551+
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
552+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
553+
POSSIBILITY OF SUCH DAMAGE.
554+
555+
--------------------------------------------------------------------------------
556+
509557
This product includes code from cppjieba.
510558

511559
* cppjieba utility in src/paimon/global_index/lucene/ directory

cmake_modules/arrow.diff

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,193 @@ index 4d3acb491e..3906ff3c59 100644
196196
int64_t pagesize_;
197197
ParquetDataPageVersion parquet_data_page_version_;
198198
ParquetVersion::type parquet_version_;
199+
200+
--- a/cpp/src/parquet/file_reader.h
201+
+++ b/cpp/src/parquet/file_reader.h
202+
@@ -210,6 +210,17 @@
203+
::arrow::Future<> WhenBuffered(const std::vector<int>& row_groups,
204+
const std::vector<int>& column_indices) const;
205+
206+
+ /// Pre-buffer arbitrary byte ranges (e.g., page-level ranges from OffsetIndex).
207+
+ /// Unlike PreBuffer(), this does NOT set the column bitmap, so
208+
+ /// GetColumnPageReader will use CachedInputStream (page-level cache path).
209+
+ void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges,
210+
+ const ::arrow::io::IOContext& ctx,
211+
+ const ::arrow::io::CacheOptions& options);
212+
+
213+
+ /// Wait for arbitrary byte ranges to be pre-buffered.
214+
+ ::arrow::Future<> WhenBufferedRanges(
215+
+ const std::vector<::arrow::io::ReadRange>& ranges) const;
216+
+
217+
private:
218+
// Holds a pointer to an instance of Contents implementation
219+
std::unique_ptr<Contents> contents_;
220+
221+
--- a/cpp/src/parquet/file_reader.cc
222+
+++ b/cpp/src/parquet/file_reader.cc
223+
@@ -207,6 +207,100 @@
224+
return {col_start, col_length};
225+
}
226+
227+
+// CachedInputStream: InputStream adapter that reads through ReadRangeCache with
228+
+// zero-cost skip for non-cached pages. Used for page-level caching where only
229+
+// specific pages are pre-buffered.
230+
+//
231+
+// Key behavior:
232+
+// - Read(): On cache hit, returns cached data. On cache miss, returns zero-filled
233+
+// buffer (zero I/O). This makes InputStream::Advance() (which calls Read() and
234+
+// discards) effectively free for skipped pages.
235+
+// - Peek(): Always falls back to source on cache miss, because PageReader uses
236+
+// Peek() to read Thrift page headers (~30 bytes) which must have real data.
237+
+class CachedInputStream : public ::arrow::io::InputStream {
238+
+ public:
239+
+ CachedInputStream(
240+
+ std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache,
241+
+ std::shared_ptr<ArrowInputFile> source,
242+
+ int64_t offset, int64_t length)
243+
+ : cache_(std::move(cache)),
244+
+ source_(std::move(source)),
245+
+ base_offset_(offset),
246+
+ length_(length) {}
247+
+
248+
+ ::arrow::Status Close() override {
249+
+ closed_ = true;
250+
+ return ::arrow::Status::OK();
251+
+ }
252+
+
253+
+ bool closed() const override { return closed_; }
254+
+
255+
+ ::arrow::Result<int64_t> Tell() const override { return position_; }
256+
+
257+
+ ::arrow::Result<std::string_view> Peek(int64_t nbytes) override {
258+
+ int64_t to_read = std::min(nbytes, length_ - position_);
259+
+ if (to_read <= 0) {
260+
+ return std::string_view();
261+
+ }
262+
+ ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
263+
+ auto result = cache_->Read(range);
264+
+ if (result.ok()) {
265+
+ peek_buffer_ = *result;
266+
+ } else {
267+
+ // Peek is used for Thrift page headers (~30 bytes) — must read real data
268+
+ ARROW_ASSIGN_OR_RAISE(peek_buffer_,
269+
+ source_->ReadAt(range.offset, range.length));
270+
+ }
271+
+ return std::string_view(
272+
+ reinterpret_cast<const char*>(peek_buffer_->data()),
273+
+ static_cast<size_t>(peek_buffer_->size()));
274+
+ }
275+
+
276+
+ ::arrow::Result<int64_t> Read(int64_t nbytes, void* out) override {
277+
+ int64_t to_read = std::min(nbytes, length_ - position_);
278+
+ if (to_read <= 0) return 0;
279+
+ ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
280+
+ auto result = cache_->Read(range);
281+
+ if (result.ok()) {
282+
+ auto& buf = *result;
283+
+ memcpy(out, buf->data(), static_cast<size_t>(buf->size()));
284+
+ position_ += buf->size();
285+
+ return buf->size();
286+
+ }
287+
+ // Cache miss: fall back to real I/O from source
288+
+ ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length));
289+
+ memcpy(out, buf->data(), static_cast<size_t>(buf->size()));
290+
+ position_ += buf->size();
291+
+ return buf->size();
292+
+ }
293+
+
294+
+ ::arrow::Result<std::shared_ptr<::arrow::Buffer>> Read(int64_t nbytes) override {
295+
+ int64_t to_read = std::min(nbytes, length_ - position_);
296+
+ if (to_read <= 0) {
297+
+ return std::make_shared<::arrow::Buffer>(nullptr, 0);
298+
+ }
299+
+ ::arrow::io::ReadRange range{base_offset_ + position_, to_read};
300+
+ auto result = cache_->Read(range);
301+
+ if (result.ok()) {
302+
+ position_ += (*result)->size();
303+
+ return *result;
304+
+ }
305+
+ // Cache miss: fall back to real I/O from source
306+
+ ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length));
307+
+ position_ += buf->size();
308+
+ return std::shared_ptr<::arrow::Buffer>(std::move(buf));
309+
+ }
310+
+
311+
+ private:
312+
+ std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache_;
313+
+ std::shared_ptr<ArrowInputFile> source_;
314+
+ int64_t base_offset_;
315+
+ int64_t length_;
316+
+ int64_t position_ = 0;
317+
+ bool closed_ = false;
318+
+ std::shared_ptr<::arrow::Buffer> peek_buffer_;
319+
+};
320+
+
321+
// RowGroupReader::Contents implementation for the Parquet file specification
322+
class SerializedRowGroup : public RowGroupReader::Contents {
323+
public:
324+
@@ -242,6 +336,11 @@
325+
// segments.
326+
PARQUET_ASSIGN_OR_THROW(auto buffer, cached_source_->Read(col_range));
327+
stream = std::make_shared<::arrow::io::BufferReader>(buffer);
328+
+ } else if (cached_source_) {
329+
+ // Page-level caching: read through cache with fallback to source.
330+
+ // Advance() is zero-cost for skipped pages via data_page_filter.
331+
+ stream = std::make_shared<CachedInputStream>(
332+
+ cached_source_, source_, col_range.offset, col_range.length);
333+
} else {
334+
stream = properties_.GetStream(source_, col_range.offset, col_range.length);
335+
}
336+
@@ -417,6 +516,26 @@
337+
return cached_source_->WaitFor(ranges);
338+
}
339+
340+
+ void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges,
341+
+ const ::arrow::io::IOContext& ctx,
342+
+ const ::arrow::io::CacheOptions& options) {
343+
+ cached_source_ =
344+
+ std::make_shared<::arrow::io::internal::ReadRangeCache>(source_, ctx, options);
345+
+ // Do NOT set prebuffered_column_chunks_ bitmap — GetColumnPageReader will
346+
+ // use CachedInputStream path instead of full-chunk BufferReader path.
347+
+ prebuffered_column_chunks_.clear();
348+
+ PARQUET_THROW_NOT_OK(cached_source_->Cache(ranges));
349+
+ }
350+
+
351+
+ ::arrow::Future<> WhenBufferedRanges(
352+
+ const std::vector<::arrow::io::ReadRange>& ranges) const {
353+
+ if (!cached_source_) {
354+
+ return ::arrow::Status::Invalid(
355+
+ "Must call PreBufferRanges before WhenBufferedRanges");
356+
+ }
357+
+ return cached_source_->WaitFor(ranges);
358+
+ }
359+
+
360+
// Metadata/footer parsing. Divided up to separate sync/async paths, and to use
361+
// exceptions for error handling (with the async path converting to Future/Status).
362+
363+
@@ -911,6 +1030,22 @@
364+
return file->WhenBuffered(row_groups, column_indices);
365+
}
366+
367+
+void ParquetFileReader::PreBufferRanges(
368+
+ const std::vector<::arrow::io::ReadRange>& ranges,
369+
+ const ::arrow::io::IOContext& ctx,
370+
+ const ::arrow::io::CacheOptions& options) {
371+
+ SerializedFile* file =
372+
+ ::arrow::internal::checked_cast<SerializedFile*>(contents_.get());
373+
+ file->PreBufferRanges(ranges, ctx, options);
374+
+}
375+
+
376+
+::arrow::Future<> ParquetFileReader::WhenBufferedRanges(
377+
+ const std::vector<::arrow::io::ReadRange>& ranges) const {
378+
+ SerializedFile* file =
379+
+ ::arrow::internal::checked_cast<SerializedFile*>(contents_.get());
380+
+ return file->WhenBufferedRanges(ranges);
381+
+}
382+
+
383+
// ----------------------------------------------------------------------
384+
// File metadata helpers
385+
199386
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
200387
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
201388
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake

include/paimon/catalog/identifier.h

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,20 @@ class PAIMON_EXPORT Identifier {
3636
explicit Identifier(const std::string& table);
3737
Identifier(const std::string& database, const std::string& table);
3838

39-
bool operator==(const Identifier& other);
39+
bool operator==(const Identifier& other) const;
4040
const std::string& GetDatabaseName() const;
4141
const std::string& GetTableName() const;
4242
Result<std::string> GetDataTableName() const;
4343
Result<std::optional<std::string>> GetBranchName() const;
4444
Result<std::string> GetBranchNameOrDefault() const;
4545
Result<std::optional<std::string>> GetSystemTableName() const;
4646
Result<bool> IsSystemTable() const;
47+
std::string GetFullName() const;
4748
std::string ToString() const;
49+
int32_t HashCode() const;
50+
51+
public:
52+
static Result<Identifier> FromString(const std::string& full_name);
4853

4954
private:
5055
Status SplitTableName() const;
@@ -58,3 +63,13 @@ class PAIMON_EXPORT Identifier {
5863
};
5964

6065
} // namespace paimon
66+
67+
namespace std {
68+
template <>
69+
struct hash<paimon::Identifier> {
70+
size_t operator()(const paimon::Identifier& identifier) const {
71+
return identifier.HashCode();
72+
}
73+
};
74+
75+
} // namespace std

include/paimon/data/blob.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ class PAIMON_EXPORT Blob {
9797
/// @param metadata A map of key-value metadata to be attached to the field.
9898
/// @return A result containing a unique pointer to the generated `ArrowSchema` or an error.
9999
static Result<std::unique_ptr<::ArrowSchema>> ArrowField(
100-
const std::string& field_name, std::unordered_map<std::string, std::string> metadata = {});
100+
const std::string& field_name, bool nullable = false,
101+
std::unordered_map<std::string, std::string> metadata = {});
101102

102103
private:
103104
class Impl;

include/paimon/defs.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ struct PAIMON_EXPORT Options {
365365
/// "partition.legacy-name" - The legacy partition name is using `ToString` for all types. If
366366
/// false, using casting to string for all types. Default value is "true".
367367
static const char PARTITION_GENERATE_LEGACY_NAME[];
368-
/// "blob-as-descriptor" - Read and write blob field using blob descriptor rather than blob
368+
/// "blob-as-descriptor" - Read blob field using blob descriptor rather than blob
369369
/// bytes. Default value is "false".
370370
static const char BLOB_AS_DESCRIPTOR[];
371371
/// "blob-field" - Specifies column names that should be stored as blob type. This is used

include/paimon/predicate/literal.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,13 @@ class PAIMON_EXPORT Literal {
8989
std::string ToString() const;
9090

9191
/// Gets the hash code for this literal.
92+
/// @note HashCode() hashes the exact bit representation (including Decimal scale), while
93+
/// operator== delegates to CompareTo() which uses numeric equality (e.g. decimals with
94+
/// different scales can compare equal). This means the hash-equality contract (equal objects
95+
/// must have equal hashes) may be violated for Decimal literals with different scales. In
96+
/// practice this is safe because all current std::unordered_map<Literal, ...> usages (bitmap
97+
/// file index) only store values from the same column, which guarantees a fixed precision and
98+
/// scale.
9299
size_t HashCode() const;
93100

94101
/// Compares this literal with another literal. The comparison follows SQL semantics for the
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Copyright 2026-present Alibaba Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#pragma once
18+
19+
#include "paimon/result.h"
20+
#include "paimon/visibility.h"
21+
22+
namespace paimon {
23+
24+
/// Reader abstraction for count queries.
25+
class PAIMON_EXPORT CountReader {
26+
public:
27+
virtual ~CountReader() = default;
28+
29+
/// Count rows for splits bound by the corresponding CreateCountReader call.
30+
virtual Result<int64_t> CountRows() = 0;
31+
};
32+
33+
} // namespace paimon

include/paimon/record_batch.h

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,22 @@ class PAIMON_EXPORT RecordBatchBuilder {
122122
/// @param data Map of partition column names to their string values.
123123
RecordBatchBuilder& SetPartition(const std::map<std::string, std::string>& data);
124124

125-
/// Set the bucket id for this record batch. If not set, default value is `-1`.
125+
/// Set the bucket id for this record batch. If not set, default value is `-2147483648`
126+
/// (i.e., `HasSpecifiedBucket()` returns false), and the bucket will be auto-resolved
127+
/// at write time based on the table's bucket option:
128+
///
129+
/// - **Unaware-bucket mode** (table option `bucket = -1`, append-only table without
130+
/// primary keys): the bucket will be auto-filled with `UNAWARE_BUCKET (0)`. If the
131+
/// caller does specify a bucket, it MUST be `UNAWARE_BUCKET (0)`, otherwise the
132+
/// write fails.
133+
/// - **Postpone-bucket mode** (table option `bucket = -2`, primary-key table whose
134+
/// bucket assignment is deferred): the bucket will be auto-filled with
135+
/// `POSTPONE_BUCKET (-2)`. If the caller does specify a bucket, it MUST be
136+
/// `POSTPONE_BUCKET (-2)`, otherwise the write fails.
137+
/// - **Fixed-bucket mode** (table option `bucket > 0`): the caller MUST explicitly
138+
/// call `SetBucket()` with a value in `[0, bucket)`; not calling `SetBucket()`
139+
/// (or passing an out-of-range value) will cause the write to fail.
140+
///
126141
/// @param bucket The bucket id for data distribution.
127142
RecordBatchBuilder& SetBucket(int32_t bucket);
128143

include/paimon/table/source/table_read.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "paimon/memory/memory_pool.h"
2424
#include "paimon/read_context.h"
2525
#include "paimon/reader/batch_reader.h"
26+
#include "paimon/reader/count_reader.h"
2627
#include "paimon/result.h"
2728
#include "paimon/table/source/split.h"
2829
#include "paimon/visibility.h"
@@ -63,6 +64,12 @@ class PAIMON_EXPORT TableRead {
6364
virtual Result<std::unique_ptr<BatchReader>> CreateReader(
6465
const std::shared_ptr<Split>& split) = 0;
6566

67+
/// Creates a `CountReader` for count queries on the specified splits.
68+
///
69+
/// Implementations may override this to provide a more efficient count path.
70+
virtual Result<std::unique_ptr<CountReader>> CreateCountReader(
71+
const std::vector<std::shared_ptr<Split>>& splits);
72+
6673
protected:
6774
explicit TableRead(const std::shared_ptr<MemoryPool>& memory_pool);
6875

0 commit comments

Comments
 (0)