Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,7 @@ macro(build_lucene)
set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install")
set(LUCENE_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
"-DLUCENE_BUILD_SHARED=OFF"
"-DENABLE_TEST=OFF"
"-DCMAKE_C_FLAGS=-pthread"
"-DCMAKE_CXX_FLAGS=-pthread"
Expand All @@ -303,7 +304,7 @@ macro(build_lucene)
"-DBoost_THREAD_FOUND=TRUE"
"-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}")

set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.so.0")
set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a")
externalproject_add(lucene_ep
${EP_COMMON_OPTIONS}
URL ${LUCENE_SOURCE_URL}
Expand All @@ -323,13 +324,14 @@ macro(build_lucene)
# The include directory must exist before it is referenced by a target.
file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}")
include_directories(SYSTEM ${LUCENE_INCLUDE_DIR} ${BOOST_INCLUDE_DIR})
add_library(lucene INTERFACE IMPORTED)
target_include_directories(lucene SYSTEM INTERFACE "${LUCENE_INCLUDE_DIR}")
target_compile_options(lucene INTERFACE -pthread)
add_library(lucene STATIC IMPORTED)
set_target_properties(lucene
PROPERTIES IMPORTED_LOCATION "${LUCENE_LIB}"
INTERFACE_INCLUDE_DIRECTORIES
"${LUCENE_INCLUDE_DIR}")

target_link_libraries(lucene
INTERFACE "${LUCENE_LIB}"
boost_date_time
INTERFACE boost_date_time
boost_filesystem
boost_regex
boost_thread
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2024-present Alibaba Inc.
* Copyright 2026-present Alibaba Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -26,25 +26,24 @@
#include "paimon/visibility.h"

namespace paimon {
/// Represents a vector search global index result that combines a Roaring bitmap of candidate row
/// Represents a scored global index result that combines a Roaring bitmap of candidate row
/// ids with an array of associated relevance scores.
///
/// **Important Ordering Note**: Despite inheriting from VectorSearchGlobalIndexResult, the results
/// are
/// **Important Ordering Note**: Inheriting from ScoredGlobalIndexResult, the results are
/// **NOT sorted by score**. Instead, both the bitmap and the score vector are ordered by
/// **ascending row id**. This design enables efficient merging and set operations while preserving
/// row id-to-score mapping.
class PAIMON_EXPORT BitmapVectorSearchGlobalIndexResult : public VectorSearchGlobalIndexResult {
class PAIMON_EXPORT BitmapScoredGlobalIndexResult : public ScoredGlobalIndexResult {
public:
BitmapVectorSearchGlobalIndexResult(RoaringBitmap64&& bitmap, std::vector<float>&& scores)
BitmapScoredGlobalIndexResult(RoaringBitmap64&& bitmap, std::vector<float>&& scores)
: bitmap_(std::move(bitmap)), scores_(std::move(scores)) {
assert(static_cast<size_t>(bitmap_.Cardinality()) == scores_.size());
}

class VectorSearchIterator : public VectorSearchGlobalIndexResult::VectorSearchIterator {
class ScoredIterator : public ScoredGlobalIndexResult::ScoredIterator {
public:
VectorSearchIterator(const RoaringBitmap64* bitmap, RoaringBitmap64::Iterator&& iter,
const float* scores)
ScoredIterator(const RoaringBitmap64* bitmap, RoaringBitmap64::Iterator&& iter,
const float* scores)
: bitmap_(bitmap), iter_(std::move(iter)), scores_(scores) {}

bool HasNext() const override {
Expand All @@ -66,8 +65,8 @@ class PAIMON_EXPORT BitmapVectorSearchGlobalIndexResult : public VectorSearchGlo

Result<std::unique_ptr<GlobalIndexResult::Iterator>> CreateIterator() const override;

Result<std::unique_ptr<VectorSearchGlobalIndexResult::VectorSearchIterator>>
CreateVectorSearchIterator() const override;
Result<std::unique_ptr<ScoredGlobalIndexResult::ScoredIterator>> CreateScoredIterator()
const override;

Result<std::shared_ptr<GlobalIndexResult>> And(
const std::shared_ptr<GlobalIndexResult>& other) override;
Expand All @@ -90,7 +89,6 @@ class PAIMON_EXPORT BitmapVectorSearchGlobalIndexResult : public VectorSearchGlo
const std::vector<float>& GetScores() const;

private:
// TODO(xinyu.lxy): may use pair<int64_t, float>
RoaringBitmap64 bitmap_;
// ordered by row id
std::vector<float> scores_;
Expand Down
9 changes: 7 additions & 2 deletions include/paimon/global_index/global_index_reader.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2024-present Alibaba Inc.
* Copyright 2026-present Alibaba Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -21,6 +21,7 @@
#include <vector>

#include "paimon/global_index/global_index_result.h"
#include "paimon/predicate/full_text_search.h"
#include "paimon/predicate/function_visitor.h"
#include "paimon/predicate/vector_search.h"
#include "paimon/visibility.h"
Expand All @@ -40,9 +41,13 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<G
/// VisitVectorSearch performs approximate vector similarity search.
/// @warning `VisitVectorSearch` may return error status when it is incorrectly invoked (e.g.,
/// BitmapGlobalIndexReader call `VisitVectorSearch`).
virtual Result<std::shared_ptr<VectorSearchGlobalIndexResult>> VisitVectorSearch(
virtual Result<std::shared_ptr<ScoredGlobalIndexResult>> VisitVectorSearch(
const std::shared_ptr<VectorSearch>& vector_search) = 0;

/// VisitFullTextSearch performs full text search.
virtual Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
const std::shared_ptr<FullTextSearch>& full_text_search) = 0;

Comment thread
lxy-9602 marked this conversation as resolved.
/// @return true if the reader is thread-safe; false otherwise.
virtual bool IsThreadSafe() const = 0;

Expand Down
20 changes: 10 additions & 10 deletions include/paimon/global_index/global_index_result.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2024-present Alibaba Inc.
* Copyright 2026-present Alibaba Inc.
*
Comment thread
lszskye marked this conversation as resolved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -76,7 +76,7 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
/// Serializes a GlobalIndexResult object into a byte array.
///
/// @note This method only supports the following concrete implementations:
/// - BitmapVectorSearchGlobalIndexResult
/// - BitmapScoredGlobalIndexResult
/// - BitmapGlobalIndexResult
///
/// @param global_index_result The GlobalIndexResult instance to serialize (must not be null).
Expand All @@ -91,7 +91,7 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
///
/// @note The concrete type of the deserialized object is determined by metadata
/// embedded in the buffer. Currently, only the following types are supported:
/// - BitmapVectorSearchGlobalIndexResult
/// - BitmapScoredGlobalIndexResult
/// - BitmapGlobalIndexResult
///
/// @param buffer Pointer to the serialized byte data (must not be null).
Expand All @@ -106,18 +106,18 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
static constexpr int32_t VERSION = 1;
};

/// Represents the result of a vector search query against a global index.
/// Represents the result with score of a query against a global index.
/// This class encapsulates a set of search candidates (row id + score pairs) and provides
/// an iterator interface to traverse them.
class PAIMON_EXPORT VectorSearchGlobalIndexResult : public GlobalIndexResult {
class PAIMON_EXPORT ScoredGlobalIndexResult : public GlobalIndexResult {
public:
/// An iterator over the vector search results, returning (row_id, score) pairs.
/// An iterator over the scored results, returning (row_id, score) pairs.
///
/// @note The results are **NOT sorted by score**. Instead, they are returned in **ascending
/// order of row_id**.
class VectorSearchIterator {
class ScoredIterator {
public:
virtual ~VectorSearchIterator() = default;
virtual ~ScoredIterator() = default;

/// Checks whether more row ids are available.
virtual bool HasNext() const = 0;
Expand All @@ -132,7 +132,7 @@ class PAIMON_EXPORT VectorSearchGlobalIndexResult : public GlobalIndexResult {
virtual std::pair<int64_t, float> NextWithScore() = 0;
};

/// Creates a new iterator for traversing the vector search results.
virtual Result<std::unique_ptr<VectorSearchIterator>> CreateVectorSearchIterator() const = 0;
/// Creates a new iterator for traversing the scored results.
virtual Result<std::unique_ptr<ScoredIterator>> CreateScoredIterator() const = 0;
};
} // namespace paimon
4 changes: 2 additions & 2 deletions src/paimon/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ set(PAIMON_COMMON_SRCS
common/fs/file_system_factory.cpp
common/global_config.cpp
common/global_index/complete_index_score_batch_reader.cpp
common/global_index/bitmap_vector_search_global_index_result.cpp
common/global_index/bitmap_scored_global_index_result.cpp
common/global_index/bitmap_global_index_result.cpp
common/global_index/global_index_result.cpp
common/global_index/global_indexer_factory.cpp
Expand Down Expand Up @@ -359,7 +359,7 @@ if(PAIMON_BUILD_TESTS)
common/global_index/global_index_result_test.cpp
common/global_index/global_indexer_factory_test.cpp
common/global_index/bitmap_global_index_result_test.cpp
common/global_index/bitmap_vector_search_global_index_result_test.cpp
common/global_index/bitmap_scored_global_index_result_test.cpp
common/global_index/bitmap/bitmap_global_index_test.cpp
common/io/byte_array_input_stream_test.cpp
common/io/data_input_output_stream_test.cpp
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2024-present Alibaba Inc.
* Copyright 2026-present Alibaba Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -14,7 +14,7 @@
* limitations under the License.
*/

#include "paimon/global_index/bitmap_vector_search_global_index_result.h"
#include "paimon/global_index/bitmap_scored_global_index_result.h"

#include "fmt/format.h"
#include "fmt/ranges.h"
Expand All @@ -41,78 +41,76 @@ std::vector<float> GetScoresFromMap(const RoaringBitmap64& bitmap,
return scores;
}
} // namespace
Result<std::unique_ptr<GlobalIndexResult::Iterator>>
BitmapVectorSearchGlobalIndexResult::CreateIterator() const {
Result<std::unique_ptr<GlobalIndexResult::Iterator>> BitmapScoredGlobalIndexResult::CreateIterator()
const {
return std::make_unique<BitmapGlobalIndexResult::Iterator>(&bitmap_, bitmap_.Begin());
}

Result<std::unique_ptr<VectorSearchGlobalIndexResult::VectorSearchIterator>>
BitmapVectorSearchGlobalIndexResult::CreateVectorSearchIterator() const {
return std::make_unique<BitmapVectorSearchGlobalIndexResult::VectorSearchIterator>(
Result<std::unique_ptr<ScoredGlobalIndexResult::ScoredIterator>>
BitmapScoredGlobalIndexResult::CreateScoredIterator() const {
return std::make_unique<BitmapScoredGlobalIndexResult::ScoredIterator>(
&bitmap_, bitmap_.Begin(), scores_.data());
}

Result<std::shared_ptr<GlobalIndexResult>> BitmapVectorSearchGlobalIndexResult::And(
Result<std::shared_ptr<GlobalIndexResult>> BitmapScoredGlobalIndexResult::And(
const std::shared_ptr<GlobalIndexResult>& other) {
auto vector_search_other =
std::dynamic_pointer_cast<BitmapVectorSearchGlobalIndexResult>(other);
if (vector_search_other) {
// If current and other result are both BitmapVectorSearchGlobalIndexResult, return
auto scored_other = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(other);
if (scored_other) {
// If current and other result are both BitmapScoredGlobalIndexResult, return
// BitmapGlobalIndexResult. Erase scores to prevent the same row id with different
// scores in current and other results.
auto supplier = [vector_search_other,
result = std::dynamic_pointer_cast<BitmapVectorSearchGlobalIndexResult>(
auto supplier = [scored_other,
result = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(
shared_from_this())]() -> Result<RoaringBitmap64> {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, vector_search_other->GetBitmap());
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, scored_other->GetBitmap());
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, result->GetBitmap());
return RoaringBitmap64::And(*r1, *r2);
};
return std::make_shared<BitmapGlobalIndexResult>(supplier);
}
auto bitmap_other = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(other);
if (bitmap_other) {
// If other bitmap is BitmapGlobalIndexResult, return BitmapVectorSearchGlobalIndexResult as
// score must exist in current vector search result.
// If other bitmap is BitmapGlobalIndexResult, return BitmapScoredGlobalIndexResult as
// score must exist in current scored result.
std::map<int64_t, float> id_to_score = CreateIdToScoreMap(bitmap_, scores_);
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* other_bitmap, bitmap_other->GetBitmap());
auto and_bitmap = RoaringBitmap64::And(bitmap_, *other_bitmap);
std::vector<float> and_scores = GetScoresFromMap(and_bitmap, id_to_score);
return std::make_shared<BitmapVectorSearchGlobalIndexResult>(std::move(and_bitmap),
std::move(and_scores));
return std::make_shared<BitmapScoredGlobalIndexResult>(std::move(and_bitmap),
std::move(and_scores));
}
return GlobalIndexResult::And(other);
}

Result<std::shared_ptr<GlobalIndexResult>> BitmapVectorSearchGlobalIndexResult::Or(
Result<std::shared_ptr<GlobalIndexResult>> BitmapScoredGlobalIndexResult::Or(
const std::shared_ptr<GlobalIndexResult>& other) {
auto vector_search_other =
std::dynamic_pointer_cast<BitmapVectorSearchGlobalIndexResult>(other);
if (vector_search_other) {
// If current and other result are both BitmapVectorSearchGlobalIndexResult, return
// BitmapVectorSearchGlobalIndexResult when current and other have has no intersection row
auto scored_other = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(other);
if (scored_other) {
// If current and other result are both BitmapScoredGlobalIndexResult, return
// BitmapScoredGlobalIndexResult when current and other have has no intersection row
// id.
std::map<int64_t, float> id_to_score = CreateIdToScoreMap(bitmap_, scores_);
size_t idx = 0;
for (auto iter = vector_search_other->bitmap_.Begin();
iter != vector_search_other->bitmap_.End(); ++iter, ++idx) {
for (auto iter = scored_other->bitmap_.Begin(); iter != scored_other->bitmap_.End();
++iter, ++idx) {
if (id_to_score.find(*iter) != id_to_score.end()) {
return Status::Invalid(
"not support two BitmapVectorSearchGlobalIndexResult or with same row id");
"not support two BitmapScoredGlobalIndexResult or with same row id");
}
id_to_score[*iter] = vector_search_other->scores_[idx];
id_to_score[*iter] = scored_other->scores_[idx];
}
auto or_bitmap = RoaringBitmap64::Or(bitmap_, vector_search_other->bitmap_);
auto or_bitmap = RoaringBitmap64::Or(bitmap_, scored_other->bitmap_);
std::vector<float> or_scores = GetScoresFromMap(or_bitmap, id_to_score);
return std::make_shared<BitmapVectorSearchGlobalIndexResult>(std::move(or_bitmap),
std::move(or_scores));
return std::make_shared<BitmapScoredGlobalIndexResult>(std::move(or_bitmap),
std::move(or_scores));
}

auto bitmap_other = std::dynamic_pointer_cast<BitmapGlobalIndexResult>(other);
if (bitmap_other) {
// If other bitmap is BitmapGlobalIndexResult, return BitmapGlobalIndexResult as
// score for union row id is unknown.
auto supplier = [bitmap_other,
result = std::dynamic_pointer_cast<BitmapVectorSearchGlobalIndexResult>(
result = std::dynamic_pointer_cast<BitmapScoredGlobalIndexResult>(
shared_from_this())]() -> Result<RoaringBitmap64> {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r1, bitmap_other->GetBitmap());
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* r2, result->GetBitmap());
Expand All @@ -123,31 +121,30 @@ Result<std::shared_ptr<GlobalIndexResult>> BitmapVectorSearchGlobalIndexResult::
return GlobalIndexResult::Or(other);
}

Result<std::shared_ptr<GlobalIndexResult>> BitmapVectorSearchGlobalIndexResult::AddOffset(
Result<std::shared_ptr<GlobalIndexResult>> BitmapScoredGlobalIndexResult::AddOffset(
int64_t offset) {
PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* bitmap, GetBitmap());
RoaringBitmap64 bitmap64;
for (auto iter = bitmap->Begin(); iter != bitmap->End(); ++iter) {
bitmap64.Add(offset + (*iter));
}
auto scores = GetScores();
return std::make_shared<BitmapVectorSearchGlobalIndexResult>(std::move(bitmap64),
std::move(scores));
return std::make_shared<BitmapScoredGlobalIndexResult>(std::move(bitmap64), std::move(scores));
}

Result<bool> BitmapVectorSearchGlobalIndexResult::IsEmpty() const {
Result<bool> BitmapScoredGlobalIndexResult::IsEmpty() const {
return bitmap_.IsEmpty();
}

Result<const RoaringBitmap64*> BitmapVectorSearchGlobalIndexResult::GetBitmap() const {
Result<const RoaringBitmap64*> BitmapScoredGlobalIndexResult::GetBitmap() const {
return &bitmap_;
}

const std::vector<float>& BitmapVectorSearchGlobalIndexResult::GetScores() const {
const std::vector<float>& BitmapScoredGlobalIndexResult::GetScores() const {
return scores_;
}

std::string BitmapVectorSearchGlobalIndexResult::ToString() const {
std::string BitmapScoredGlobalIndexResult::ToString() const {
std::vector<std::string> formatted_scores;
formatted_scores.reserve(scores_.size());
for (const auto& score : scores_) {
Expand Down
Loading
Loading