feat: update lumina lib to v0.3.0-rc1 (alibaba#372)

Author: lxy-9602

src/paimon/global_index/lumina/lumina_api_test.cpp

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 #include "lumina/api/LuminaBuilder.h"
+#include "lumina/api/LuminaSearcher.h"
 #include "lumina/core/Constants.h"
 #include "lumina/core/Status.h"
 #include "lumina/core/Types.h"
@@ -133,7 +134,7 @@ class LuminaInterfaceTest : public ::testing::Test {
         ASSERT_GT(paimon_pool->MaxMemoryUsage(), 0);
     }
 
-    void CheckResult(const std::vector<::lumina::api::LuminaSearcher::SearchHit>& search_result,
+    void CheckResult(const std::vector<::lumina::api::SearchHit>& search_result,
                      const std::vector<::lumina::core::vector_id_t>& expected_row_ids,
                      const std::vector<float>& expected_distances) const {
         ASSERT_EQ(search_result.size(), expected_row_ids.size());

src/paimon/global_index/lumina/lumina_global_index.h

@@ -23,6 +23,7 @@
 #include <vector>
 
 #include "arrow/api.h"
+#include "lumina/api/LuminaSearcher.h"
 #include "lumina/api/Options.h"
 #include "lumina/extensions/SearchWithFilterExtension.h"
 #include "paimon/global_index/bitmap_global_index_result.h"

third_party/lumina/VERSION

@@ -1,2 +1,2 @@
-tag: v0.2.2
-89ea85ed8ef350455a15e1a2519271007ae15807
+tag: v0.3.0-rc1
+304227808425be9b2db0ded1e2e43be0ffba42b2

third_party/lumina/include/lumina/api/Dataset.h

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+
 #pragma once
 
 #include <cstdint>

third_party/lumina/include/lumina/api/Extension.h

@@ -22,27 +22,41 @@
 #include <lumina/api/Query.h>
 #include <lumina/core/NoCopyable.h>
 #include <lumina/core/Status.h>
+#include <memory>
 #include <string>
 #include <string_view>
-#include <unordered_map>
 #include <vector>
 
 namespace lumina::api {
 
 // Note: Attach only registers capabilities; the framework does not own extension lifetimes.
 // Callers must ensure thread safety and lifetime covers the search/build process.
-class ISearchExtension : public core::NoCopyable
+class ISearchExtension : public core::NoCopyable, public core::NoMoveable
 {
 public:
     virtual ~ISearchExtension() = default;
     virtual std::string_view Name() const noexcept = 0;
 };
 
-class IBuildExtension : public core::NoCopyable
+class BuilderStatusManager;
+
+class IBuildExtension : public core::NoCopyable, public core::NoMoveable
 {
 public:
     virtual ~IBuildExtension() = default;
     virtual std::string_view Name() const noexcept = 0;
+
+    /// Called by Impl::Attach() to inject the status manager that the extension
+    /// uses to coordinate state transitions (e.g., tag insert).
+    /// Default implementation is a no-op for extensions that don't need it.
+    virtual void SetBuilderStatusManager(std::shared_ptr<BuilderStatusManager> /*manager*/) noexcept {}
+};
+
+class IStreamExtension : public core::NoCopyable, public core::NoMoveable
+{
+public:
+    virtual ~IStreamExtension() = default;
+    virtual std::string_view Name() const noexcept = 0;
 };
 
 } // namespace lumina::api

third_party/lumina/include/lumina/api/LuminaBuilder.h

@@ -46,6 +46,7 @@ class LuminaBuilder final : public core::NoCopyable
 public:
     class Impl;
     LuminaBuilder(LuminaBuilder&&) noexcept;
+    LuminaBuilder& operator=(LuminaBuilder&&) noexcept;
     LuminaBuilder(std::unique_ptr<Impl> impl) noexcept;
     ~LuminaBuilder() noexcept;
     // Typical offline build: create a Builder by preset (e.g. "ivf", "diskann", "bruteforce", "demo")

third_party/lumina/include/lumina/api/LuminaSearcher.h

@@ -20,6 +20,7 @@
 #include <lumina/api/Extension.h>
 #include <lumina/api/Options.h>
 #include <lumina/api/Query.h>
+#include <lumina/api/SearchResult.h>
 #include <lumina/core/MemoryResource.h>
 #include <lumina/core/NoCopyable.h>
 #include <lumina/core/Result.h>
@@ -46,6 +47,7 @@ class LuminaSearcher final : public core::NoCopyable
     LuminaSearcher(std::unique_ptr<Impl> impl) noexcept;
     // -- Semantics: movable, not copyable --
     LuminaSearcher(LuminaSearcher&&) noexcept;
+    LuminaSearcher& operator=(LuminaSearcher&&) noexcept;
     ~LuminaSearcher() noexcept;
 
     static core::Result<LuminaSearcher> Create(const SearcherOptions& options) noexcept;
@@ -55,20 +57,13 @@ class LuminaSearcher final : public core::NoCopyable
     core::Status Open(const IOOptions& ioOptions) noexcept;
     core::Status Open(std::unique_ptr<io::FileReader> reader, const IOOptions& ioOptions) noexcept;
 
-    struct SearchHit {
-        core::vector_id_t id {0};
-        float distance {0.0f};
-    };
-
-    struct SearchResult {
-        std::vector<SearchHit> topk;
-        std::unordered_map<std::string, std::string> searchStats;
-    };
+    using SearchHit = api::SearchHit;
+    using SearchResult = api::SearchResult;
 
     // Index info: basic searcher metadata
     struct IndexInfo {
-        uint64_t count {0}; // Total vectors
-        core::dimension_t dim {0};   // Vector dimension
+        uint64_t count {0};        // Total vectors
+        core::dimension_t dim {0}; // Vector dimension
     };
 
     core::Result<SearchResult> Search(const Query& q, const SearchOptions& options) noexcept;

third_party/lumina/include/lumina/api/LuminaStreamer.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2025-present Alibaba Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include <cstddef>
+#include <cstdint>
+#include <lumina/api/Extension.h>
+#include <lumina/api/Options.h>
+#include <lumina/api/Query.h>
+#include <lumina/api/SearchResult.h>
+#include <lumina/core/MemoryResource.h>
+#include <lumina/core/NoCopyable.h>
+#include <lumina/core/Result.h>
+#include <lumina/core/Status.h>
+#include <lumina/core/Types.h>
+#include <memory>
+#include <memory_resource>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace lumina::api {
+
+// Real-time, in-memory vector indexing. Single-writer Insert, concurrent Search.
+class LuminaStreamer final : public core::NoCopyable
+{
+public:
+    class Impl;
+
+    LuminaStreamer(LuminaStreamer&&) noexcept;
+    LuminaStreamer& operator=(LuminaStreamer&&) noexcept;
+    explicit LuminaStreamer(std::unique_ptr<Impl> impl) noexcept;
+    ~LuminaStreamer() noexcept;
+
+    static core::Result<LuminaStreamer> Create(const StreamerOptions& options) noexcept;
+    static core::Result<LuminaStreamer> Create(const StreamerOptions& options,
+                                               const core::MemoryResourceConfig& memoryConfig) noexcept;
+
+    /** Insert a single vector.
+     *  @param data  Non-null pointer to exactly dim floats; returns InvalidArgument if null.
+     *  @param id    Caller-assigned identifier. Uniqueness is NOT enforced. */
+    core::Status Insert(const float* data, core::vector_id_t id) noexcept;
+
+    using SearchHit = api::SearchHit;
+    using SearchResult = api::SearchResult;
+
+    // Search is `const`: it does not mutate any observable state of this
+    // instance. Safe to call concurrently with other Search/GetMeta calls and
+    // with a single in-flight Insert (see class-level thread-safety note).
+    core::Result<SearchResult> Search(const Query& q, const SearchOptions& options) const noexcept;
+    core::Result<SearchResult> Search(const Query& q, const SearchOptions& options,
+                                      std::pmr::memory_resource& sessionPool) const noexcept;
+
+    struct IndexInfo {
+        uint64_t count {0};        // Total vectors currently visible to Search.
+        core::dimension_t dim {0}; // Vector dimension
+    };
+
+    IndexInfo GetMeta() const noexcept;
+
+    // -- Extension attach (per instance) --
+    core::Status Attach(IStreamExtension& ext) noexcept;
+
+private:
+    std::unique_ptr<Impl> _p;
+};
+
+} // namespace lumina::api

third_party/lumina/include/lumina/api/Options.h

@@ -19,10 +19,6 @@
 #include <algorithm>
 #include <cstddef>
 #include <cstdint>
-#include <lumina/core/Result.h>
-#include <lumina/core/Status.h>
-#include <lumina/mpl/Concepts.h>
-#include <lumina/telemetry/Log.h>
 #include <optional>
 #include <sstream>
 #include <string>
@@ -32,6 +28,12 @@
 #include <utility>
 #include <variant>
 
+#include <lumina/core/Macro.h>
+#include <lumina/core/Result.h>
+#include <lumina/core/Status.h>
+#include <lumina/mpl/Concepts.h>
+#include <lumina/telemetry/Log.h>
+
 namespace lumina::api {
 
 enum class OptionsType {
@@ -40,14 +42,14 @@ enum class OptionsType {
     Builder,
     Quantizer,
     IO,
+    Streamer,
 };
 
 template <OptionsType T>
 class Options
 {
 public:
     Options() noexcept = default;
-    ~Options() noexcept = default;
 
     using Value = std::variant<int64_t, double, bool, std::string>;
     using Map = std::unordered_map<std::string, Value>;
@@ -122,7 +124,7 @@ class Options
     }
 
     // Direct access to all key/value pairs (read-only)
-    const Map& Values() const noexcept { return _values; }
+    const Map& Values() const noexcept LUMINA_LIFETIME_BOUND { return _values; }
 
     // Debug string representation of all options
     std::string DebugString() const noexcept
@@ -177,6 +179,7 @@ class Options
 
 using SearchOptions = Options<OptionsType::Search>;
 using SearcherOptions = Options<OptionsType::Searcher>;
+using StreamerOptions = Options<OptionsType::Streamer>;
 using BuilderOptions = Options<OptionsType::Builder>;
 using QuantizerOptions = Options<OptionsType::Quantizer>;
 using IOOptions = Options<OptionsType::IO>;

third_party/lumina/include/lumina/api/OptionsNormalize.h

@@ -25,11 +25,12 @@
 namespace lumina::api {
 
 // Weakly-typed C++ entry: normalize string->string maps into typed Options.
-core::Result<BuilderOptions> NormalizeBuilderOptions(const std::unordered_map<std::string, std::string>& raw);
-core::Result<SearcherOptions> NormalizeSearcherOptions(const std::unordered_map<std::string, std::string>& raw);
+core::Result<BuilderOptions> NormalizeBuilderOptions(const std::unordered_map<std::string, std::string>& raw) noexcept;
+core::Result<SearcherOptions>
+NormalizeSearcherOptions(const std::unordered_map<std::string, std::string>& raw) noexcept;
 core::Result<SearchOptions> NormalizeSearchOptions(const std::string& indexType,
-    const std::unordered_map<std::string, std::string>& raw);
+                                                   const std::unordered_map<std::string, std::string>& raw) noexcept;
 // check global search options
-core::Status ValidateSearchOptions(const api::SearchOptions& options);
+core::Status ValidateSearchOptions(const api::SearchOptions& options) noexcept;
 
 } // namespace lumina::api