feat: add iceberg_data library alongside iceberg (apache#631)

Move data writers, deletes/, and puffin/ into a separate `iceberg_data`
library that links the existing `iceberg` target. `delete_file_index`
stays in `iceberg` because manifest_group embeds
DeleteFileIndex::Builder with only core dependencies.

* `iceberg` — unchanged target name for metadata/planning, expressions,
manifests, catalog (incl. in-memory), utilities, file I/O abstractions,
and delete_file_index.

* `iceberg_data` — data/, deletes/, puffin/; links `iceberg`.

`iceberg_bundle` links `iceberg_data` when the bundle is built.
`iceberg_rest` links `iceberg` and cpr only.

Author: zhjwpku

example/CMakeLists.txt

@@ -22,7 +22,7 @@ project(example)
 
 set(CMAKE_CXX_STANDARD 23)
 
-find_package(iceberg CONFIG REQUIRED)
+find_package(iceberg CONFIG REQUIRED COMPONENTS bundle rest)
 
 add_executable(demo_example demo_example.cc)
 

src/iceberg/CMakeLists.txt

@@ -20,14 +20,7 @@ set(ICEBERG_INCLUDES "$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src>"
 set(ICEBERG_SOURCES
     arrow_c_data_guard_internal.cc
     catalog/memory/in_memory_catalog.cc
-    data/data_writer.cc
-    data/delete_loader.cc
-    data/equality_delete_writer.cc
-    data/position_delete_writer.cc
-    data/writer.cc
     delete_file_index.cc
-    deletes/roaring_position_bitmap.cc
-    deletes/position_delete_index.cc
     expression/aggregate.cc
     expression/binder.cc
     expression/evaluator.cc
@@ -66,9 +59,6 @@ set(ICEBERG_SOURCES
     partition_field.cc
     partition_spec.cc
     partition_summary.cc
-    puffin/file_metadata.cc
-    puffin/puffin_format.cc
-    puffin/json_serde.cc
     row/arrow_array_wrapper.cc
     row/manifest_wrapper.cc
     row/partition_values.cc
@@ -134,24 +124,22 @@ list(APPEND
      ICEBERG_STATIC_BUILD_INTERFACE_LIBS
      "$<IF:$<BOOL:${NANOARROW_VENDORED}>,nanoarrow::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_static>,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>"
      nlohmann_json::nlohmann_json
-     roaring::roaring
      ZLIB::ZLIB)
 list(APPEND
      ICEBERG_SHARED_BUILD_INTERFACE_LIBS
      "$<IF:$<BOOL:${NANOARROW_VENDORED}>,nanoarrow::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_shared>,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>"
      nlohmann_json::nlohmann_json
-     roaring::roaring
      ZLIB::ZLIB)
 list(APPEND
      ICEBERG_STATIC_INSTALL_INTERFACE_LIBS
      "$<IF:$<BOOL:${NANOARROW_VENDORED}>,iceberg::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_static>,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>"
      "$<IF:$<BOOL:${NLOHMANN_JSON_VENDORED}>,iceberg::nlohmann_json,$<IF:$<TARGET_EXISTS:nlohmann_json::nlohmann_json>,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>"
-     "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+)
 list(APPEND
      ICEBERG_SHARED_INSTALL_INTERFACE_LIBS
      "$<IF:$<BOOL:${NANOARROW_VENDORED}>,iceberg::nanoarrow_static,$<IF:$<TARGET_EXISTS:nanoarrow::nanoarrow_shared>,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>"
      "$<IF:$<BOOL:${NLOHMANN_JSON_VENDORED}>,iceberg::nlohmann_json,$<IF:$<TARGET_EXISTS:nlohmann_json::nlohmann_json>,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>"
-     "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+)
 
 add_iceberg_lib(iceberg
                 SOURCES
@@ -169,6 +157,52 @@ add_iceberg_lib(iceberg
                 OUTPUTS
                 ICEBERG_LIBRARIES)
 
+set(ICEBERG_DATA_SOURCES
+    data/data_writer.cc
+    data/delete_loader.cc
+    data/equality_delete_writer.cc
+    data/position_delete_writer.cc
+    data/writer.cc
+    deletes/position_delete_index.cc
+    deletes/roaring_position_bitmap.cc
+    puffin/file_metadata.cc
+    puffin/json_serde.cc
+    puffin/puffin_format.cc)
+
+set(ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS)
+set(ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS)
+set(ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS)
+set(ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS)
+
+list(APPEND ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
+     roaring::roaring)
+list(APPEND ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
+     roaring::roaring)
+list(APPEND
+     ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:iceberg::iceberg_static>,iceberg::iceberg_static,iceberg::iceberg_shared>"
+     "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+list(APPEND
+     ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS
+     "$<IF:$<TARGET_EXISTS:iceberg::iceberg_shared>,iceberg::iceberg_shared,iceberg::iceberg_static>"
+     "$<IF:$<BOOL:${CROARING_VENDORED}>,iceberg::roaring,roaring::roaring>")
+
+add_iceberg_lib(iceberg_data
+                SOURCES
+                ${ICEBERG_DATA_SOURCES}
+                EXTRA_INCLUDES
+                ${ICEBERG_INCLUDES}
+                SHARED_LINK_LIBS
+                ${ICEBERG_DATA_SHARED_BUILD_INTERFACE_LIBS}
+                STATIC_LINK_LIBS
+                ${ICEBERG_DATA_STATIC_BUILD_INTERFACE_LIBS}
+                STATIC_INSTALL_INTERFACE_LIBS
+                ${ICEBERG_DATA_STATIC_INSTALL_INTERFACE_LIBS}
+                SHARED_INSTALL_INTERFACE_LIBS
+                ${ICEBERG_DATA_SHARED_INSTALL_INTERFACE_LIBS})
+
 iceberg_install_all_headers(iceberg)
 
 add_subdirectory(catalog)
@@ -210,29 +244,29 @@ if(ICEBERG_BUILD_BUNDLE)
 
   list(APPEND
        ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS
-       "$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
+       "$<IF:$<TARGET_EXISTS:iceberg_data_static>,iceberg_data_static,iceberg_data_shared>"
        "$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>"
        "$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>"
        "$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_static>,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>"
   )
   list(APPEND
        ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS
-       "$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
+       "$<IF:$<TARGET_EXISTS:iceberg_data_shared>,iceberg_data_shared,iceberg_data_static>"
        "$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>"
        "$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>"
        "$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>"
   )
 
   list(APPEND
        ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS
-       "$<IF:$<TARGET_EXISTS:iceberg::iceberg_static>,iceberg::iceberg_static,iceberg::iceberg_shared>"
+       "$<IF:$<TARGET_EXISTS:iceberg::iceberg_data_static>,iceberg::iceberg_data_static,iceberg::iceberg_data_shared>"
        "$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>>"
        "$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>>"
        "$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_static>,avro-cpp::avrocpp_static,avro-cpp::avrocpp_shared>>"
   )
   list(APPEND
        ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS
-       "$<IF:$<TARGET_EXISTS:iceberg::iceberg_shared>,iceberg::iceberg_shared,iceberg::iceberg_static>"
+       "$<IF:$<TARGET_EXISTS:iceberg::iceberg_data_shared>,iceberg::iceberg_data_shared,iceberg::iceberg_data_static>"
        "$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>>"
        "$<IF:$<BOOL:${ARROW_VENDORED}>,iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>>"
        "$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>"

src/iceberg/arrow_c_data_guard_internal.h

@@ -22,10 +22,11 @@
 #include <nanoarrow/nanoarrow.h>
 
 #include "iceberg/arrow_c_data.h"
+#include "iceberg/iceberg_export.h"
 
 namespace iceberg::internal {
 
-class ArrowArrayGuard {
+class ICEBERG_EXPORT ArrowArrayGuard {
  public:
   explicit ArrowArrayGuard(ArrowArray* array) : array_(array) {}
   ~ArrowArrayGuard();
@@ -34,7 +35,7 @@ class ArrowArrayGuard {
   ArrowArray* array_;
 };
 
-class ArrowSchemaGuard {
+class ICEBERG_EXPORT ArrowSchemaGuard {
  public:
   explicit ArrowSchemaGuard(ArrowSchema* schema) : schema_(schema) {}
   ~ArrowSchemaGuard();
@@ -43,7 +44,7 @@ class ArrowSchemaGuard {
   ArrowSchema* schema_;
 };
 
-class ArrowArrayViewGuard {
+class ICEBERG_EXPORT ArrowArrayViewGuard {
  public:
   explicit ArrowArrayViewGuard(ArrowArrayView* view) : view_(view) {}
   ~ArrowArrayViewGuard();
@@ -52,7 +53,7 @@ class ArrowArrayViewGuard {
   ArrowArrayView* view_;
 };
 
-class ArrowArrayBufferGuard {
+class ICEBERG_EXPORT ArrowArrayBufferGuard {
  public:
   explicit ArrowArrayBufferGuard(ArrowBuffer* buffer) : buffer_(buffer) {}
   ~ArrowArrayBufferGuard();

src/iceberg/data/data_writer.h

@@ -31,15 +31,15 @@
 #include "iceberg/arrow_c_data.h"
 #include "iceberg/data/writer.h"
 #include "iceberg/file_format.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
 #include "iceberg/result.h"
 #include "iceberg/row/partition_values.h"
 #include "iceberg/type_fwd.h"
 
 namespace iceberg {
 
 /// \brief Options for creating a DataWriter.
-struct ICEBERG_EXPORT DataWriterOptions {
+struct ICEBERG_DATA_EXPORT DataWriterOptions {
   std::string path;
   std::shared_ptr<Schema> schema;
   std::shared_ptr<PartitionSpec> spec;
@@ -51,7 +51,7 @@ struct ICEBERG_EXPORT DataWriterOptions {
 };
 
 /// \brief Writer for Iceberg data files.
-class ICEBERG_EXPORT DataWriter : public FileWriter {
+class ICEBERG_DATA_EXPORT DataWriter : public FileWriter {
  public:
   ~DataWriter() override;
 

src/iceberg/data/delete_loader.h

@@ -26,14 +26,14 @@
 #include <span>
 #include <string_view>
 
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
 #include "iceberg/result.h"
 #include "iceberg/type_fwd.h"
 
 namespace iceberg {
 
 /// \brief Loads delete files and constructs in-memory delete indexes.
-class ICEBERG_EXPORT DeleteLoader {
+class ICEBERG_DATA_EXPORT DeleteLoader {
  public:
   /// \brief Create a DeleteLoader.
   /// \param io FileIO instance for reading delete files

src/iceberg/data/equality_delete_writer.h

@@ -32,15 +32,15 @@
 #include "iceberg/arrow_c_data.h"
 #include "iceberg/data/writer.h"
 #include "iceberg/file_format.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
 #include "iceberg/result.h"
 #include "iceberg/row/partition_values.h"
 #include "iceberg/type_fwd.h"
 
 namespace iceberg {
 
 /// \brief Options for creating an EqualityDeleteWriter.
-struct ICEBERG_EXPORT EqualityDeleteWriterOptions {
+struct ICEBERG_DATA_EXPORT EqualityDeleteWriterOptions {
   std::string path;
   std::shared_ptr<Schema> schema;
   std::shared_ptr<PartitionSpec> spec;
@@ -54,7 +54,7 @@ struct ICEBERG_EXPORT EqualityDeleteWriterOptions {
 };
 
 /// \brief Writer for Iceberg equality delete files.
-class ICEBERG_EXPORT EqualityDeleteWriter : public FileWriter {
+class ICEBERG_DATA_EXPORT EqualityDeleteWriter : public FileWriter {
  public:
   ~EqualityDeleteWriter() override;
 

src/iceberg/data/meson.build

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+install_headers(
+    [
+        'data_writer.h',
+        'delete_loader.h',
+        'equality_delete_writer.h',
+        'position_delete_writer.h',
+        'writer.h',
+    ],
+    subdir: 'iceberg/data',
+)

src/iceberg/data/position_delete_writer.h

@@ -31,15 +31,15 @@
 #include "iceberg/arrow_c_data.h"
 #include "iceberg/data/writer.h"
 #include "iceberg/file_format.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
 #include "iceberg/result.h"
 #include "iceberg/row/partition_values.h"
 #include "iceberg/type_fwd.h"
 
 namespace iceberg {
 
 /// \brief Options for creating a PositionDeleteWriter.
-struct ICEBERG_EXPORT PositionDeleteWriterOptions {
+struct ICEBERG_DATA_EXPORT PositionDeleteWriterOptions {
   std::string path;
   std::shared_ptr<Schema> schema;
   std::shared_ptr<PartitionSpec> spec;
@@ -51,7 +51,7 @@ struct ICEBERG_EXPORT PositionDeleteWriterOptions {
 };
 
 /// \brief Writer for Iceberg position delete files.
-class ICEBERG_EXPORT PositionDeleteWriter : public FileWriter {
+class ICEBERG_DATA_EXPORT PositionDeleteWriter : public FileWriter {
  public:
   ~PositionDeleteWriter() override;
 

src/iceberg/data/writer.h

@@ -27,14 +27,14 @@
 #include <vector>
 
 #include "iceberg/arrow_c_data.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
 #include "iceberg/result.h"
 #include "iceberg/type_fwd.h"
 
 namespace iceberg {
 
 /// \brief Base interface for data file writers.
-class ICEBERG_EXPORT FileWriter {
+class ICEBERG_DATA_EXPORT FileWriter {
  public:
   virtual ~FileWriter();
 
@@ -49,7 +49,7 @@ class ICEBERG_EXPORT FileWriter {
   virtual Status Close() = 0;
 
   /// \brief File metadata for all files produced by this writer.
-  struct ICEBERG_EXPORT WriteResult {
+  struct ICEBERG_DATA_EXPORT WriteResult {
     /// Usually a writer produces a single data or delete file.
     /// Position delete writer may produce multiple file-scoped delete files.
     /// In the future, multiple files can be produced if file rolling is supported.

src/iceberg/deletes/position_delete_index.h

@@ -26,7 +26,7 @@
 #include <memory>
 
 #include "iceberg/deletes/roaring_position_bitmap.h"
-#include "iceberg/iceberg_export.h"
+#include "iceberg/iceberg_data_export.h"
 
 namespace iceberg {
 
@@ -35,7 +35,7 @@ namespace iceberg {
 /// This class provides a domain-specific API for position deletes
 /// in Iceberg MOR (merge-on-read) tables. Positions are 0-based
 /// row indices within a data file.
-class ICEBERG_EXPORT PositionDeleteIndex {
+class ICEBERG_DATA_EXPORT PositionDeleteIndex {
  public:
   PositionDeleteIndex() = default;
   ~PositionDeleteIndex() = default;