Skip to content

Commit 24cbe84

Browse files
authored
feat(catalog): add SQL catalog (#693)
Add a relational database backed SqlCatalog with a CatalogStore abstraction, built-in sqlpp23 stores for SQLite, PostgreSQL, and MySQL, and Java-compatible catalog tables.
1 parent 7d352f5 commit 24cbe84

27 files changed

Lines changed: 3225 additions & 3 deletions

.github/workflows/cpp-linter.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,20 @@ jobs:
4141
persist-credentials: false
4242
- name: Install dependencies
4343
shell: bash
44-
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
44+
run: |
45+
sudo apt-get update
46+
sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev
4547
- name: Run build
4648
env:
4749
CC: gcc-14
4850
CXX: g++-14
4951
run: |
5052
mkdir build && cd build
51-
cmake .. -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
53+
cmake .. -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
54+
-DICEBERG_BUILD_SQL_CATALOG=ON \
55+
-DICEBERG_SQL_SQLITE=ON \
56+
-DICEBERG_SQL_POSTGRESQL=ON \
57+
-DICEBERG_SQL_MYSQL=ON
5258
cmake --build .
5359
- uses: cpp-linter/cpp-linter-action@0f6d1b8d7e38b584cbee606eb23d850c217d54f8 # v2.15.1
5460
id: linter
@@ -66,7 +72,7 @@ jobs:
6672
database: build
6773
verbosity: 'debug'
6874
# need '-fno-builtin-std-forward_like', see https://github.com/llvm/llvm-project/issues/101614
69-
extra-args: '-std=c++23 -I$PWD/src -I$PWD/build/src -fno-builtin-std-forward_like'
75+
extra-args: '-std=c++23 -I$PWD/src -I$PWD/build/src -I$PWD/build/_deps/sqlpp23-src/include -I/usr/include/postgresql -I/usr/include/mysql -fno-builtin-std-forward_like'
7076
- name: Fail fast?!
7177
if: steps.linter.outputs.checks-failed != 0
7278
run: |
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: SQL Catalog Tests
19+
20+
on:
21+
push:
22+
branches:
23+
- '**'
24+
- '!dependabot/**'
25+
tags:
26+
- '**'
27+
pull_request:
28+
types: [opened, synchronize, reopened, ready_for_review]
29+
30+
concurrency:
31+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
32+
cancel-in-progress: true
33+
34+
permissions:
35+
contents: read
36+
37+
env:
38+
ICEBERG_HOME: /tmp/iceberg
39+
40+
jobs:
41+
sql-catalog:
42+
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
43+
name: SQL Catalog (${{ matrix.title }})
44+
runs-on: ${{ matrix.runs-on }}
45+
timeout-minutes: 45
46+
strategy:
47+
fail-fast: false
48+
matrix:
49+
include:
50+
- title: AMD64 Ubuntu 24.04
51+
runs-on: ubuntu-24.04
52+
CC: gcc-14
53+
CXX: g++-14
54+
cmake_build_type: Debug
55+
cmake_extra_args: ""
56+
- title: AArch64 macOS 26
57+
runs-on: macos-26
58+
cmake_build_type: Debug
59+
cmake_extra_args: ""
60+
- title: AMD64 Windows 2025
61+
runs-on: windows-2025
62+
cmake_build_type: Release
63+
cmake_extra_args: -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake
64+
steps:
65+
- name: Checkout iceberg-cpp
66+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
67+
with:
68+
persist-credentials: false
69+
- name: Set up MSVC Developer Command Prompt
70+
if: ${{ startsWith(matrix.runs-on, 'windows') }}
71+
uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
72+
with:
73+
arch: x64
74+
- name: Install dependencies on Ubuntu
75+
if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
76+
shell: bash
77+
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
78+
- name: Set Ubuntu Compilers
79+
if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
80+
run: |
81+
echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV
82+
echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV
83+
- name: Install dependencies on macOS
84+
if: ${{ startsWith(matrix.runs-on, 'macos') }}
85+
shell: bash
86+
run: |
87+
brew install fmt
88+
echo "CMAKE_PREFIX_PATH=$(brew --prefix fmt):${CMAKE_PREFIX_PATH:-}" >> "${GITHUB_ENV}"
89+
- name: Install dependencies on Windows
90+
if: ${{ startsWith(matrix.runs-on, 'windows') }}
91+
shell: pwsh
92+
run: |
93+
vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
94+
- name: Configure Iceberg
95+
shell: bash
96+
run: |
97+
cmake -S . -B build -G Ninja \
98+
-DCMAKE_INSTALL_PREFIX="${ICEBERG_HOME}" \
99+
-DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} \
100+
-DICEBERG_BUILD_STATIC=ON \
101+
-DICEBERG_BUILD_SHARED=ON \
102+
-DICEBERG_BUILD_REST=OFF \
103+
-DICEBERG_BUILD_SQL_CATALOG=ON \
104+
-DICEBERG_SQL_SQLITE=ON \
105+
${{ matrix.cmake_extra_args }}
106+
- name: Build SQL catalog tests
107+
shell: bash
108+
run: cmake --build build --target sql_catalog_test
109+
- name: Run SQL catalog tests
110+
shell: bash
111+
run: ctest --test-dir build -R '^sql_catalog_test$' --output-on-failure -C ${{ matrix.cmake_build_type }}

.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@ cmake-build-release/
2828
.vscode
2929
.cache
3030

31+
# mkdocs generated output
32+
/mkdocs/site/
33+
/mkdocs/docs/api/
34+
3135
# devcontainer
3236
.devcontainer/*
3337
!.devcontainer/*.template

CMakeLists.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON)
4646
option(ICEBERG_BUILD_REST "Build rest catalog client" ON)
4747
option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF)
4848
option(ICEBERG_BUILD_HIVE "Build hive (HMS) catalog client" OFF)
49+
option(ICEBERG_BUILD_SQL_CATALOG "Build SQL catalog client" OFF)
50+
# Built-in SQL catalog database connectors. Disable all of them to build a SQL
51+
# catalog that only works with a user-supplied CatalogStore.
52+
option(ICEBERG_SQL_SQLITE "Build the SQLite connector for the SQL catalog" OFF)
53+
option(ICEBERG_SQL_POSTGRESQL "Build the PostgreSQL connector for the SQL catalog" OFF)
54+
option(ICEBERG_SQL_MYSQL "Build the MySQL connector for the SQL catalog" OFF)
4955
option(ICEBERG_S3 "Build with S3 support" OFF)
5056
option(ICEBERG_ENABLE_ASAN "Enable Address Sanitizer" OFF)
5157
option(ICEBERG_ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)

README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,10 @@ cmake --install build
9797
| `ICEBERG_BUILD_REST` | `ON` | Build REST catalog client |
9898
| `ICEBERG_BUILD_REST_INTEGRATION_TESTS` | `OFF` | Build REST catalog integration tests |
9999
| `ICEBERG_BUILD_HIVE` | `OFF` | Build Hive (HMS) catalog client |
100+
| `ICEBERG_BUILD_SQL_CATALOG` | `OFF` | Build SQL catalog client |
101+
| `ICEBERG_SQL_SQLITE` | `OFF` | Build the SQLite connector for the SQL catalog |
102+
| `ICEBERG_SQL_POSTGRESQL` | `OFF` | Build the PostgreSQL connector for the SQL catalog |
103+
| `ICEBERG_SQL_MYSQL` | `OFF` | Build the MySQL connector for the SQL catalog |
100104
| `ICEBERG_ENABLE_ASAN` | `OFF` | Enable Address Sanitizer |
101105
| `ICEBERG_ENABLE_UBSAN` | `OFF` | Enable Undefined Behavior Sanitizer |
102106

cmake_modules/IcebergThirdpartyToolchain.cmake

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,90 @@ function(resolve_cpr_dependency)
511511
PARENT_SCOPE)
512512
endfunction()
513513

514+
# ----------------------------------------------------------------------
515+
# SQL catalog database connectors (sqlpp23)
516+
#
517+
# The SQL catalog talks to the database through a semantic `CatalogStore`
518+
# interface (src/iceberg/catalog/sql/catalog_store.h). The built-in stores are
519+
# implemented on top of sqlpp23, a header-only, compile-time type-safe SQL
520+
# library. Each connector is opt-in and pulls in its native client library:
521+
#
522+
# ICEBERG_SQL_SQLITE -> sqlpp23::sqlite3 (SQLite::SQLite3)
523+
# ICEBERG_SQL_POSTGRESQL -> sqlpp23::postgresql (PostgreSQL::PostgreSQL)
524+
# ICEBERG_SQL_MYSQL -> sqlpp23::mysql (MySQL::MySQL)
525+
#
526+
# Users who inject their own `CatalogStore` do not need sqlpp23 or any connector.
527+
528+
function(resolve_sql_catalog_dependencies)
529+
if(NOT ICEBERG_SQL_SQLITE
530+
AND NOT ICEBERG_SQL_POSTGRESQL
531+
AND NOT ICEBERG_SQL_MYSQL)
532+
message(STATUS "SQL catalog: no built-in connectors enabled")
533+
return()
534+
endif()
535+
536+
if(CMAKE_VERSION VERSION_LESS 3.28)
537+
message(FATAL_ERROR "Built-in SQL catalog connectors require CMake >= 3.28; disable "
538+
"ICEBERG_SQL_SQLITE, ICEBERG_SQL_POSTGRESQL, and ICEBERG_SQL_MYSQL "
539+
"or use CMake >= 3.28")
540+
endif()
541+
542+
prepare_fetchcontent()
543+
544+
# sqlpp23 requires C++23 and CMake >= 3.28.
545+
set(CMAKE_CXX_STANDARD 23)
546+
# Header-only consumption; do not scan for C++20 modules.
547+
set(BUILD_WITH_MODULES OFF)
548+
# Let sqlpp23 verify and locate the native client libraries for the connectors
549+
# we enable, exposing the sqlpp23::<connector> targets.
550+
set(BUILD_SQLITE3_CONNECTOR ${ICEBERG_SQL_SQLITE})
551+
set(BUILD_POSTGRESQL_CONNECTOR ${ICEBERG_SQL_POSTGRESQL})
552+
set(BUILD_MYSQL_CONNECTOR ${ICEBERG_SQL_MYSQL})
553+
554+
if(DEFINED ENV{ICEBERG_SQLPP23_URL})
555+
set(SQLPP23_URL "$ENV{ICEBERG_SQLPP23_URL}")
556+
else()
557+
set(SQLPP23_URL "https://github.com/rbock/sqlpp23/archive/refs/tags/0.69.tar.gz")
558+
endif()
559+
560+
fetchcontent_declare(sqlpp23
561+
${FC_DECLARE_COMMON_OPTIONS}
562+
URL ${SQLPP23_URL}
563+
FIND_PACKAGE_ARGS
564+
NAMES
565+
Sqlpp23
566+
CONFIG)
567+
fetchcontent_makeavailable(sqlpp23)
568+
569+
# sqlpp23 locates the native client libraries within its own subdirectory
570+
# scope. Re-run find_package with GLOBAL so the imported targets are visible
571+
# where the SQL catalog library is defined, and record them as downstream
572+
# system dependencies for the installed interface.
573+
if(ICEBERG_SQL_SQLITE)
574+
find_package(SQLite3 REQUIRED GLOBAL)
575+
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES SQLite3)
576+
message(STATUS "SQL catalog: SQLite connector enabled (sqlpp23::sqlite3)")
577+
endif()
578+
if(ICEBERG_SQL_POSTGRESQL)
579+
find_package(PostgreSQL REQUIRED GLOBAL)
580+
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES PostgreSQL)
581+
message(STATUS "SQL catalog: PostgreSQL connector enabled (sqlpp23::postgresql)")
582+
endif()
583+
if(ICEBERG_SQL_MYSQL)
584+
# MySQL has no standard CMake module; reuse the one sqlpp23 ships.
585+
if(sqlpp23_SOURCE_DIR)
586+
list(APPEND CMAKE_MODULE_PATH "${sqlpp23_SOURCE_DIR}/cmake/modules")
587+
endif()
588+
find_package(MySQL REQUIRED GLOBAL)
589+
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES MySQL)
590+
message(STATUS "SQL catalog: MySQL connector enabled (sqlpp23::mysql)")
591+
endif()
592+
593+
set(ICEBERG_SYSTEM_DEPENDENCIES
594+
${ICEBERG_SYSTEM_DEPENDENCIES}
595+
PARENT_SCOPE)
596+
endfunction()
597+
514598
# ----------------------------------------------------------------------
515599
# Zstd
516600

@@ -539,3 +623,7 @@ endif()
539623
if(ICEBERG_BUILD_REST)
540624
resolve_cpr_dependency()
541625
endif()
626+
627+
if(ICEBERG_BUILD_SQL_CATALOG)
628+
resolve_sql_catalog_dependencies()
629+
endif()

mkdocs/docs/getting-started.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,11 @@ cmake --install build
7474
| `ICEBERG_BUILD_BUNDLE` | `ON` | Build the battery-included library |
7575
| `ICEBERG_BUILD_REST` | `ON` | Build REST catalog client |
7676
| `ICEBERG_BUILD_REST_INTEGRATION_TESTS` | `OFF` | Build REST catalog integration tests |
77+
| `ICEBERG_BUILD_HIVE` | `OFF` | Build Hive (HMS) catalog client |
78+
| `ICEBERG_BUILD_SQL_CATALOG` | `OFF` | Build SQL catalog client |
79+
| `ICEBERG_SQL_SQLITE` | `OFF` | Build the SQLite connector for the SQL catalog |
80+
| `ICEBERG_SQL_POSTGRESQL` | `OFF` | Build the PostgreSQL connector for the SQL catalog |
81+
| `ICEBERG_SQL_MYSQL` | `OFF` | Build the MySQL connector for the SQL catalog |
7782
| `ICEBERG_ENABLE_ASAN` | `OFF` | Enable Address Sanitizer |
7883
| `ICEBERG_ENABLE_UBSAN` | `OFF` | Enable Undefined Behavior Sanitizer |
7984

mkdocs/docs/sql-catalog.md

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<!--
2+
~ Licensed to the Apache Software Foundation (ASF) under one
3+
~ or more contributor license agreements. See the NOTICE file
4+
~ distributed with this work for additional information
5+
~ regarding copyright ownership. The ASF licenses this file
6+
~ to you under the Apache License, Version 2.0 (the
7+
~ "License"); you may not use this file except in compliance
8+
~ with the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing,
13+
~ software distributed under the License is distributed on an
14+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
~ KIND, either express or implied. See the License for the
16+
~ specific language governing permissions and limitations
17+
~ under the License.
18+
-->
19+
20+
# SQL Catalog
21+
22+
`SqlCatalog` implements the Iceberg `Catalog` API on top of a relational
23+
database. Its schema is compatible with the Apache Iceberg Java `JdbcCatalog`
24+
and stores catalog rows in `iceberg_tables` and
25+
`iceberg_namespace_properties`.
26+
27+
## Build
28+
29+
The SQL catalog is currently available through the CMake build only. Meson does
30+
not build or install it yet.
31+
32+
Enable the catalog at configure time:
33+
34+
```bash
35+
cmake -S . -B build -DICEBERG_BUILD_SQL_CATALOG=ON
36+
```
37+
38+
Built-in connectors are optional:
39+
40+
| CMake option | Default | Native dependency |
41+
|--------------|---------|-------------------|
42+
| `ICEBERG_SQL_SQLITE` | `OFF` | SQLite3 |
43+
| `ICEBERG_SQL_POSTGRESQL` | `OFF` | libpq |
44+
| `ICEBERG_SQL_MYSQL` | `OFF` | libmysqlclient |
45+
46+
The built-in connectors use
47+
[sqlpp23](https://github.com/rbock/sqlpp23), which is fetched by CMake when a
48+
connector is enabled. Projects can also supply their own `CatalogStore`
49+
implementation and disable all built-in connectors.
50+
51+
## Usage
52+
53+
```cpp
54+
#include "iceberg/catalog/sql/sql_catalog.h"
55+
56+
using iceberg::sql::SqlCatalog;
57+
using iceberg::sql::SqlCatalogConfig;
58+
59+
SqlCatalogConfig config{
60+
.name = "prod",
61+
.uri = "/var/lib/iceberg/catalog.db",
62+
.warehouse_location = "s3://my-bucket/warehouse",
63+
};
64+
65+
auto catalog = SqlCatalog::MakeSqliteCatalog(config, file_io).value();
66+
```
67+
68+
Connector factories are always declared in the public headers. If a connector
69+
was not built, its factory returns `ErrorKind::kNotSupported`.

mkdocs/mkdocs.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ nav:
5858
- Release Process: release-process.md
5959
- Verify a Release Candidate: verify-rc.md
6060
- API Documentation: api/index.html
61+
- SQL Catalog: sql-catalog.md
6162

6263
extra:
6364
social:

src/iceberg/catalog/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,7 @@ endif()
2424
if(ICEBERG_BUILD_HIVE)
2525
add_subdirectory(hive)
2626
endif()
27+
28+
if(ICEBERG_BUILD_SQL_CATALOG)
29+
add_subdirectory(sql)
30+
endif()

0 commit comments

Comments
 (0)