test: Implement REST Catalog Integration Tests with Docker in CI Pipeline

Author: HeartLinked

.github/workflows/test.yml

@@ -56,17 +56,24 @@ jobs:
         shell: bash
         run: ci/scripts/build_example.sh $(pwd)/example
   macos:
-    name: AArch64 macOS 26
-    runs-on: macos-26
+    name: AArch64 macOS 15
+    runs-on: macos-15
     timeout-minutes: 30
     strategy:
       fail-fast: false
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
+      - name: Setup Docker on macOS
+        id: setup-docker
+        uses: douglascamata/setup-docker-macos-action@v1.0.1
+        with:
+          lima: v1.2.1
+          colima: v0.9.1
+          colima-network-address: false
       - name: Build Iceberg
         shell: bash
-        run: ci/scripts/build_iceberg.sh $(pwd)
+        run: ci/scripts/build_iceberg.sh $(pwd) ON
       - name: Build Example
         shell: bash
         run: ci/scripts/build_example.sh $(pwd)/example

CMakeLists.txt

@@ -42,6 +42,7 @@ option(ICEBERG_BUILD_SHARED "Build shared library" OFF)
 option(ICEBERG_BUILD_TESTS "Build tests" ON)
 option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON)
 option(ICEBERG_BUILD_REST "Build rest catalog client" ON)
+option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF)
 option(ICEBERG_ENABLE_ASAN "Enable Address Sanitizer" OFF)
 option(ICEBERG_ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)
 

ci/scripts/build_iceberg.sh

@@ -21,6 +21,7 @@ set -eux
 
 source_dir=${1}
 build_dir=${1}/build
+build_rest_integration_test=${2:-OFF}
 
 mkdir ${build_dir}
 pushd ${build_dir}
@@ -34,6 +35,7 @@ CMAKE_ARGS=(
     "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
     "-DICEBERG_BUILD_STATIC=ON"
     "-DICEBERG_BUILD_SHARED=ON"
+    "-DICEBERG_BUILD_TESTS=${build_rest_integration_test}"
 )
 
 if is_windows; then

src/iceberg/test/CMakeLists.txt

@@ -15,28 +15,21 @@
 # specific language governing permissions and limitations
 # under the License.
 
-fetchcontent_declare(cpp-httplib
-                     GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git
-                     GIT_TAG 89c932f313c6437c38f2982869beacc89c2f2246 #release-0.26.0
-)
-
 fetchcontent_declare(googletest
                      GIT_REPOSITORY https://github.com/google/googletest.git
                      GIT_TAG b514bdc898e2951020cbdca1304b75f5950d1f59 # release-1.15.2
                      FIND_PACKAGE_ARGS
                      NAMES
                      GTest)
 
-if(ICEBERG_BUILD_REST)
-  fetchcontent_makeavailable(cpp-httplib googletest)
-else()
-  fetchcontent_makeavailable(googletest)
-endif()
+fetchcontent_makeavailable(googletest)
 
 set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/src/iceberg/test/resources")
 
 configure_file("test_config.h.in" "test_config.h")
 
+add_subdirectory(util)
+
 function(add_iceberg_test test_name)
   set(options USE_BUNDLE)
   set(oneValueArgs)
@@ -54,10 +47,10 @@ function(add_iceberg_test test_name)
 
   if(ARG_USE_BUNDLE)
     target_link_libraries(${test_name} PRIVATE iceberg_bundle_static GTest::gtest_main
-                                               GTest::gmock)
+                                               GTest::gmock iceberg_test_util)
   else()
     target_link_libraries(${test_name} PRIVATE iceberg_static GTest::gtest_main
-                                               GTest::gmock)
+                                               GTest::gmock iceberg_test_util)
   endif()
 
   add_test(NAME ${test_name} COMMAND ${test_name})
@@ -178,15 +171,14 @@ if(ICEBERG_BUILD_REST)
     target_include_directories(${test_name} PRIVATE "${CMAKE_BINARY_DIR}/iceberg/test/")
     target_sources(${test_name} PRIVATE ${ARG_SOURCES})
     target_link_libraries(${test_name} PRIVATE GTest::gtest_main GTest::gmock
-                                               iceberg_rest_static)
+                                               iceberg_rest_static iceberg_test_util)
     add_test(NAME ${test_name} COMMAND ${test_name})
   endfunction()
 
-  add_rest_iceberg_test(rest_catalog_test
-                        SOURCES
-                        rest_catalog_test.cc
-                        rest_json_internal_test.cc
+  add_rest_iceberg_test(rest_catalog_test SOURCES rest_json_internal_test.cc
                         rest_util_test.cc)
 
-  target_include_directories(rest_catalog_test PRIVATE ${cpp-httplib_SOURCE_DIR})
+  if(ICEBERG_BUILD_REST_INTEGRATION_TESTS)
+    add_rest_iceberg_test(rest_catalog_integration_test SOURCES rest_catalog_test.cc)
+  endif()
 endif()

src/iceberg/test/meson.build

@@ -27,6 +27,9 @@ configure_file(
     install_dir: get_option('includedir') / 'iceberg/test',
 )
 
+# Add test utilities subdirectory
+subdir('util')
+
 iceberg_tests = {
     'schema_test': {
         'sources': files(
@@ -104,7 +107,7 @@ foreach test_name, values : iceberg_tests
     exc = executable(
         test_name,
         sources: values['sources'],
-        dependencies: [iceberg_dep, gmock_main_dep] + values.get(
+        dependencies: [iceberg_dep, gmock_main_dep, iceberg_test_util_dep] + values.get(
             'dependencies',
             [],
         ),

src/iceberg/test/resources/docker-compose.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+networks:
+  rest_bridge:
+
+services:
+  rest:
+    image: apache/iceberg-rest-fixture:1.10.0
+    environment:
+      - AWS_ACCESS_KEY_ID=admin
+      - AWS_SECRET_ACCESS_KEY=password
+      - AWS_REGION=us-east-1
+      - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog
+      - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory
+      - CATALOG_WAREHOUSE=file:///tmp/iceberg_warehouse
+    networks:
+      rest_bridge:
+    ports:
+      - "8181:8181"