feat(cmake): reuse Arrow's bundled AWS SDK for SigV4 (no system SDK for S3+SigV4)

Author: plusplusjiajia

.github/workflows/aws_test.yml

@@ -94,14 +94,14 @@ jobs:
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
       - name: Cache vcpkg packages
-        if: ${{ matrix.sigv4 == 'ON' }}
+        if: ${{ matrix.sigv4 == 'ON' && matrix.s3 == 'OFF' }}
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         id: vcpkg-cache
         with:
           path: /usr/local/share/vcpkg/installed
           key: vcpkg-x64-linux-aws-sdk-cpp-s3-${{ matrix.s3 }}-sigv4-${{ matrix.sigv4 }}-${{ hashFiles('.github/workflows/aws_test.yml') }}
       - name: Install AWS SDK via vcpkg
-        if: ${{ matrix.sigv4 == 'ON' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
+        if: ${{ matrix.sigv4 == 'ON' && matrix.s3 == 'OFF' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
         shell: bash
         # Retry to ride out transient GitHub/mirror download failures (504s).
         run: |
@@ -132,7 +132,7 @@ jobs:
       - name: Build and test Iceberg
         shell: bash
         env:
-          CMAKE_TOOLCHAIN_FILE: ${{ matrix.sigv4 == 'ON' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
+          CMAKE_TOOLCHAIN_FILE: ${{ matrix.sigv4 == 'ON' && matrix.s3 == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
         run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ${{ matrix.s3 }} ${{ matrix.sigv4 }}
 
   # Exercise the Meson build with SigV4 enabled (resolves aws-cpp-sdk-core via

CMakeLists.txt

@@ -54,6 +54,13 @@ option(ICEBERG_SQL_POSTGRESQL "Build the PostgreSQL connector for the SQL catalo
 option(ICEBERG_SQL_MYSQL "Build the MySQL connector for the SQL catalog" OFF)
 option(ICEBERG_S3 "Build with S3 support" OFF)
 option(ICEBERG_SIGV4 "Build SigV4 authentication support (requires AWS SDK)" OFF)
+set(ICEBERG_AWSSDK_SOURCE
+    "AUTO"
+    CACHE STRING
+          "AWS SDK source for SigV4: AUTO (reuse Arrow's bundled AWS SDK when \
+ICEBERG_S3 is ON, otherwise SYSTEM), SYSTEM (find an installed AWS SDK), or \
+BUNDLED (reuse Arrow's bundled AWS SDK; requires ICEBERG_S3)")
+set_property(CACHE ICEBERG_AWSSDK_SOURCE PROPERTY STRINGS AUTO SYSTEM BUNDLED)
 option(ICEBERG_ENABLE_ASAN "Enable Address Sanitizer" OFF)
 option(ICEBERG_ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)
 

cmake_modules/IcebergThirdpartyToolchain.cmake

@@ -20,6 +20,22 @@
 set(ICEBERG_SYSTEM_DEPENDENCIES)
 set(ICEBERG_ARROW_INSTALL_INTERFACE_LIBS)
 
+if(ICEBERG_SIGV4)
+  set(ICEBERG_AWSSDK_SOURCE_RESOLVED "${ICEBERG_AWSSDK_SOURCE}")
+  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "AUTO")
+    if(ICEBERG_S3)
+      set(ICEBERG_AWSSDK_SOURCE_RESOLVED "BUNDLED")
+    else()
+      set(ICEBERG_AWSSDK_SOURCE_RESOLVED "SYSTEM")
+    endif()
+  endif()
+  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "BUNDLED" AND NOT ICEBERG_S3)
+    message(FATAL_ERROR "ICEBERG_AWSSDK_SOURCE=BUNDLED requires ICEBERG_S3=ON: "
+                        "the bundled AWS SDK is provided by Arrow's S3 support.")
+  endif()
+  message(STATUS "AWS SDK source for SigV4: ${ICEBERG_AWSSDK_SOURCE_RESOLVED}")
+endif()
+
 # ----------------------------------------------------------------------
 # Versions and URLs for toolchain builds
 #
@@ -110,9 +126,7 @@ function(resolve_arrow_dependency)
   set(ARROW_RUNTIME_SIMD_LEVEL "NONE")
   set(ARROW_POSITION_INDEPENDENT_CODE ON)
   set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
-  # With SigV4 also on, make Arrow's S3 reuse the system AWS SDK (the one SigV4
-  # finds) instead of bundling its own, so only one AWS SDK is linked (no ODR).
-  if(ICEBERG_S3 AND ICEBERG_SIGV4)
+  if(ICEBERG_S3 AND ICEBERG_SIGV4 AND ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "SYSTEM")
     set(AWSSDK_SOURCE "SYSTEM")
   endif()
   set(ARROW_WITH_ZLIB ON)
@@ -638,6 +652,10 @@ endif()
 # AWS SDK for C++
 
 function(resolve_aws_sdk_dependency)
+  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "BUNDLED")
+    message(STATUS "SigV4 reuses Arrow's bundled AWS SDK (aws-cpp-sdk-core)")
+    return()
+  endif()
   find_package(AWSSDK REQUIRED COMPONENTS core)
   list(APPEND ICEBERG_SYSTEM_DEPENDENCIES AWSSDK)
   set(ICEBERG_SYSTEM_DEPENDENCIES

src/iceberg/catalog/rest/CMakeLists.txt

@@ -58,8 +58,10 @@ list(APPEND
 if(ICEBERG_SIGV4)
   list(APPEND ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS aws-cpp-sdk-core)
   list(APPEND ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS aws-cpp-sdk-core)
-  list(APPEND ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS aws-cpp-sdk-core)
-  list(APPEND ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS aws-cpp-sdk-core)
+  if(ICEBERG_AWSSDK_SOURCE_RESOLVED STREQUAL "SYSTEM")
+    list(APPEND ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS aws-cpp-sdk-core)
+    list(APPEND ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS aws-cpp-sdk-core)
+  endif()
 endif()
 
 add_iceberg_lib(iceberg_rest