ci: cache the sccache directory across C++ test builds (#765)

abnobdoss · Abanoub Doss · web-flow · commit f38090e0c7b1 · 2026-06-23T12:06:08.000+08:00
## What
Turn on compiler caching (sccache) for the Linux and macOS builds in
`test`, `aws_test`, `sanitizer_test`, and `sql_catalog_test`, and switch
the Windows `test` build to the same setup. `main` builds once and saves
the cache; pull requests reuse it without writing back.

## Why
Right now only the Windows builds reuse compiled output — every Linux
and macOS build recompiles the whole bundled Arrow/Parquet/Avro/Boost
stack from scratch, even though it never changes between PRs. Building
it once and reusing it removes most of that repeated work. Saving the
cache as a single file (instead of one upload per compiled file) also
avoids the upload rate limit that causes "cache write error" spam.

## Validation
On a warm pull-request run, every build reused the cache: 99.6–99.9% of
files came from cache, zero write errors. The heavy builds drop from
~10–27 min to ~1.5–5 min.

---------

Co-authored-by: Abanoub Doss &lt;abanoub.doss@gmail.com&gt;
diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml
@@ -74,6 +74,8 @@ jobs:
       AWS_DEFAULT_REGION: us-east-1
       AWS_ENDPOINT_URL: http://127.0.0.1:9000
       AWS_EC2_METADATA_DISABLED: "TRUE"
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -113,11 +115,29 @@ jobs:
         if: ${{ matrix.s3 == 'ON' }}
         shell: bash
         run: bash ci/scripts/start_minio.sh
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }}
+          restore-keys: |
+            sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
       - name: Build and test Iceberg
         shell: bash
         env:
           CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
-        run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
+        run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
+      - name: Show sccache stats
+        shell: bash
+        run: sccache --show-stats
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }}
 
   # Exercise the Meson build with SigV4 enabled (resolves aws-cpp-sdk-core via
   # its CMake config, not pkg-config whose Cflags force -std=c++11).
@@ -126,6 +146,9 @@ jobs:
     name: Meson SigV4 (AMD64 Ubuntu 24.04)
     runs-on: ubuntu-24.04
     timeout-minutes: 45
+    env:
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     steps:
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
@@ -161,14 +184,32 @@ jobs:
           echo "::error::vcpkg install failed after 3 attempts"
           exit 1
       - name: Set Ubuntu Compilers
+        # Wrap the compiler with sccache: Meson uses an explicit CC/CXX verbatim,
+        # so the launcher must be prepended here to route compiles through sccache.
         run: |
-          echo "CC=gcc-14" >> $GITHUB_ENV
-          echo "CXX=g++-14" >> $GITHUB_ENV
+          echo "CC=sccache gcc-14" >> $GITHUB_ENV
+          echo "CXX=sccache g++-14" >> $GITHUB_ENV
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-meson-sigv4-${{ github.run_id }}
+          restore-keys: |
+            sccache-meson-sigv4-
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
       - name: Build and test Iceberg
         shell: bash
         env:
           CMAKE_PREFIX_PATH: /usr/local/share/vcpkg/installed/x64-linux
         run: |
           meson setup builddir -Dsigv4=enabled
           meson compile -C builddir
+          sccache --show-stats
           meson test -C builddir --timeout-multiplier 0 --print-errorlogs
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-meson-sigv4-${{ github.run_id }}
diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml
@@ -39,6 +39,9 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
     name: "ASAN and UBSAN Tests"
     runs-on: ubuntu-24.04
+    env:
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -47,14 +50,33 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-sanitizer-ubuntu-${{ github.run_id }}
+          restore-keys: |
+            sccache-sanitizer-ubuntu-
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
       - name: Configure and Build with ASAN & UBSAN
         env:
           CC: gcc-14
           CXX: g++-14
         run: |
           mkdir build && cd build
-          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON
+          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
           cmake --build . --verbose
+      - name: Show sccache stats
+        shell: bash
+        run: sccache --show-stats
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-sanitizer-ubuntu-${{ github.run_id }}
       - name: Run Tests
         working-directory: build
         env:
diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml
@@ -61,6 +61,9 @@ jobs:
             runs-on: windows-2025
             cmake_build_type: Release
             cmake_extra_args: -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake
+    env:
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -92,6 +95,15 @@ jobs:
         shell: pwsh
         run: |
           vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-sqlcatalog-${{ matrix.runs-on }}-${{ github.run_id }}
+          restore-keys: |
+            sccache-sqlcatalog-${{ matrix.runs-on }}-
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
       - name: Configure Iceberg
         shell: bash
         run: |
@@ -103,10 +115,21 @@ jobs:
             -DICEBERG_BUILD_REST=OFF \
             -DICEBERG_BUILD_SQL_CATALOG=ON \
             -DICEBERG_SQL_SQLITE=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
             ${{ matrix.cmake_extra_args }}
       - name: Build SQL catalog tests
         shell: bash
         run: cmake --build build --target sql_catalog_test
+      - name: Show sccache stats
+        shell: bash
+        run: sccache --show-stats
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-sqlcatalog-${{ matrix.runs-on }}-${{ github.run_id }}
       - name: Run SQL catalog tests
         shell: bash
         run: ctest --test-dir build -R '^sql_catalog_test$' --output-on-failure -C ${{ matrix.cmake_build_type }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -45,6 +45,9 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
+    env:
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -53,12 +56,30 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-test-ubuntu-${{ github.run_id }}
+          restore-keys: |
+            sccache-test-ubuntu-
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
       - name: Build Iceberg
         shell: bash
         env:
           CC: gcc-14
           CXX: g++-14
-        run: ci/scripts/build_iceberg.sh $(pwd) ON
+        run: ci/scripts/build_iceberg.sh $(pwd) ON ON
+      - name: Show sccache stats
+        shell: bash
+        run: sccache --show-stats
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-test-ubuntu-${{ github.run_id }}
       - name: Build Example
         shell: bash
         env:
@@ -72,14 +93,35 @@ jobs:
     timeout-minutes: 30
     strategy:
       fail-fast: false
+    env:
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           persist-credentials: false
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-test-macos-${{ github.run_id }}
+          restore-keys: |
+            sccache-test-macos-
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
       - name: Build Iceberg
         shell: bash
-        run: ci/scripts/build_iceberg.sh $(pwd)
+        run: ci/scripts/build_iceberg.sh $(pwd) OFF ON
+      - name: Show sccache stats
+        shell: bash
+        run: sccache --show-stats
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-test-macos-${{ github.run_id }}
       - name: Build Example
         shell: bash
         run: ci/scripts/build_example.sh $(pwd)/example
@@ -90,6 +132,9 @@ jobs:
     timeout-minutes: 60
     strategy:
       fail-fast: false
+    env:
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -110,17 +155,28 @@ jobs:
         shell: pwsh
         run: |
           vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-test-windows-${{ github.run_id }}
+          restore-keys: |
+            sccache-test-windows-
       - name: Setup sccache
         uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
       - name: Build Iceberg
         shell: pwsh
-        env:
-          SCCACHE_GHA_ENABLED: "true"
         run: |
           $ErrorActionPreference = "Stop"
           bash -lc 'ci/scripts/build_iceberg.sh $(pwd) OFF ON'
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           sccache --show-stats
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-test-windows-${{ github.run_id }}
       - name: Build Example
         shell: pwsh
         run: |
@@ -131,6 +187,9 @@ jobs:
     name: Meson - ${{ matrix.title }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 30
+    env:
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
+      SCCACHE_CACHE_SIZE: "2G"
     strategy:
       max-parallel: 15
       fail-fast: false
@@ -159,24 +218,34 @@ jobs:
           python3 -m pip install --upgrade pip
           python3 -m pip install -r requirements.txt
       - name: Set Ubuntu Compilers
+        # Wrap the compiler with sccache: Meson auto-detects sccache for the
+        # default compiler (macOS/Windows), but uses an explicit CC/CXX verbatim,
+        # so the launcher must be prepended here to route Ubuntu compiles through it.
         if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
         run: |
-          echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV
-          echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV
+          echo "CC=sccache ${{ matrix.CC }}" >> $GITHUB_ENV
+          echo "CXX=sccache ${{ matrix.CXX }}" >> $GITHUB_ENV
+      - name: Restore sccache cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }}
+          restore-keys: |
+            sccache-meson-${{ matrix.runs-on }}-
       - name: Setup sccache
-        if: ${{ startsWith(matrix.runs-on, 'windows') }}
         uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
-      - name: Enable sccache
-        if: ${{ startsWith(matrix.runs-on, 'windows') }}
-        shell: bash
-        run: echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
       - name: Build Iceberg
         run: |
           meson setup builddir ${{ matrix.meson-setup-args || '' }}
           meson compile -C builddir
       - name: Show sccache stats
-        if: ${{ startsWith(matrix.runs-on, 'windows') }}
         run: sccache --show-stats
+      - name: Save sccache cache
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/.sccache
+          key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }}
       - name: Test Iceberg
         run: |
           meson test -C builddir --timeout-multiplier 0 --print-errorlogs
