Skip to content
build

Sync with base repo #2

Sign in to view logs

Sync with base repo

Sync with base repo #2

Workflow file for this run

.github/workflows/build.yml at 600fca2
name: build
on: [push, pull_request, merge_group]
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
# Supply chain security: all cargo commands that resolve dependencies use --locked to
# enforce the committed Cargo.lock. This prevents CI from silently resolving a newer
# (potentially compromised) dependency version. If Cargo.lock is out of sync with
# Cargo.toml, the build fails immediately. Any dependency change must be an explicit,
# reviewable update to Cargo.lock in the PR. Commands that skip --locked: cargo fmt
# (no dep resolution), cargo msrv verify/show (wrapper tool), cargo miri setup (tooling).
#
# Swatinem/rust-cache caches the cargo registry and target directory (~450MB per job).
# save-if restricts cache writes to main pushes only. PRs read from main's cache but
# never write their own entries.
#
# The key insight: Cargo.lock changes infrequently, so main's cache key almost always
# matches. PRs download and compile zero dependencies on cache hit. By only writing on
# main, we keep main's cache entries alive (no LRU eviction from PR churn), and every
# PR benefits from them.
#
# Without this, GHA's ref-scoped caching works against us: each PR writes ~6.3GB of
# cache entries (14 jobs x ~450MB) that only that PR can read. A handful of active PRs
# fills the 10GB cache budget, LRU evicts main's shared entries, and every subsequent
# PR compiles from scratch.
#
# The save-if condition checks both event_name == 'push' and ref == main because
# pull_request_target events set github.ref to the base branch (main), not the PR
# branch. Without the event_name check, those workflows would write cache entries on
# every PR.
#
# Note: actions-rust-lang/setup-rust-toolchain has built-in Swatinem/rust-cache that
# writes on every run with no save-if support. We disable it with cache: false and
# manage caching explicitly via the Swatinem/rust-cache steps below.
jobs:
format:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Install minimal stable with rustfmt
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
components: rustfmt
- name: format
run: cargo fmt -- --check
msrv:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Install minimal stable and cargo msrv
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- uses: taiki-e/install-action@7bc99eee1f1b8902a125006cf790a1f4c8461e63 # v2.69.8
with:
tool: cargo-msrv
- name: verify-msrv
run: |
cargo msrv --path kernel/ verify --all-features
cargo msrv --path derive-macros/ verify --all-features
cargo msrv --path ffi/ verify --all-features
cargo msrv --path ffi-proc-macros/ verify --all-features
msrv-run-tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Install minimal stable and cargo msrv
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- uses: taiki-e/install-action@7bc99eee1f1b8902a125006cf790a1f4c8461e63 # v2.69.8
with:
tool: cargo-msrv
- uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest
- name: Get rust-version from Cargo.toml
id: rust-version
run: echo "RUST_VERSION=$(cargo msrv show --path kernel/ --output-format minimal)" >> $GITHUB_ENV
- name: Install specified rust version
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
toolchain: ${{ env.RUST_VERSION }}
- name: run tests
run: |
pushd kernel
echo "Testing with $(cargo msrv show --output-format minimal)"
cargo +$(cargo msrv show --output-format minimal) nextest run --locked
docs:
runs-on: ubuntu-latest
env:
RUSTDOCFLAGS: -D warnings
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Install minimal stable
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- name: build docs
run: cargo doc --locked --workspace --all-features --no-deps
# When we run cargo { build, clippy } --no-default-features, we want to build/lint the kernel to
# ensure that we can build the kernel without any features enabled. Unfortunately, due to how
# cargo resolves features, if we have a workspace member that depends on the kernel with features
# enabled, the kernel will be compiled with those features (even if we specify
# --no-default-features).
#
# To cope with this, we split build/clippy --no-default-features into two runs:
# 1. build/clippy all packages that depend on the kernel with some features enabled:
# - acceptance
# - test_utils
# - feature_tests
# (and examples)
# - inspect-table
# - read-table-changes
# - read-table-multi-threaded
# - read-table-single-threaded
# 2. build/clippy all packages that only have no-feature kernel dependency
# - delta_kernel
# - delta_kernel_derive
# - delta_kernel_ffi
# - delta_kernel_ffi_macros
build:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- macOS-latest
- ubuntu-latest
- windows-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Install minimal stable with clippy
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
components: clippy
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- name: build and lint with clippy
run: cargo clippy --locked --benches --tests --all-features -- -D warnings
- name: lint without default features - packages which depend on kernel with features enabled
run: cargo clippy --locked --workspace --no-default-features --exclude delta_kernel --exclude delta_kernel_ffi --exclude delta_kernel_derive --exclude delta_kernel_ffi_macros -- -D warnings
- name: lint without default features - packages which don't depend on kernel with features enabled
run: cargo clippy --locked --no-default-features --package delta_kernel --package delta_kernel_ffi --package delta_kernel_derive --package delta_kernel_ffi_macros -- -D warnings
- name: check kernel builds with default-engine-native-tls
run: cargo build --locked -p feature_tests --features default-engine-native-tls
- name: test native-tls backend has no crypto provider conflict
run: cargo test --locked -p feature_tests --features default-engine-native-tls
- name: check kernel builds with default-engine-rustls
run: cargo build --locked -p feature_tests --features default-engine-rustls
- name: test rustls TLS backend feature-tests
run: cargo test --locked -p feature_tests --features default-engine-rustls
test:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- macOS-latest
- ubuntu-latest
- windows-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: filter
with:
filters: |
ffi:
- 'ffi/src/handle.rs'
- 'ffi-proc-macros/**'
- name: Install minimal stable with clippy and rustfmt
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest
- name: test
run: cargo nextest run --locked --workspace --all-features -E 'not test(read_table_version_hdfs) and not test(invalid_handle_code)'
- name: trybuild tests
if: steps.filter.outputs.ffi == 'true'
run: cargo test --locked --package delta_kernel_ffi --features internal-api -- invalid_handle_code
ffi_test:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- macOS-latest
- ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Setup cmake
uses: jwlawson/actions-setup-cmake@0d6a7d60b009d01c9e7523be22153ff8f19460d3 # v2.2.0
with:
cmake-version: "3.30.x"
- name: Install arrow-glib-linux
run: |
if [ "$RUNNER_OS" == "Linux" ]; then
sudo apt update
sudo apt install -y -V ca-certificates lsb-release wget
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb.sha512
sha512sum -c apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb.sha512 || exit 1
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt update
sudo apt install -y -V libarrow-dev # For C++
sudo apt install -y -V libarrow-glib-dev # For GLib (C)
sudo apt install -y -V valgrind # For memory leak test
fi
- name: Install arrow-glib-macOS
if: runner.os == 'macOS'
uses: ./.github/actions/use-homebrew-tools
with:
tools: "apache-arrow apache-arrow-glib"
- name: Install minimal stable
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- name: Set output on fail
run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV"
- name: Build kernel
run: |
pushd acceptance
cargo build --locked
popd
pushd ffi
cargo build --locked --features default-engine-rustls,test-ffi,tracing,delta-kernel-unity-catalog
popd
- name: build and run read-table test
run: |
pushd ffi/examples/read-table
mkdir build
pushd build
cmake ..
make
make test
- name: build and run visit-expression test
run: |
pushd ffi/examples/visit-expression
mkdir build
pushd build
cmake ..
make
make test
- name: build and run delta-kernel-unity-catalog-ffi test
run: |
pushd ffi/examples/delta-kernel-unity-catalog-example
mkdir build
pushd build
cmake ..
make
make test
miri:
name: "Miri (shard ${{ matrix.partition }}/3)"
runs-on: ubuntu-latest
strategy:
matrix:
partition: [1, 2, 3]
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Install Miri
run: |
rustup toolchain install nightly --component miri
rustup override set nightly
cargo miri setup
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest
- name: Test with Miri
run: |
pushd ffi
MIRIFLAGS=-Zmiri-disable-isolation cargo miri nextest run --locked --features default-engine-rustls,delta-kernel-unity-catalog --partition slice:${{ matrix.partition }}/3
coverage:
runs-on: ubuntu-latest
env:
CARGO_TERM_COLOR: always
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Install rust
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
with:
cache: false
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@2d15d02e710b40b6332201aba6af30d595b5cd96 # cargo-llvm-cov
- name: Generate code coverage
run: cargo llvm-cov --locked --all-features --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs
- name: Upload coverage to Codecov
uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
with:
files: codecov.json
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}