Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .github/actions/setup-submodules/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
name: "Setup Submodules"
description: "Initialize and update git submodules for testing"
runs:
using: "composite"
steps:
- name: Initialize and update submodules
shell: bash
run: |
git config --global --add safe.directory $GITHUB_WORKSPACE
git submodule init
# Override update=none setting for CI
git config submodule.testing.update checkout
git config submodule.parquet-testing.update checkout
git config submodule.datafusion-testing.update checkout
git submodule update --depth 1
3 changes: 2 additions & 1 deletion .github/workflows/dependencies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ jobs:
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/docs_pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ jobs:
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
- name: Install doc dependencies
Expand Down
37 changes: 34 additions & 3 deletions .github/workflows/extended.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,34 @@ permissions:
checks: write

jobs:

# Check crate compiles and base cargo check passes
linux-build-lib:
name: linux build test
runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
# note: do not use amd/rust container to preserve disk space
steps:
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Install Rust
run: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
source $HOME/.cargo/env
rustup toolchain install
- name: Install Protobuf Compiler
run: |
sudo apt-get update
sudo apt-get install -y protobuf-compiler
- name: Prepare cargo build
run: |
cargo check --profile ci --all-targets
cargo clean

# Run extended tests (with feature 'extended_tests')
linux-test-extended:
name: cargo test 'extended_tests' (amd64)
Expand All @@ -67,8 +95,9 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
- name: Install Rust
Expand Down Expand Up @@ -114,8 +143,9 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand All @@ -136,8 +166,9 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
# Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
- name: Install protobuf compiler
run: |
Expand Down
36 changes: 24 additions & 12 deletions .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand Down Expand Up @@ -325,8 +326,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
run: rustup toolchain install stable
- name: Rust Dependency Cache
Expand Down Expand Up @@ -357,8 +359,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand Down Expand Up @@ -388,8 +391,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand Down Expand Up @@ -451,8 +455,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand Down Expand Up @@ -499,8 +504,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand All @@ -524,8 +530,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand Down Expand Up @@ -567,8 +574,9 @@ jobs:
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-macos-aarch64-builder
- name: Run datafusion-ffi tests
Expand Down Expand Up @@ -660,8 +668,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand All @@ -685,8 +694,9 @@ jobs:
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand All @@ -707,8 +717,9 @@ jobs:
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
Expand Down Expand Up @@ -742,8 +753,9 @@ jobs:
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
fetch-depth: 1
- name: Setup Submodules
uses: ./.github/actions/setup-submodules

- name: Mark repository as safe for git
# Required for git commands inside container (avoids "dubious ownership" error)
Expand Down
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
[submodule "parquet-testing"]
path = parquet-testing
url = https://github.com/apache/parquet-testing.git
update = none
[submodule "testing"]
path = testing
url = https://github.com/apache/arrow-testing
update = none
[submodule "datafusion-testing"]
path = datafusion-testing
url = https://github.com/apache/datafusion-testing.git
branch = main
update = none
Loading