Skip to content

Commit 7c6eb79

Browse files
committed
merge main
2 parents 9f0f660 + 533ef35 commit 7c6eb79

728 files changed

Lines changed: 68333 additions & 14411 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.ai/skills/datafusion-ffi/SKILL.md

Lines changed: 360 additions & 0 deletions
Large diffs are not rendered by default.

.asf.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,29 @@ github:
6060
- "Spell Check with Typos"
6161
- "Circular Dependency Check"
6262
- "Detect Unused Dependencies"
63+
- "linux build test"
64+
- "cargo check datafusion-common features"
65+
- "cargo check datafusion-substrait features"
66+
- "cargo check datafusion-proto features"
67+
- "cargo check datafusion features"
68+
- "cargo check datafusion-functions features"
69+
- "cargo test (amd64)"
70+
- "cargo test datafusion-cli (amd64)"
71+
- "cargo examples (amd64)"
72+
- "cargo test doc (amd64)"
73+
- "cargo doc"
74+
- "build and run with wasm-pack"
75+
- "verify benchmark results (amd64)"
76+
- "Run sqllogictest with Postgres runner"
77+
- "Run sqllogictest in Substrait round-trip mode"
78+
- "cargo test (macos-aarch64)"
79+
- "Verify Vendored Code"
80+
- "Check cargo fmt"
81+
- "clippy"
82+
- "check Cargo.toml formatting"
83+
- "check configs.md and ***_functions.md is up-to-date"
84+
- "check example README is up-to-date"
85+
- "Verify MSRV (Min Supported Rust Version)"
6386
# needs to be updated as part of the release process
6487
# .asf.yaml doesn't support wildcard branch protection rules, only exact branch names
6588
# https://github.com/apache/infrastructure-asfyaml?tab=readme-ov-file#branch-protection
@@ -74,6 +97,12 @@ github:
7497
branch-52:
7598
required_pull_request_reviews:
7699
required_approving_review_count: 1
100+
branch-53:
101+
required_pull_request_reviews:
102+
required_approving_review_count: 1
103+
branch-54:
104+
required_pull_request_reviews:
105+
required_approving_review_count: 1
77106
pull_requests:
78107
# enable updating head branches of pull requests
79108
allow_update_branch: true

.github/workflows/audit.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ jobs:
4545
steps:
4646
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
4747
- name: Install cargo-audit
48-
uses: taiki-e/install-action@481c34c1cf3a84c68b5e46f4eccfc82af798415a # v2.75.23
48+
uses: taiki-e/install-action@920ab1831fbf4fb3ef75c8ead83556c918bb7290 # v2.79.8
4949
with:
5050
tool: cargo-audit
5151
- name: Run audit check

.github/workflows/breaking_changes_detector.yml

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,21 +59,22 @@ jobs:
5959
with:
6060
fetch-depth: 0
6161

62-
# For fork PRs, `origin` points to the fork, not the upstream repo.
63-
# Explicitly fetch the base branch from the upstream repo so we have
64-
# a valid baseline ref for both diff and semver-checks.
62+
# `origin` may point at a fork (when a contributor runs this locally) or
63+
# at a stale ref. Fetch the base branch from the PR's upstream repo into
64+
# a dedicated `apache/<base>` ref so the baseline is unambiguous and the
65+
# same ref name works locally (`git remote add apache ...`) and in CI.
6566
- name: Fetch base branch
6667
env:
6768
BASE_REF: ${{ github.base_ref }}
6869
REPO: ${{ github.repository }}
69-
run: git fetch "https://github.com/${REPO}.git" "${BASE_REF}:refs/remotes/origin/${BASE_REF}"
70+
run: git fetch "https://github.com/${REPO}.git" "${BASE_REF}:refs/remotes/apache/${BASE_REF}"
7071

7172
- name: Determine changed crates
7273
id: changed_crates
7374
env:
7475
BASE_REF: ${{ github.base_ref }}
7576
run: |
76-
PACKAGES=$(ci/scripts/changed_crates.sh changed-crates "origin/${BASE_REF}")
77+
PACKAGES=$(ci/scripts/changed_crates.sh changed-crates "apache/${BASE_REF}")
7778
echo "packages=$PACKAGES" >> "$GITHUB_OUTPUT"
7879
echo "Changed crates: $PACKAGES"
7980
@@ -88,7 +89,7 @@ jobs:
8889
8990
- name: Install cargo-semver-checks
9091
if: steps.changed_crates.outputs.packages != ''
91-
uses: taiki-e/install-action@94cb46f8d6e437890146ffbd78a778b78e623fb2 # v2.74.0
92+
uses: taiki-e/install-action@920ab1831fbf4fb3ef75c8ead83556c918bb7290 # v2.79.8
9293
with:
9394
tool: cargo-semver-checks
9495

@@ -102,7 +103,8 @@ jobs:
102103
set +e
103104
# `tee` lets cargo's output stream live into the Actions log
104105
# while we also keep a copy for the PR comment.
105-
ci/scripts/changed_crates.sh semver-check "origin/${BASE_REF}" $PACKAGES \
106+
# Using `apache` remote here to point to the repository the pull request is against
107+
ci/scripts/changed_crates.sh semver-check "apache/${BASE_REF}" $PACKAGES \
106108
2>&1 | tee /tmp/semver-output.txt
107109
EXIT_CODE=${PIPESTATUS[0]}
108110
# Pass the result through an output instead of failing the job:

.github/workflows/breaking_changes_detector_comment.yml

Lines changed: 79 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@ on:
5050
permissions:
5151
contents: read
5252

53+
# A dedicated label, separate from the existing `api change` label.
54+
# `api change` may be applied manually for behavioral changes that aren't
55+
# strictly API changes, so we can't safely auto-remove it when this check
56+
# passes. This auto-managed label is fully owned by the workflow.
57+
env:
58+
BREAKING_CHANGE_LABEL: "auto detected api change"
59+
5360
jobs:
5461
comment-on-pr:
5562
name: Comment on pull request
@@ -97,36 +104,84 @@ jobs:
97104
echo "${DELIM}"
98105
} >> "$GITHUB_OUTPUT"
99106
100-
# The marker `<!-- semver-check-comment -->` is what makes the comment
101-
# "sticky": maintain-one-comment uses it to find and replace (or
102-
# delete) the existing comment instead of stacking new ones.
107+
108+
# Find any existing sticky comment by its hidden marker so we can update
109+
# or delete it instead of stacking new ones.
110+
- name: Find existing sticky comment
111+
id: find
112+
env:
113+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
114+
REPO: ${{ github.repository }}
115+
PR_NUMBER: ${{ steps.read.outputs.pr_number }}
116+
run: |
117+
COMMENT_ID=$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments" \
118+
--jq '.[] | select(.body | contains("<!-- semver-check-comment -->")) | .id' \
119+
| head -n1)
120+
echo "comment_id=${COMMENT_ID}" >> "$GITHUB_OUTPUT"
121+
122+
# update the existing comment found above, or create a new one. The hidden
123+
# marker `<!-- semver-check-comment -->` stays in the body so the next run
124+
# finds it again. LOGS is interpolated via a shell parameter expansion,
125+
# whose result bash does not re-scan, so untrusted log content cannot
126+
# inject further commands.
103127
- name: Upsert sticky comment
104128
if: steps.read.outputs.result != 'success'
105-
uses: actions-cool/maintain-one-comment@909842216bc8e8658364c572ec52100f4c2cc50a # v3.3.0
106-
with:
107-
token: ${{ secrets.GITHUB_TOKEN }}
108-
number: ${{ steps.read.outputs.pr_number }}
109-
body-include: '<!-- semver-check-comment -->'
110-
body: |
111-
<!-- semver-check-comment -->
112-
Thank you for opening this pull request!
129+
env:
130+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
131+
REPO: ${{ github.repository }}
132+
PR_NUMBER: ${{ steps.read.outputs.pr_number }}
133+
COMMENT_ID: ${{ steps.find.outputs.comment_id }}
134+
LOGS: ${{ steps.read.outputs.logs }}
135+
run: |
136+
set -euo pipefail
137+
BODY="<!-- semver-check-comment -->
138+
Thank you for opening this pull request!
113139
114-
Reviewer note: [cargo-semver-checks](https://github.com/obi1kenobi/cargo-semver-checks) reported the current version number is not SemVer-compatible with the changes in this pull request (compared against the base branch).
140+
Reviewer note: [cargo-semver-checks](https://github.com/obi1kenobi/cargo-semver-checks) reported the current version number is not SemVer-compatible with the changes in this pull request (compared against the base branch).
115141
116-
<details>
117-
<summary>Details</summary>
142+
<details>
143+
<summary>Details</summary>
118144
119-
```
120-
${{ steps.read.outputs.logs }}
121-
```
145+
\`\`\`
146+
${LOGS}
147+
\`\`\`
122148
123-
</details>
149+
</details>"
150+
151+
# Use --raw-field (not --field): always sends the value as a literal string. while --field would treat a leading `@` as a file to read
152+
# (even though the body does not start with user input we are being cautious)
153+
if [ -n "$COMMENT_ID" ]; then
154+
gh api "repos/${REPO}/issues/comments/${COMMENT_ID}" --method PATCH --raw-field body="$BODY"
155+
else
156+
gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" --method POST --raw-field body="$BODY"
157+
fi
124158
159+
# Clear a stale comment once the breaking change is resolved.
125160
- name: Delete sticky comment
161+
if: steps.read.outputs.result == 'success' && steps.find.outputs.comment_id != ''
162+
env:
163+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
164+
REPO: ${{ github.repository }}
165+
COMMENT_ID: ${{ steps.find.outputs.comment_id }}
166+
run: gh api -X DELETE "repos/${REPO}/issues/comments/${COMMENT_ID}"
167+
168+
- name: Add "auto detected api change" label
169+
if: steps.read.outputs.result != 'success'
170+
env:
171+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
172+
REPO: ${{ github.repository }}
173+
PR_NUMBER: ${{ steps.read.outputs.pr_number }}
174+
run: |
175+
gh pr edit "$PR_NUMBER" --repo "$REPO" \
176+
--add-label "$BREAKING_CHANGE_LABEL"
177+
178+
- name: Remove "auto detected api change" label
126179
if: steps.read.outputs.result == 'success'
127-
uses: actions-cool/maintain-one-comment@909842216bc8e8658364c572ec52100f4c2cc50a # v3.3.0
128-
with:
129-
token: ${{ secrets.GITHUB_TOKEN }}
130-
number: ${{ steps.read.outputs.pr_number }}
131-
body-include: '<!-- semver-check-comment -->'
132-
delete: true
180+
env:
181+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
182+
REPO: ${{ github.repository }}
183+
PR_NUMBER: ${{ steps.read.outputs.pr_number }}
184+
run: |
185+
# No-op when the label isn't currently applied.
186+
gh pr edit "$PR_NUMBER" --repo "$REPO" \
187+
--remove-label "$BREAKING_CHANGE_LABEL" || true

.github/workflows/codeql.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ jobs:
4545
persist-credentials: false
4646

4747
- name: Initialize CodeQL
48-
uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
48+
uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4
4949
with:
5050
languages: actions
5151

5252
- name: Perform CodeQL Analysis
53-
uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
53+
uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4
5454
with:
5555
category: "/language:actions"

.github/workflows/dev.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
source ci/scripts/utils/tool_versions.sh
6565
echo "LYCHEE_VERSION=${LYCHEE_VERSION}" >> "$GITHUB_ENV"
6666
- name: Install lychee
67-
uses: taiki-e/install-action@481c34c1cf3a84c68b5e46f4eccfc82af798415a # v2.75.23
67+
uses: taiki-e/install-action@920ab1831fbf4fb3ef75c8ead83556c918bb7290 # v2.79.8
6868
with:
6969
tool: lychee@${{ env.LYCHEE_VERSION }}
7070
- name: Run markdown link check

.github/workflows/extended.yml

Lines changed: 6 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -57,41 +57,13 @@ permissions:
5757
checks: write
5858

5959
jobs:
60-
61-
# Check crate compiles and base cargo check passes
62-
linux-build-lib:
63-
name: linux build test
64-
runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
65-
# note: do not use amd/rust container to preserve disk space
66-
steps:
67-
- uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
68-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
69-
with:
70-
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
71-
submodules: true
72-
fetch-depth: 1
73-
- name: Install Rust
74-
run: |
75-
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
76-
source $HOME/.cargo/env
77-
rustup toolchain install
78-
- name: Install Protobuf Compiler
79-
run: |
80-
sudo apt-get update
81-
sudo apt-get install -y protobuf-compiler
82-
- name: Prepare cargo build
83-
run: |
84-
cargo check --profile ci --all-targets
85-
cargo clean
86-
8760
# Run extended tests (with feature 'extended_tests')
8861
linux-test-extended:
8962
name: cargo test 'extended_tests' (amd64)
90-
needs: [linux-build-lib]
91-
runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
63+
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
9264
# note: do not use amd/rust container to preserve disk space
9365
steps:
94-
- uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
66+
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
9567
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
9668
with:
9769
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
@@ -134,11 +106,11 @@ jobs:
134106
# Check answers are correct when hash values collide
135107
hash-collisions:
136108
name: cargo test hash collisions (amd64)
137-
runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
109+
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
138110
container:
139111
image: amd64/rust
140112
steps:
141-
- uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
113+
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
142114
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
143115
with:
144116
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
@@ -156,11 +128,11 @@ jobs:
156128
157129
sqllogictest-sqlite:
158130
name: "Run sqllogictests with the sqlite test suite"
159-
runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
131+
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
160132
container:
161133
image: amd64/rust
162134
steps:
163-
- uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
135+
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
164136
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
165137
with:
166138
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push

.github/workflows/labeler.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
github.event_name == 'pull_request_target' &&
4545
(github.event.action == 'opened' ||
4646
github.event.action == 'synchronize')
47-
uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
47+
uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0
4848
with:
4949
repo-token: ${{ secrets.GITHUB_TOKEN }}
5050
configuration-path: .github/workflows/labeler/labeler-config.yml

0 commit comments

Comments
 (0)