Skip to content

Commit 563a088

Browse files
andygroveandygrove
committed
upmerge
2 parents c565e4e + 53f4cf7 commit 563a088

49 files changed

Lines changed: 2562 additions & 938 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.asf.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ github:
4545
main:
4646
required_pull_request_reviews:
4747
required_approving_review_count: 1
48-
48+
pull_requests:
49+
allow_update_branch: true
4950
# publishes the content of the `asf-site` branch to
5051
# https://datafusion.apache.org/comet/
5152
publish:

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ concurrency:
2323

2424
on:
2525
push:
26+
branches:
27+
- main
2628
paths-ignore:
2729
- "doc/**"
2830
- "docs/**"
@@ -69,9 +71,9 @@ jobs:
6971
~/.cargo/registry
7072
~/.cargo/git
7173
native/target
72-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
74+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
7375
restore-keys: |
74-
${{ runner.os }}-cargo-ci-
76+
${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
7577
7678
- name: Build native library
7779
# Use CI profile for faster builds (no LTO) and to share cache with pr_build_linux.yml.
@@ -88,7 +90,7 @@ jobs:
8890
~/.cargo/registry
8991
~/.cargo/git
9092
native/target
91-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
93+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
9294

9395
- name: Upload native library
9496
uses: actions/upload-artifact@v6

‎.github/workflows/miri.yml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,25 @@ name: Run Miri Safety Checks
1919

2020
on:
2121
push:
22+
branches:
23+
- main
2224
paths-ignore:
2325
- "doc/**"
2426
- "docs/**"
2527
- "**.md"
2628
- "native/core/benches/**"
2729
- "native/spark-expr/benches/**"
2830
- "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
29-
pull_request:
30-
paths-ignore:
31-
- "doc/**"
32-
- "docs/**"
33-
- "**.md"
34-
- "native/core/benches/**"
35-
- "native/spark-expr/benches/**"
36-
- "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
31+
# Disabled until Miri compatibility is restored
32+
# https://github.com/apache/datafusion-comet/issues/3499
33+
# pull_request:
34+
# paths-ignore:
35+
# - "doc/**"
36+
# - "docs/**"
37+
# - "**.md"
38+
# - "native/core/benches/**"
39+
# - "native/spark-expr/benches/**"
40+
# - "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
3741
# manual trigger
3842
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
3943
workflow_dispatch:

‎.github/workflows/pr_benchmark_check.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ concurrency:
2626

2727
on:
2828
push:
29+
branches:
30+
- main
2931
paths:
3032
- "native/core/benches/**"
3133
- "native/spark-expr/benches/**"

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ concurrency:
2323

2424
on:
2525
push:
26+
branches:
27+
- main
2628
paths-ignore:
2729
- "doc/**"
2830
- "docs/**"
@@ -84,9 +86,9 @@ jobs:
8486
~/.cargo/registry
8587
~/.cargo/git
8688
native/target
87-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
89+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
8890
restore-keys: |
89-
${{ runner.os }}-cargo-ci-
91+
${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
9092
9193
- name: Build native library (CI profile)
9294
run: |
@@ -112,7 +114,7 @@ jobs:
112114
~/.cargo/registry
113115
~/.cargo/git
114116
native/target
115-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
117+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
116118

117119
# Run Rust tests (runs in parallel with build-native, uses debug builds)
118120
linux-test-rust:
@@ -138,9 +140,9 @@ jobs:
138140
~/.cargo/git
139141
native/target
140142
# Note: Java version intentionally excluded - Rust target is JDK-independent
141-
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
143+
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
142144
restore-keys: |
143-
${{ runner.os }}-cargo-debug-
145+
${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
144146
145147
- name: Rust test steps
146148
uses: ./.github/actions/rust-test
@@ -153,7 +155,7 @@ jobs:
153155
~/.cargo/registry
154156
~/.cargo/git
155157
native/target
156-
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
158+
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
157159

158160
linux-test:
159161
needs: build-native

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ concurrency:
2323

2424
on:
2525
push:
26+
branches:
27+
- main
2628
paths-ignore:
2729
- "doc/**"
2830
- "docs/**"
@@ -84,9 +86,9 @@ jobs:
8486
~/.cargo/registry
8587
~/.cargo/git
8688
native/target
87-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
89+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
8890
restore-keys: |
89-
${{ runner.os }}-cargo-ci-
91+
${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
9092
9193
- name: Build native library (CI profile)
9294
run: |
@@ -112,7 +114,7 @@ jobs:
112114
~/.cargo/registry
113115
~/.cargo/git
114116
native/target
115-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
117+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
116118

117119
macos-aarch64-test:
118120
needs: build-native

‎.github/workflows/spark_sql_test.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ concurrency:
2323

2424
on:
2525
push:
26+
branches:
27+
- main
2628
paths-ignore:
2729
- "doc/**"
2830
- "docs/**"
@@ -75,9 +77,9 @@ jobs:
7577
~/.cargo/registry
7678
~/.cargo/git
7779
native/target
78-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
80+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
7981
restore-keys: |
80-
${{ runner.os }}-cargo-ci-
82+
${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-
8183
8284
- name: Build native library (CI profile)
8385
run: |
@@ -101,7 +103,7 @@ jobs:
101103
~/.cargo/registry
102104
~/.cargo/git
103105
native/target
104-
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}
106+
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
105107

106108
spark-sql-test:
107109
needs: build-native

‎.github/workflows/stale.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: "Close stale PRs"
19+
on:
20+
schedule:
21+
- cron: "30 1 * * *"
22+
23+
jobs:
24+
close-stale-prs:
25+
runs-on: ubuntu-latest
26+
permissions:
27+
issues: write
28+
pull-requests: write
29+
steps:
30+
- uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1
31+
with:
32+
stale-pr-message: "Thank you for your contribution. Unfortunately, this pull request is stale because it has been open 60 days with no activity. Please remove the stale label or comment or this will be closed in 7 days."
33+
days-before-pr-stale: 60
34+
days-before-pr-close: 7
35+
# do not close stale issues
36+
days-before-issue-stale: -1
37+
days-before-issue-close: -1
38+
repo-token: ${{ secrets.GITHUB_TOKEN }}

‎common/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ object CometConf extends ShimCometConf {
5454
private val TRACING_GUIDE = "For more information, refer to the Comet Tracing " +
5555
"Guide (https://datafusion.apache.org/comet/contributor-guide/tracing.html)"
5656

57+
private val DEBUGGING_GUIDE = "For more information, refer to the Comet Debugging " +
58+
"Guide (https://datafusion.apache.org/comet/contributor-guide/debugging.html)"
59+
5760
/** List of all configs that is used for generating documentation */
5861
val allConfs = new ListBuffer[ConfigEntry[_]]
5962

@@ -549,6 +552,13 @@ object CometConf extends ShimCometConf {
549552
.booleanConf
550553
.createWithDefault(false)
551554

555+
val COMET_DEBUG_MEMORY_ENABLED: ConfigEntry[Boolean] =
556+
conf(s"$COMET_PREFIX.debug.memory")
557+
.category(CATEGORY_TESTING)
558+
.doc(s"When enabled, log all native memory pool interactions. $DEBUGGING_GUIDE.")
559+
.booleanConf
560+
.createWithDefault(false)
561+
552562
val COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE = "verbose"
553563
val COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK = "fallback"
554564

0 commit comments

Comments
 (0)