Skip to content

Commit 58d42be

Browse files
0lai00lai0
authored
Merge branch 'apache:main' into native_engine_panics_on_time
2 parents 9d95891 + 1a4eef6 commit 58d42be

45 files changed

Lines changed: 2251 additions & 1124 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ jobs:
9595
key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }}
9696

9797
- name: Upload native library
98-
uses: actions/upload-artifact@v6
98+
uses: actions/upload-artifact@v7
9999
with:
100100
name: native-lib-iceberg
101101
path: native/target/ci/libcomet.so
@@ -126,7 +126,7 @@ jobs:
126126
rust-version: ${{env.RUST_VERSION}}
127127
jdk-version: ${{ matrix.java-version }}
128128
- name: Download native library
129-
uses: actions/download-artifact@v7
129+
uses: actions/download-artifact@v8
130130
with:
131131
name: native-lib-iceberg
132132
path: native/target/release/
@@ -170,7 +170,7 @@ jobs:
170170
rust-version: ${{env.RUST_VERSION}}
171171
jdk-version: ${{ matrix.java-version }}
172172
- name: Download native library
173-
uses: actions/download-artifact@v7
173+
uses: actions/download-artifact@v8
174174
with:
175175
name: native-lib-iceberg
176176
path: native/target/release/
@@ -214,7 +214,7 @@ jobs:
214214
rust-version: ${{env.RUST_VERSION}}
215215
jdk-version: ${{ matrix.java-version }}
216216
- name: Download native library
217-
uses: actions/download-artifact@v7
217+
uses: actions/download-artifact@v8
218218
with:
219219
name: native-lib-iceberg
220220
path: native/target/release/
@@ -258,7 +258,7 @@ jobs:
258258
rust-version: ${{env.RUST_VERSION}}
259259
jdk-version: ${{ matrix.java-version }}
260260
- name: Download native library
261-
uses: actions/download-artifact@v7
261+
uses: actions/download-artifact@v8
262262
with:
263263
name: native-lib-iceberg
264264
path: native/target/release/
@@ -302,7 +302,7 @@ jobs:
302302
rust-version: ${{env.RUST_VERSION}}
303303
jdk-version: ${{ matrix.java-version }}
304304
- name: Download native library
305-
uses: actions/download-artifact@v7
305+
uses: actions/download-artifact@v8
306306
with:
307307
name: native-lib-iceberg
308308
path: native/target/release/
@@ -346,7 +346,7 @@ jobs:
346346
rust-version: ${{env.RUST_VERSION}}
347347
jdk-version: ${{ matrix.java-version }}
348348
- name: Download native library
349-
uses: actions/download-artifact@v7
349+
uses: actions/download-artifact@v8
350350
with:
351351
name: native-lib-iceberg
352352
path: native/target/release/

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ jobs:
102102
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3"
103103

104104
- name: Upload native library
105-
uses: actions/upload-artifact@v6
105+
uses: actions/upload-artifact@v7
106106
with:
107107
name: native-lib-linux
108108
path: native/target/ci/libcomet.so
@@ -294,7 +294,7 @@ jobs:
294294
jdk-version: ${{ matrix.profile.java_version }}
295295

296296
- name: Download native library
297-
uses: actions/download-artifact@v7
297+
uses: actions/download-artifact@v8
298298
with:
299299
name: native-lib-linux
300300
# Download to release/ since Maven's -Prelease expects libcomet.so there
@@ -340,7 +340,7 @@ jobs:
340340
jdk-version: 11
341341

342342
- name: Download native library
343-
uses: actions/download-artifact@v7
343+
uses: actions/download-artifact@v8
344344
with:
345345
name: native-lib-linux
346346
path: native/target/release/
@@ -398,7 +398,7 @@ jobs:
398398
jdk-version: 11
399399

400400
- name: Download native library
401-
uses: actions/download-artifact@v7
401+
uses: actions/download-artifact@v8
402402
with:
403403
name: native-lib-linux
404404
path: native/target/release/

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ jobs:
102102
RUSTFLAGS: "-Ctarget-cpu=apple-m1"
103103

104104
- name: Upload native library
105-
uses: actions/upload-artifact@v6
105+
uses: actions/upload-artifact@v7
106106
with:
107107
name: native-lib-macos
108108
path: native/target/ci/libcomet.dylib
@@ -233,7 +233,7 @@ jobs:
233233
protoc-architecture: aarch_64
234234

235235
- name: Download native library
236-
uses: actions/download-artifact@v7
236+
uses: actions/download-artifact@v8
237237
with:
238238
name: native-lib-macos
239239
# Download to release/ since Maven's -Prelease expects libcomet.dylib there

‎.github/workflows/spark_sql_test.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ jobs:
9191
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3"
9292

9393
- name: Upload native library
94-
uses: actions/upload-artifact@v6
94+
uses: actions/upload-artifact@v7
9595
with:
9696
name: native-lib-linux
9797
path: native/target/ci/libcomet.so
@@ -145,7 +145,7 @@ jobs:
145145
rust-version: ${{env.RUST_VERSION}}
146146
jdk-version: ${{ matrix.config.java }}
147147
- name: Download native library
148-
uses: actions/download-artifact@v7
148+
uses: actions/download-artifact@v8
149149
with:
150150
name: native-lib-linux
151151
path: native/target/release/
@@ -168,7 +168,7 @@ jobs:
168168
LC_ALL: "C.UTF-8"
169169
- name: Upload fallback log
170170
if: ${{ github.event.inputs.collect-fallback-logs == 'true' }}
171-
uses: actions/upload-artifact@v6
171+
uses: actions/upload-artifact@v7
172172
with:
173173
name: fallback-log-spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}-spark-${{ matrix.config.spark-full }}
174174
path: "**/fallback.log"
@@ -180,14 +180,14 @@ jobs:
180180
runs-on: ubuntu-24.04
181181
steps:
182182
- name: Download fallback log artifacts
183-
uses: actions/download-artifact@v7
183+
uses: actions/download-artifact@v8
184184
with:
185185
path: fallback-logs/
186186
- name: Merge fallback logs
187187
run: |
188188
find ./fallback-logs/ -type f -name "fallback.log" -print0 | xargs -0 cat | sort -u > all_fallback.log
189189
- name: Upload merged fallback log
190-
uses: actions/upload-artifact@v6
190+
uses: actions/upload-artifact@v7
191191
with:
192192
name: all-fallback-log
193193
path: all_fallback.log

‎dev/diffs/4.0.1.diff‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2960,6 +2960,30 @@ index 09ed6955a51..236a4e99824 100644
29602960
)
29612961
}
29622962
test(s"parquet widening conversion $fromType -> $toType") {
2963+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
2964+
index 458b5dfc0f4..d209f3c85bc 100644
2965+
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
2966+
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala
2967+
@@ -26,7 +26,7 @@ import org.apache.parquet.hadoop.util.HadoopInputFile
2968+
import org.apache.parquet.schema.{LogicalTypeAnnotation, PrimitiveType}
2969+
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
2970+
2971+
-import org.apache.spark.sql.{QueryTest, Row}
2972+
+import org.apache.spark.sql.{IgnoreCometSuite, QueryTest, Row}
2973+
import org.apache.spark.sql.internal.SQLConf
2974+
import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
2975+
import org.apache.spark.sql.test.SharedSparkSession
2976+
@@ -35,7 +35,9 @@ import org.apache.spark.unsafe.types.VariantVal
2977+
/**
2978+
* Test shredding Variant values in the Parquet reader/writer.
2979+
*/
2980+
-class ParquetVariantShreddingSuite extends QueryTest with ParquetTest with SharedSparkSession {
2981+
+class ParquetVariantShreddingSuite extends QueryTest with ParquetTest with SharedSparkSession
2982+
+ // TODO enable tests once https://github.com/apache/datafusion-comet/issues/2209 is fixed
2983+
+ with IgnoreCometSuite {
2984+
2985+
private def testWithTempDir(name: String)(block: File => Unit): Unit = test(name) {
2986+
withTempDir { dir =>
29632987
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
29642988
index b8f3ea3c6f3..bbd44221288 100644
29652989
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala

0 commit comments

Comments
 (0)