diff --git a/.github/workflows/delta_build_gate.yml b/.github/workflows/delta_build_gate.yml new file mode 100644 index 0000000000..163b1af344 --- /dev/null +++ b/.github/workflows/delta_build_gate.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build-gate verification for the optional contrib/delta integration. +# +# A single cheap job that runs dev/verify-contrib-delta-gate.sh, which proves: +# - the DEFAULT cargo / mvn / dylib build carries ZERO Delta surface +# (no comet-contrib-delta / delta_kernel in the cargo tree, no io.delta in +# the effective pom, no Delta symbols in libcomet); and +# - the gated build (`--features contrib-delta` / `-Pcontrib-delta`) compiles +# and pulls the right delta-spark per Spark profile. +# +# The full contrib Scala suite + regression workflows land later (A.6a/A.6b); +# this minimal gate is all the build-gate + inert-wiring unit needs. + +name: Delta Contrib Build Gate + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + branches: + - main + paths-ignore: + - "benchmarks/**" + - "doc/**" + - "docs/**" + - "**.md" + - "dev/changelog/*.md" + - "native/core/benches/**" + - "native/spark-expr/benches/**" + - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala" + - "spark-integration/**" + pull_request: + paths-ignore: + - "benchmarks/**" + - "doc/**" + - "docs/**" + - "**.md" + - "dev/changelog/*.md" + - "native/core/benches/**" + - "native/spark-expr/benches/**" + - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala" + - "spark-integration/**" + workflow_dispatch: + +permissions: + contents: read + +env: + RUST_VERSION: stable + RUST_BACKTRACE: 1 + # Force GNU ld on Linux to match the rest of Comet's CI (rust-lld can't + # resolve -ljvm against the Zulu JDK layout installed by setup-java). + RUSTFLAGS: "-Clink-arg=-fuse-ld=bfd" + +jobs: + delta-build-gate: + name: Build-gate verification (default builds exclude Delta) + runs-on: ubuntu-24.04 + container: + image: amd64/rust + env: + SPARK_LOCAL_IP: localhost + steps: + - uses: actions/checkout@v6 + + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.RUST_VERSION }} + jdk-version: 17 + + - name: Run dev/verify-contrib-delta-gate.sh + run: | + dev/verify-contrib-delta-gate.sh diff --git a/contrib/delta/native/Cargo.lock b/contrib/delta/native/Cargo.lock new file mode 100644 index 0000000000..671a24016e --- /dev/null +++ b/contrib/delta/native/Cargo.lock @@ -0,0 +1,2757 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", +] + +[[package]] +name = "arrow-json" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +dependencies = [ + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "comet-contrib-delta" +version = "0.18.0" +dependencies = [ + "datafusion", + "datafusion-comet-proto", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet", + "rand", + "regex", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion-catalog" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools", + "log", + "object_store", +] + +[[package]] +name = "datafusion-comet-proto" +version = "0.18.0" +dependencies = [ + "prost", + "prost-build", +] + +[[package]] +name = "datafusion-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "chrono", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "libc", + "log", + "object_store", + "parquet", + "paste", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "glob", + "itertools", + "log", + "object_store", + "rand", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "serde_json", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" + +[[package]] +name = "datafusion-execution" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "rand", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "itertools", + "paste", + "serde_json", +] + +[[package]] +name = "datafusion-expr-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap", + "itertools", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "hex", + "itertools", + "log", + "memchr", + "num-traits", + "rand", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "num-traits", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-table" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +dependencies = [ + "datafusion-doc", + "quote", + "syn", +] + +[[package]] +name = "datafusion-optimizer" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap", + "itertools", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "parking_lot", + "paste", + "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +dependencies = [ + "ahash", + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "parking_lot", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools", +] + +[[package]] +name = "datafusion-physical-plan" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +dependencies = [ + "ahash", + "arrow", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-pruning" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools", + "log", +] + +[[package]] +name = "datafusion-session" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +dependencies = [ + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools", + "parking_lot", + "percent-encoding", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/contrib/delta/native/Cargo.toml b/contrib/delta/native/Cargo.toml new file mode 100644 index 0000000000..d390171e6d --- /dev/null +++ b/contrib/delta/native/Cargo.toml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Standalone Cargo.toml -- this crate is outside the `native/` workspace root +# (`native/Cargo.toml` lists `exclude = ["../contrib"]`), so it cannot use +# `{ workspace = true }` inheritance. Versions are kept in sync with the rest of the +# repo by convention; the path deps anchor against the same on-disk crates. +# +# BUILD-GATE STUB: this is the inert wiring unit. The crate exists only so that a +# `--features contrib-delta` build of core compiles and links, and so the gate-verify +# script can prove the default build carries zero Delta surface. The real kernel-read +# implementation (delta_kernel, object_store, the scan/engine/dv modules, ...) lands in +# later units; here `planner::plan_delta_scan` returns a clean "not implemented" error so +# any Delta read that reaches native falls back to vanilla Spark. + +[package] +name = "comet-contrib-delta" +description = "delta-kernel-rs integration for Comet (build-gate stub; native read path lands in a later unit). Linked into libcomet via core's `contrib-delta` Cargo feature flag." +version = "0.18.0" +edition = "2021" +rust-version = "1.86.0" +publish = false +license = "Apache-2.0" + +[lib] +# rlib: linked INTO `libcomet` when `contrib-delta` is enabled on core. Never a cdylib +# on its own -- there's no separate Delta library to ship. +crate-type = ["rlib"] + +[dependencies] +# Typed Delta proto messages live in core's proto crate (alongside IcebergScan, ...) so the +# dispatcher arm has direct access; the stub planner names `DeltaScan` / `DeltaScanCommon` from here. +datafusion-comet-proto = { path = "../../../native/proto" } +# For `ExecutionPlan` (return type), `SchemaRef` (the shim hands these in), and `DataFusionError` +# (the not-implemented error). Versions/features mirror core so the shared lockfile resolves cleanly. +datafusion = { version = "53.1.0", default-features = false, features = ["parquet"] } diff --git a/contrib/delta/native/src/lib.rs b/contrib/delta/native/src/lib.rs new file mode 100644 index 0000000000..cbd1ff1728 --- /dev/null +++ b/contrib/delta/native/src/lib.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `comet-contrib-delta`: delta-kernel-rs integration for Comet. +//! +//! BUILD-GATE STUB. This unit establishes the crate boundary and the one symbol core's +//! `planner::delta_scan` shim calls -- [`planner::plan_delta_scan`] -- so a +//! `--features contrib-delta` build compiles end to end. The function returns a clean +//! "not implemented" error today; the real kernel-read scan path (log replay, deletion +//! vectors, column mapping, row tracking) lands in later units. Linked into `libcomet` +//! only when core's `contrib-delta` feature is on; default builds carry zero Delta code. + +pub mod planner; diff --git a/contrib/delta/native/src/planner.rs b/contrib/delta/native/src/planner.rs new file mode 100644 index 0000000000..072dd700a7 --- /dev/null +++ b/contrib/delta/native/src/planner.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Delta scan planning entry point. +//! +//! BUILD-GATE STUB: matches the exact signature core's `planner::delta_scan` shim calls +//! (`plan_delta_scan(&DeltaScan, &DeltaScanCommon, &SchemaRef, &SchemaRef)`), but returns a +//! clean `DataFusionError::NotImplemented` instead of building a scan. The shim maps that error +//! to a `GeneralError`, which surfaces on the JVM side as a fallback so the query runs on vanilla +//! Spark. The real kernel-read implementation replaces this in a later unit. + +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::error::DataFusionError; +use datafusion::physical_plan::ExecutionPlan; + +use datafusion_comet_proto::spark_operator::{DeltaScan, DeltaScanCommon}; + +/// Plan a native Delta scan. Stub: not yet implemented (build-gate unit). +/// +/// The signature mirrors the real implementation so core's shim and the dispatcher arm compile +/// unchanged when the read path lands. `required_schema` / `partition_schema` are computed by core +/// (which owns the proto -> arrow converter) and handed in. +pub fn plan_delta_scan( + _scan: &DeltaScan, + _common: &DeltaScanCommon, + _required_schema: &SchemaRef, + _partition_schema: &SchemaRef, +) -> Result, DataFusionError> { + Err(DataFusionError::NotImplemented( + "contrib-delta native Delta read path is not yet implemented (build-gate stub)".to_string(), + )) +} diff --git a/contrib/delta/src/main/scala/org/apache/comet/contrib/delta/DeltaConf.scala b/contrib/delta/src/main/scala/org/apache/comet/contrib/delta/DeltaConf.scala new file mode 100644 index 0000000000..02d2ab3f39 --- /dev/null +++ b/contrib/delta/src/main/scala/org/apache/comet/contrib/delta/DeltaConf.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import org.apache.comet.{ConfigBuilder, ConfigEntry} + +/** + * Contrib-local config entries for the Delta integration. Lives in the contrib's package rather + * than in core's `CometConf` so PR1 stays format-agnostic. Side-effect of object construction is + * registering the entries with `CometConf.allConfs` (via the `ConfigBuilder` machinery), so they + * show up in the generated user-guide docs and `SQLConf` resolution works the usual way. + */ +object DeltaConf { + + // CometConf.register asserts every config has a non-empty category — used for grouping + // entries in the generated user-guide docs. "scan" matches the existing core + // CATEGORY_SCAN string (CATEGORY_* constants in CometConf are `private val` so contribs + // can't reference the symbol; the assertion only checks `nonEmpty`). + private val CATEGORY = "scan" + + val COMET_DELTA_NATIVE_ENABLED: ConfigEntry[Boolean] = + ConfigBuilder("spark.comet.scan.deltaNative.enabled") + .category(CATEGORY) + .doc( + "Whether to enable native Delta table scans via delta-kernel-rs. When enabled, " + + "Delta tables are read directly through Comet's tuned ParquetSource + " + + "DV-filter wrapper, bypassing Spark's Delta reader for better performance.") + .booleanConf + .createWithDefault(true) + + val COMET_DELTA_FALLBACK_ON_UNSUPPORTED_FEATURE: ConfigEntry[Boolean] = + ConfigBuilder("spark.comet.scan.deltaNative.fallbackOnUnsupportedFeature") + .category(CATEGORY) + .doc( + "When true (default), the Delta contrib falls back to Spark's Delta reader on " + + "any Delta protocol feature it doesn't yet support. When false, the contrib " + + "raises an error instead -- useful for tests that want to assert the native " + + "path is reachable for a particular query.") + .booleanConf + .createWithDefault(true) + + val COMET_DELTA_DATA_FILE_CONCURRENCY_LIMIT: ConfigEntry[Int] = + ConfigBuilder("spark.comet.scan.deltaNative.dataFileConcurrencyLimit") + .category(CATEGORY) + .doc( + "Per-Spark-task concurrency when reading Delta data files. Higher values " + + "improve throughput on tables with many small files at the cost of memory. " + + "Values between 2 and 8 are typical.") + .intConf + .checkValue(v => v > 0, "Data file concurrency limit must be positive") + .createWithDefault(1) + + val COMET_DELTA_CDF_MAX_PARTITIONS: ConfigEntry[Int] = + ConfigBuilder("spark.comet.delta.cdf.maxPartitions") + .category(CATEGORY) + .doc( + "Maximum number of Spark partitions a Change Data Feed (readChangeFeed) read is split " + + "into. The inclusive version range is chunked into up to this many contiguous " + + "sub-ranges, each read by an independent native delta-kernel TableChanges call, so a " + + "multi-version CDF read parallelizes across tasks instead of reading the whole range " + + "on one task. Capped by the number of commits in the range.") + .intConf + .checkValue(v => v > 0, "CDF max partitions must be positive") + .createWithDefault(8) +} diff --git a/dev/verify-contrib-delta-gate.sh b/dev/verify-contrib-delta-gate.sh new file mode 100755 index 0000000000..df0427b54e --- /dev/null +++ b/dev/verify-contrib-delta-gate.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Verify the `contrib-delta` build gate keeps Delta surface out of default builds. +# +# Three independent layers are checked: +# 1. Cargo: default `cargo build` doesn't compile `comet-contrib-delta` and +# doesn't pull `delta_kernel` into the dependency tree. +# 2. Maven: default `mvn ... package` doesn't compile any +# `org/apache/comet/contrib/` classes and doesn't pull `io.delta:*` deps. +# 3. Symbol/size: the resulting `libcomet` (`.so` on Linux, `.dylib` on macOS) from the default build is +# meaningfully smaller than the contrib-enabled build, and carries no +# `comet_contrib_delta`/`delta_kernel`/etc. external symbols. +# +# Exit non-zero on the first failure. Designed to be wired into CI so a future +# change that leaks Delta into core gets caught immediately. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +NATIVE_DIR="$ROOT/native" +SPARK_DIR="$ROOT/spark" + +# CI containers check out the repo as a different user than the job runs as, so git refuses to +# operate on it ("detected dubious ownership") -- which can make Maven plugins that read git +# metadata fail fast with no useful output. Mark the tree safe up front (best-effort). +git config --global --add safe.directory "$ROOT" 2>/dev/null || true +git config --global --add safe.directory '*' 2>/dev/null || true + +# Use the repo's Maven wrapper, not a bare `mvn`. The CI gate job runs in a bare +# `amd64/rust` container that has no system Maven on PATH, so `mvn` would fail to +# launch, produce no dependency output, and trip the anti-vacuous guard below. +# `./mvnw` self-provisions the pinned Maven version regardless of the image. +MVNW="$ROOT/mvnw" + +red() { printf '\033[31m%s\033[0m\n' "$*"; } +green() { printf '\033[32m%s\033[0m\n' "$*"; } +hdr() { printf '\n\033[36m==> %s\033[0m\n' "$*"; } + +# ---- Cargo gate ----------------------------------------------------------- + +hdr "Cargo: default build does not depend on comet-contrib-delta / delta_kernel" +cd "$NATIVE_DIR" +TREE_DEFAULT="$(cargo tree -p datafusion-comet --no-default-features 2>/dev/null)" +# Anti-vacuous (mirrors the Maven gate below): a failing `cargo tree` yields empty output, and the +# command-substitution failure doesn't trip `set -e` in an assignment -- so assert the root crate we +# KNOW is always present before concluding "no Delta deps", otherwise a broken cargo-tree run would +# pass the leak check vacuously. (`datafusion-comet ` with a trailing space matches only the root +# crate line, not `datafusion-comet-proto`/`-common`.) +if ! grep -q 'datafusion-comet ' <<<"$TREE_DEFAULT"; then + red "FAIL: default cargo tree produced no datafusion-comet entry (cargo tree likely failed;" + red " refusing to conclude 'no Delta deps' vacuously)" + exit 1 +fi +if grep -qE 'comet-contrib-delta|delta_kernel|delta-kernel' <<<"$TREE_DEFAULT"; then + red "FAIL: default cargo tree contains Delta-related deps:" + grep -E 'comet-contrib-delta|delta_kernel|delta-kernel' <<<"$TREE_DEFAULT" + exit 1 +fi +green "OK: cargo tree default is clean of contrib + kernel" + +TREE_CONTRIB="$(cargo tree -p datafusion-comet --features contrib-delta 2>/dev/null)" +# The build-gate unit ships a STUB contrib crate, so the gated tree pulls in +# `comet-contrib-delta` but not yet the heavy `delta_kernel` (that arrives with the +# native read-path unit). Require the contrib crate to be present; the symbol check +# below guards against grep-pattern drift either way. +CONTRIB_HITS="$(printf '%s\n' "$TREE_CONTRIB" | grep -cE 'comet-contrib-delta' || true)" +if [[ "$CONTRIB_HITS" -lt 1 ]]; then + red "FAIL: --features contrib-delta tree missing the comet-contrib-delta crate (hits=$CONTRIB_HITS)" + exit 1 +fi +green "OK: cargo tree with contrib-delta correctly pulls comet-contrib-delta" + +# ---- Maven gate ----------------------------------------------------------- + +hdr "Maven: default profile excludes io.delta:* dependencies" +cd "$ROOT" +# `dependency:list` can't run in a fresh CI checkout: it needs the sibling reactor JARs +# (comet-common, the shims) which aren't built, so it fails with a resolution error and no +# output. `help:effective-pom` only merges POM models (no artifact resolution), so it works +# without a build. We extract the ACTIVE top-level -- after , +# before the listing -- which is exactly what dependency:list would have shown for the +# active profiles (and excludes the inactive contrib-delta profile's own io.delta declaration). +# Last effective-pom invocation's combined output, kept so the anti-vacuous guard can SHOW why +# mvn failed instead of swallowing it. +EPOM_LOG="$(mktemp)" +delta_active_deps() { # args: -P / -D flags + local epom + epom="$(mktemp)" + "$MVNW" help:effective-pom -Djava.version=17 -Dmaven.gitcommitid.skip -pl spark \ + -Doutput="$epom" "$@" >"$EPOM_LOG" 2>&1 || true + awk '/<\/dependencyManagement>/{f=1} //{f=0} f' "$epom" + rm -f "$epom" +} +# Resolved delta-spark version from the active deps (empty if absent). +delta_spark_version() { # args: -P flags + delta_active_deps "$@" | + grep -A2 'artifactId>delta-spark' | + grep -oE '[^<]+' | sed -n '1s///p' +} + +DEPS_DEFAULT="$(delta_active_deps -Pspark-4.1)" +# Anti-vacuous: a broken mvn run yields empty output; assert a dep we KNOW is always present so a +# broken run fails loudly instead of "passing" the io.delta check by finding nothing. +if ! grep -q 'org.apache.spark' <<<"$DEPS_DEFAULT"; then + red "FAIL: default effective-pom produced no org.apache.spark deps (mvn likely failed;" + red " refusing to conclude 'zero io.delta' vacuously)" + red " --- mvnw: $MVNW (java=${JAVA_HOME:-unset}) ---" + red " --- effective-pom output (last 60 lines) ---" + tail -60 "$EPOM_LOG" >&2 || true + exit 1 +fi +if grep -q 'io.delta' <<<"$DEPS_DEFAULT"; then + red "FAIL: default Maven build pulls io.delta dependencies:" + echo "$DEPS_DEFAULT" | grep -A2 'io.delta' + exit 1 +fi +green "OK: default Maven build has zero io.delta dependencies" + +# Per-Spark Delta version pinning: spark-4.1 -> delta-spark 4.1.x, spark-3.5 -> 3.x, spark-4.0 -> +# 4.0.x (Delta 4.1 needs Spark 4.1 internals; 4.0 must stay on 4.0.x to avoid a runtime +# NoSuchMethodError on ParserInterface.$init$). +V41="$(delta_spark_version -Pspark-4.1,contrib-delta)" +case "$V41" in + 4.1.*) green "OK: -Pcontrib-delta + spark-4.1 correctly pulls delta-spark $V41" ;; + *) red "FAIL: -Pcontrib-delta + spark-4.1 expected delta-spark 4.1.x, got '${V41:-}'"; exit 1 ;; +esac +V35="$(delta_spark_version -Pspark-3.5,contrib-delta)" +case "$V35" in + 3.*) green "OK: -Pcontrib-delta + spark-3.5 correctly pulls delta-spark $V35" ;; + *) red "FAIL: -Pcontrib-delta + spark-3.5 expected delta-spark 3.x, got '${V35:-}'"; exit 1 ;; +esac +V40="$(delta_spark_version -Pspark-4.0,contrib-delta)" +case "$V40" in + 4.0.*) green "OK: -Pcontrib-delta + spark-4.0 correctly pulls delta-spark $V40" ;; + *) red "FAIL: -Pcontrib-delta + spark-4.0 expected delta-spark 4.0.x, got '${V40:-}'"; exit 1 ;; +esac + +# ---- Compiled-class gate -------------------------------------------------- + +hdr "Compiled classes: no contrib/delta classes in default build" +cd "$ROOT" +"$MVNW" -Pspark-4.1 -Djava.version=17 -Dmaven.compiler.source=17 -Dmaven.compiler.target=17 -Dmaven.gitcommitid.skip -pl spark -am test-compile -q -DskipTests=true >/dev/null 2>&1 +LEAK_CLASSES="$(find spark/target/classes -path '*comet/contrib*' -name '*.class' 2>/dev/null)" +if [[ -n "$LEAK_CLASSES" ]]; then + red "FAIL: default Maven build compiled contrib classes:" + echo "$LEAK_CLASSES" + exit 1 +fi +DELTA_IMPL_LEAKS="$(find spark/target/classes \ + \( -name 'CometDeltaNativeScan*' \ + -o -name 'CometDeltaNativeScanExec*' \ + -o -name 'DeltaScanRule*' \ + -o -name 'DeltaReflection*' \) \ + 2>/dev/null)" +if [[ -n "$DELTA_IMPL_LEAKS" ]]; then + red "FAIL: default Maven build compiled Delta-implementation classes:" + echo "$DELTA_IMPL_LEAKS" + exit 1 +fi +green "OK: only the always-present DeltaIntegration reflection bridge in default classes" + +# ---- libcomet symbol/size gate ------------------------------------------- + +hdr "libcomet: default build is smaller and has no Delta symbols" +cd "$NATIVE_DIR" +# The cdylib extension is platform-specific: `libcomet.so` on Linux (CI), `libcomet.dylib` on +# macOS. Find whichever the build produced; `stat`/`nm` flags also differ across the two. +# Echo whichever lib exists (empty if neither). MUST return 0: `ls a b` exits non-zero when one of +# the two paths is absent (the normal case -- only one extension exists per platform), and under +# `set -euo pipefail` that would kill the script at `LIB=$(comet_lib)` even though it found the lib. +comet_lib() { + local f + for f in target/debug/libcomet.so target/debug/libcomet.dylib; do + if [[ -f "$f" ]]; then + echo "$f" + return 0 + fi + done + return 0 +} +lib_size() { stat -c%s "$1" 2>/dev/null || stat -f%z "$1"; } +# Plain `nm` on the (unstripped debug) lib lists the full symbol table on both platforms, so a +# name grep finds the Delta symbols whether they're exported or internal. +delta_syms() { + nm "$1" 2>/dev/null | grep -ciE 'comet_contrib_delta|delta_kernel|deltadvfilter|deltasynthetic' || true +} + +cargo clean -p comet-contrib-delta -p datafusion-comet >/dev/null 2>&1 || true +cargo build -j 4 -p datafusion-comet >/dev/null 2>&1 +LIB_DEFAULT="$(comet_lib)" +if [[ -z "$LIB_DEFAULT" ]]; then + red "FAIL: default build produced no libcomet.{so,dylib} under $NATIVE_DIR/target/debug" + exit 1 +fi +SIZE_DEFAULT="$(lib_size "$LIB_DEFAULT")" +if command -v nm >/dev/null 2>&1; then + EXT_SYMS="$(delta_syms "$LIB_DEFAULT")" +else + EXT_SYMS=0 +fi +if [[ "$EXT_SYMS" -ne 0 ]]; then + red "FAIL: default libcomet contains $EXT_SYMS Delta-related symbols" + exit 1 +fi +green "OK: default libcomet has 0 Delta symbols (size=$SIZE_DEFAULT bytes)" + +cargo build -j 4 -p datafusion-comet --features contrib-delta >/dev/null 2>&1 +LIB_CONTRIB="$(comet_lib)" +SIZE_CONTRIB="$(lib_size "$LIB_CONTRIB")" +if [[ "$SIZE_CONTRIB" -le "$SIZE_DEFAULT" ]]; then + red "FAIL: contrib-enabled libcomet (size=$SIZE_CONTRIB) is not larger than default (size=$SIZE_DEFAULT)" + red " (would indicate contrib was being linked into default build too)" + exit 1 +fi +# Sanity check: the contrib-enabled libcomet MUST contain Delta-related symbols. +# Without this, a future Rust toolchain that mangles symbols differently (so our +# grep pattern stops matching) would silently make the default-build check a no-op +# while still passing -- the gate would lie about being enforced. Asserting both +# "default has 0" AND "contrib has >0" catches grep-pattern drift. +if command -v nm >/dev/null 2>&1; then + CONTRIB_SYMS="$(delta_syms "$LIB_CONTRIB")" + if [[ "$CONTRIB_SYMS" -lt 1 ]]; then + red "FAIL: contrib-enabled libcomet has 0 Delta-related symbols matching our grep pattern." + red " This means the symbol-name pattern in this script has drifted from what" + red " Rust currently emits, and the default-build check above is now a no-op." + red " Inspect the dylib's exports and update the grep pattern." + exit 1 + fi +fi +DIFF_MB=$(( (SIZE_CONTRIB - SIZE_DEFAULT) / 1024 / 1024 )) +green "OK: contrib-enabled libcomet is ${DIFF_MB} MB larger than default (size=$SIZE_CONTRIB bytes)" + +# ---- Summary -------------------------------------------------------------- + +hdr "All gate checks passed" +echo " default cargo: no comet-contrib-delta, no delta_kernel" +echo " default mvn: no io.delta:*, no contrib/delta classes" +echo " default dylib: ${DIFF_MB} MB smaller than contrib build, 0 Delta symbols" +echo +echo "Run with: dev/verify-contrib-delta-gate.sh" diff --git a/native/Cargo.lock b/native/Cargo.lock index 1ce78adfdb..8afda613b2 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1515,6 +1515,14 @@ dependencies = [ "memchr", ] +[[package]] +name = "comet-contrib-delta" +version = "0.18.0" +dependencies = [ + "datafusion", + "datafusion-comet-proto", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1990,6 +1998,7 @@ dependencies = [ "aws-config", "aws-credential-types", "bytes", + "comet-contrib-delta", "criterion", "datafusion", "datafusion-comet-common", diff --git a/native/Cargo.toml b/native/Cargo.toml index 29cc70e806..43272db27f 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -18,6 +18,10 @@ [workspace] default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"] members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "hdfs", "fs-hdfs"] +# The contrib crate at ../contrib/delta/native is intentionally NOT a workspace member +# (workspace members must live hierarchically under the workspace root). It's pulled in +# as a path dep by `core/Cargo.toml` when the `contrib-delta` feature is enabled. +exclude = ["../contrib"] resolver = "2" [workspace.package] diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 5cb970c762..802930e1eb 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -69,6 +69,9 @@ aws-config = { workspace = true } aws-credential-types = { workspace = true } parking_lot = "0.12.5" datafusion-comet-objectstore-hdfs = { path = "../hdfs", optional = true, default-features = false, features = ["hdfs"] } +# Optional Delta Lake contrib (enabled by the `contrib-delta` feature). Source lives +# under `contrib/delta/native/` so non-Delta committers can ignore it. +comet-contrib-delta = { path = "../../contrib/delta/native", optional = true } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2"] } object_store_opendal = { version = "0.57.0", optional = true } hdfs-sys = {version = "0.3", optional = true, features = ["hdfs_3_3"]} @@ -100,6 +103,10 @@ default = ["hdfs-opendal"] hdfs = ["datafusion-comet-objectstore-hdfs"] hdfs-opendal = ["opendal", "object_store_opendal", "hdfs-sys"] jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] +# Delta Lake integration. When enabled, links the `comet-contrib-delta` crate +# into `libcomet` and activates the `OpStruct::DeltaScan` dispatcher arm. +# Default builds carry zero Delta surface. +contrib-delta = ["dep:comet-contrib-delta"] # exclude optional packages from cargo machete verifications [package.metadata.cargo-machete] diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 4a7a21006d..420eb9c0b9 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -219,7 +219,13 @@ pub fn get_runtime() -> &'static Runtime { TOKIO_RUNTIME.get_or_init(|| build_runtime(None)) } -/// Returns a short name for an OpStruct variant. +/// Returns a short name for an OpStruct variant. Used for tracing event names; +/// no contrib-specific logic. The `OpStruct::DeltaScan` arm stays unconditional +/// even in non-`contrib-delta` builds because the proto enum is generated +/// regardless of cargo feature flags and Rust requires an exhaustive match -- +/// returning a name string here keeps default-build error messages (e.g. "Received +/// a DeltaScan operator but core was built without the `contrib-delta` Cargo +/// feature") able to identify the offender by name. fn op_name(op: &OpStruct) -> &'static str { match op { OpStruct::Scan(_) => "Scan", @@ -240,6 +246,7 @@ fn op_name(op: &OpStruct) -> &'static str { OpStruct::CsvScan(_) => "CsvScan", OpStruct::ShuffleScan(_) => "ShuffleScan", OpStruct::BroadcastNestedLoopJoin(_) => "BroadcastNestedLoopJoin", + OpStruct::DeltaScan(_) => "DeltaScan", } } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index c07a92d700..133db8066f 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -21,6 +21,15 @@ pub mod expression_registry; pub mod macros; pub mod operator_registry; +// Glue that wires the optional Delta integration into core's plan-tree builder. +// Compiled only under `--features contrib-delta`; default builds carry zero Delta +// surface. The bulk of the Delta logic lives in the `comet-contrib-delta` crate -- +// this module is just the bridge that reaches into core's `pub(crate)` planner +// helpers (`create_expr`, `init_datasource_exec`, `prepare_object_store_with_configs`) +// and calls into that crate. +#[cfg(feature = "contrib-delta")] +mod delta_scan; + use crate::execution::operators::init_csv_datasource_exec; use crate::execution::operators::AlignedArrowStreamReader; use crate::execution::operators::IcebergScanExec; @@ -1597,6 +1606,27 @@ impl PhysicalPlanner { )), )) } + OpStruct::DeltaScan(scan) => { + // Delta Lake scan -- handled by the optional `contrib/delta/` integration. + // The dispatcher arm exists unconditionally so a default build that receives + // a Delta-shaped plan from a misconfigured driver gets a clear error instead + // of a "no match" decode failure. + #[cfg(not(feature = "contrib-delta"))] + { + let _ = scan; + Err(GeneralError( + "Received a DeltaScan operator but core was built without the \ + `contrib-delta` Cargo feature. Rebuild with both \ + `-Pcontrib-delta` (Maven) and `--features contrib-delta` (Cargo) \ + to enable Delta Lake support." + .into(), + )) + } + #[cfg(feature = "contrib-delta")] + { + delta_scan::plan_delta_scan(self, spark_plan, scan) + } + } OpStruct::ShuffleWriter(writer) => { assert_eq!(children.len(), 1); let (scans, shuffle_scans, child) = @@ -3194,7 +3224,13 @@ pub fn from_protobuf_eval_mode(value: i32) -> Result SchemaRef { let arrow_fields = spark_types diff --git a/native/core/src/execution/planner/delta_scan.rs b/native/core/src/execution/planner/delta_scan.rs new file mode 100644 index 0000000000..3ef4f351ad --- /dev/null +++ b/native/core/src/execution/planner/delta_scan.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `OpStruct::DeltaScan` planner body, feature-gated behind `contrib-delta`. +//! +//! Thin bridge between core's plan-tree builder and the [`comet_contrib_delta`] crate. ALL +//! Delta-specific scan planning lives in `comet_contrib_delta::planner::plan_delta_scan`; this +//! shim only computes the requested/partition Arrow schemas (core owns the proto -> arrow type +//! converter, used across the planner) and wraps the contrib's `ExecutionPlan` in a `SparkPlan`. +//! It lives in core (not the contrib crate) because the converter is core's `pub(crate)` and a +//! `contrib -> core` dependency would cycle with core's optional `contrib-delta` dep. Compiled +//! only under `--features contrib-delta`; default builds carry zero Delta surface. + +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion_comet_proto::spark_operator::{DeltaScan, Operator}; + +use crate::execution::operators::ExecutionError::GeneralError; +use crate::execution::planner::convert_spark_types_to_arrow_schema; +use crate::execution::planner::PhysicalPlanner; +use crate::execution::planner::PlanCreationResult; +use crate::execution::spark_plan::SparkPlan; + +pub(crate) fn plan_delta_scan( + // The kernel-read path doesn't need the planner (the old ParquetSource path used it to build + // pushed-down filter exprs); kept in the signature for the dispatcher call site. + _planner: &PhysicalPlanner, + spark_plan: &Operator, + scan: &DeltaScan, +) -> PlanCreationResult { + let common = scan + .common + .as_ref() + .ok_or_else(|| GeneralError("DeltaScan missing common data".into()))?; + + // Core owns the proto -> arrow schema converter (`to_arrow_datatype`, used across the planner), + // so compute the requested + partition schemas here and hand them to the contrib, which owns all + // Delta-specific scan planning (#77). + let required_schema: SchemaRef = + convert_spark_types_to_arrow_schema(common.required_schema.as_slice()); + let partition_schema: SchemaRef = + convert_spark_types_to_arrow_schema(common.partition_schema.as_slice()); + + let exec = comet_contrib_delta::planner::plan_delta_scan( + scan, + common, + &required_schema, + &partition_schema, + ) + .map_err(|e| GeneralError(format!("{e}")))?; + + Ok(( + vec![], + vec![], + Arc::new(SparkPlan::new(spark_plan.plan_id, exec, vec![])), + )) +} diff --git a/native/core/src/execution/planner/operator_registry.rs b/native/core/src/execution/planner/operator_registry.rs index dc98a32fb0..5cb5c5acb2 100644 --- a/native/core/src/execution/planner/operator_registry.rs +++ b/native/core/src/execution/planner/operator_registry.rs @@ -152,5 +152,11 @@ fn get_operator_type(spark_operator: &Operator) -> Option { OpStruct::CsvScan(_) => Some(OperatorType::CsvScan), OpStruct::ShuffleScan(_) => None, // Not yet in OperatorType enum OpStruct::BroadcastNestedLoopJoin(_) => None, + // Optional contrib; not in OperatorType enum. The arm stays unconditional + // even in non-`contrib-delta` builds because the proto enum is generated + // regardless of cargo features and Rust requires an exhaustive match. No + // contrib-specific logic lives here -- we just signal "no OperatorType + // mapping" so the supports-mixed-codegen check skips it. + OpStruct::DeltaScan(_) => None, } } diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 15265f1d86..788c1834c1 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -54,6 +54,10 @@ message Operator { CsvScan csv_scan = 115; ShuffleScan shuffle_scan = 116; BroadcastNestedLoopJoin broadcast_nested_loop_join = 117; + // Delta Lake scan. Wire format used by `contrib/delta/`. Only decoded when + // core is built with `--features contrib-delta`; in default builds the + // dispatcher arm is `#[cfg]`-stubbed out so the contrib has zero runtime cost. + DeltaScan delta_scan = 118; } } @@ -276,6 +280,224 @@ message IcebergDeleteFile { uint64 file_size_in_bytes = 5; } +// ===================================================================================== +// Delta Lake scan messages -- consumed by the optional `contrib/delta/` integration. +// Field numbers must remain stable; older serialised plans encode them on the wire. +// ===================================================================================== + +// Per-scan invariants. Lives at the head of every Delta scan operator payload. +message DeltaScanCommon { + // Required (output) schema -- pure-logical at every nesting level. The kernel-read path + // physicalises it via `column_mappings`; partition columns are split out by name. + repeated SparkStructField required_schema = 1; + // Partition columns (stored in add actions, not in file paths). + repeated SparkStructField partition_schema = 3; + // Fields 2 (data_schema), 4 (data_filters), 5 (projection_vector) belonged to the removed + // legacy ParquetSource read path; the kernel-read path reads only `required_schema` + + // `column_mappings`. Reserved so the numbers aren't reused. + reserved 2, 4, 5; + reserved "data_schema", "data_filters", "projection_vector"; + string session_timezone = 6; + bool case_sensitive = 7; + // Cloud storage credentials / options, keyed by Hadoop-style names. + map object_store_options = 8; + // Root path of the Delta table (file:// or s3://...). + string table_root = 9; + // Optional snapshot version; unset = latest. + optional uint64 snapshot_version = 10; + // Per-file parallelism knob within a single Spark task. + uint32 data_file_concurrency_limit = 11; + // Informational label for debug output. + string source = 12; + // Field 13 was `column_mappings` (logical->physical): the executor's only consumer was the + // `physicalise_field` schema rebuild, removed once the kernel-read path ships kernel's own + // `scan.physical_schema()`/`logical_schema()`. Partition-value key translation happens + // driver-side (see `DeltaScanTaskList.column_mappings`), so nothing executor-side needs it. + reserved 13; + optional string materialized_row_id_column_name = 14; + optional string materialized_row_commit_version_column_name = 15; + // When true the DataSourceExec swallows object-store NotFound errors as empty + // streams; matches Spark's `spark.sql.files.ignoreMissingFiles=true` semantics. + bool ignore_missing_files = 16; + // When true the native parquet reader matches Spark schema fields to parquet + // file fields by `PARQUET:field_id` metadata rather than by name. Used by + // Delta tables with `delta.columnMapping.mode = id` (the contrib copies + // `delta.columnMapping.id` to `PARQUET:field_id` on each schema field). + bool use_field_id = 17; + // Delta synthetic columns to append to the scan output via + // `DeltaSyntheticColumnsExec`. The corresponding columns must NOT appear in + // `data_schema` or `required_schema` -- they're synthesised at exec time + // from row positions / DV bitmaps and inserted after the parquet read. + bool emit_row_index = 18; + bool emit_is_row_deleted = 19; + // Row-tracking synthesis (row_id = baseRowId + physical_row_index, + // row_commit_version = defaultRowCommitVersion). Per-task baseRowId / + // defaultRowCommitVersion travel on `DeltaScanTask` (fields 6 + 7). + bool emit_row_id = 20; + bool emit_row_commit_version = 21; + // When synthetic columns aren't a contiguous suffix of `required_schema`, this + // sequence tells the native dispatcher how to project the final output. Each + // entry is an index into [parquet_output, appended_synthetics] -- positive + // means a column from the (stripped) parquet read at that index; the synthetic + // tail comes after, in the canonical row_index, is_row_deleted, row_id, + // row_commit_version order. Empty = synthetics are already a suffix; no + // reorder needed. + repeated int32 final_output_indices = 22; + // Override for the row_index column name. Defaults to "__delta_internal_row_index" + // when empty. Delta plans that read `_metadata.row_index` from row-tracking-enabled + // tables expose the row index as `_tmp_metadata_row_index` (the Delta-internal + // physical name from `DeltaParquetFileFormat.TMP_METADATA_ROW_INDEX_COLUMN_NAME`) + // rather than the canonical `__delta_internal_row_index`. Setting this lets the + // contrib produce a column with the alternate name from the same native synthesis. + string row_index_column_alias = 23; + // Spark `_metadata.*` virtual columns the scan must surface. Delta's + // `PreprocessTableWithDVs` strategy and the row-tracking projection both inject + // these into the FileScan output: `file_path`, `file_name`, `file_size`, + // `file_block_start`, `file_block_length`, `file_modification_time`. Each + // appears as a flat top-level column in the scan's output schema. Native + // synthesizes them per-task from `DeltaScanTask` data (file_path, file_size, + // modification_time) and per-batch derivations. Empty list = no metadata + // columns to emit. Names are emitted in the order given, appended AFTER + // synthetic columns (row_index / is_row_deleted / row_id / row_commit_version) + // unless `final_output_indices` reorders them. + repeated string metadata_column_names = 24; + // Field 25 was `bool kernel_read`: a transitional toggle from when the legacy ParquetSource read + // path still co-existed with the kernel-read path. The legacy path is gone (#50) and kernel-read + // is the only read path, so the toggle (and its `spark.comet.delta.kernelRead.enabled` config) + // were removed -- the driver no longer sends it and the native planner no longer gates on it. + reserved 25; + reserved "kernel_read"; + // Kernel-built data-column schemas for the kernel-read path, produced by the driver from + // `delta-kernel-rs`'s `Scan` (`scan.physical_schema()` / `scan.logical_schema()`), serialized as + // Arrow IPC schema messages. The physical schema carries the correct physical names AND field-ids + // at EVERY nesting level (kernel's own column-mapping resolution); the logical schema is the + // matching pure-logical shape. The executor reads parquet with the physical schema and relabels + // physical->logical via the schema pair -- no Comet-side physicalisation. Empty when the driver + // didn't build them (e.g. the non-kernel-read add-files path), in which case the executor falls + // back to physicalising `required_schema` via `column_mappings`. + bytes kernel_physical_schema = 26; + bytes kernel_logical_schema = 27; + // Stage B/C: when true, DeltaKernelScanExec produces ALL synthetic columns itself (row_index / + // row_id / file_path via kernel metadata columns; is_row_deleted by inverting the decoded DV; + // row_commit_version + Spark `_metadata.*` as per-file constants) and assembles the full output by + // name -- so the standalone DeltaSyntheticColumnsExec is NOT stacked on top and no + // `final_output_indices` reorder is needed. Default false (legacy DeltaSyntheticColumnsExec path). + // When set, `required_schema` is the FULL output (synthetics NOT stripped) and the per-file + // synthesis inputs ride on DeltaScanTask (base_row_id / default_row_commit_version / + // modification_time / byte range / dv). + bool synthesize_in_worker = 28; + // Change Data Feed (CDF / readChangeFeed): when true, DeltaKernelScanExec reads table changes via + // delta-kernel-rs's `TableChanges` API (single partition reconstructs TableChanges(start, end) and + // calls `execute()`), emitting the data columns plus `_change_type` / `_commit_version` / + // `_commit_timestamp`. The version range is below; `required_schema` is the full CDF output + // (assembled by name). No per-file tasks -- the worker enumerates + reads the whole range itself + // (kernel's CDF per-file API is pub(crate); only execute() is public). Default false. + bool cdf_read = 29; + // Inclusive start version of the CDF range. + uint64 cdf_start_version = 30; + // Inclusive end version of the CDF range. `has_cdf_end_version` distinguishes "latest" (unset). + optional uint64 cdf_end_version = 31; + // Delta's test-only deletion-vector filename prefix + // (`spark.databricks.delta.testOnly.dvFileNamePrefix`, default `test%dv%prefix-` under + // `Utils.isTesting`, empty in production). Delta prepends it to the DV filename at write time + // (`deletion_vector_.bin`) but delta-kernel-rs resolves the un-prefixed name, so + // executor-side DV reads splice it back in for "u"-storage DVs. Empty = no-op (production). + string dv_file_name_prefix = 32; +} + +// Operator payload for a Delta scan. +message DeltaScan { + DeltaScanCommon common = 1; + repeated DeltaScanTask tasks = 2; + // Absolute URL of the Delta table root, used by the executor to resolve relative + // ("u" storage-type) DV paths via kernel's `absolute_path` -- kernel joins + // `_delta_log/deletion_vectors/` onto this, so it MUST end with `/` (the + // driver normalises before sending). Empty for tables with no DVs (executor + // simply never decodes), but populate it whenever known. + string table_root = 3; +} + +message DeltaScanTask { + string file_path = 1; + uint64 file_size = 2; + optional uint64 record_count = 3; + repeated DeltaPartitionValue partition_values = 4; + // Field 5 (repeated uint64 deleted_row_indexes) removed in the Iceberg-style DV refactor: + // the driver used to materialise the FULL deletion-vector bitmap to a Vec here, which + // could reach ~1 GB for a single DV with ~99 M deleted rows (DELETE/MERGE on huge tables). + // The executor now resolves the DV itself via the `dv` field below -- matches the contrib's + // own Iceberg integration (IcebergScanCommon.delete_files_pool + IcebergFileScanTask.delete_files_idx). + reserved 5; + reserved "deleted_row_indexes"; + optional int64 base_row_id = 6; + optional int64 default_row_commit_version = 7; + // File-splitting byte range. Both unset = read whole file. + optional uint64 byte_range_start = 8; + optional uint64 byte_range_end = 9; + // File modification time in epoch milliseconds. Surfaced through Spark's + // `_metadata.file_modification_time` column when emit_metadata_columns + // includes "file_modification_time". Unset = 0 (rare; AddFile usually + // carries one). + optional int64 modification_time = 10; + // Deletion-vector descriptor for this file (absent = no DV applies). Mirrors + // delta_kernel::actions::deletion_vector::DeletionVectorDescriptor. The executor reads + // the actual DV bitmap on-task via dv_reader::read_dv_indexes -- the driver only ships + // the path/offset/size, so the per-scan-exec heap stays KB-scale regardless of DV size. + optional DeltaDvDescriptor dv = 11; + // Kernel's own fully-resolved physical->logical transform for THIS file, serialized as JSON + // (serde of `delta_kernel::expressions::Expression`). The driver gets it from kernel's scan + // enumeration (`ScanMetadata.scan_file_transforms`, one per scan file), where partition values + // and `baseRowId` (for row_id `GenerateRowId`) are already baked in as literals. The executor + // deserializes it and applies `transform_to_logical` -- so partition injection, column-mapping + // relabel (incl. nested), and row-tracking come from kernel rather than Comet-side reconstruction. + // Empty when the driver has no kernel transform for this file (e.g. a plain table with no mapping + // / partitions / row-tracking -> identity), in which case the executor uses a pass-through read. + bytes transform_json = 12; +} + +// Mirrors delta_kernel::actions::deletion_vector::DeletionVectorDescriptor. Carries everything +// an executor needs to read + decode the DV bitmap, without materialising it on the driver. +message DeltaDvDescriptor { + // "u" (UUID-named, relative to /_delta_log/deletion_vectors/) + // "p" (absolute URI in path_or_inline_dv) + // "i" (inline -- bytes carried in inline_bytes) + string storage_type = 1; + // For "u": Delta-encoded UUID path component. For "p": absolute URI. For "i": empty. + string path_or_inline_dv = 2; + // Byte offset within the DV file ("u"/"p" only). Absent ≡ 0. + optional uint64 offset = 3; + // Size in bytes of the serialised RoaringBitmap entry. + uint64 size_in_bytes = 4; + // Number of deleted rows (descriptor metadata; the executor doesn't need to count). + uint64 cardinality = 5; + // Inline-DV bytes (storage_type="i" only). Empty for "u"/"p". + bytes inline_bytes = 6; +} + +message DeltaPartitionValue { + string name = 1; + optional string value = 2; +} + +// JNI return value from planDeltaScan -- the list of tasks for a given snapshot. +message DeltaScanTaskList { + uint64 snapshot_version = 1; + string table_root = 2; + repeated DeltaScanTask tasks = 3; + repeated string unsupported_features = 4; + // Field 5 was `column_mappings`: column mapping is now fully resolved by the kernel-built schemas + // shipped on `DeltaScanCommon` (kernel resolves physical names + field-ids at every nesting + // level), and partition-value key translation happens driver-side, so nothing on the wire needs + // a column-mapping tree. + reserved 5; + // Kernel-built data-column schemas (Arrow IPC), produced by the driver from the projected + // `Scan` (`scan.physical_schema()` / `scan.logical_schema()`). Carried back to Scala and copied + // onto `DeltaScanCommon.kernel_physical_schema` / `kernel_logical_schema`. Empty when the driver + // had no projection to build the scan against. + bytes physical_schema = 6; + bytes logical_schema = 7; +} + message Projection { repeated spark.spark_expression.Expr project_list = 1; } diff --git a/pom.xml b/pom.xml index f53b153d28..7ffaa350b6 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,14 @@ under the License. UTF-8 UTF-8 11 + + 4.1.0 ${java.version} ${java.version} 3.11.0 diff --git a/spark/pom.xml b/spark/pom.xml index 780ecb63aa..49a95d09d3 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -122,6 +122,18 @@ under the License. + + + com.google.guava + failureaccess + 1.0.2 + test + org.codehaus.jackson jackson-mapper-asl @@ -231,6 +243,13 @@ under the License. spark-3.5 + + + 3.3.2 + org.apache.iceberg @@ -256,6 +275,15 @@ under the License. spark-4.0 + + + 4.0.0 + org.apache.iceberg @@ -283,6 +311,10 @@ under the License. true + + + 4.1.0 + @@ -331,6 +363,69 @@ under the License. + + + contrib-delta + + + + io.delta + delta-spark_${scala.binary.version} + ${delta.version} + provided + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-contrib-delta-source + generate-sources + + add-source + + + + ${project.parent.basedir}/contrib/delta/src/main/scala + + + + + add-contrib-delta-test-source + generate-test-sources + + add-test-source + + + + ${project.parent.basedir}/contrib/delta/src/test/scala + + + + + + + + + generate-docs diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 50cd0927b4..64dda8a3b9 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -265,6 +265,16 @@ case class CometExecRule(session: SparkSession) // spotless:on private def transform(plan: SparkPlan): SparkPlan = { def convertNode(op: SparkPlan): SparkPlan = op match { + // Delta scan marker produced by the optional contrib/delta integration. Matched by type + // (no compile-time dependency on the contrib) -- present only when -Pcontrib-delta was + // activated. The marker wraps the original, link-bearing scan, so the produced exec's + // originalPlan keeps its logicalLink with no workaround. If conversion declines, the marker + // itself falls back to the vanilla Spark Delta scan, so leaving it in the plan is safe. + case scan if DeltaIntegration.isDeltaScanMarker(scan) => + DeltaIntegration.scanHandler + .flatMap(handler => convertToComet(scan, handler)) + .getOrElse(scan) + // Fully native scan for V1. CometScanExec must always convert to a native scan; the JVM // fallback path has been removed. If conversion fails, fall back to the original Spark scan. case scan: CometScanExec => diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index af68f9530a..1f42c4029c 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -88,14 +88,6 @@ case class CometScanRule(session: SparkSession) case _ => false } - def hasMetadataCol(plan: SparkPlan): Boolean = { - plan.expressions.exists(_.exists { - case a: Attribute => - a.isMetadataCol - case _ => false - }) - } - def isIcebergMetadataTable(scanExec: BatchScanExec): Boolean = { // List of Iceberg metadata tables: // https://iceberg.apache.org/docs/latest/spark-queries/#inspecting-tables @@ -127,12 +119,15 @@ case class CometScanRule(session: SparkSession) case scan if !CometConf.COMET_NATIVE_SCAN_ENABLED.get(conf) => withFallbackReason(scan, "Comet Scan is not enabled") - case scan if hasMetadataCol(scan) => - withFallbackReason(scan, "Metadata column is not supported") - - // data source V1 + // V1 scans go through `transformV1Scan` which itself first delegates to any + // available V1 contrib (today: Delta) and only then applies generic Comet + // bailouts like the metadata-column rejection. This keeps the metadata-col + // guard in place for V2 and non-contrib V1 paths without referencing any + // specific contrib class from this outer match. case scanExec: FileSourceScanExec => transformV1Scan(fullPlan, scanExec) + case scan if hasMetadataCol(scan) => + withFallbackReason(scan, "Metadata column is not supported") // data source V2 case scanExec: BatchScanExec => @@ -148,6 +143,14 @@ case class CometScanRule(session: SparkSession) } } + /** True when any expression in `plan` references a metadata column. */ + private def hasMetadataCol(plan: SparkPlan): Boolean = { + plan.expressions.exists(_.exists { + case a: Attribute => a.isMetadataCol + case _ => false + }) + } + private def transformV1Scan(plan: SparkPlan, scanExec: FileSourceScanExec): SparkPlan = { // On Spark 3.4, injectQueryStageOptimizerRule is unavailable, so @@ -165,9 +168,27 @@ case class CometScanRule(session: SparkSession) scanExec.relation match { case r: HadoopFsRelation => + // Try the optional Delta contrib first. When this build wasn't compiled with + // `-Pcontrib-delta`, the bridge returns None and we fall through to the + // vanilla scan path. When the Delta classes are on the classpath, the contrib + // either claims the scan (returning a CometScanExec marker) or declines via + // its own `withFallbackReason` fallback message. + DeltaIntegration.transformV1IfDelta(plan, session, scanExec, r) match { + case Some(handled) => return handled + case None => // proceed with vanilla logic + } + // Metadata-col bailout moved here so V1 contribs (Delta) get first crack + // at scans with synthetic metadata columns before generic Comet rejects + // them. For non-contrib V1 scans this is equivalent to the outer check. + if (hasMetadataCol(scanExec)) { + return withFallbackReason(scanExec, "Metadata column is not supported") + } if (!CometScanExec.isFileFormatSupported(r.fileFormat)) { return withFallbackReason(scanExec, s"Unsupported file format ${r.fileFormat}") } + // NOTE: the object_store scheme gate lives in `nativeScan` (below), shared with the + // non-contrib path. The Delta delegation above runs before it, so contrib scans are + // unaffected; vanilla V1 scans hit the gate when this method calls `nativeScan`. val hadoopConf = r.sparkSession.sessionState.newHadoopConfWithOptions(r.options) // TODO is this restriction valid for all native scan types? diff --git a/spark/src/main/scala/org/apache/comet/rules/DeltaIntegration.scala b/spark/src/main/scala/org/apache/comet/rules/DeltaIntegration.scala new file mode 100644 index 0000000000..12c4fce6b8 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/rules/DeltaIntegration.scala @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.rules + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} +import org.apache.spark.sql.execution.datasources.HadoopFsRelation + +import org.apache.comet.serde.CometOperatorSerde +import org.apache.comet.util.ClassLoaders + +/** + * Reflection-based bridge to the optional `contrib/delta/` integration. + * + * On default builds the contrib classes don't exist on the classpath, so the reflective class + * lookups fail and every method here returns the "not handled" sentinel. On builds compiled with + * `-Pcontrib-delta` (Maven) + `--features contrib-delta` (Cargo), the contrib classes are present + * and the lookups resolve, dispatching the call into the contrib helpers. + * + * Keeping this bridge as one small file in core lets the Delta detection block in `CometScanRule` + * and the serde dispatch in `CometExecRule` stay ~10 lines each -- exactly the shape Parth's + * review on #4339 asked for. + * + * No `SPI`, no `ServiceLoader`, no registry: the contrib provides its own static helper objects + * with stable names; this bridge just calls them. + */ +object DeltaIntegration extends org.apache.spark.internal.Logging { + + // Scala compiles `object Foo` into BOTH `Foo.class` (a static-forwarder class) AND + // `Foo$.class` (the actual module class). Only the latter has the `MODULE$` singleton + // field that the reflection bridge dereferences. Looking up the unqualified name + // returns the forwarder, where `getField("MODULE$")` throws -- and the surrounding + // try/catch silently turns that into `None`, making every Delta scan fall through to + // Spark's reader. (This bug shipped silently until the test suite caught it; the Delta + // regression suite was passing because Delta's own tests don't depend on Comet + // engaging.) The trailing `$` selects the module class explicitly. + private val ScanRuleClass = "org.apache.comet.contrib.delta.DeltaScanRule$" + private val SerdeClass = "org.apache.comet.contrib.delta.CometDeltaNativeScan$" + + // Fully-qualified name of the contrib's planning-time marker (CometDeltaScanMarker), matched by + // class name so core carries no compile-time dependency on the contrib. CometExecRule routes any + // node of this type to the Delta serde handler. Type-based dispatch survives node copies / AQE + // re-planning (a TreeNodeTag would not), and the marker wraps the original, link-bearing scan as + // the produced exec's originalPlan -- so no logicalLink workaround is needed (mirrors Iceberg's + // CometBatchScanExec + nativeIcebergScanMetadata field). + private val MarkerClass = "org.apache.spark.sql.comet.CometDeltaScanMarker" + + /** True when `plan` is the Delta contrib's scan marker. */ + def isDeltaScanMarker(plan: SparkPlan): Boolean = plan.getClass.getName == MarkerClass + + // Lazy class lookups -- single reflection cost per JVM, cached either as the + // class handle or as the empty option if the contrib wasn't bundled. + @volatile private var scanRuleLookup: Option[Option[Class[AnyRef]]] = None + @volatile private var serdeLookup: Option[Option[Class[AnyRef]]] = None + + private def scanRuleCls: Option[Class[AnyRef]] = + scanRuleLookup.getOrElse { + // ClassLoaders.loadClass resolves via Spark's thread-context ClassLoader first (where + // user/vendor jars are wired), matching IcebergReflection's convention, then caches the + // result so the lookup is a single reflection cost per JVM. + val cls = + try { + Some(ClassLoaders.loadClass(ScanRuleClass).asInstanceOf[Class[AnyRef]]) + } catch { case _: ClassNotFoundException => None } + scanRuleLookup = Some(cls) + cls + } + + private def serdeCls: Option[Class[AnyRef]] = + serdeLookup.getOrElse { + val cls = + try { + Some(ClassLoaders.loadClass(SerdeClass).asInstanceOf[Class[AnyRef]]) + } catch { case _: ClassNotFoundException => None } + serdeLookup = Some(cls) + cls + } + + /** True when the Delta contrib was bundled into this build. */ + def isAvailable: Boolean = scanRuleCls.isDefined + + /** + * Delegate the V1 scan transform to the Delta contrib when both (a) the contrib is on the + * classpath, AND (b) the relation's file format is `DeltaParquetFileFormat`. + * + * Returns `Some(plan)` if the contrib handled the scan (either with a transformed + * `CometScanExec` marker or by explicitly declining via the `withFallbackReason` path); `None` + * to indicate "not a Delta scan, proceed with the vanilla CometScanRule path". + */ + // Cached reflective binding: resolved once per JVM. The contrib's + // `transformV1IfDelta` is invoked for every V1 scan in every plan, even + // non-Delta ones; resolving the Method on each call would be a per-scan + // reflection round-trip just to find we don't apply. + @volatile private var transformV1IfDeltaBindingCache + : Option[Option[(AnyRef, java.lang.reflect.Method)]] = None + + private def transformV1IfDeltaBinding: Option[(AnyRef, java.lang.reflect.Method)] = + transformV1IfDeltaBindingCache.getOrElse { + val binding = scanRuleCls.flatMap { cls => + try { + val module = cls.getField("MODULE$").get(null) + val m = cls.getMethod( + "transformV1IfDelta", + classOf[SparkPlan], + classOf[SparkSession], + classOf[FileSourceScanExec], + classOf[HadoopFsRelation]) + Some((module, m)) + } catch { + // Only swallow true reflection-binding failures (signature/access drift). + // Other exceptions (linkage errors, init failures) should surface. + case _: NoSuchMethodException | _: NoSuchFieldException | _: IllegalAccessException => + None + } + } + transformV1IfDeltaBindingCache = Some(binding) + binding + } + + def transformV1IfDelta( + plan: SparkPlan, + session: SparkSession, + scanExec: FileSourceScanExec, + relation: HadoopFsRelation): Option[SparkPlan] = { + transformV1IfDeltaBinding.flatMap { case (module, m) => + try { + Option(m.invoke(module, plan, session, scanExec, relation)) + .map(_.asInstanceOf[Option[SparkPlan]]) + .flatten + } catch { + // Reflection invocation errors (signature drift / access) -> silent None. + case _: IllegalAccessException | _: IllegalArgumentException => + None + // The contrib's transform threw a real exception -- DON'T silently swallow; + // surface it as a log warning + decline. Without this, kernel-rs IO errors, + // CCE on a Delta version bump, NPE in the CM-id translator etc. would + // silently fall back to vanilla and the user would never know Comet declined. + case e: java.lang.reflect.InvocationTargetException => + logWarning( + "CometDeltaNativeScan.transformV1IfDelta threw, declining to vanilla Delta " + + "for this scan", + Option(e.getCause).getOrElse(e)) + None + } + } + } + + // Change Data Feed support (`isCdfRelation` / `transformCdf` + their reflective binding) lands + // with the CDF unit that also wires the `readChangeFeed` hook into CometExecRule -- shipping it + // here would be dead code with no caller. + + // Cached Delta scan handler, resolved once per JVM (mirrors the binding caches above): the + // contrib's serde companion is looked up reflectively and the (`None` vs handler) result is + // memoised so CometExecRule pays no reflective field round-trip per Delta marker node. + @volatile private var scanHandlerCache: Option[Option[CometOperatorSerde[_]]] = None + + /** + * The Delta scan handler, resolved via reflection from the contrib's `CometDeltaNativeScan` + * companion object. Returns `None` when the contrib isn't bundled into this build. + * `CometExecRule` calls this and passes the result through the standard `convertToComet(scan, + * handler)` path so the Delta scan flows through the same code as `CometNativeScan` etc. + */ + def scanHandler: Option[CometOperatorSerde[_]] = + scanHandlerCache.getOrElse { + // Annotate the result type explicitly: Scala 2.12 (Spark 3.4 builds) cannot infer the + // existential `CometOperatorSerde[_]` as `flatMap`'s type parameter without an expected + // type here (2.13 can). The original un-cached `def` got it from the method return type. + val handler: Option[CometOperatorSerde[_]] = serdeCls.flatMap { cls => + try { + val module = cls.getField("MODULE$").get(null) + Some(module.asInstanceOf[CometOperatorSerde[_]]) + } catch { + // Only swallow true reflection-binding failures. A ClassCastException here + // would mean the contrib's serde object doesn't implement the expected trait + // -- a real bug worth surfacing. + case _: NoSuchFieldException | _: IllegalAccessException => None + } + } + scanHandlerCache = Some(handler) + handler + } +}