diff --git a/.github/workflows/delta_build_gate.yml b/.github/workflows/delta_build_gate.yml new file mode 100644 index 0000000000..163b1af344 --- /dev/null +++ b/.github/workflows/delta_build_gate.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build-gate verification for the optional contrib/delta integration. +# +# A single cheap job that runs dev/verify-contrib-delta-gate.sh, which proves: +# - the DEFAULT cargo / mvn / dylib build carries ZERO Delta surface +# (no comet-contrib-delta / delta_kernel in the cargo tree, no io.delta in +# the effective pom, no Delta symbols in libcomet); and +# - the gated build (`--features contrib-delta` / `-Pcontrib-delta`) compiles +# and pulls the right delta-spark per Spark profile. +# +# The full contrib Scala suite + regression workflows land later (A.6a/A.6b); +# this minimal gate is all the build-gate + inert-wiring unit needs. + +name: Delta Contrib Build Gate + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + branches: + - main + paths-ignore: + - "benchmarks/**" + - "doc/**" + - "docs/**" + - "**.md" + - "dev/changelog/*.md" + - "native/core/benches/**" + - "native/spark-expr/benches/**" + - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala" + - "spark-integration/**" + pull_request: + paths-ignore: + - "benchmarks/**" + - "doc/**" + - "docs/**" + - "**.md" + - "dev/changelog/*.md" + - "native/core/benches/**" + - "native/spark-expr/benches/**" + - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala" + - "spark-integration/**" + workflow_dispatch: + +permissions: + contents: read + +env: + RUST_VERSION: stable + RUST_BACKTRACE: 1 + # Force GNU ld on Linux to match the rest of Comet's CI (rust-lld can't + # resolve -ljvm against the Zulu JDK layout installed by setup-java). + RUSTFLAGS: "-Clink-arg=-fuse-ld=bfd" + +jobs: + delta-build-gate: + name: Build-gate verification (default builds exclude Delta) + runs-on: ubuntu-24.04 + container: + image: amd64/rust + env: + SPARK_LOCAL_IP: localhost + steps: + - uses: actions/checkout@v6 + + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.RUST_VERSION }} + jdk-version: 17 + + - name: Run dev/verify-contrib-delta-gate.sh + run: | + dev/verify-contrib-delta-gate.sh diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 174eae89c6..5aa9241614 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -358,6 +358,7 @@ jobs: org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite org.apache.spark.sql.comet.PlanDataInjectorSuite org.apache.spark.sql.comet.CometDecimalArithmeticViewSuite + org.apache.spark.sql.comet.CometScanWithPlanDataSuite org.apache.spark.sql.comet.util.UtilsSuite org.apache.comet.objectstore.NativeConfigSuite org.apache.spark.sql.CometToPrettyStringSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 2294e00363..b499ac2c87 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -174,6 +174,7 @@ jobs: org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite org.apache.spark.sql.comet.PlanDataInjectorSuite org.apache.spark.sql.comet.CometDecimalArithmeticViewSuite + org.apache.spark.sql.comet.CometScanWithPlanDataSuite org.apache.spark.sql.comet.util.UtilsSuite org.apache.comet.objectstore.NativeConfigSuite org.apache.spark.sql.CometToPrettyStringSuite diff --git a/contrib/delta/native/Cargo.lock b/contrib/delta/native/Cargo.lock new file mode 100644 index 0000000000..5796639553 --- /dev/null +++ b/contrib/delta/native/Cargo.lock @@ -0,0 +1,4205 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" + +[[package]] +name = "arrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", +] + +[[package]] +name = "arrow-json" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +dependencies = [ + "bitflags", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comet-contrib-delta" +version = "0.18.0" +dependencies = [ + "arrow", + "datafusion", + "datafusion-comet-jni-bridge", + "datafusion-comet-proto", + "delta_kernel", + "jni", + "log", + "object_store", + "prost", + "serde_json", + "tempfile", + "thiserror", + "url", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet", + "rand 0.9.4", + "regex", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion-catalog" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools", + "log", + "object_store", +] + +[[package]] +name = "datafusion-comet-common" +version = "0.18.0" +dependencies = [ + "arrow", + "datafusion", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "datafusion-comet-jni-bridge" +version = "0.18.0" +dependencies = [ + "arrow", + "datafusion", + "datafusion-comet-common", + "jni", + "lazy_static", + "once_cell", + "parquet", + "paste", + "prost", + "regex", + "thiserror", +] + +[[package]] +name = "datafusion-comet-proto" +version = "0.18.0" +dependencies = [ + "prost", + "prost-build", +] + +[[package]] +name = "datafusion-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "chrono", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "libc", + "log", + "object_store", + "parquet", + "paste", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "glob", + "itertools", + "log", + "object_store", + "rand 0.9.4", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "serde_json", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" + +[[package]] +name = "datafusion-execution" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.4", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "itertools", + "paste", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap", + "itertools", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "hex", + "itertools", + "log", + "md-5", + "memchr", + "num-traits", + "rand 0.9.4", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "num-traits", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.16.1", + "itertools", + "itoa", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +dependencies = [ + "datafusion-doc", + "quote", + "syn", +] + +[[package]] +name = "datafusion-optimizer" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap", + "itertools", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "parking_lot", + "paste", + "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +dependencies = [ + "ahash", + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "parking_lot", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools", +] + +[[package]] +name = "datafusion-physical-plan" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +dependencies = [ + "ahash", + "arrow", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-pruning" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools", + "log", +] + +[[package]] +name = "datafusion-session" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +dependencies = [ + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap", + "log", + "regex", + "sqlparser", +] + +[[package]] +name = "delta_kernel" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bbaa9c975772c718e55ed1e502976a25d353a290b46b00ad1514751514be46d" +dependencies = [ + "arrow", + "bytes", + "chrono", + "crc", + "delta_kernel_derive", + "futures", + "indexmap", + "itertools", + "object_store", + "parquet", + "percent-encoding", + "rand 0.9.4", + "reqwest 0.13.4", + "roaring", + "rustc_version", + "serde", + "serde_json", + "strum", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "url", + "uuid", + "z85", +] + +[[package]] +name = "delta_kernel_derive" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "797369f8300adc409361bba541ed868a7cade2637691a22af521ad0909be2454" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.1", + "reqwest 0.12.28", + "ring", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store", + "parquet-variant", + "parquet-variant-compute", + "parquet-variant-json", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet-variant" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74c8db065291f088a2aad8ab831853eae1871c0d311c8d0b83bbc3b7e735d0fc" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap", + "num-traits", + "simdutf8", + "uuid", +] + +[[package]] +name = "parquet-variant-compute" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a530e8d5b5e14efcb39c9a6ec55432ad11f6afb7dc4455a79be0dc615fe3cc31" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap", + "parquet-variant", + "parquet-variant-json", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet-variant-json" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00ed89908289f67caa2ca078f9ff9aacd6229a313ec92b12bf4f48f613dc2b97" +dependencies = [ + "arrow-schema", + "base64", + "chrono", + "parquet-variant", + "serde_json", + "uuid", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sqlparser" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "rand 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "z85" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e61e59a957b7ccee15d2049f86e8bfd6f66968fcd88f018950662d9b86e675" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/contrib/delta/native/Cargo.toml b/contrib/delta/native/Cargo.toml new file mode 100644 index 0000000000..b3f150f5cf --- /dev/null +++ b/contrib/delta/native/Cargo.toml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Standalone Cargo.toml -- this crate is outside the `native/` workspace root +# (`native/Cargo.toml` lists `exclude = ["../contrib"]`), so it cannot use +# `{ workspace = true }` inheritance. Versions are kept in sync with the rest of the +# repo by convention; the path deps anchor against the same on-disk crates. +# +# This unit ships the DRIVER side (log replay, predicate pushdown, scan-task assembly, +# JNI). It declares only the deps the driver modules (error/engine/predicate/scan/jni) +# actually use; the executor-side read path lands in the next unit and brings its own +# deps (parquet, roaring, datafusion-datasource, futures, chrono*, comet-common, ...). +# `planner` is still the build-gate stub until then. + +[package] +name = "comet-contrib-delta" +description = "delta-kernel-rs integration for Comet. Reads Delta tables via kernel-rs's log replay and DataFusion's parquet scan. Linked into libcomet via core's `contrib-delta` Cargo feature flag." +version = "0.18.0" +edition = "2021" +rust-version = "1.86.0" +publish = false +license = "Apache-2.0" + +[lib] +# rlib: linked INTO `libcomet` when `contrib-delta` is enabled on core. Never a cdylib +# on its own -- there's no separate Delta library to ship. +crate-type = ["rlib"] + +[dependencies] +# Typed Delta proto messages live in core's proto crate (alongside IcebergScan, ...) so the +# dispatcher arm has direct access. Re-exported as `crate::proto::*`. +datafusion-comet-proto = { path = "../../../native/proto" } +# JNI helpers (CometError, CometResult, try_unwrap_or_throw) for the driver-side JNI entry +# points. jni-bridge is a leaf crate -- depending on it drags no Comet logic into the contrib. +datafusion-comet-jni-bridge = { path = "../../../native/jni-bridge" } +# Heavy Delta dep -- intentionally lives ONLY in this contrib, never in core. delta_kernel +# 0.24 pins the SAME arrow (58) as Comet via the `arrow-58` feature, so kernel RecordBatches +# flow straight into Comet plans with no arrow-version bridge. +delta_kernel = { version = "0.24", default-features = false, features = ["default-engine-rustls", "arrow-58"] } +# DataFusion / Arrow versions chosen to match core's pinned values. The driver only needs +# `ExecutionPlan` / `SchemaRef` / `DataFusionError` (the planner stub) -- no `parquet` feature +# here; the executor unit, which actually scans parquet, adds it back. +datafusion = { version = "53.1.0", default-features = false } +arrow = { version = "58.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } +# Match core Comet's object_store features (native/Cargo.toml) so the Azure/GCS builders +# reachable via `object_store::parse_url` mirror core's non-S3 read path. +object_store = { version = "0.13.1", features = ["aws", "azure", "gcp", "http"] } +url = "2.5.4" +thiserror = "2" +prost = "0.14.3" +# Parse the analysis-time Delta schema JSON (shipped from the JVM) into a kernel `StructType` +# for `ScanBuilder::with_schema`, so kernel resolves column-mapping physical names against the +# schema the query was PLANNED with (correct for schema-change-since-analysis). +serde_json = "1" +jni = "0.22.4" +log = "0.4" + +[dev-dependencies] +# Used by unit tests under #[cfg(test)] in scan.rs to materialise a Delta `_delta_log` +# in a tempdir without polluting the real filesystem. +tempfile = "3" diff --git a/contrib/delta/native/src/engine.rs b/contrib/delta/native/src/engine.rs new file mode 100644 index 0000000000..910ee53cb9 --- /dev/null +++ b/contrib/delta/native/src/engine.rs @@ -0,0 +1,598 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Construction of a delta-kernel-rs `DefaultEngine` backed by `object_store`. +//! +//! Ported from tantivy4java's `delta_reader/engine.rs` (Apache-2.0) with +//! minor changes: uses Comet's error type instead of `anyhow`, and uses the +//! `object_store` 0.13 dependency that kernel +//! requires. Comet's main `object_store = "0.13"` tree is untouched. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; +use url::Url; + +use delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor; +use delta_kernel::engine::default::DefaultEngine; +use object_store::aws::AmazonS3Builder; +use object_store::local::LocalFileSystem; +use object_store::ObjectStore; + +use super::error::{DeltaError, DeltaResult}; + +/// Concrete engine type returned by [`get_or_create_engine`]. +pub type DeltaEngine = DefaultEngine; + +/// S3 credentials used to construct kernel's engine. +/// +/// Only S3 is field-per-knob: like core Comet (`parquet_support::prepare_object_store_with_configs`), +/// the Hadoop `fs.s3a.*` keys (incl. per-bucket overrides) are bridged explicitly because +/// `object_store::parse_url` cannot read them. Azure / GCS / other schemes are built straight +/// through `object_store::parse_url`, which sources credentials from the ambient environment +/// (`AZURE_*` / `GOOGLE_*` / ADC / instance metadata) -- matching core's non-S3 path -- so they +/// need no fields here. +#[derive(Clone, Default, Hash, PartialEq, Eq)] +pub struct DeltaStorageConfig { + pub aws_access_key: Option, + pub aws_secret_key: Option, + pub aws_session_token: Option, + pub aws_region: Option, + pub aws_endpoint: Option, + pub aws_force_path_style: bool, + // Azure (abfs/abfss/wasb/wasbs/az). Bridged from `fs.azure.*` so the native reader uses the + // SAME credentials Spark would, instead of falling back to ambient `AZURE_*` env on executors. + pub azure_account_name: Option, + pub azure_account_key: Option, + pub azure_sas_token: Option, + // GCS (gs/gcs). Bridged from `fs.gs.*`. + pub gcs_service_account_path: Option, + pub gcs_service_account_key: Option, +} + +impl DeltaStorageConfig { + /// `object_store`-style config key/value pairs for the Azure store, derived from the bridged + /// Hadoop creds. Empty when nothing was bridged -- the caller then falls back to ambient creds. + /// (`object_store` reads the account/container from the abfss/wasb URL authority; for the + /// `az://` scheme the account name is supplied here when known.) + pub fn azure_object_store_options(&self) -> Vec<(&'static str, String)> { + let mut o = Vec::new(); + if let Some(v) = &self.azure_account_name { + o.push(("azure_storage_account_name", v.clone())); + } + if let Some(v) = &self.azure_account_key { + o.push(("azure_storage_account_key", v.clone())); + } + if let Some(v) = &self.azure_sas_token { + o.push(("azure_storage_sas_key", v.clone())); + } + o + } + + /// `object_store`-style config key/value pairs for the GCS store. Empty = fall back to ambient. + pub fn gcs_object_store_options(&self) -> Vec<(&'static str, String)> { + let mut o = Vec::new(); + if let Some(v) = &self.gcs_service_account_path { + o.push(("google_service_account", v.clone())); + } + if let Some(v) = &self.gcs_service_account_key { + o.push(("google_service_account_key", v.clone())); + } + o + } +} + +// Hand-written `Debug` (NOT derived) so a stray `{:?}` -- a future debug log, an error wrapper -- +// can never leak credential material. Secret fields render as `` / `None`; non-secret fields +// (region, endpoint, account name) stay visible for diagnosability. Defense-in-depth: nothing on +// the read path Debug-prints this today. +impl std::fmt::Debug for DeltaStorageConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn redact(o: &Option) -> &'static str { + if o.is_some() { + "" + } else { + "None" + } + } + f.debug_struct("DeltaStorageConfig") + .field("aws_access_key", &redact(&self.aws_access_key)) + .field("aws_secret_key", &redact(&self.aws_secret_key)) + .field("aws_session_token", &redact(&self.aws_session_token)) + .field("aws_region", &self.aws_region) + .field("aws_endpoint", &self.aws_endpoint) + .field("aws_force_path_style", &self.aws_force_path_style) + .field("azure_account_name", &self.azure_account_name) + .field("azure_account_key", &redact(&self.azure_account_key)) + .field("azure_sas_token", &redact(&self.azure_sas_token)) + .field("gcs_service_account_path", &self.gcs_service_account_path) + .field("gcs_service_account_key", &redact(&self.gcs_service_account_key)) + .finish() + } +} + +/// Build an `ObjectStore` for the given URL and credentials. +/// +/// `s3://` / `s3a://` are built from the bridged `fs.s3a.*` config; `file://` is local. +/// Azure (`az` / `azure` / `abfs` / `abfss` / `wasb` / `wasbs`) and GCS (`gs` / `gcs`) are +/// built via `object_store::parse_url` (ambient/env credentials), mirroring core Comet's +/// non-S3 read path. Any other scheme is rejected with [`DeltaError::UnsupportedScheme`]. +pub fn create_object_store( + url: &Url, + config: &DeltaStorageConfig, +) -> DeltaResult> { + let scheme = url.scheme(); + + let store: Arc = match scheme { + "s3" | "s3a" => { + let bucket = url.host_str().ok_or_else(|| DeltaError::MissingBucket { + url: url.to_string(), + })?; + let mut builder = AmazonS3Builder::new().with_bucket_name(bucket); + + if let Some(ref key) = config.aws_access_key { + builder = builder.with_access_key_id(key); + } + if let Some(ref secret) = config.aws_secret_key { + builder = builder.with_secret_access_key(secret); + } + if let Some(ref token) = config.aws_session_token { + builder = builder.with_token(token); + } + if let Some(ref region) = config.aws_region { + builder = builder.with_region(region); + } + if let Some(ref endpoint) = config.aws_endpoint { + builder = builder.with_endpoint(endpoint); + } + if config.aws_force_path_style { + builder = builder.with_virtual_hosted_style_request(false); + } + // Allow HTTP endpoints (MinIO, LocalStack, custom S3-compat) + if config + .aws_endpoint + .as_ref() + .is_some_and(|e| e.starts_with("http://")) + { + builder = builder.with_allow_http(true); + } + + Arc::new(builder.build()?) + } + "az" | "azure" | "abfs" | "abfss" | "wasb" | "wasbs" => { + // Build the Azure store from the credentials Spark bridged from `fs.azure.*` + // (account key / SAS) so executors use the SAME identity Spark would, instead of + // ambient `AZURE_*` env / managed identity. `object_store` reads the account and + // container from the abfss/wasb URL authority. With no bridged creds, fall back to + // `parse_url` (ambient), parity with core's non-S3 read path. + let opts = config.azure_object_store_options(); + let (store, _path) = if opts.is_empty() { + object_store::parse_url(url)? + } else { + object_store::parse_url_opts(url, opts)? + }; + Arc::from(store) + } + "gs" | "gcs" => { + // Build the GCS store from the service account Spark bridged from `fs.gs.*`; with no + // bridged creds, fall back to `parse_url` (ambient `GOOGLE_*` / ADC), parity with core. + let opts = config.gcs_object_store_options(); + let (store, _path) = if opts.is_empty() { + object_store::parse_url(url)? + } else { + object_store::parse_url_opts(url, opts)? + }; + Arc::from(store) + } + "file" | "" => Arc::new(LocalFileSystem::new()), + other => { + return Err(DeltaError::UnsupportedScheme { + scheme: other.to_string(), + url: url.to_string(), + }); + } + }; + + Ok(store) +} + +/// Process-wide cache of constructed engines, keyed by (scheme, authority, config). +/// +/// Each `DefaultEngine` owns a `TokioBackgroundExecutor` which spawns one std::thread +/// running a current_thread tokio runtime; the runtime's blocking pool (used by +/// kernel for parquet/object_store IO) holds spawned threads for `thread_keep_alive` +/// (~10s) after each spawn_blocking call. Constructing a fresh engine per JNI +/// `planDeltaScan` call therefore accumulates OS threads during regression runs that +/// hit kernel hundreds of times per minute, eventually tripping the per-process +/// thread cap (e.g. `pthread_create EAGAIN` aborts on macOS where `ulimit -u` +/// defaults to ~1300). Sharing one engine per (scheme, authority, config) bounds the +/// thread count by table-storage diversity instead of by request count. +/// +/// **LRU-bounded**: the cache holds at most `MAX_CACHE_ENTRIES` engines. When full, +/// the least-recently-used entry is evicted and its `Arc` drops -- +/// `DefaultEngine`'s `TokioBackgroundExecutor` joins its OS thread on drop, so the +/// thread count stabilizes even when long-running drivers rotate credentials (e.g. +/// hourly STS / IRSA rotations on production). Without this bound, every rotation +/// produced a new cache entry (because `DeltaStorageConfig` is part of the key and +/// `aws_session_token` rotates) and leaked one tokio thread per rotation -- a +/// production-grade memory + thread leak over days. +/// +/// `Arc` is handed out so callers don't hold the mutex while using the +/// engine; concurrent in-flight scans against an evicted engine keep it alive until +/// they complete. +const MAX_CACHE_ENTRIES: usize = 32; + +type EngineKey = (String, String, DeltaStorageConfig); + +/// Cache state: maps key to (engine, monotonic last-use counter). The counter is the +/// LRU recency stamp; we bump it on every hit AND on every insert, and evict the +/// entry with the smallest counter when full. +struct EngineCacheState { + map: HashMap, u64)>, + counter: u64, +} + +fn engine_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| { + Mutex::new(EngineCacheState { + map: HashMap::new(), + counter: 0, + }) + }) +} + +fn engine_key(url: &Url, config: &DeltaStorageConfig) -> EngineKey { + let scheme = url.scheme().to_string(); + // host+port form the storage target (e.g. S3 bucket, ABFS account); for file:// + // the authority is empty which collapses every local table to a single entry. + let authority = match (url.host_str(), url.port()) { + (Some(h), Some(p)) => format!("{h}:{p}"), + (Some(h), None) => h.to_string(), + _ => String::new(), + }; + (scheme, authority, config.clone()) +} + +/// Return a shared `DeltaEngine` for the given URL+config, building one on first use. +/// +/// LRU-bounded: when the cache is full, the least-recently-used entry is evicted. +/// In-flight users of an evicted engine keep it alive via their `Arc` clone until +/// they're done; only THEN does the evicted entry's `TokioBackgroundExecutor` join +/// its OS thread. +pub fn get_or_create_engine( + table_url: &Url, + config: &DeltaStorageConfig, +) -> DeltaResult> { + let key = engine_key(table_url, config); + // Mutex is held only across the (cheap) HashMap lookup and, on miss, the engine + // construction. Multi-threaded JNI callers serialize here on first miss per key + // but proceed lock-free on subsequent hits via the returned Arc clone. + let mut cache = engine_cache().lock().unwrap_or_else(|e| e.into_inner()); + cache.counter = cache.counter.wrapping_add(1); + let stamp = cache.counter; + if let Some(entry) = cache.map.get_mut(&key) { + // Hit: bump the LRU stamp and return the existing Arc. + entry.1 = stamp; + return Ok(Arc::clone(&entry.0)); + } + // Miss: build a fresh engine. If the cache is at capacity, evict the LRU entry + // first so the bound is respected. + if cache.map.len() >= MAX_CACHE_ENTRIES { + if let Some(victim_key) = cache + .map + .iter() + .min_by_key(|(_, (_, s))| *s) + .map(|(k, _)| k.clone()) + { + cache.map.remove(&victim_key); + } + } + let store = create_object_store(table_url, config)?; + let engine = Arc::new(DefaultEngine::builder(store).build()); + cache.map.insert(key, (Arc::clone(&engine), stamp)); + Ok(engine) +} + +/// Test-only: clear the cache so tests don't see entries from prior tests. +#[cfg(test)] +pub(crate) fn _clear_cache_for_tests() { + let mut cache = engine_cache().lock().unwrap_or_else(|e| e.into_inner()); + cache.map.clear(); + cache.counter = 0; +} + +#[cfg(test)] +mod tests { + use super::*; + + fn url(s: &str) -> Url { + Url::parse(s).unwrap() + } + + fn empty_config() -> DeltaStorageConfig { + DeltaStorageConfig::default() + } + + #[test] + fn debug_redacts_credential_material() { + // A stray `{:?}` on the storage config (e.g. a future debug log) must NOT leak the + // secret key / session token / access key into logs. Non-secret fields stay visible + // for diagnosability. + let c = DeltaStorageConfig { + aws_access_key: Some("AKIDEXAMPLE".to_string()), + aws_secret_key: Some("SUPERSECRETKEYVALUE".to_string()), + aws_session_token: Some("SESSIONTOKENVALUE".to_string()), + aws_region: Some("us-west-2".to_string()), + aws_endpoint: Some("https://s3.example".to_string()), + aws_force_path_style: true, + ..Default::default() + }; + let s = format!("{c:?}"); + assert!(!s.contains("SUPERSECRETKEYVALUE"), "secret key leaked: {s}"); + assert!(!s.contains("SESSIONTOKENVALUE"), "session token leaked: {s}"); + assert!(!s.contains("AKIDEXAMPLE"), "access key leaked: {s}"); + // Non-secret fields remain visible. + assert!(s.contains("us-west-2"), "region should be visible: {s}"); + assert!(s.contains("s3.example"), "endpoint should be visible: {s}"); + } + + #[test] + fn create_object_store_local_file() { + let store = create_object_store(&url("file:///tmp/x"), &empty_config()).unwrap(); + // Just verify Arc construction succeeded; LocalFileSystem doesn't expose + // anything we can usefully assert on without doing IO. + assert!(format!("{store:?}").contains("LocalFileSystem")); + } + + #[test] + fn create_object_store_empty_scheme_is_local() { + // The "file" | "" arm maps the empty-scheme case (URL like `relative/path` + // wouldn't actually parse, but the arm exists for code paths that hand us + // a Url with an empty scheme). + let mut u = url("file:///x"); + u.set_scheme("").ok(); // best-effort; if it fails, the file:// arm still hits + let store = create_object_store(&u, &empty_config()).unwrap(); + assert!(format!("{store:?}").contains("LocalFileSystem")); + } + + #[test] + fn create_object_store_s3_requires_bucket() { + // `s3://` with empty host is rejected as MissingBucket. + // url::Url::parse("s3:///x") gives host=None. + let bad = url("s3:///just-a-path"); + let err = create_object_store(&bad, &empty_config()).unwrap_err(); + match err { + DeltaError::MissingBucket { .. } => {} + other => panic!("expected MissingBucket, got {other:?}"), + } + } + + #[test] + fn create_object_store_s3_builds_with_full_creds() { + let cfg = DeltaStorageConfig { + aws_access_key: Some("AKIA…".into()), + aws_secret_key: Some("secret".into()), + aws_session_token: Some("token".into()), + aws_region: Some("us-west-2".into()), + aws_endpoint: Some("https://s3.example.com".into()), + aws_force_path_style: true, + ..Default::default() + }; + let store = create_object_store(&url("s3://my-bucket/path"), &cfg).unwrap(); + assert!(format!("{store:?}").contains("AmazonS3") || format!("{store:?}").contains("S3")); + } + + #[test] + fn create_object_store_s3_http_endpoint_allows_http() { + let cfg = DeltaStorageConfig { + aws_access_key: Some("k".into()), + aws_secret_key: Some("s".into()), + aws_endpoint: Some("http://localhost:9000".into()), + aws_force_path_style: true, + ..Default::default() + }; + // MinIO-style: endpoint starts with http:// → builder enables allow_http. + // We can't introspect the builder's flag, but ensuring construction + // succeeds covers the branch. + create_object_store(&url("s3://minio-bucket"), &cfg).unwrap(); + } + + #[test] + fn create_object_store_azure_via_parse_url() { + // Azure schemes are built through object_store::parse_url (parity with core's + // non-S3 path). The account comes from the abfss host; credentials resolve from + // the ambient environment at request time, so construction succeeds with no config. + let u = url("abfss://container@myacct.dfs.core.windows.net/path"); + create_object_store(&u, &empty_config()).expect("azure store builds via parse_url"); + } + + #[test] + fn create_object_store_gcs_via_parse_url() { + // GCS is likewise built through object_store::parse_url; ADC / GOOGLE_* resolve + // lazily, so construction succeeds with no explicit config (was UnsupportedScheme + // before this gained parity with core). + create_object_store(&url("gs://my-bucket/path"), &empty_config()) + .expect("gcs store builds via parse_url"); + } + + #[test] + fn create_object_store_azure_with_explicit_key() { + // With bridged fs.azure.* creds, the store is built via parse_url_opts using the + // explicit account key (not ambient env). Construction is lazy, so no network. + let cfg = DeltaStorageConfig { + azure_account_name: Some("myacct".to_string()), + azure_account_key: Some("dGVzdGtleQ==".to_string()), + ..Default::default() + }; + let u = url("abfss://container@myacct.dfs.core.windows.net/path"); + let store = create_object_store(&u, &cfg).expect("azure store builds with explicit key"); + let dbg = format!("{store:?}").to_lowercase(); + assert!(dbg.contains("azure") || dbg.contains("microsoft"), "got: {dbg}"); + } + + // Note: an engine-level GCS explicit-creds test is omitted because object_store's + // `google_service_account` reads + parses the keyfile at build time, so it can't be + // exercised with a fake path. The bridging is covered by jni::tests::gcs_creds_bridged_from_hadoop + // (extraction -> gcs_object_store_options) and exercised end-to-end against real GCS. + + #[test] + fn create_object_store_unsupported_scheme() { + // A scheme outside the S3 / Azure / GCS / file arms is rejected before reaching + // object_store. `ftp` is never a Delta storage backend. + let err = create_object_store(&url("ftp://host/p"), &empty_config()).unwrap_err(); + match err { + DeltaError::UnsupportedScheme { scheme, .. } => assert_eq!(scheme, "ftp"), + other => panic!("expected UnsupportedScheme, got {other:?}"), + } + } + + #[test] + fn engine_key_collapses_local_paths() { + let cfg = empty_config(); + let a = engine_key(&url("file:///tmp/a"), &cfg); + let b = engine_key(&url("file:///tmp/b/c/d"), &cfg); + assert_eq!(a, b, "all local file:// URLs share one engine entry"); + } + + #[test] + fn engine_key_distinguishes_s3_buckets() { + let cfg = empty_config(); + let a = engine_key(&url("s3://bucket-a/path"), &cfg); + let b = engine_key(&url("s3://bucket-b/path"), &cfg); + assert_ne!(a, b); + } + + #[test] + fn engine_key_includes_port() { + let cfg = empty_config(); + let a = engine_key(&url("s3://host:9000/p"), &cfg); + let b = engine_key(&url("s3://host:9001/p"), &cfg); + let c = engine_key(&url("s3://host/p"), &cfg); + assert_ne!(a, b); + assert_ne!(a, c); + assert_ne!(b, c); + } + + #[test] + fn engine_key_distinguishes_credentials() { + let cfg_a = DeltaStorageConfig { + aws_access_key: Some("AKIA1".into()), + ..Default::default() + }; + let cfg_b = DeltaStorageConfig { + aws_access_key: Some("AKIA2".into()), + ..Default::default() + }; + let a = engine_key(&url("s3://bucket/p"), &cfg_a); + let b = engine_key(&url("s3://bucket/p"), &cfg_b); + assert_ne!(a, b, "different credentials must NOT share a cached engine"); + } + + #[test] + fn engine_key_path_does_not_affect_key() { + let cfg = empty_config(); + let a = engine_key(&url("s3://bucket/path/a"), &cfg); + let b = engine_key(&url("s3://bucket/path/b/c"), &cfg); + assert_eq!(a, b, "paths within the same bucket share one engine"); + } + + #[test] + fn get_or_create_engine_returns_same_arc_on_hit() { + let cfg = empty_config(); + let u = url("file:///tmp/cache-test"); + let e1 = get_or_create_engine(&u, &cfg).unwrap(); + let e2 = get_or_create_engine(&u, &cfg).unwrap(); + assert!( + Arc::ptr_eq(&e1, &e2), + "second call must return the cached Arc, not a fresh engine" + ); + } + + #[test] + fn get_or_create_engine_distinct_keys_yield_distinct_engines() { + let cfg = empty_config(); + let e_file = get_or_create_engine(&url("file:///tmp/distinct-a"), &cfg).unwrap(); + // s3:// would actually try to set up an AWS client; use a different file path + // which collapses to the same key per `engine_key_collapses_local_paths`. So we + // exercise a distinct-key case via a different cred config. + let cfg_b = DeltaStorageConfig { + aws_access_key: Some("dummy".into()), + ..Default::default() + }; + let e_creds = get_or_create_engine(&url("file:///tmp/distinct-a"), &cfg_b).unwrap(); + assert!( + !Arc::ptr_eq(&e_file, &e_creds), + "differing config keys must yield distinct engines" + ); + } + + #[test] + fn get_or_create_engine_evicts_lru_when_full() { + _clear_cache_for_tests(); + // Build MAX_CACHE_ENTRIES distinct engines, each with a distinct credential + // tuple so they all key uniquely against the same local URL. + let urls_and_engines: Vec<(String, Arc)> = (0..MAX_CACHE_ENTRIES) + .map(|i| { + let cfg = DeltaStorageConfig { + aws_access_key: Some(format!("key-{i}")), + ..Default::default() + }; + let url_s = format!("file:///tmp/lru-{i}"); + let eng = get_or_create_engine(&url(&url_s), &cfg).unwrap(); + (format!("key-{i}"), eng) + }) + .collect(); + + // Cache is now exactly full. + assert_eq!( + engine_cache().lock().unwrap().map.len(), + MAX_CACHE_ENTRIES, + "cache should be at capacity after filling it" + ); + + // Touch entry 1 so it becomes most-recently-used. Entry 0 is now LRU. + let cfg_1 = DeltaStorageConfig { + aws_access_key: Some(urls_and_engines[1].0.clone()), + ..Default::default() + }; + let _hit_1 = get_or_create_engine(&url("file:///tmp/lru-1"), &cfg_1).unwrap(); + + // Insert one more -- entry 0 (LRU after the touch) should be evicted. + let cfg_new = DeltaStorageConfig { + aws_access_key: Some("key-new".into()), + ..Default::default() + }; + let _new = get_or_create_engine(&url("file:///tmp/lru-new"), &cfg_new).unwrap(); + + assert_eq!( + engine_cache().lock().unwrap().map.len(), + MAX_CACHE_ENTRIES, + "cache size should stay at capacity after eviction" + ); + + // Hitting key-0 again should construct a fresh engine (not return the original Arc). + let cfg_0 = DeltaStorageConfig { + aws_access_key: Some(urls_and_engines[0].0.clone()), + ..Default::default() + }; + let fresh_0 = get_or_create_engine(&url("file:///tmp/lru-0"), &cfg_0).unwrap(); + assert!( + !Arc::ptr_eq(&urls_and_engines[0].1, &fresh_0), + "key-0 was LRU and should have been evicted -> fresh engine on re-insert" + ); + } +} diff --git a/contrib/delta/native/src/error.rs b/contrib/delta/native/src/error.rs new file mode 100644 index 0000000000..6120daf6fa --- /dev/null +++ b/contrib/delta/native/src/error.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Error types for the delta module. +//! +//! Kept local rather than folded into `CometError` because `delta_kernel` +//! lives in an isolated dep subtree — we don't want kernel's error type +//! leaking into `errors.rs` and coupling the main error path to kernel. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum DeltaError { + #[error("invalid delta table URL '{url}': {source}")] + InvalidUrl { + url: String, + #[source] + source: url::ParseError, + }, + + #[error("cannot resolve local path '{path}': {source}")] + PathResolution { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("cannot convert path to URL: {path}")] + PathToUrl { path: String }, + + #[error("unsupported URL scheme '{scheme}' for delta table: {url}")] + UnsupportedScheme { scheme: String, url: String }, + + #[error("missing bucket/container in URL: {url}")] + MissingBucket { url: String }, + + #[error("object store construction failed: {0}")] + ObjectStore(#[from] object_store::Error), + + #[error("delta kernel error: {0}")] + Kernel(#[from] delta_kernel::Error), + + #[error("{0}")] + Internal(String), +} + +pub type DeltaResult = std::result::Result; diff --git a/contrib/delta/native/src/jni.rs b/contrib/delta/native/src/jni.rs new file mode 100644 index 0000000000..d14f20f445 --- /dev/null +++ b/contrib/delta/native/src/jni.rs @@ -0,0 +1,739 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Driver-side JNI entry point for Delta log replay. +//! +//! Exposes `Java_org_apache_comet_contrib_delta_Native_planDeltaScan`. The Scala driver +//! calls this once per query to ask kernel for the active file list at a +//! given snapshot version, then distributes the returned tasks across +//! Spark executors via Comet's usual split-mode serialization. + +use jni::{ + objects::{JByteArray, JClass, JMap, JObject, JString}, + sys::{jbyteArray, jlong}, + Env, EnvUnowned, +}; +use prost::Message; + +use crate::scan::plan_delta_scan_with_predicate; +use crate::DeltaStorageConfig; +use datafusion_comet_jni_bridge::errors::{try_unwrap_or_throw, CometError, CometResult}; +// Proto types now live in this contrib's own proto module (was core's +// datafusion_comet_proto::spark_operator). +use crate::proto::{DeltaPartitionValue, DeltaScanTask, DeltaScanTaskList}; + +/// `Java_org_apache_comet_contrib_delta_Native_planDeltaScan`. +/// +/// # Arguments (JNI wire order) +/// 1. `table_url` — absolute URL or bare path of the Delta table root +/// 2. `snapshot_version` — `-1` for latest, otherwise the exact version +/// 3. `storage_options` — a `java.util.Map` of cloud +/// credentials. **Phase 1 currently only consumes a small subset** (the +/// AWS / Azure keys listed in `DeltaStorageConfig`); unknown keys are +/// silently ignored. Full options-map plumbing lands with Phase 2. +/// +/// # Returns +/// A Java `byte[]` containing a prost-encoded [`DeltaScanTaskList`] +/// message, or `null` on error (with a `CometNativeException` thrown on +/// the JVM side via `try_unwrap_or_throw`). +/// +/// # Safety +/// Inherently unsafe because it dereferences raw JNI pointers. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDeltaScan( + e: EnvUnowned, + _class: JClass, + table_url: JString, + snapshot_version: jlong, + storage_options: JObject, + predicate_bytes: JByteArray, + column_names: jni::objects::JObjectArray, + // The query's data-read schema as Delta schema JSON (`StructType.json`, carrying + // `delta.columnMapping.physicalName` + `id` from the analysis-time or snapshot schema). Drives + // `scan.with_schema(...)` so kernel resolves the physical names the query was PLANNED with + // (correct under schema-change-since-analysis). Empty => full-table scan, no kernel schemas. + projected_schema_json: JString, +) -> jbyteArray { + try_unwrap_or_throw(&e, |env| { + let url_str: String = table_url.try_to_string(env)?; + let version = if snapshot_version < 0 { + None + } else { + Some(snapshot_version as u64) + }; + // S3 bucket (URL host) for per-bucket credential resolution; None for non-S3. + let s3_bucket = url::Url::parse(&url_str) + .ok() + .filter(|u| matches!(u.scheme(), "s3" | "s3a")) + .and_then(|u| u.host_str().map(|h| h.to_string())); + let config = if storage_options.is_null() { + DeltaStorageConfig::default() + } else { + let jmap: JMap<'_> = env.cast_local::(storage_options)?; + // S3 static keys (global + per-bucket) are bridged here; Azure / GCS go + // through object_store::parse_url + ambient creds in `create_object_store`. + // Residual (08-known-limitations.md A2e): Hadoop's explicit S3 + // credential-provider classes (`fs.s3a.aws.credentials.provider` = + // AssumedRole / WebIdentity / ...) are not honored -- object_store's own + // default chain (static keys here, else IMDS / ECS / env) is used instead. + extract_storage_config(env, &jmap, s3_bucket.as_deref())? + }; + + // Phase 2: read column names for BoundReference resolution. + // storageOptions map carries Hadoop-style keys (fs.s3a.access.key, + // fs.s3a.secret.key, fs.s3a.endpoint, fs.s3a.path.style.access, + // fs.s3a.endpoint.region, fs.s3a.session.token) extracted by + // NativeConfig.extractObjectStoreOptions on the Scala side. + // extract_storage_config below maps these to kernel's DeltaStorageConfig. + let col_names = read_string_array(env, &column_names)?; + + // Phase 2: deserialize the Catalyst predicate (if provided) for + // kernel's stats-based file pruning. Empty bytes = no predicate. + let _predicate_proto: Option> = if predicate_bytes.is_null() { + None + } else { + let bytes = env.convert_byte_array(predicate_bytes)?; + if bytes.is_empty() { + None + } else { + Some(bytes) + } + }; + + // Phase 2: translate Catalyst predicate proto to kernel Predicate for + // stats-based file pruning during log replay. Pass column names for + // BoundReference index-to-name resolution. + let kernel_predicate = _predicate_proto.and_then(|bytes| { + use prost::Message; + match datafusion_comet_proto::spark_expression::Expr::decode(bytes.as_slice()) { + Ok(expr) => Some(crate::predicate::catalyst_to_kernel_predicate_with_names( + &expr, &col_names, + )), + Err(e) => { + log::warn!( + "Failed to decode predicate for Delta file pruning: {e}; \ + scanning all files" + ); + None + } + } + }); + + // The data-read schema (Delta JSON) for kernel's `with_schema`. Absent => full-table scan. + let projected_schema_json = decode_jstring(env, &projected_schema_json)?; + + let plan = plan_delta_scan_with_predicate( + &url_str, + &config, + version, + kernel_predicate, + projected_schema_json, + ) + .map_err(|e| CometError::Internal(format!("delta_kernel log replay failed: {e}")))?; + + // Under column mapping, kernel returns partition_values keyed by the + // PHYSICAL column name (e.g. `col-`), but `partition_schema` + // (and therefore `build_delta_partitioned_files`'s lookup) uses the + // LOGICAL name. Build the inverse lookup so we can translate keys + // back to logical names on the wire. + let physical_to_logical: std::collections::HashMap = plan + .column_mappings + .iter() + .map(|(logical, physical)| (physical.clone(), logical.clone())) + .collect(); + + let tasks: Vec = plan + .entries + .into_iter() + .map(|entry| DeltaScanTask { + file_path: resolve_file_path(&url_str, &entry.path), + file_size: entry.size as u64, + record_count: entry.num_records, + // Partition values are produced by kernel as an + // unordered `HashMap` per file. Translate + // physical -> logical when a column mapping is present so + // `build_delta_partitioned_files` can match by logical name. + partition_values: entry + .partition_values + .into_iter() + .map(|(name, value)| { + let logical_name = physical_to_logical.get(&name).cloned().unwrap_or(name); + DeltaPartitionValue { + name: logical_name, + value: Some(value), + } + }) + .collect(), + // Deletion-vector descriptor (path/offset/size) -- the executor's + // `DeltaSyntheticColumnsExec` calls `kernel::DeletionVectorDescriptor::read` + // to decode the bitmap on-task. The driver-side `plan_delta_scan` + // no longer materialises the indexes (task #218 / Iceberg-style + // refactor: per-scan-exec heap stays KB-scale regardless of DV size). + dv: entry.dv_descriptor, + // Row tracking: extracted from each scan-files RecordBatch's + // `fileConstantValues.baseRowId` / `defaultRowCommitVersion` columns + // in scan.rs (see `extract_row_tracking_for_selected`). Kernel's + // `visit_scan_files` callback doesn't surface these; we read them + // directly. `None` when the table doesn't have row tracking enabled. + base_row_id: entry.base_row_id, + default_row_commit_version: entry.default_row_commit_version, + // Splitting is done on the Scala side just before serialization, + // not here on the kernel-driver path. Leave unset. + byte_range_start: None, + byte_range_end: None, + // Surface the kernel-provided modification time (epoch millis) so + // `_metadata.file_modification_time` matches Spark on the kernel-driver + // path too. The value is already populated on `DeltaFileEntry` from the + // scan-file metadata; the BatchFileIndex path sets it from + // AddFile.modificationTime equivalently. + modification_time: Some(entry.modification_time), + // Kernel's fully-resolved physical->logical transform for this file (serde JSON), + // extracted in `plan_delta_scan` from `scan_file.transform`. The executor applies it + // via `transform_to_logical` (partition injection / column-mapping relabel / + // row-tracking baked in). Empty = identity (plain pass-through). + transform_json: entry.transform_json, + }) + .collect(); + + // `plan.column_mappings` is consumed above (partition-value key translation) and is NOT put + // on the wire: column mapping is fully resolved by the kernel schemas shipped on + // `DeltaScanCommon`, so the executor needs no `column_mappings`. + + let msg = DeltaScanTaskList { + snapshot_version: plan.version, + table_root: url_str, + tasks, + unsupported_features: plan.unsupported_features, + // Kernel-built projected schemas (Arrow IPC) -- Scala copies these onto + // `DeltaScanCommon.kernel_physical_schema` / `kernel_logical_schema`. Empty when no + // projection was supplied. + physical_schema: plan.physical_schema_ipc, + logical_schema: plan.logical_schema_ipc, + }; + + let bytes = msg.encode_to_vec(); + let result = env.byte_array_from_slice(&bytes)?; + Ok(result.into_raw()) + }) +} + +/// Schema-only companion to `planDeltaScan` for the batch-file-index read path (the file list comes +/// from Delta `AddFile`s on the Scala side; only kernel's resolved physical/logical schemas are +/// needed). Builds the snapshot at `snapshot_version` + a projected `Scan` and returns a +/// `DeltaScanTaskList` with ONLY `physical_schema` / `logical_schema` set (Arrow IPC, field-ids at +/// every nesting level). Returns an empty (no schemas) message when `projected_schema_ipc` is +/// absent/empty -- the executor then falls back to physicalising `required_schema`. +/// +/// # Safety +/// Inherently unsafe because it dereferences raw JNI pointers. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDeltaReadSchemas( + e: EnvUnowned, + _class: JClass, + table_url: JString, + snapshot_version: jlong, + storage_options: JObject, + projected_schema_json: JString, +) -> jbyteArray { + try_unwrap_or_throw(&e, |env| { + let url_str: String = table_url.try_to_string(env)?; + let version = if snapshot_version < 0 { + None + } else { + Some(snapshot_version as u64) + }; + let s3_bucket = url::Url::parse(&url_str) + .ok() + .filter(|u| matches!(u.scheme(), "s3" | "s3a")) + .and_then(|u| u.host_str().map(|h| h.to_string())); + let config = if storage_options.is_null() { + DeltaStorageConfig::default() + } else { + let jmap: JMap<'_> = env.cast_local::(storage_options)?; + extract_storage_config(env, &jmap, s3_bucket.as_deref())? + }; + + // The data-read schema (Delta JSON) for kernel's `with_schema`. Empty => zero data columns + // => no kernel schemas needed. + let (physical_schema, logical_schema) = match decode_jstring(env, &projected_schema_json)? { + Some(json) => crate::scan::plan_delta_read_schemas(&url_str, &config, version, json) + .map_err(|e| { + CometError::Internal(format!("delta kernel read-schema build failed: {e}")) + })?, + None => (Vec::new(), Vec::new()), + }; + + let msg = DeltaScanTaskList { + snapshot_version: version.unwrap_or(0), + table_root: url_str, + tasks: Vec::new(), + unsupported_features: Vec::new(), + physical_schema, + logical_schema, + }; + let bytes = msg.encode_to_vec(); + let result = env.byte_array_from_slice(&bytes)?; + Ok(result.into_raw()) + }) +} + +/// Join `entry.path` (Delta add-action path, usually relative to the +/// table root) with `table_root` to yield an absolute URL the native-side +/// `build_delta_partitioned_files` can feed straight into +/// `object_store::path::Path::from_url_path`. +fn resolve_file_path(table_root: &str, relative: &str) -> String { + // Fully-qualified paths (kernel surfaces these for some tables, e.g. after + // MERGE, REPLACE, or SHALLOW CLONE) pass through untouched. Accept both + // `file:///abs` (authority form) and `file:/abs` (Hadoop `Path.toUri` form, + // which SHALLOW CLONE uses when it stores absolute paths in AddFile.path). + if has_uri_scheme(relative) { + return relative.to_string(); + } + + if table_root.ends_with('/') { + format!("{table_root}{relative}") + } else { + format!("{table_root}/{relative}") + } +} + +/// True if `s` starts with a URI scheme — `^[A-Za-z][A-Za-z0-9+.-]*:` per RFC 3986. +/// We check the scheme only (not whether a `//` authority follows) because Hadoop's +/// `Path.toUri.toString` emits `file:/abs` (single slash) for local absolute paths +/// and Delta stores that form verbatim in AddFile.path for SHALLOW CLONE tables. +fn has_uri_scheme(s: &str) -> bool { + let bytes = s.as_bytes(); + if bytes.is_empty() || !bytes[0].is_ascii_alphabetic() { + return false; + } + for (i, &b) in bytes.iter().enumerate().skip(1) { + if b == b':' { + return i >= 1; + } + if !(b.is_ascii_alphanumeric() || b == b'+' || b == b'-' || b == b'.') { + return false; + } + } + false +} + +/// Walk a `java.util.Map` of storage options into the S3 +/// [`DeltaStorageConfig`]. `bucket` is the table's S3 bucket (URL host), used to +/// prefer per-bucket Hadoop keys over global ones; pass `None` for non-S3 tables. +/// +/// Reads the JMap into a Rust `HashMap` and delegates to +/// [`delta_storage_config_from_map`] for the actual key mapping. Splitting +/// the JNI traversal from the mapping logic lets the mapping be unit-tested +/// without a JVM. +fn extract_storage_config( + env: &mut Env, + jmap: &JMap<'_>, + bucket: Option<&str>, +) -> CometResult { + let m = jmap_to_hashmap(env, jmap)?; + Ok(delta_storage_config_from_map(&m, bucket)) +} + +/// Pure (JVM-free) mapping from a generic options map to the S3 [`DeltaStorageConfig`]. +/// +/// Only S3 credentials are bridged here -- accepting kernel-style keys +/// (`aws_access_key_id`, ...) AND the Hadoop-style keys Comet's +/// `NativeConfig.extractObjectStoreOptions` produces (`fs.s3a.access.key`, ...). +/// The kernel-style key wins when both are present. Per-bucket Hadoop keys +/// (`fs.s3a.bucket..`) override the global `fs.s3a.` when +/// `bucket` (the table's S3 bucket) is supplied -- matching Hadoop S3A / core Comet. +/// +/// Azure and GCS are intentionally NOT bridged here: `create_object_store` builds +/// them via `object_store::parse_url`, which sources credentials from the ambient +/// environment (`AZURE_*` / `GOOGLE_*` / ADC / instance metadata), exactly as core +/// Comet's non-S3 read path does. So `fs.azure.*` / `fs.gs.*` keys are not consulted. +/// +/// Residual (tracked in `08-known-limitations.md` A2e): Hadoop's explicit S3 +/// credential-provider classes (`fs.s3a.aws.credentials.provider` = +/// AssumedRole / WebIdentity / ...) are not honored; object_store's own default +/// chain (static keys here, else IMDS / ECS / env) is used instead. +pub fn delta_storage_config_from_map( + m: &std::collections::HashMap, + bucket: Option<&str>, +) -> DeltaStorageConfig { + // A per-bucket Hadoop key (`fs.s3a.bucket..`) overrides the + // global `fs.s3a.`. + fn s3_hadoop( + m: &std::collections::HashMap, + bucket: Option<&str>, + suffix: &str, + ) -> Option { + bucket + .and_then(|b| m.get(&format!("fs.s3a.bucket.{b}.{suffix}")).cloned()) + .or_else(|| m.get(&format!("fs.s3a.{suffix}")).cloned()) + } + // Kernel-style (object_store) key wins over the Hadoop form. + let kernel_or_s3 = |kernel: &str, suffix: &str| { + m.get(kernel) + .cloned() + .or_else(|| s3_hadoop(m, bucket, suffix)) + }; + // First map entry whose key starts with `prefix`, returning (key, value). Used for the Hadoop + // Azure forms that embed the account/container in the key name + // (`fs.azure.account.key..dfs.core.windows.net`, `fs.azure.sas.....`). + let first_with_prefix = |prefix: &str| -> Option<(&String, &String)> { + m.iter().find(|(k, _)| k.starts_with(prefix)) + }; + // Azure account key: kernel-style `azure_storage_account_key` wins, else any `fs.azure.account.key*`. + let azure_account_key = m + .get("azure_storage_account_key") + .cloned() + .or_else(|| first_with_prefix("fs.azure.account.key").map(|(_, v)| v.clone())); + // Azure account name: kernel-style key, else parsed from the `fs.azure.account.key..*` + // key name (object_store otherwise reads it from the abfss/wasb URL authority). + let azure_account_name = m + .get("azure_storage_account_name") + .cloned() + .or_else(|| { + first_with_prefix("fs.azure.account.key.").and_then(|(k, _)| { + k.strip_prefix("fs.azure.account.key.") + .and_then(|rest| rest.split('.').next()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + }) + }); + // Azure SAS: kernel-style `azure_storage_sas_key`, else any `fs.azure.sas*` (fixed or scoped). + let azure_sas_token = m + .get("azure_storage_sas_key") + .cloned() + .or_else(|| first_with_prefix("fs.azure.sas").map(|(_, v)| v.clone())); + // GCS service account: kernel-style path, else the Hadoop JSON keyfile path. + let gcs_service_account_path = m + .get("google_service_account") + .cloned() + .or_else(|| m.get("fs.gs.auth.service.account.json.keyfile").cloned()); + let gcs_service_account_key = m.get("google_service_account_key").cloned(); + + DeltaStorageConfig { + aws_access_key: kernel_or_s3("aws_access_key_id", "access.key"), + aws_secret_key: kernel_or_s3("aws_secret_access_key", "secret.key"), + aws_session_token: kernel_or_s3("aws_session_token", "session.token"), + aws_region: m + .get("aws_region") + .cloned() + .or_else(|| s3_hadoop(m, bucket, "endpoint.region")) + .or_else(|| s3_hadoop(m, bucket, "region")), + aws_endpoint: kernel_or_s3("aws_endpoint", "endpoint"), + aws_force_path_style: m + .get("aws_force_path_style") + .cloned() + .or_else(|| s3_hadoop(m, bucket, "path.style.access")) + .map(|s| s == "true") + .unwrap_or(false), + azure_account_name, + azure_account_key, + azure_sas_token, + gcs_service_account_path, + gcs_service_account_key, + } +} + +/// Decode a Java `String` into `Option`: `None` for null/empty. +fn decode_jstring(env: &mut Env, s: &JString) -> CometResult> { + if s.is_null() { + return Ok(None); + } + let v = s.try_to_string(env)?; + Ok(if v.is_empty() { None } else { Some(v) }) +} + +/// Read a Java `String[]` into a `Vec`. Returns empty vec for null arrays. +fn read_string_array(env: &mut Env, arr: &jni::objects::JObjectArray) -> CometResult> { + if arr.is_null() { + return Ok(Vec::new()); + } + let len = arr.len(env)?; + let mut result = Vec::with_capacity(len); + for i in 0..len { + let obj = arr.get_element(env, i)?; + // Checked downcast (matches `jmap_to_hashmap`): the array is declared + // `String[]`, but use the safe cast rather than `JString::from_raw` so an + // unexpected element type surfaces as a clean error instead of UB. + let jstr: JString = env.cast_local::(obj)?; + result.push(jstr.try_to_string(env)?); + } + Ok(result) +} + +/// Iterate a `java.util.Map` into a Rust `HashMap`. Used when we need to +/// pass the full Hadoop config map to a downstream consumer (e.g., +/// `s3::resolve_static_credentials`) that walks its own provider chain. +/// +/// Uses `env.cast_local::(...)` to safely downcast each key/value entry rather +/// than the `unsafe { JString::from_raw(..., into_raw()) }` shortcut used elsewhere in +/// this file -- the runtime cast performs the same JNI-side type check the JLS implies +/// for `Map` but without the unchecked transmute. +fn jmap_to_hashmap( + env: &mut Env, + jmap: &JMap<'_>, +) -> CometResult> { + let mut out = std::collections::HashMap::new(); + jmap.iter(env).and_then(|mut iter| { + while let Some(entry) = iter.next(env)? { + let k = entry.key(env)?; + let v = entry.value(env)?; + let kstr: JString = env.cast_local::(k)?; + let key = kstr.try_to_string(env)?; + if !v.is_null() { + let vstr: JString = env.cast_local::(v)?; + let value = vstr.try_to_string(env)?; + out.insert(key, value); + } + } + Ok(()) + })?; + Ok(out) +} + +// Re-export the test helpers so the integration_tests module can verify +// `resolve_file_path` without exposing it in the public API surface. +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolve_file_path_joins_with_slash() { + assert_eq!( + resolve_file_path("file:///tmp/t/", "part-0.parquet"), + "file:///tmp/t/part-0.parquet" + ); + assert_eq!( + resolve_file_path("file:///tmp/t", "part-0.parquet"), + "file:///tmp/t/part-0.parquet" + ); + } + + #[test] + fn resolve_file_path_passes_through_absolute() { + assert_eq!( + resolve_file_path("file:///tmp/t/", "s3://bucket/data/part-0.parquet"), + "s3://bucket/data/part-0.parquet" + ); + } + + /// Cred-audit regression: full matrix of storage-option keys that the + /// Scala side (`NativeConfig.extractObjectStoreOptions` + + /// `augmentWithResolvedAwsCredentials`) is allowed to send, mapped to + /// the expected [`DeltaStorageConfig`] field values. When new + /// translation branches land here, extend this matrix so a future + /// silent drop is caught. + #[test] + fn extract_storage_config_matrix() { + use std::collections::HashMap; + // Case 1: Hadoop-style S3A keys (the Comet path produces these). + let mut hadoop_s3 = HashMap::new(); + hadoop_s3.insert("fs.s3a.access.key".into(), "AK".into()); + hadoop_s3.insert("fs.s3a.secret.key".into(), "SK".into()); + hadoop_s3.insert("fs.s3a.session.token".into(), "TOK".into()); + hadoop_s3.insert("fs.s3a.endpoint.region".into(), "us-west-2".into()); + hadoop_s3.insert("fs.s3a.endpoint".into(), "https://s3.example".into()); + hadoop_s3.insert("fs.s3a.path.style.access".into(), "true".into()); + let cfg = delta_storage_config_from_map(&hadoop_s3, None); + assert_eq!(cfg.aws_access_key.as_deref(), Some("AK")); + assert_eq!(cfg.aws_secret_key.as_deref(), Some("SK")); + assert_eq!(cfg.aws_session_token.as_deref(), Some("TOK")); + assert_eq!(cfg.aws_region.as_deref(), Some("us-west-2")); + assert_eq!(cfg.aws_endpoint.as_deref(), Some("https://s3.example")); + assert!(cfg.aws_force_path_style); + + // Case 2: Kernel-style keys WIN when both forms are present. + let mut both = HashMap::new(); + both.insert("aws_access_key_id".into(), "KERNEL_AK".into()); + both.insert("fs.s3a.access.key".into(), "HADOOP_AK".into()); + both.insert("aws_secret_access_key".into(), "KERNEL_SK".into()); + both.insert("fs.s3a.secret.key".into(), "HADOOP_SK".into()); + let cfg = delta_storage_config_from_map(&both, None); + assert_eq!(cfg.aws_access_key.as_deref(), Some("KERNEL_AK")); + assert_eq!(cfg.aws_secret_key.as_deref(), Some("KERNEL_SK")); + + // Case 3: `fs.s3a.region` is the deepest-fallback for region + // (after both `aws_region` and `fs.s3a.endpoint.region`). + let mut region_fallback = HashMap::new(); + region_fallback.insert("fs.s3a.region".into(), "eu-central-1".into()); + let cfg = delta_storage_config_from_map(®ion_fallback, None); + assert_eq!(cfg.aws_region.as_deref(), Some("eu-central-1")); + + // Case 4: per-bucket S3 key overrides the global one for the table's bucket + // (A2b). A different bucket -- or no bucket -- falls back to global. + let mut per_bucket = HashMap::new(); + per_bucket.insert("fs.s3a.access.key".into(), "GLOBAL".into()); + per_bucket.insert("fs.s3a.bucket.my-bucket.access.key".into(), "PERBKT".into()); + assert_eq!( + delta_storage_config_from_map(&per_bucket, Some("my-bucket")) + .aws_access_key + .as_deref(), + Some("PERBKT") + ); + assert_eq!( + delta_storage_config_from_map(&per_bucket, Some("other")) + .aws_access_key + .as_deref(), + Some("GLOBAL") + ); + assert_eq!( + delta_storage_config_from_map(&per_bucket, None) + .aws_access_key + .as_deref(), + Some("GLOBAL") + ); + + // Case 5: empty map -> all defaults (no creds, force_path_style=false). + let cfg = delta_storage_config_from_map(&HashMap::new(), None); + assert!(cfg.aws_access_key.is_none()); + assert!(cfg.aws_secret_key.is_none()); + assert!(cfg.aws_session_token.is_none()); + assert!(cfg.aws_region.is_none()); + assert!(cfg.aws_endpoint.is_none()); + assert!(!cfg.aws_force_path_style); + } + + /// Azure / GCS credentials are deliberately NOT carried in `DeltaStorageConfig`: + /// `create_object_store` builds those via `object_store::parse_url` + ambient + /// credentials (parity with core's non-S3 path). This guards against accidentally + /// re-introducing config bridging that leaks `fs.azure.*` / `fs.gs.*` into the S3 + /// fields. (A2e residual -- Hadoop S3 credential-provider classes -- is documented + /// in `08-known-limitations.md`, not asserted here.) + #[test] + fn azure_and_gcs_keys_do_not_leak_into_s3_config() { + use std::collections::HashMap; + let mut m = HashMap::new(); + m.insert( + "fs.azure.account.key.myacct.dfs.core.windows.net".into(), + "HADOOP_AZKEY".into(), + ); + m.insert("fs.azure.account.oauth2.client.id".into(), "CLIENT".into()); + m.insert( + "fs.gs.auth.service.account.json.keyfile".into(), + "/tmp/key.json".into(), + ); + let cfg = delta_storage_config_from_map(&m, None); + assert!(cfg.aws_access_key.is_none()); + assert!(cfg.aws_secret_key.is_none()); + assert!(cfg.aws_endpoint.is_none()); + } + + #[test] + fn azure_creds_bridged_from_hadoop() { + use std::collections::HashMap; + let mut m = HashMap::new(); + m.insert( + "fs.azure.account.key.myacct.dfs.core.windows.net".into(), + "HADOOP_AZKEY".into(), + ); + let cfg = delta_storage_config_from_map(&m, None); + // Before this bridge the executor discarded fs.azure.* and used ambient AZURE_* env. + assert_eq!(cfg.azure_account_key.as_deref(), Some("HADOOP_AZKEY")); + assert_eq!(cfg.azure_account_name.as_deref(), Some("myacct")); + let opts = cfg.azure_object_store_options(); + assert!( + opts.iter() + .any(|(k, v)| *k == "azure_storage_account_key" && v == "HADOOP_AZKEY"), + "azure account key must reach object_store: {opts:?}" + ); + assert!( + opts.iter() + .any(|(k, _)| *k == "azure_storage_account_name"), + "azure account name must reach object_store: {opts:?}" + ); + // S3 fields stay empty (non-leakage the other direction). + assert!(cfg.aws_access_key.is_none()); + assert!(cfg.gcs_service_account_path.is_none()); + } + + #[test] + fn azure_sas_bridged_from_hadoop() { + use std::collections::HashMap; + let mut m = HashMap::new(); + m.insert("fs.azure.sas.fixed.token".into(), "SAS_TOKEN_VALUE".into()); + let cfg = delta_storage_config_from_map(&m, None); + assert_eq!(cfg.azure_sas_token.as_deref(), Some("SAS_TOKEN_VALUE")); + assert!(cfg + .azure_object_store_options() + .iter() + .any(|(k, v)| *k == "azure_storage_sas_key" && v == "SAS_TOKEN_VALUE")); + } + + #[test] + fn gcs_creds_bridged_from_hadoop() { + use std::collections::HashMap; + let mut m = HashMap::new(); + m.insert( + "fs.gs.auth.service.account.json.keyfile".into(), + "/etc/gcs/key.json".into(), + ); + let cfg = delta_storage_config_from_map(&m, None); + assert_eq!( + cfg.gcs_service_account_path.as_deref(), + Some("/etc/gcs/key.json") + ); + assert!(cfg + .gcs_object_store_options() + .iter() + .any(|(k, v)| *k == "google_service_account" && v == "/etc/gcs/key.json")); + // S3/Azure fields stay empty. + assert!(cfg.aws_access_key.is_none()); + assert!(cfg.azure_account_key.is_none()); + } + + #[test] + fn s3_keys_do_not_leak_into_azure_or_gcs() { + use std::collections::HashMap; + let mut m = HashMap::new(); + m.insert("fs.s3a.access.key".into(), "AK".into()); + m.insert("fs.s3a.secret.key".into(), "SK".into()); + m.insert("fs.s3a.session.token".into(), "TOK".into()); + let cfg = delta_storage_config_from_map(&m, Some("my-bucket")); + assert_eq!(cfg.aws_access_key.as_deref(), Some("AK")); // sanity: S3 still extracted + assert!(cfg.azure_account_key.is_none()); + assert!(cfg.azure_account_name.is_none()); + assert!(cfg.azure_sas_token.is_none()); + assert!(cfg.gcs_service_account_path.is_none()); + assert!(cfg.gcs_service_account_key.is_none()); + assert!(cfg.azure_object_store_options().is_empty()); + assert!(cfg.gcs_object_store_options().is_empty()); + } + + #[test] + fn resolve_file_path_passes_through_single_slash_file_uri() { + // SHALLOW CLONE stores paths as Hadoop `Path.toUri.toString` which uses + // single-slash form `file:/abs/...`. Must not be concat'd onto the clone root. + assert_eq!( + resolve_file_path( + "file:/tmp/clonetable/", + "file:/tmp/parquet_table/part-0.parquet" + ), + "file:/tmp/parquet_table/part-0.parquet" + ); + } + + #[test] + fn has_uri_scheme_matches_schemes() { + assert!(has_uri_scheme("file:/abs")); + assert!(has_uri_scheme("file:///abs")); + assert!(has_uri_scheme("s3://bucket/k")); + assert!(has_uri_scheme("hdfs://nn/path")); + assert!(!has_uri_scheme("part-0.parquet")); + assert!(!has_uri_scheme("/abs/path")); + assert!(!has_uri_scheme("1bad:/scheme")); // must start with letter + assert!(!has_uri_scheme("")); + } +} diff --git a/contrib/delta/native/src/lib.rs b/contrib/delta/native/src/lib.rs new file mode 100644 index 0000000000..0fbdb06745 --- /dev/null +++ b/contrib/delta/native/src/lib.rs @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Delta Lake integration for Apache DataFusion Comet. +//! +//! Enabled in core via `--features contrib-delta`. Default builds carry zero +//! Delta surface; this crate is not linked unless the feature is on. +//! +//! Surfaces: +//! - JNI: `Java_org_apache_comet_contrib_delta_Native_planDeltaScan` (driver-side +//! log replay via delta-kernel-rs; returns a `DeltaScanTaskList` proto) +//! - [`scan::plan_delta_scan`] / `scan::plan_delta_scan_with_predicate`: driver-side +//! helpers that replay the log and assemble the scan-task list. (Core's planner +//! dispatcher calls `planner::plan_delta_scan`, which is still a build-gate stub in +//! this unit and becomes the real entry point in the executor unit.) +//! +//! No `#[ctor]` registration, no contrib-private operator-planner registry; this +//! crate exposes plain Rust functions that core calls directly under +//! `#[cfg(feature = "contrib-delta")]`. +//! +//! This unit ships the driver side (log replay, predicate pushdown, scan-task +//! assembly, JNI). The executor-side read path (`kernel_scan`, `dv_reader`) and the +//! real `planner` land in the next unit; until then `planner` is a build-gate stub. + +pub mod engine; +pub mod error; +pub mod jni; +pub mod planner; +pub mod predicate; +pub mod scan; + +/// Re-export of the Delta proto messages the driver modules reference via +/// `crate::proto::Delta...`. The messages themselves live in core's proto crate. (The +/// `DeltaScan` / `DeltaScanCommon` envelopes are consumed only by the planner, which names +/// them directly from `datafusion_comet_proto`, so they're added to this re-export by the +/// executor unit rather than here.) +pub mod proto { + pub use datafusion_comet_proto::spark_operator::{ + DeltaDvDescriptor, DeltaPartitionValue, DeltaScanTask, DeltaScanTaskList, + }; +} + +pub use engine::DeltaStorageConfig; +pub use error::{DeltaError, DeltaResult}; +pub use scan::{list_delta_files, plan_delta_scan, DeltaFileEntry, DeltaScanPlan}; diff --git a/contrib/delta/native/src/planner.rs b/contrib/delta/native/src/planner.rs new file mode 100644 index 0000000000..072dd700a7 --- /dev/null +++ b/contrib/delta/native/src/planner.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Delta scan planning entry point. +//! +//! BUILD-GATE STUB: matches the exact signature core's `planner::delta_scan` shim calls +//! (`plan_delta_scan(&DeltaScan, &DeltaScanCommon, &SchemaRef, &SchemaRef)`), but returns a +//! clean `DataFusionError::NotImplemented` instead of building a scan. The shim maps that error +//! to a `GeneralError`, which surfaces on the JVM side as a fallback so the query runs on vanilla +//! Spark. The real kernel-read implementation replaces this in a later unit. + +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::error::DataFusionError; +use datafusion::physical_plan::ExecutionPlan; + +use datafusion_comet_proto::spark_operator::{DeltaScan, DeltaScanCommon}; + +/// Plan a native Delta scan. Stub: not yet implemented (build-gate unit). +/// +/// The signature mirrors the real implementation so core's shim and the dispatcher arm compile +/// unchanged when the read path lands. `required_schema` / `partition_schema` are computed by core +/// (which owns the proto -> arrow converter) and handed in. +pub fn plan_delta_scan( + _scan: &DeltaScan, + _common: &DeltaScanCommon, + _required_schema: &SchemaRef, + _partition_schema: &SchemaRef, +) -> Result, DataFusionError> { + Err(DataFusionError::NotImplemented( + "contrib-delta native Delta read path is not yet implemented (build-gate stub)".to_string(), + )) +} diff --git a/contrib/delta/native/src/predicate.rs b/contrib/delta/native/src/predicate.rs new file mode 100644 index 0000000000..7cbf02e5fb --- /dev/null +++ b/contrib/delta/native/src/predicate.rs @@ -0,0 +1,697 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Translates Catalyst-proto `Expr` to delta-kernel `Predicate` for +//! stats-based file pruning. +//! +//! Supported operators: =, !=, <, <=, >, >=, AND, OR, NOT, IS NULL, +//! IS NOT NULL, IN (including NOT IN). Cast wrappers are unwrapped +//! (kernel stats don't need type coercion). Anything else becomes +//! `Predicate::unknown()`, which disables data skipping for that +//! subtree but is never incorrect. + +use datafusion_comet_proto::spark_expression::{self, expr::ExprStruct, literal, Expr}; +use delta_kernel::expressions::{ArrayData, BinaryPredicateOp, Expression, Predicate, Scalar}; +use delta_kernel::schema::{ArrayType, DataType}; + +/// Translate with column name resolution for BoundReferences. +pub fn catalyst_to_kernel_predicate_with_names(expr: &Expr, column_names: &[String]) -> Predicate { + translate_predicate(expr, column_names) +} + +/// Try to translate a Catalyst-proto `Expr` into a kernel `Predicate` +/// (without column name resolution — BoundReferences become Unknown). +pub fn catalyst_to_kernel_predicate(expr: &Expr) -> Predicate { + translate_predicate(expr, &[]) +} + +fn translate_predicate(expr: &Expr, names: &[String]) -> Predicate { + let to_expr = |e: &Expr| catalyst_to_kernel_expression_with_names(e, names); + match expr.expr_struct.as_ref() { + Some(ExprStruct::IsNull(unary)) => match unary.child.as_deref() { + Some(child) => Predicate::is_null(to_expr(child)), + None => Predicate::unknown("missing_child"), + }, + Some(ExprStruct::IsNotNull(unary)) => match unary.child.as_deref() { + Some(child) => Predicate::is_not_null(to_expr(child)), + None => Predicate::unknown("missing_child"), + }, + Some(ExprStruct::Eq(binary)) => binary_pred_n( + Predicate::eq, + binary.left.as_deref(), + binary.right.as_deref(), + names, + ), + Some(ExprStruct::Neq(binary)) => binary_pred_n( + Predicate::ne, + binary.left.as_deref(), + binary.right.as_deref(), + names, + ), + Some(ExprStruct::Lt(binary)) => binary_pred_n( + Predicate::lt, + binary.left.as_deref(), + binary.right.as_deref(), + names, + ), + Some(ExprStruct::LtEq(binary)) => binary_pred_n( + Predicate::le, + binary.left.as_deref(), + binary.right.as_deref(), + names, + ), + Some(ExprStruct::Gt(binary)) => binary_pred_n( + Predicate::gt, + binary.left.as_deref(), + binary.right.as_deref(), + names, + ), + Some(ExprStruct::GtEq(binary)) => binary_pred_n( + Predicate::ge, + binary.left.as_deref(), + binary.right.as_deref(), + names, + ), + Some(ExprStruct::And(binary)) => match (binary.left.as_deref(), binary.right.as_deref()) { + (Some(l), Some(r)) => { + Predicate::and(translate_predicate(l, names), translate_predicate(r, names)) + } + _ => Predicate::unknown("and_missing_child"), + }, + Some(ExprStruct::Or(binary)) => match (binary.left.as_deref(), binary.right.as_deref()) { + (Some(l), Some(r)) => { + Predicate::or(translate_predicate(l, names), translate_predicate(r, names)) + } + _ => Predicate::unknown("or_missing_child"), + }, + Some(ExprStruct::Not(unary)) => match unary.child.as_deref() { + Some(child) => Predicate::not(translate_predicate(child, names)), + None => Predicate::unknown("not_missing_child"), + }, + Some(ExprStruct::In(in_expr)) => translate_in(in_expr, names), + // Unwrap Cast: kernel stats don't need type coercion, pass child through + Some(ExprStruct::Cast(cast)) => match cast.child.as_deref() { + Some(child) => translate_predicate(child, names), + None => Predicate::unknown("cast_missing_child"), + }, + _ => Predicate::unknown("unsupported_catalyst_expr"), + } +} + +fn translate_in(in_expr: &spark_expression::In, names: &[String]) -> Predicate { + let value = match in_expr.in_value.as_deref() { + Some(v) => catalyst_to_kernel_expression_with_names(v, names), + None => return Predicate::unknown("in_missing_value"), + }; + + let scalars: Vec = in_expr + .lists + .iter() + .filter_map(catalyst_literal_to_scalar) + .collect(); + + if scalars.is_empty() { + return Predicate::unknown("in_no_literal_values"); + } + + let kernel_type = scalar_to_kernel_type(&scalars[0]); + let array_data = match ArrayData::try_new(ArrayType::new(kernel_type, true), scalars) { + Ok(ad) => ad, + Err(_) => return Predicate::unknown("in_array_type_mismatch"), + }; + let array = Expression::literal(Scalar::Array(array_data)); + + let pred = Predicate::binary(BinaryPredicateOp::In, value, array); + if in_expr.negated { + Predicate::not(pred) + } else { + pred + } +} + +fn scalar_to_kernel_type(s: &Scalar) -> DataType { + match s { + Scalar::Boolean(_) => DataType::BOOLEAN, + Scalar::Byte(_) => DataType::BYTE, + Scalar::Short(_) => DataType::SHORT, + Scalar::Integer(_) => DataType::INTEGER, + Scalar::Long(_) => DataType::LONG, + Scalar::Float(_) => DataType::FLOAT, + Scalar::Double(_) => DataType::DOUBLE, + Scalar::String(_) => DataType::STRING, + Scalar::Timestamp(_) => DataType::TIMESTAMP, + Scalar::TimestampNtz(_) => DataType::TIMESTAMP_NTZ, + Scalar::Date(_) => DataType::DATE, + _ => DataType::STRING, + } +} + +fn catalyst_literal_to_scalar(expr: &Expr) -> Option { + use spark_expression::data_type::DataTypeId; + match expr.expr_struct.as_ref() { + Some(ExprStruct::Literal(lit)) => { + // Spark stores TIMESTAMP / TIMESTAMP_NTZ as int64 micros and DATE as int32 days, all in + // the int value slots. Honor the declared `datatype` so the kernel scalar carries the + // right TYPE: a bare `Scalar::Long`/`Scalar::Integer` makes kernel reject the comparison + // against a temporal column ("Invalid comparison: Timestamp < Int64") and abort the scan. + let type_id = lit.datatype.as_ref().map(|d| d.type_id); + match &lit.value { + Some(literal::Value::BoolVal(b)) => Some(Scalar::Boolean(*b)), + Some(literal::Value::ByteVal(v)) => Some(Scalar::Byte(*v as i8)), + Some(literal::Value::ShortVal(v)) => Some(Scalar::Short(*v as i16)), + Some(literal::Value::IntVal(v)) if type_id == Some(DataTypeId::Date as i32) => { + Some(Scalar::Date(*v)) + } + Some(literal::Value::IntVal(v)) => Some(Scalar::Integer(*v)), + Some(literal::Value::LongVal(v)) + if type_id == Some(DataTypeId::Timestamp as i32) => + { + Some(Scalar::Timestamp(*v)) + } + Some(literal::Value::LongVal(v)) + if type_id == Some(DataTypeId::TimestampNtz as i32) => + { + Some(Scalar::TimestampNtz(*v)) + } + Some(literal::Value::LongVal(v)) => Some(Scalar::Long(*v)), + Some(literal::Value::FloatVal(v)) => Some(Scalar::Float(*v)), + Some(literal::Value::DoubleVal(v)) => Some(Scalar::Double(*v)), + Some(literal::Value::StringVal(s)) => Some(Scalar::String(s.clone())), + _ => None, + } + } + _ => None, + } +} + +fn binary_pred_n( + builder: impl Fn(Expression, Expression) -> Predicate, + left: Option<&Expr>, + right: Option<&Expr>, + names: &[String], +) -> Predicate { + match (left, right) { + (Some(l), Some(r)) => builder( + catalyst_to_kernel_expression_with_names(l, names), + catalyst_to_kernel_expression_with_names(r, names), + ), + _ => Predicate::unknown("binary_missing_child"), + } +} + +/// Translate a Catalyst-proto `Expr` into a kernel value `Expression`. +/// +/// `column_names` maps BoundReference indices to column names. When +/// empty, BoundReferences become unknown expressions (disabling file +/// skipping for that sub-expression but never producing wrong results). +pub fn catalyst_to_kernel_expression_with_names( + expr: &Expr, + column_names: &[String], +) -> Expression { + match expr.expr_struct.as_ref() { + Some(ExprStruct::Bound(bound)) => { + let idx = bound.index as usize; + if idx < column_names.len() { + Expression::column([column_names[idx].as_str()]) + } else { + Expression::unknown("bound_ref_out_of_range") + } + } + Some(ExprStruct::Literal(lit)) => catalyst_literal_to_kernel(lit), + // Unwrap Cast: pass child expression through for kernel stats evaluation + Some(ExprStruct::Cast(cast)) => match cast.child.as_deref() { + Some(child) => catalyst_to_kernel_expression_with_names(child, column_names), + None => Expression::unknown("cast_missing_child"), + }, + _ => Expression::unknown("unsupported_expr_operand"), + } +} + +fn catalyst_literal_to_kernel(lit: &spark_expression::Literal) -> Expression { + use spark_expression::data_type::DataTypeId; + // Spark stores TIMESTAMP / TIMESTAMP_NTZ as int64 micros and DATE as int32 days, in the int + // value slots. Honor the declared `datatype` so the kernel literal carries the right TYPE -- a + // bare LONG/INTEGER literal makes kernel reject the comparison against a temporal column + // ("Invalid comparison: Timestamp(µs) < Int64") and abort the whole scan. This is the operand + // path for binary comparisons (=, <, <=, >, >=, !=); IN-lists use `catalyst_literal_to_scalar`. + let type_id = lit.datatype.as_ref().map(|d| d.type_id); + match &lit.value { + Some(literal::Value::BoolVal(b)) => Expression::literal(*b), + Some(literal::Value::ByteVal(v)) => Expression::literal(*v), + Some(literal::Value::ShortVal(v)) => Expression::literal(*v), + Some(literal::Value::IntVal(v)) if type_id == Some(DataTypeId::Date as i32) => { + Expression::literal(Scalar::Date(*v)) + } + Some(literal::Value::IntVal(v)) => Expression::literal(*v), + Some(literal::Value::LongVal(v)) if type_id == Some(DataTypeId::Timestamp as i32) => { + Expression::literal(Scalar::Timestamp(*v)) + } + Some(literal::Value::LongVal(v)) if type_id == Some(DataTypeId::TimestampNtz as i32) => { + Expression::literal(Scalar::TimestampNtz(*v)) + } + Some(literal::Value::LongVal(v)) => Expression::literal(*v), + Some(literal::Value::FloatVal(v)) => Expression::literal(*v), + Some(literal::Value::DoubleVal(v)) => Expression::literal(*v), + Some(literal::Value::StringVal(s)) => Expression::literal(s.as_str()), + _ => Expression::null_literal(DataType::STRING), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion_comet_proto::spark_expression::{ + BinaryExpr, BoundReference, Cast, In, Literal, UnaryExpr, + }; + + // ---- builders for proto Exprs ---- + + fn mk_expr(es: ExprStruct) -> Expr { + Expr { + expr_struct: Some(es), + query_context: None, + expr_id: None, + } + } + + fn lit_int(v: i32) -> Expr { + mk_expr(ExprStruct::Literal(Literal { + value: Some(literal::Value::IntVal(v)), + ..Default::default() + })) + } + + fn lit_string(s: &str) -> Expr { + mk_expr(ExprStruct::Literal(Literal { + value: Some(literal::Value::StringVal(s.to_string())), + ..Default::default() + })) + } + + fn bound_ref(idx: i32) -> Expr { + mk_expr(ExprStruct::Bound(BoundReference { + index: idx, + ..Default::default() + })) + } + + fn binary(struct_fn: impl FnOnce(Box) -> ExprStruct, l: Expr, r: Expr) -> Expr { + let be = BinaryExpr { + left: Some(Box::new(l)), + right: Some(Box::new(r)), + }; + mk_expr(struct_fn(Box::new(be))) + } + + fn unary(struct_fn: impl FnOnce(Box) -> ExprStruct, child: Expr) -> Expr { + let ue = UnaryExpr { + child: Some(Box::new(child)), + }; + mk_expr(struct_fn(Box::new(ue))) + } + + fn pred_str(p: &Predicate) -> String { + format!("{p:?}") + } + + // ---- literal coverage ---- + + #[test] + fn literal_kinds_all_translate() { + let cases: Vec<(literal::Value, &str)> = vec![ + (literal::Value::BoolVal(true), "true"), + (literal::Value::ByteVal(5), "5"), + (literal::Value::ShortVal(6), "6"), + (literal::Value::IntVal(7), "7"), + (literal::Value::LongVal(8), "8"), + (literal::Value::FloatVal(1.5), "1.5"), + (literal::Value::DoubleVal(2.5), "2.5"), + (literal::Value::StringVal("hi".to_string()), "hi"), + ]; + for (val, needle) in cases { + let lit = Literal { + value: Some(val), + ..Default::default() + }; + let expr = catalyst_literal_to_kernel(&lit); + assert!( + format!("{expr:?}").contains(needle), + "literal didn't translate: needle={needle}, got={expr:?}" + ); + } + } + + // Temporal literals must carry their kernel TYPE so a predicate against a TIMESTAMP/DATE column + // type-checks in kernel. Before the fix a TIMESTAMP `LongVal` became a bare `Scalar::Long`, + // making kernel reject "Timestamp(µs) < Int64" and abort the scan (gencol partition-filter + // repro). This is the binary-operand path (`catalyst_literal_to_kernel`). + #[test] + fn temporal_literals_carry_kernel_type() { + use spark_expression::data_type::DataTypeId; + use spark_expression::DataType as ProtoDataType; + let mk = |val: literal::Value, tid: DataTypeId| Literal { + value: Some(val), + datatype: Some(ProtoDataType { + type_id: tid as i32, + ..Default::default() + }), + ..Default::default() + }; + // TIMESTAMP (int64 micros) -> Scalar::Timestamp, NOT Long. + let ts = catalyst_literal_to_kernel(&mk( + literal::Value::LongVal(1_600_000_000_000_000), + DataTypeId::Timestamp, + )); + assert!( + format!("{ts:?}").contains("Timestamp"), + "timestamp literal must be a Timestamp scalar, got {ts:?}" + ); + // TIMESTAMP_NTZ -> Scalar::TimestampNtz. + let tsntz = + catalyst_literal_to_kernel(&mk(literal::Value::LongVal(123), DataTypeId::TimestampNtz)); + assert!( + format!("{tsntz:?}").contains("TimestampNtz"), + "timestamp_ntz literal must be a TimestampNtz scalar, got {tsntz:?}" + ); + // DATE (int32 days) -> Scalar::Date. + let d = catalyst_literal_to_kernel(&mk(literal::Value::IntVal(18_000), DataTypeId::Date)); + assert!( + format!("{d:?}").contains("Date"), + "date literal must be a Date scalar, got {d:?}" + ); + // A plain LONG (non-temporal) stays Long. + let l = catalyst_literal_to_kernel(&mk(literal::Value::LongVal(8), DataTypeId::Int64)); + let ls = format!("{l:?}"); + assert!( + ls.contains('8') && !ls.contains("Timestamp"), + "plain long literal must stay Long, got {ls}" + ); + } + + #[test] + fn literal_unsupported_becomes_null() { + let lit = Literal { + value: None, + ..Default::default() + }; + let expr = catalyst_literal_to_kernel(&lit); + // Should be a NULL literal (DataType::STRING). Just sanity-check it's not panicking. + let _ = format!("{expr:?}"); + } + + #[test] + fn literal_to_scalar_extracts_all_kinds() { + let exprs = vec![ + (lit_int(7), Scalar::Integer(7)), + (lit_string("foo"), Scalar::String("foo".into())), + ]; + for (expr, want) in exprs { + let got = catalyst_literal_to_scalar(&expr).unwrap(); + assert_eq!(format!("{got:?}"), format!("{want:?}")); + } + } + + #[test] + fn literal_to_scalar_non_literal_returns_none() { + assert!(catalyst_literal_to_scalar(&bound_ref(0)).is_none()); + } + + // ---- binary operators ---- + + #[test] + fn binary_eq_with_bound_ref_and_literal() { + let expr = binary(ExprStruct::Eq, bound_ref(0), lit_int(7)); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["x".to_string()]); + let s = pred_str(&p); + // Should mention "Eq" or "=" plus column name x and value 7 + assert!(s.contains("Eq") || s.contains("Equal"), "got: {s}"); + assert!(s.contains("x"), "no column name: {s}"); + } + + #[test] + fn binary_all_relops_translate() { + let kinds: Vec) -> ExprStruct> = vec![ + ExprStruct::Eq, + ExprStruct::Neq, + ExprStruct::Lt, + ExprStruct::LtEq, + ExprStruct::Gt, + ExprStruct::GtEq, + ]; + for k in kinds { + let expr = binary(k, bound_ref(0), lit_int(1)); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["c".to_string()]); + let s = pred_str(&p); + assert!( + !s.contains("Unknown") && !s.contains("unknown"), + "operator translated to unknown: {s}" + ); + } + } + + #[test] + fn binary_missing_child_falls_back_to_unknown() { + let be = BinaryExpr { + left: Some(Box::new(bound_ref(0))), + // right is None + ..Default::default() + }; + let expr = mk_expr(ExprStruct::Eq(Box::new(be))); + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("binary_missing_child")); + } + + // ---- logical operators ---- + + #[test] + fn and_combines_two_children() { + let l = binary(ExprStruct::Eq, bound_ref(0), lit_int(1)); + let r = binary(ExprStruct::Eq, bound_ref(1), lit_int(2)); + let expr = binary(ExprStruct::And, l, r); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["a".to_string(), "b".to_string()]); + let s = pred_str(&p); + assert!(s.contains("a") && s.contains("b"), "{s}"); + } + + #[test] + fn or_combines_two_children() { + let l = binary(ExprStruct::Eq, bound_ref(0), lit_int(1)); + let r = binary(ExprStruct::Eq, bound_ref(0), lit_int(2)); + let expr = binary(ExprStruct::Or, l, r); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["a".to_string()]); + let _ = pred_str(&p); // just check it doesn't panic + } + + #[test] + fn and_missing_child_falls_back_to_unknown() { + let be = BinaryExpr { + left: Some(Box::new(bound_ref(0))), + ..Default::default() + }; + let expr = mk_expr(ExprStruct::And(Box::new(be))); + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("and_missing_child")); + } + + #[test] + fn or_missing_child_falls_back_to_unknown() { + let be = BinaryExpr { + right: Some(Box::new(bound_ref(0))), + ..Default::default() + }; + let expr = mk_expr(ExprStruct::Or(Box::new(be))); + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("or_missing_child")); + } + + // ---- unary operators ---- + + #[test] + fn is_null_translates() { + let expr = unary(ExprStruct::IsNull, bound_ref(0)); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["c".to_string()]); + let s = pred_str(&p); + assert!(s.contains("Null") || s.contains("null"), "{s}"); + } + + #[test] + fn is_not_null_translates() { + let expr = unary(ExprStruct::IsNotNull, bound_ref(0)); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["c".to_string()]); + let s = pred_str(&p); + assert!(s.contains("Null") || s.contains("null"), "{s}"); + } + + #[test] + fn not_translates() { + let inner = binary(ExprStruct::Eq, bound_ref(0), lit_int(1)); + let expr = unary(ExprStruct::Not, inner); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["c".to_string()]); + let s = pred_str(&p); + assert!(s.contains("Not") || s.contains("not"), "{s}"); + } + + #[test] + fn unary_missing_child_falls_back_to_unknown() { + let expr = mk_expr(ExprStruct::IsNull(Box::default())); + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("missing_child")); + } + + // ---- IN ---- + + #[test] + fn in_translates_with_literal_list() { + let in_expr = In { + in_value: Some(Box::new(bound_ref(0))), + lists: vec![lit_int(1), lit_int(2), lit_int(3)], + negated: false, + }; + let expr = mk_expr(ExprStruct::In(Box::new(in_expr))); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["c".to_string()]); + let s = pred_str(&p); + assert!(!s.contains("unknown"), "got: {s}"); + } + + #[test] + fn in_negated_wraps_with_not() { + let in_expr = In { + in_value: Some(Box::new(bound_ref(0))), + lists: vec![lit_int(1)], + negated: true, + }; + let expr = mk_expr(ExprStruct::In(Box::new(in_expr))); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["c".to_string()]); + let s = pred_str(&p); + assert!(s.contains("Not") || s.contains("not"), "{s}"); + } + + #[test] + fn in_empty_list_falls_back_to_unknown() { + let in_expr = In { + in_value: Some(Box::new(bound_ref(0))), + lists: vec![], + ..Default::default() + }; + let expr = mk_expr(ExprStruct::In(Box::new(in_expr))); + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("in_no_literal_values")); + } + + #[test] + fn in_missing_value_falls_back_to_unknown() { + let in_expr = In::default(); // in_value is None + let expr = mk_expr(ExprStruct::In(Box::new(in_expr))); + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("in_missing_value")); + } + + // ---- Cast unwrap ---- + + #[test] + fn cast_unwraps_in_predicate_context() { + let cast = Cast { + child: Some(Box::new(binary(ExprStruct::Eq, bound_ref(0), lit_int(1)))), + ..Default::default() + }; + let expr = mk_expr(ExprStruct::Cast(Box::new(cast))); + let p = catalyst_to_kernel_predicate_with_names(&expr, &["c".to_string()]); + let s = pred_str(&p); + assert!(!s.contains("unsupported"), "Cast didn't unwrap: {s}"); + } + + #[test] + fn cast_unwraps_in_expression_context() { + let cast = Cast { + child: Some(Box::new(bound_ref(0))), + ..Default::default() + }; + let expr = mk_expr(ExprStruct::Cast(Box::new(cast))); + let kernel_expr = catalyst_to_kernel_expression_with_names(&expr, &["x".to_string()]); + // After unwrap: should resolve to column "x" + assert!(format!("{kernel_expr:?}").contains("x")); + } + + #[test] + fn cast_missing_child_falls_back_to_unknown() { + let expr = mk_expr(ExprStruct::Cast(Box::default())); + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("cast_missing_child")); + } + + // ---- BoundReference resolution ---- + + #[test] + fn bound_ref_resolves_to_column_name() { + let expr = bound_ref(1); + let kernel_expr = catalyst_to_kernel_expression_with_names( + &expr, + &["a".to_string(), "b".to_string(), "c".to_string()], + ); + assert!(format!("{kernel_expr:?}").contains("b")); + } + + #[test] + fn bound_ref_out_of_range_yields_unknown() { + let expr = bound_ref(5); + let kernel_expr = + catalyst_to_kernel_expression_with_names(&expr, &["only_one".to_string()]); + assert!(format!("{kernel_expr:?}").contains("bound_ref_out_of_range")); + } + + #[test] + fn bound_ref_with_empty_names_yields_unknown() { + let expr = bound_ref(0); + let kernel_expr = catalyst_to_kernel_expression_with_names(&expr, &[]); + assert!(format!("{kernel_expr:?}").contains("bound_ref_out_of_range")); + } + + // ---- unsupported fallback ---- + + #[test] + fn unsupported_expr_kind_falls_back_to_unknown_predicate() { + // An Expr with no expr_struct at all -> unsupported_catalyst_expr. + let expr = Expr { + expr_struct: None, + query_context: None, + expr_id: None, + }; + let p = catalyst_to_kernel_predicate(&expr); + assert!(pred_str(&p).contains("unsupported_catalyst_expr")); + } + + // ---- scalar type inference ---- + + #[test] + fn scalar_to_kernel_type_round_trips() { + let cases = vec![ + (Scalar::Boolean(true), "BOOLEAN"), + (Scalar::Integer(0), "INTEGER"), + (Scalar::Long(0), "LONG"), + (Scalar::String("".into()), "STRING"), + ]; + for (scalar, name) in cases { + let dt = scalar_to_kernel_type(&scalar); + // DataType::Display for these is "boolean"/"integer"/"long"/"string". + // Just verify it's not panicking + roughly matches. + assert!( + format!("{dt:?}").to_uppercase().contains(name) || format!("{dt:?}").contains(name), + "type mismatch for {scalar:?}: got {dt:?}" + ); + } + } +} diff --git a/contrib/delta/native/src/scan.rs b/contrib/delta/native/src/scan.rs new file mode 100644 index 0000000000..43f975c372 --- /dev/null +++ b/contrib/delta/native/src/scan.rs @@ -0,0 +1,779 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Delta log replay: given a table URL, return the list of active parquet +//! files with partition values, record-count stats, and deletion-vector +//! flags. +//! +//! Ported from tantivy4java's `delta_reader/scan.rs`. The API is the +//! smallest possible surface that still proves end-to-end kernel +//! integration: `Snapshot::builder_for(url)` → `scan_builder().build()` → +//! `scan_metadata(&engine)` → `visit_scan_files(...)`. +//! +//! **Critical gotcha** preserved from the reference implementation: kernel +//! internally does `table_root.join("_delta_log/")`, and `Url::join` will +//! *replace* the last path segment if the base URL does not end in `/`. So +//! `normalize_url` always appends a trailing slash. + +use std::collections::HashMap; +use std::sync::Arc; +use url::Url; + +use delta_kernel::snapshot::Snapshot; + +use super::engine::{get_or_create_engine, DeltaStorageConfig}; +use super::error::{DeltaError, DeltaResult}; + +/// Metadata for a single active parquet file in a Delta table. +/// +/// Plain Rust types only — no arrow / parquet / object_store types. This is +/// the boundary at which kernel's isolated dep subtree meets the rest of +/// Comet. +#[derive(Debug, Clone)] +pub struct DeltaFileEntry { + /// Parquet file path, relative to the table root. + pub path: String, + /// File size in bytes. + pub size: i64, + /// Last-modified time as epoch millis. + pub modification_time: i64, + /// Record count from log stats, if known. + pub num_records: Option, + /// Partition column → value mapping from the add action. + pub partition_values: HashMap, + /// Deletion-vector descriptor for this file, when one is in use. `None` + /// when the file has no DV. Carries everything the EXECUTOR needs to read + /// the DV bitmap on-task -- the driver no longer materialises the deleted + /// row indexes (which could reach ~1 GB long[] for a 99 M-row DV on the + /// 2 B-row "huge table delete" test). See task #218 / the Iceberg-style + /// refactor: the driver ships KB-scale descriptors, the executor calls + /// `kernel::DeletionVectorDescriptor::read` once per partition. + pub dv_descriptor: Option, + /// `AddFile.baseRowId` for row-tracking-enabled tables. `None` when the + /// table doesn't have row tracking. `row_id` for any row in this file is + /// `base_row_id + physical_row_index`. + pub base_row_id: Option, + /// `AddFile.defaultRowCommitVersion` for row-tracking-enabled tables. + /// `None` when the table doesn't have row tracking. Constant per file. + pub default_row_commit_version: Option, + /// Kernel's fully-resolved physical->logical transform for this file, serialized as JSON + /// (serde of `delta_kernel::expressions::Expression`). Partition values + `baseRowId` are baked + /// in as literals by kernel's `get_transform_expr`. Empty = no transform (plain pass-through). + /// The executor deserializes + applies it via `transform_to_logical`, so partition injection / + /// column-mapping relabel / row-tracking come from kernel rather than Comet-side reconstruction. + pub transform_json: Vec, +} + +impl DeltaFileEntry { + /// True if this entry has a deletion vector in use. + pub fn has_deletion_vector(&self) -> bool { + self.dv_descriptor.is_some() + } +} + +/// Result of planning a Delta scan: the active file list plus the pinned +/// snapshot version plus a list of reader features that Comet's native path +/// doesn't yet handle. The Scala side uses the feature list to decide +/// whether to fall back to Spark's vanilla Delta reader. +#[derive(Debug, Clone)] +pub struct DeltaScanPlan { + pub entries: Vec, + pub version: u64, + pub unsupported_features: Vec, + /// Top-level `(logical_name, physical_name)` pairs for column-mapped tables. Driver-internal + /// only -- used to translate kernel's physical-keyed partition values back to logical names. + /// Empty when column_mapping_mode is None. (Nested column mapping is handled by the kernel-built + /// schemas, not this list.) + pub column_mappings: Vec<(String, String)>, + /// Kernel-built data-column schemas (Arrow IPC schema messages) for the kernel-read path, + /// produced from the projected `Scan` (`scan.physical_schema()` / `scan.logical_schema()`). + /// The physical schema carries correct physical names + field-ids at every nesting level + /// (kernel's own column-mapping resolution); the executor reads parquet with it and relabels + /// physical->logical via the schema pair -- no Comet-side physicalisation. Empty when the + /// driver had no projected schema to build the scan against. + pub physical_schema_ipc: Vec, + pub logical_schema_ipc: Vec, +} + +/// Serialize an arrow-58 schema to an encapsulated IPC schema message (continuation + length + +/// flatbuffer `Message` whose header is the Schema). Symmetric with the executor's +/// `arrow_ipc::convert::try_schema_from_ipc_buffer` decode and with Arrow-Java's +/// `Schema.serializeAsMessage()` (used for the inbound projection). Field metadata -- including +/// `PARQUET:field_id` -- is preserved. +fn arrow_schema_to_ipc_bytes(schema: &arrow::datatypes::Schema) -> DeltaResult> { + let mut buf = Vec::new(); + { + let mut writer = arrow::ipc::writer::StreamWriter::try_new(&mut buf, schema) + .map_err(|e| DeltaError::Internal(format!("arrow IPC schema serialize: {e}")))?; + writer + .finish() + .map_err(|e| DeltaError::Internal(format!("arrow IPC schema finish: {e}")))?; + } + Ok(buf) +} + +/// `(physical_schema_ipc, logical_schema_ipc)` for a built `Scan`: kernel's own projected +/// physical/logical data-column schemas (Arrow IPC, field-ids preserved at every nesting level). +fn scan_schemas_to_ipc(scan: &delta_kernel::scan::Scan) -> DeltaResult<(Vec, Vec)> { + use delta_kernel::engine::arrow_conversion::TryIntoArrow; + let physical_arrow: arrow::datatypes::Schema = scan + .physical_schema() + .as_ref() + .try_into_arrow() + .map_err(|e| DeltaError::Internal(format!("kernel physical schema->arrow: {e}")))?; + let logical_arrow: arrow::datatypes::Schema = + scan.logical_schema() + .as_ref() + .try_into_arrow() + .map_err(|e| DeltaError::Internal(format!("kernel logical schema->arrow: {e}")))?; + Ok(( + arrow_schema_to_ipc_bytes(&physical_arrow)?, + arrow_schema_to_ipc_bytes(&logical_arrow)?, + )) +} + +/// Parse the data-read schema (Delta schema JSON) into a kernel `SchemaRef` for +/// `ScanBuilder::with_schema`. +/// +/// The JVM ships the query's data columns drawn from the ANALYSIS-TIME schema (falling back to the +/// snapshot schema), so each field carries `delta.columnMapping.physicalName` + `delta.columnMapping.id` +/// at every nesting level -- the same Delta-JSON format kernel uses for the snapshot schema in the +/// log. Feeding kernel this schema is what makes schema-change-since-analysis correct: kernel's +/// `make_physical` resolves physical names from THESE annotations and its field-id matching +/// null-fills any column whose id changed since analysis (Delta's schema-on-read escape hatch), +/// instead of reading the LIVE snapshot's current names. +fn read_schema_from_json(json: &str) -> DeltaResult { + let st: delta_kernel::schema::StructType = serde_json::from_str(json) + .map_err(|e| DeltaError::Internal(format!("parse data-read schema JSON: {e}")))?; + Ok(Arc::new(st)) +} + +/// Schema-only kernel scan for the batch-file-index read path (`PreparedDeltaFileIndex` / +/// `TahoeBatchFileIndex` / ...), where the file list comes from Delta's `AddFile`s (NOT kernel log +/// replay, for correctness) but the kernel-read executor still needs kernel's resolved +/// physical/logical schemas. Builds the snapshot at `version` + a `Scan` projected to +/// `projected_schema_json` and returns its `(physical_schema_ipc, logical_schema_ipc)`. Does NOT +/// enumerate files. +pub fn plan_delta_read_schemas( + url_str: &str, + config: &DeltaStorageConfig, + version: Option, + projected_schema_json: String, +) -> DeltaResult<(Vec, Vec)> { + let url = normalize_url(url_str)?; + let engine = get_or_create_engine(&url, config)?; + let snapshot = { + let mut builder = Snapshot::builder_for(url); + if let Some(v) = version { + builder = builder.at_version(v); + } + builder.build(&*engine)? + }; + let read_schema = read_schema_from_json(&projected_schema_json)?; + // `Snapshot::build()` already returns `Arc` (= `SnapshotRef`); `scan_builder` consumes + // it by value. + let scan = snapshot.scan_builder().with_schema(read_schema).build()?; + scan_schemas_to_ipc(&scan) +} + +/// Build the top-level Delta column mappings (`logical_name` -> `physical_name`) for a column-mapped +/// table. Used driver-side only, to translate kernel's physical-keyed partition values back to +/// logical names (partition columns are top-level). Nested column mapping no longer needs a recursive +/// tree here -- the executor reads with kernel's own `scan.physical_schema()` / `logical_schema()` +/// (shipped on `DeltaScanCommon`), which resolve physical names + field-ids at every nesting level. +fn build_struct_column_mappings(st: &delta_kernel::schema::StructType) -> Vec<(String, String)> { + use delta_kernel::schema::{ColumnMetadataKey, MetadataValue}; + st.fields() + .filter_map(|field| { + let physical = match field + .metadata + .get(ColumnMetadataKey::ColumnMappingPhysicalName.as_ref()) + { + Some(MetadataValue::String(p)) => p.clone(), + _ => return None, + }; + Some((field.name().clone(), physical)) + }) + .collect() +} + +/// List every active parquet file in a Delta table at the given version. +/// +/// Returns `(entries, actual_version)` where `actual_version` is the +/// snapshot version that was actually read — equal to `version` when +/// specified, or the latest version otherwise. +/// +/// Thin wrapper around [`plan_delta_scan`] that drops the feature list. +/// New code should call `plan_delta_scan` directly so it can honor the +/// unsupported-feature gate. +pub fn list_delta_files( + url_str: &str, + config: &DeltaStorageConfig, + version: Option, +) -> DeltaResult<(Vec, u64)> { + let plan = plan_delta_scan(url_str, config, version)?; + Ok((plan.entries, plan.version)) +} + +/// Plan a Delta scan against the given URL + optional snapshot version. +/// +/// This is the full-fat variant of [`list_delta_files`]: it also reports +/// which reader features are *in use* for this snapshot and NOT yet +/// supported by Comet's native path. +/// +/// Feature detection blends two signals: +/// 1. [`delta_kernel::snapshot::Snapshot::table_properties`] — the +/// protocol-level flags (`column_mapping_mode`, `enable_type_widening`, +/// `enable_row_tracking`). +/// 2. The per-file `ScanFile::dv_info.has_vector()` flag — set to true +/// only when the specific file actually has a deletion vector attached. +/// This is tighter than the `enable_deletion_vectors` table property +/// because a DV-enabled table with no deletes yet is still safe for +/// Comet to read natively. +pub fn plan_delta_scan( + url_str: &str, + config: &DeltaStorageConfig, + version: Option, +) -> DeltaResult { + plan_delta_scan_with_predicate(url_str, config, version, None, None) +} + +/// `projected_schema_json`: the query's data-read columns as Delta schema JSON (the analysis-time +/// schema, or the snapshot schema, carrying `delta.columnMapping.physicalName` + `id`). When present, +/// the scan is built with `with_schema(it)` so kernel resolves the physical names the query was +/// PLANNED with, and `scan.physical_schema()` / `scan.logical_schema()` are returned for the +/// executor's kernel-read path (the intended `delta-kernel-rs` distributed-read pattern: ship the +/// scan's schemas to workers rather than reconstructing them). `None` => full-table scan, no schemas. +pub fn plan_delta_scan_with_predicate( + url_str: &str, + config: &DeltaStorageConfig, + version: Option, + kernel_predicate: Option, + projected_schema_json: Option, +) -> DeltaResult { + let url = normalize_url(url_str)?; + let engine = get_or_create_engine(&url, config)?; + + let snapshot = { + let mut builder = Snapshot::builder_for(url); + if let Some(v) = version { + builder = builder.at_version(v); + } + builder.build(&*engine)? + }; + let actual_version = snapshot.version(); + + // Protocol-level feature gate. Collect the names of features we don't + // yet handle so the Scala side can decide to fall back. Note that we + // explicitly do NOT treat the following as fallback-worthy: + // - `change_data_feed`: only affects CDF queries, not regular reads + // - `in_commit_timestamps`: regular reads work fine + // - `iceberg_compat_v1/v2`: doesn't change Delta read correctness + // - `append_only`: write-side constraint, reads are unaffected + let unsupported_features: Vec = Vec::new(); + let props = snapshot.table_properties(); + // columnMapping is now handled by Phase 4 — no longer a fallback trigger. + // typeWidening: DataFusion's parquet schema adapter handles widening reads + // (parquet stores the file's original type; the adapter casts to the table's + // current widened type at read time). Removed from the gate; verified by + // TypeWidening{TableFeature,Metadata,...}Suite in the Delta regression. + // rowTracking: tables with `enable_row_tracking=true` are scannable + // natively. Queries that explicitly select `_metadata.row_id` / + // `_metadata.row_commit_version` are handled in CometScanRule's + // `applyRowTrackingRewrite` (it rewrites the scan to read the materialized + // physical column, or declines when no materialized name is available). + // No need to gate the whole table's scan path here. + + // Phase 4: extract logical→physical column name mapping from schema metadata. + // For column_mapping_mode = id or name, each StructField carries a + // `delta.columnMapping.physicalName` metadata entry that tells us what the + // parquet file's column name actually is. + let column_mappings: Vec<(String, String)> = if props.column_mapping_mode.is_some() { + build_struct_column_mappings(snapshot.schema().as_ref()) + } else { + Vec::new() + }; + + // `Snapshot::build()` returns `Arc`, and `scan_builder` consumes + // it. Clone the Arc so we keep a stable handle through scan construction + // (driver no longer needs `table_root()` here -- DV decode now happens on the + // executor via `dv_reader::read_dv_indexes` -- but the Arc retention is still + // wanted for any future post-scan-build kernel API that wants the snapshot). + let snapshot_arc: Arc<_> = snapshot; + let mut scan_builder = Arc::clone(&snapshot_arc).scan_builder(); + if let Some(pred) = kernel_predicate { + scan_builder = scan_builder.with_predicate(Arc::new(pred)); + } + // Project the scan to the query's data columns so `scan.physical_schema()` / + // `scan.logical_schema()` carry the projected shape kernel resolves -- from the analysis-time + // schema so schema-change-since-analysis reads correctly (see `read_schema_from_json`). + let projected = projected_schema_json.is_some(); + if let Some(json) = &projected_schema_json { + scan_builder = scan_builder.with_schema(read_schema_from_json(json)?); + } + let scan = scan_builder.build()?; + + // Kernel-read schemas: serialize the scan's own physical/logical schemas (Arrow IPC, field-ids + // preserved) for the executor. Only when we projected -- a full-table scan's schemas aren't the + // query's read shape, and the executor's non-kernel-read path doesn't consume them. + let (physical_schema_ipc, logical_schema_ipc) = if projected { + scan_schemas_to_ipc(&scan)? + } else { + (Vec::new(), Vec::new()) + }; + + // Per-row state extracted directly from each scan_metadata RecordBatch -- avoids + // both `DvInfo` (whose `deletion_vector` field is `pub(crate)`) and the per-DV + // `get_row_indexes` driver-side read that previously materialised a Vec. + // + // Two parallel vecs (indexed by visit order over SELECTED rows): + // - row_tracking: (baseRowId, defaultRowCommitVersion). `ScanFile` doesn't + // surface these; we extract from the `fileConstantValues` struct column. + // - dv_descriptors: per-row DV descriptor as a proto message. None = no DV. + // The executor calls `kernel::DeletionVectorDescriptor::read` on-task -- + // the driver no longer holds the expanded indexes. + // + // Comet's native synthetic-columns exec uses base_row_id / default_row_commit_version + // to synthesise Delta's logical `row_id` and `row_commit_version`. + struct RawEntry { + path: String, + size: i64, + modification_time: i64, + num_records: Option, + partition_values: HashMap, + dv_descriptor: Option, + base_row_id: Option, + default_row_commit_version: Option, + transform_json: Vec, + } + + // Kernel's `visit_scan_files` requires a `fn` callback (not `FnMut`), so any + // per-call state must live in the `context` we pass in. Use a struct that carries + // both the accumulator AND the per-row lookups for the current batch. + struct RawEntryAcc { + entries: Vec, + row_tracking: Vec<(Option, Option)>, + dv_descriptors: Vec>, + next_idx: usize, + // First transform-serialization error, surfaced after the walk (the `visit_scan_files` + // callback is a plain `fn() -> ()`, so it can't propagate a `Result` directly). + transform_err: Option, + } + let mut acc = RawEntryAcc { + entries: Vec::new(), + row_tracking: Vec::new(), + dv_descriptors: Vec::new(), + next_idx: 0, + transform_err: None, + }; + let scan_metadata = scan.scan_metadata(&*engine)?; + + for meta_result in scan_metadata { + let meta: delta_kernel::scan::ScanMetadata = meta_result?; + // Pre-extract per-row state for the SELECTED rows in this batch. Kernel's + // `visit_scan_files` walks selected rows in order; we build parallel vecs + // indexed by visit order, so the callback pulls each row's values via a + // shared counter. + acc.row_tracking = extract_row_tracking_for_selected(&meta)?; + acc.dv_descriptors = extract_dv_descriptors_for_selected(&meta)?; + acc.next_idx = 0; + acc = meta.visit_scan_files( + acc, + |acc: &mut RawEntryAcc, scan_file: delta_kernel::scan::state::ScanFile| { + let num_records = scan_file.stats.as_ref().map(|s| s.num_records); + let (base_row_id, default_row_commit_version) = acc + .row_tracking + .get(acc.next_idx) + .copied() + .unwrap_or((None, None)); + let dv_descriptor = acc + .dv_descriptors + .get(acc.next_idx) + .cloned() + .unwrap_or(None); + acc.next_idx += 1; + // Kernel already resolved this file's transform (partition literals + baseRowId + // baked in) onto `scan_file.transform`. Serialize it as JSON for the executor; empty + // when kernel has no transform (plain pass-through). Capture the first error to + // surface after the walk. + let transform_json = match &scan_file.transform { + Some(expr) => match serde_json::to_vec(expr.as_ref()) { + Ok(bytes) => bytes, + Err(e) => { + if acc.transform_err.is_none() { + acc.transform_err = Some(format!( + "serialize kernel transform for {}: {e}", + scan_file.path + )); + } + Vec::new() + } + }, + None => Vec::new(), + }; + acc.entries.push(RawEntry { + path: scan_file.path, + size: scan_file.size, + modification_time: scan_file.modification_time, + num_records, + partition_values: scan_file.partition_values, + dv_descriptor, + base_row_id, + default_row_commit_version, + transform_json, + }); + }, + )?; + if let Some(msg) = acc.transform_err.take() { + return Err(DeltaError::Internal(msg)); + } + } + let raw = acc.entries; + + // No more driver-side DV materialisation -- just forward the descriptor. The + // executor (`dv_reader::read_dv_indexes` invoked from `DeltaSyntheticColumnsExec`) + // reads + decodes the RoaringBitmap on-task. Pre-refactor this loop called + // `DvInfo::get_row_indexes` and produced a `Vec` per file, which on the + // 99 M-row "huge table delete" DV reached ~800 MB per scan exec (task #218). + let mut entries: Vec = Vec::with_capacity(raw.len()); + for r in raw { + entries.push(DeltaFileEntry { + path: r.path, + size: r.size, + modification_time: r.modification_time, + num_records: r.num_records, + partition_values: r.partition_values, + dv_descriptor: r.dv_descriptor, + base_row_id: r.base_row_id, + default_row_commit_version: r.default_row_commit_version, + transform_json: r.transform_json, + }); + } + + Ok(DeltaScanPlan { + entries, + version: actual_version, + unsupported_features, + column_mappings, + physical_schema_ipc, + logical_schema_ipc, + }) +} + +/// Normalize a table URL so kernel's `table_root.join("_delta_log/")` +/// appends rather than replaces. Bare paths become `file://` URLs. +/// +/// Accepts three shapes: +/// 1. `s3://`, `s3a://`, `az://`, `azure://`, `abfs://`, `abfss://`, +/// `file://` — already-formed URLs, parsed directly. +/// 2. `file:/Users/...` — Hadoop's `Path.toUri.toString` output, which +/// uses a *single* slash and is NOT a valid `Url::parse` input. We +/// rewrite this to `file://` before parsing. +/// 3. Bare local paths — canonicalized and turned into `file://` via +/// `Url::from_directory_path`. +/// +/// NOTE: this intentionally does NOT reuse core's scan URL parsing +/// (`prepare_object_store_with_configs` / `get_partitioned_files`). Those do a bare +/// `Url::parse` on already-`file://`-formed, URL-encoded *data-file* paths Spark supplies. +/// The Delta *table root* instead arrives from `DeltaReflection.extractTableRoot` as +/// Hadoop's `Path.toUri.toString` (the single-slash `file:/...` form `Url::parse` rejects) +/// -- hence the rewrite below -- and kernel additionally requires the trailing-slash +/// invariant core has no reason to enforce. Percent-escapes in the path body are passed +/// through verbatim (the rewrite only splices the `file:` prefix); kernel decodes them once, +/// matching the double-encode applied in `DeltaReflection.pathToSingleEncodedUri`. The +/// executor-side sibling `dv_reader::normalize_table_root` enforces the same trailing-slash +/// invariant on an already-`file://`-formed root; keep the two in lock-step. +pub(crate) fn normalize_url(url_str: &str) -> DeltaResult { + // Hadoop's java.net.URI.toString emits `file:/path/to/t` (one slash) + // for local files. Rewrite into the `file:///path` form that + // `Url::parse` understands. + if url_str.starts_with("file:/") && !url_str.starts_with("file://") { + let rewritten = format!("file://{}", &url_str["file:".len()..]); + let mut url = Url::parse(&rewritten).map_err(|e| DeltaError::InvalidUrl { + url: url_str.to_string(), + source: e, + })?; + ensure_trailing_slash(&mut url); + return Ok(url); + } + + if url_str.starts_with("s3://") + || url_str.starts_with("s3a://") + || url_str.starts_with("az://") + || url_str.starts_with("azure://") + || url_str.starts_with("abfs://") + || url_str.starts_with("abfss://") + || url_str.starts_with("file://") + { + let mut url = Url::parse(url_str).map_err(|e| DeltaError::InvalidUrl { + url: url_str.to_string(), + source: e, + })?; + ensure_trailing_slash(&mut url); + Ok(url) + } else { + let abs_path = std::path::Path::new(url_str).canonicalize().map_err(|e| { + DeltaError::PathResolution { + path: url_str.to_string(), + source: e, + } + })?; + Url::from_directory_path(&abs_path).map_err(|_| DeltaError::PathToUrl { + path: abs_path.display().to_string(), + }) + } +} + +fn ensure_trailing_slash(url: &mut Url) { + let path = url.path().to_string(); + if !path.ends_with('/') { + url.set_path(&format!("{path}/")); + } +} + +/// Extract `(baseRowId, defaultRowCommitVersion)` per SELECTED row from a `ScanMetadata` +/// batch's underlying `RecordBatch`. Kernel's `visit_scan_files` callback receives a +/// `ScanFile` that does NOT surface these row-tracking values; they live in the raw +/// `fileConstantValues` struct column on the underlying arrow batch. +/// +/// `kernel/src/scan/log_replay.rs::SCAN_ROW_SCHEMA` defines the schema: +/// { path, size, modificationTime, stats, deletionVector, +/// fileConstantValues: { partitionValues, baseRowId, defaultRowCommitVersion, tags } } +/// So the fileConstantValues struct is the 6th top-level field (index 5), and within it +/// baseRowId is at field index 1 and defaultRowCommitVersion at field index 2. +/// +/// Returns one `(Option, Option)` per SELECTED row, in visit_scan_files order. +/// Rows where row tracking isn't enabled have `(None, None)`. +fn extract_row_tracking_for_selected( + meta: &delta_kernel::scan::ScanMetadata, +) -> DeltaResult, Option)>> { + use delta_kernel::arrow::array::{Array, Int64Array, StructArray}; + use delta_kernel::engine::arrow_data::ArrowEngineData; + let engine_data = meta.scan_files.data(); + let arrow = match engine_data.any_ref().downcast_ref::() { + Some(a) => a, + // Non-Arrow engine (shouldn't happen for our DefaultEngine path); return empty + // so downstream sees (None, None) per row and the row-tracking decline gate + // takes over. + None => return Ok(Vec::new()), + }; + let batch = arrow.record_batch(); + let total_rows = batch.num_rows(); + + let file_constants = batch + .column_by_name("fileConstantValues") + .and_then(|c| c.as_any().downcast_ref::()); + let (base_arr, default_arr): (Option<&Int64Array>, Option<&Int64Array>) = match file_constants { + Some(s) => ( + s.column_by_name("baseRowId") + .and_then(|c| c.as_any().downcast_ref::()), + s.column_by_name("defaultRowCommitVersion") + .and_then(|c| c.as_any().downcast_ref::()), + ), + None => (None, None), + }; + + let sel = meta.scan_files.selection_vector(); + // FilteredEngineData::try_new asserts `sel.len() <= data.len()`; rows beyond + // sel.len() are treated as not-selected. visit_scan_files visits only rows that ARE + // selected, so any rows past sel.len() won't appear in the callback and our parallel + // vec stays aligned. The explicit bound below makes the contract obvious. + let bounded_rows = total_rows.min(sel.len()); + let mut out: Vec<(Option, Option)> = + Vec::with_capacity(sel.iter().filter(|b| **b).count()); + for (i, &keep) in sel.iter().enumerate().take(bounded_rows) { + if !keep { + continue; + } + let b = base_arr.and_then(|a| if a.is_null(i) { None } else { Some(a.value(i)) }); + let d = default_arr.and_then(|a| if a.is_null(i) { None } else { Some(a.value(i)) }); + out.push((b, d)); + } + Ok(out) +} + +/// Per-row DV descriptors extracted from kernel's `scan_metadata.scan_files` +/// RecordBatch, indexed by SELECTED row position (parallel to row_tracking, +/// consumed in `visit_scan_files` order). +/// +/// `None` for rows without a DV (no deletion vector attached to that AddFile). +/// +/// We extract directly from the RecordBatch instead of via `ScanFile.dv_info` +/// because kernel's `DvInfo` only exposes `has_vector()` + `get_row_indexes()` +/// publicly (the descriptor itself is `pub(crate)`), and `get_row_indexes()` +/// materialises the full bitmap on the DRIVER -- which was the 1 GB long[] +/// retention bug we're fixing (task #218). Reading the fields from the kernel +/// scan_files schema directly lets us ship the descriptor instead of the indices. +/// +/// Schema reference: `delta_kernel::scan::log_replay::SCAN_ROW_SCHEMA` includes a +/// `deletionVector` struct column with fields `storageType` (utf8), +/// `pathOrInlineDv` (utf8), `offset` (int32, nullable), `sizeInBytes` (int32), +/// `cardinality` (int64). When the AddFile has no DV the whole struct is null. +fn extract_dv_descriptors_for_selected( + meta: &delta_kernel::scan::ScanMetadata, +) -> DeltaResult>> { + use delta_kernel::arrow::array::{Array, Int32Array, Int64Array, StringArray, StructArray}; + use delta_kernel::engine::arrow_data::ArrowEngineData; + let engine_data = meta.scan_files.data(); + let arrow = match engine_data.any_ref().downcast_ref::() { + Some(a) => a, + // Non-Arrow engine (shouldn't happen for our DefaultEngine path); return + // empty so downstream sees None per row -- which matches the no-DV case. + None => return Ok(Vec::new()), + }; + let batch = arrow.record_batch(); + let total_rows = batch.num_rows(); + + let dv_struct = batch + .column_by_name("deletionVector") + .and_then(|c| c.as_any().downcast_ref::()); + let (storage_arr, path_arr, offset_arr, size_arr, card_arr) = match dv_struct { + Some(s) => ( + s.column_by_name("storageType") + .and_then(|c| c.as_any().downcast_ref::()), + s.column_by_name("pathOrInlineDv") + .and_then(|c| c.as_any().downcast_ref::()), + s.column_by_name("offset") + .and_then(|c| c.as_any().downcast_ref::()), + s.column_by_name("sizeInBytes") + .and_then(|c| c.as_any().downcast_ref::()), + s.column_by_name("cardinality") + .and_then(|c| c.as_any().downcast_ref::()), + ), + None => (None, None, None, None, None), + }; + + let sel = meta.scan_files.selection_vector(); + let bounded_rows = total_rows.min(sel.len()); + let mut out: Vec> = + Vec::with_capacity(sel.iter().filter(|b| **b).count()); + for (i, &keep) in sel.iter().enumerate().take(bounded_rows) { + if !keep { + continue; + } + // A row has a DV iff the outer struct is non-null AND storageType is present + // and non-null. Kernel materialises the entire struct as null when there's + // no DV; the field-level null check is a belt-and-braces for engines that + // might emit a non-null struct with null fields. + let struct_null = dv_struct.map(|s| s.is_null(i)).unwrap_or(true); + if struct_null { + out.push(None); + continue; + } + let storage_null = storage_arr.map(|a| a.is_null(i)).unwrap_or(true); + if storage_null { + out.push(None); + continue; + } + let storage_type = storage_arr.unwrap().value(i).to_string(); + let path_or_inline_dv = path_arr.map(|a| a.value(i).to_string()).unwrap_or_default(); + // offset is `optional uint64` on the proto side -- preserve the null/non-null + // distinction from the source (Delta inline DVs sometimes lack an offset). + let offset = offset_arr.and_then(|a| { + if a.is_null(i) { + None + } else { + Some(a.value(i) as u64) + } + }); + let size_in_bytes = size_arr.map(|a| a.value(i) as u64).unwrap_or(0); + let cardinality = card_arr.map(|a| a.value(i) as u64).unwrap_or(0); + out.push(Some(crate::proto::DeltaDvDescriptor { + storage_type, + path_or_inline_dv, + offset, + size_in_bytes, + cardinality, + // inline_bytes is reserved for a future optimisation where the driver + // pre-decodes inline DVs (rare + already small). For now the executor + // decodes via `path_or_inline_dv` (kernel's + // `DeletionVectorDescriptor::read` handles all three storage types + // uniformly when reconstructed with the original `path_or_inline_dv`). + inline_bytes: Vec::new(), + })); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_url_trailing_slash() { + let url = normalize_url("file:///tmp/my_table").unwrap(); + assert!(url.path().ends_with('/'), "URL should end with /: {url}"); + assert_eq!(url.as_str(), "file:///tmp/my_table/"); + + let url = normalize_url("file:///tmp/my_table/").unwrap(); + assert_eq!(url.as_str(), "file:///tmp/my_table/"); + + let url = normalize_url("s3://bucket/path/to/table").unwrap(); + assert!(url.path().ends_with('/'), "URL should end with /: {url}"); + } + + #[test] + fn test_normalize_url_hadoop_single_slash_form() { + // Hadoop's Path.toUri.toString produces `file:/path` (single slash), + // not `file:///path`. Must be normalized to a Url::parse-able form. + let url = normalize_url("file:/Users/alice/tmp/t").unwrap(); + assert_eq!(url.as_str(), "file:///Users/alice/tmp/t/"); + + let url = normalize_url("file:/tmp/t/").unwrap(); + assert_eq!(url.as_str(), "file:///tmp/t/"); + } + + #[test] + fn test_normalize_url_join_behavior() { + // The critical invariant: joining `_delta_log/` onto a normalized + // URL must *append*, not replace the last segment. + let url = normalize_url("file:///tmp/my_table").unwrap(); + let log_url = url.join("_delta_log/").unwrap(); + assert_eq!(log_url.as_str(), "file:///tmp/my_table/_delta_log/"); + } + + #[test] + fn test_list_delta_files_local() { + // Hand-build a minimal Delta table in a tempdir: one protocol action, + // one metadata action, one add action. No Parquet data needed — + // we're exercising the log-replay path only. + let tmp = tempfile::tempdir().unwrap(); + let table_dir = tmp.path().join("test_delta"); + let delta_log = table_dir.join("_delta_log"); + std::fs::create_dir_all(&delta_log).unwrap(); + + let commit0 = [ + r#"{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}"#, + r#"{"metaData":{"id":"test-id","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1700000000000}}"#, + r#"{"add":{"path":"part-00000.parquet","partitionValues":{},"size":5000,"modificationTime":1700000000000,"dataChange":true,"stats":"{\"numRecords\":50}"}}"#, + ] + .join("\n"); + std::fs::write(delta_log.join("00000000000000000000.json"), &commit0).unwrap(); + std::fs::write(table_dir.join("part-00000.parquet"), [0u8]).unwrap(); + + let config = DeltaStorageConfig::default(); + let (entries, version) = + list_delta_files(table_dir.to_str().unwrap(), &config, None).unwrap(); + + assert_eq!(version, 0); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].path, "part-00000.parquet"); + assert_eq!(entries[0].size, 5000); + assert_eq!(entries[0].num_records, Some(50)); + assert!(!entries[0].has_deletion_vector()); + } +} diff --git a/contrib/delta/src/main/scala/org/apache/comet/contrib/delta/DeltaConf.scala b/contrib/delta/src/main/scala/org/apache/comet/contrib/delta/DeltaConf.scala new file mode 100644 index 0000000000..02d2ab3f39 --- /dev/null +++ b/contrib/delta/src/main/scala/org/apache/comet/contrib/delta/DeltaConf.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import org.apache.comet.{ConfigBuilder, ConfigEntry} + +/** + * Contrib-local config entries for the Delta integration. Lives in the contrib's package rather + * than in core's `CometConf` so PR1 stays format-agnostic. Side-effect of object construction is + * registering the entries with `CometConf.allConfs` (via the `ConfigBuilder` machinery), so they + * show up in the generated user-guide docs and `SQLConf` resolution works the usual way. + */ +object DeltaConf { + + // CometConf.register asserts every config has a non-empty category — used for grouping + // entries in the generated user-guide docs. "scan" matches the existing core + // CATEGORY_SCAN string (CATEGORY_* constants in CometConf are `private val` so contribs + // can't reference the symbol; the assertion only checks `nonEmpty`). + private val CATEGORY = "scan" + + val COMET_DELTA_NATIVE_ENABLED: ConfigEntry[Boolean] = + ConfigBuilder("spark.comet.scan.deltaNative.enabled") + .category(CATEGORY) + .doc( + "Whether to enable native Delta table scans via delta-kernel-rs. When enabled, " + + "Delta tables are read directly through Comet's tuned ParquetSource + " + + "DV-filter wrapper, bypassing Spark's Delta reader for better performance.") + .booleanConf + .createWithDefault(true) + + val COMET_DELTA_FALLBACK_ON_UNSUPPORTED_FEATURE: ConfigEntry[Boolean] = + ConfigBuilder("spark.comet.scan.deltaNative.fallbackOnUnsupportedFeature") + .category(CATEGORY) + .doc( + "When true (default), the Delta contrib falls back to Spark's Delta reader on " + + "any Delta protocol feature it doesn't yet support. When false, the contrib " + + "raises an error instead -- useful for tests that want to assert the native " + + "path is reachable for a particular query.") + .booleanConf + .createWithDefault(true) + + val COMET_DELTA_DATA_FILE_CONCURRENCY_LIMIT: ConfigEntry[Int] = + ConfigBuilder("spark.comet.scan.deltaNative.dataFileConcurrencyLimit") + .category(CATEGORY) + .doc( + "Per-Spark-task concurrency when reading Delta data files. Higher values " + + "improve throughput on tables with many small files at the cost of memory. " + + "Values between 2 and 8 are typical.") + .intConf + .checkValue(v => v > 0, "Data file concurrency limit must be positive") + .createWithDefault(1) + + val COMET_DELTA_CDF_MAX_PARTITIONS: ConfigEntry[Int] = + ConfigBuilder("spark.comet.delta.cdf.maxPartitions") + .category(CATEGORY) + .doc( + "Maximum number of Spark partitions a Change Data Feed (readChangeFeed) read is split " + + "into. The inclusive version range is chunked into up to this many contiguous " + + "sub-ranges, each read by an independent native delta-kernel TableChanges call, so a " + + "multi-version CDF read parallelizes across tasks instead of reading the whole range " + + "on one task. Capped by the number of commits in the range.") + .intConf + .checkValue(v => v > 0, "CDF max partitions must be positive") + .createWithDefault(8) +} diff --git a/dev/verify-contrib-delta-gate.sh b/dev/verify-contrib-delta-gate.sh new file mode 100755 index 0000000000..8ea02ebc95 --- /dev/null +++ b/dev/verify-contrib-delta-gate.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Verify the `contrib-delta` build gate keeps Delta surface out of default builds. +# +# Three independent layers are checked: +# 1. Cargo: default `cargo build` doesn't compile `comet-contrib-delta` and +# doesn't pull `delta_kernel` into the dependency tree. +# 2. Maven: default `mvn ... package` doesn't compile any +# `org/apache/comet/contrib/` classes and doesn't pull `io.delta:*` deps. +# 3. Symbol/size: the resulting `libcomet` (`.so` on Linux, `.dylib` on macOS) from the default build is +# meaningfully smaller than the contrib-enabled build, and carries no +# `comet_contrib_delta`/`delta_kernel`/etc. external symbols. +# +# Exit non-zero on the first failure. Designed to be wired into CI so a future +# change that leaks Delta into core gets caught immediately. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +NATIVE_DIR="$ROOT/native" +SPARK_DIR="$ROOT/spark" + +# CI containers check out the repo as a different user than the job runs as, so git refuses to +# operate on it ("detected dubious ownership") -- which can make Maven plugins that read git +# metadata fail fast with no useful output. Mark the tree safe up front (best-effort). +git config --global --add safe.directory "$ROOT" 2>/dev/null || true +git config --global --add safe.directory '*' 2>/dev/null || true + +# Use the repo's Maven wrapper, not a bare `mvn`. The CI gate job runs in a bare +# `amd64/rust` container that has no system Maven on PATH, so `mvn` would fail to +# launch, produce no dependency output, and trip the anti-vacuous guard below. +# `./mvnw` self-provisions the pinned Maven version regardless of the image. +MVNW="$ROOT/mvnw" + +red() { printf '\033[31m%s\033[0m\n' "$*"; } +green() { printf '\033[32m%s\033[0m\n' "$*"; } +hdr() { printf '\n\033[36m==> %s\033[0m\n' "$*"; } + +# ---- Cargo gate ----------------------------------------------------------- + +hdr "Cargo: default build does not depend on comet-contrib-delta / delta_kernel" +cd "$NATIVE_DIR" +TREE_DEFAULT="$(cargo tree -p datafusion-comet --no-default-features 2>/dev/null)" +# Anti-vacuous (mirrors the Maven gate below): a failing `cargo tree` yields empty output, and the +# command-substitution failure doesn't trip `set -e` in an assignment -- so assert the root crate we +# KNOW is always present before concluding "no Delta deps", otherwise a broken cargo-tree run would +# pass the leak check vacuously. (`datafusion-comet ` with a trailing space matches only the root +# crate line, not `datafusion-comet-proto`/`-common`.) +if ! grep -q 'datafusion-comet ' <<<"$TREE_DEFAULT"; then + red "FAIL: default cargo tree produced no datafusion-comet entry (cargo tree likely failed;" + red " refusing to conclude 'no Delta deps' vacuously)" + exit 1 +fi +if grep -qE 'comet-contrib-delta|delta_kernel|delta-kernel' <<<"$TREE_DEFAULT"; then + red "FAIL: default cargo tree contains Delta-related deps:" + grep -E 'comet-contrib-delta|delta_kernel|delta-kernel' <<<"$TREE_DEFAULT" + exit 1 +fi +green "OK: cargo tree default is clean of contrib + kernel" + +TREE_CONTRIB="$(cargo tree -p datafusion-comet --features contrib-delta 2>/dev/null)" +# The gated tree must pull in both the contrib crate AND the heavy `delta_kernel` it +# depends on (driver-side log replay). Require >=2 hits; the symbol check below guards +# against grep-pattern drift either way. +CONTRIB_HITS="$(printf '%s\n' "$TREE_CONTRIB" | grep -cE 'comet-contrib-delta|delta_kernel|delta-kernel' || true)" +if [[ "$CONTRIB_HITS" -lt 2 ]]; then + red "FAIL: --features contrib-delta tree missing expected Delta-related entries (hits=$CONTRIB_HITS)" + exit 1 +fi +green "OK: cargo tree with contrib-delta correctly pulls comet-contrib-delta + delta_kernel" + +# ---- Maven gate ----------------------------------------------------------- + +hdr "Maven: default profile excludes io.delta:* dependencies" +cd "$ROOT" +# `dependency:list` can't run in a fresh CI checkout: it needs the sibling reactor JARs +# (comet-common, the shims) which aren't built, so it fails with a resolution error and no +# output. `help:effective-pom` only merges POM models (no artifact resolution), so it works +# without a build. We extract the ACTIVE top-level -- after , +# before the listing -- which is exactly what dependency:list would have shown for the +# active profiles (and excludes the inactive contrib-delta profile's own io.delta declaration). +# Last effective-pom invocation's combined output, kept so the anti-vacuous guard can SHOW why +# mvn failed instead of swallowing it. +EPOM_LOG="$(mktemp)" +delta_active_deps() { # args: -P / -D flags + local epom + epom="$(mktemp)" + "$MVNW" help:effective-pom -Djava.version=17 -Dmaven.gitcommitid.skip -pl spark \ + -Doutput="$epom" "$@" >"$EPOM_LOG" 2>&1 || true + awk '/<\/dependencyManagement>/{f=1} //{f=0} f' "$epom" + rm -f "$epom" +} +# Resolved delta-spark version from the active deps (empty if absent). +delta_spark_version() { # args: -P flags + delta_active_deps "$@" | + grep -A2 'artifactId>delta-spark' | + grep -oE '[^<]+' | sed -n '1s///p' +} + +DEPS_DEFAULT="$(delta_active_deps -Pspark-4.1)" +# Anti-vacuous: a broken mvn run yields empty output; assert a dep we KNOW is always present so a +# broken run fails loudly instead of "passing" the io.delta check by finding nothing. +if ! grep -q 'org.apache.spark' <<<"$DEPS_DEFAULT"; then + red "FAIL: default effective-pom produced no org.apache.spark deps (mvn likely failed;" + red " refusing to conclude 'zero io.delta' vacuously)" + red " --- mvnw: $MVNW (java=${JAVA_HOME:-unset}) ---" + red " --- effective-pom output (last 60 lines) ---" + tail -60 "$EPOM_LOG" >&2 || true + exit 1 +fi +if grep -q 'io.delta' <<<"$DEPS_DEFAULT"; then + red "FAIL: default Maven build pulls io.delta dependencies:" + echo "$DEPS_DEFAULT" | grep -A2 'io.delta' + exit 1 +fi +green "OK: default Maven build has zero io.delta dependencies" + +# Per-Spark Delta version pinning: spark-4.1 -> delta-spark 4.1.x, spark-3.5 -> 3.x, spark-4.0 -> +# 4.0.x (Delta 4.1 needs Spark 4.1 internals; 4.0 must stay on 4.0.x to avoid a runtime +# NoSuchMethodError on ParserInterface.$init$). +V41="$(delta_spark_version -Pspark-4.1,contrib-delta)" +case "$V41" in + 4.1.*) green "OK: -Pcontrib-delta + spark-4.1 correctly pulls delta-spark $V41" ;; + *) red "FAIL: -Pcontrib-delta + spark-4.1 expected delta-spark 4.1.x, got '${V41:-}'"; exit 1 ;; +esac +V35="$(delta_spark_version -Pspark-3.5,contrib-delta)" +case "$V35" in + 3.*) green "OK: -Pcontrib-delta + spark-3.5 correctly pulls delta-spark $V35" ;; + *) red "FAIL: -Pcontrib-delta + spark-3.5 expected delta-spark 3.x, got '${V35:-}'"; exit 1 ;; +esac +V40="$(delta_spark_version -Pspark-4.0,contrib-delta)" +case "$V40" in + 4.0.*) green "OK: -Pcontrib-delta + spark-4.0 correctly pulls delta-spark $V40" ;; + *) red "FAIL: -Pcontrib-delta + spark-4.0 expected delta-spark 4.0.x, got '${V40:-}'"; exit 1 ;; +esac + +# ---- Compiled-class gate -------------------------------------------------- + +hdr "Compiled classes: no contrib/delta classes in default build" +cd "$ROOT" +"$MVNW" -Pspark-4.1 -Djava.version=17 -Dmaven.compiler.source=17 -Dmaven.compiler.target=17 -Dmaven.gitcommitid.skip -pl spark -am test-compile -q -DskipTests=true >/dev/null 2>&1 +LEAK_CLASSES="$(find spark/target/classes -path '*comet/contrib*' -name '*.class' 2>/dev/null)" +if [[ -n "$LEAK_CLASSES" ]]; then + red "FAIL: default Maven build compiled contrib classes:" + echo "$LEAK_CLASSES" + exit 1 +fi +DELTA_IMPL_LEAKS="$(find spark/target/classes \ + \( -name 'CometDeltaNativeScan*' \ + -o -name 'CometDeltaNativeScanExec*' \ + -o -name 'DeltaScanRule*' \ + -o -name 'DeltaReflection*' \) \ + 2>/dev/null)" +if [[ -n "$DELTA_IMPL_LEAKS" ]]; then + red "FAIL: default Maven build compiled Delta-implementation classes:" + echo "$DELTA_IMPL_LEAKS" + exit 1 +fi +green "OK: only the always-present DeltaIntegration reflection bridge in default classes" + +# ---- libcomet symbol/size gate ------------------------------------------- + +hdr "libcomet: default build is smaller and has no Delta symbols" +cd "$NATIVE_DIR" +# The cdylib extension is platform-specific: `libcomet.so` on Linux (CI), `libcomet.dylib` on +# macOS. Find whichever the build produced; `stat`/`nm` flags also differ across the two. +# Echo whichever lib exists (empty if neither). MUST return 0: `ls a b` exits non-zero when one of +# the two paths is absent (the normal case -- only one extension exists per platform), and under +# `set -euo pipefail` that would kill the script at `LIB=$(comet_lib)` even though it found the lib. +comet_lib() { + local f + for f in target/debug/libcomet.so target/debug/libcomet.dylib; do + if [[ -f "$f" ]]; then + echo "$f" + return 0 + fi + done + return 0 +} +lib_size() { stat -c%s "$1" 2>/dev/null || stat -f%z "$1"; } +# Plain `nm` on the (unstripped debug) lib lists the full symbol table on both platforms, so a +# name grep finds the Delta symbols whether they're exported or internal. +delta_syms() { + nm "$1" 2>/dev/null | grep -ciE 'comet_contrib_delta|delta_kernel|deltadvfilter|deltasynthetic' || true +} + +cargo clean -p comet-contrib-delta -p datafusion-comet >/dev/null 2>&1 || true +cargo build -j 4 -p datafusion-comet >/dev/null 2>&1 +LIB_DEFAULT="$(comet_lib)" +if [[ -z "$LIB_DEFAULT" ]]; then + red "FAIL: default build produced no libcomet.{so,dylib} under $NATIVE_DIR/target/debug" + exit 1 +fi +SIZE_DEFAULT="$(lib_size "$LIB_DEFAULT")" +if command -v nm >/dev/null 2>&1; then + EXT_SYMS="$(delta_syms "$LIB_DEFAULT")" +else + EXT_SYMS=0 +fi +if [[ "$EXT_SYMS" -ne 0 ]]; then + red "FAIL: default libcomet contains $EXT_SYMS Delta-related symbols" + exit 1 +fi +green "OK: default libcomet has 0 Delta symbols (size=$SIZE_DEFAULT bytes)" + +cargo build -j 4 -p datafusion-comet --features contrib-delta >/dev/null 2>&1 +LIB_CONTRIB="$(comet_lib)" +SIZE_CONTRIB="$(lib_size "$LIB_CONTRIB")" +if [[ "$SIZE_CONTRIB" -le "$SIZE_DEFAULT" ]]; then + red "FAIL: contrib-enabled libcomet (size=$SIZE_CONTRIB) is not larger than default (size=$SIZE_DEFAULT)" + red " (would indicate contrib was being linked into default build too)" + exit 1 +fi +# Sanity check: the contrib-enabled libcomet MUST contain Delta-related symbols. +# Without this, a future Rust toolchain that mangles symbols differently (so our +# grep pattern stops matching) would silently make the default-build check a no-op +# while still passing -- the gate would lie about being enforced. Asserting both +# "default has 0" AND "contrib has >0" catches grep-pattern drift. +if command -v nm >/dev/null 2>&1; then + CONTRIB_SYMS="$(delta_syms "$LIB_CONTRIB")" + if [[ "$CONTRIB_SYMS" -lt 1 ]]; then + red "FAIL: contrib-enabled libcomet has 0 Delta-related symbols matching our grep pattern." + red " This means the symbol-name pattern in this script has drifted from what" + red " Rust currently emits, and the default-build check above is now a no-op." + red " Inspect the dylib's exports and update the grep pattern." + exit 1 + fi +fi +DIFF_MB=$(( (SIZE_CONTRIB - SIZE_DEFAULT) / 1024 / 1024 )) +green "OK: contrib-enabled libcomet is ${DIFF_MB} MB larger than default (size=$SIZE_CONTRIB bytes)" + +# ---- Summary -------------------------------------------------------------- + +hdr "All gate checks passed" +echo " default cargo: no comet-contrib-delta, no delta_kernel" +echo " default mvn: no io.delta:*, no contrib/delta classes" +echo " default dylib: ${DIFF_MB} MB smaller than contrib build, 0 Delta symbols" +echo +echo "Run with: dev/verify-contrib-delta-gate.sh" diff --git a/native/Cargo.lock b/native/Cargo.lock index 1ce78adfdb..1cb826a9cd 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1515,6 +1515,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "comet-contrib-delta" +version = "0.18.0" +dependencies = [ + "arrow", + "datafusion", + "datafusion-comet-jni-bridge", + "datafusion-comet-proto", + "delta_kernel", + "jni 0.22.4", + "log", + "object_store", + "prost", + "serde_json", + "thiserror 2.0.18", + "url", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1593,6 +1611,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1636,6 +1664,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32c" version = "0.6.8" @@ -1990,6 +2033,7 @@ dependencies = [ "aws-config", "aws-credential-types", "bytes", + "comet-contrib-delta", "criterion", "datafusion", "datafusion-comet-common", @@ -2748,6 +2792,50 @@ dependencies = [ "uuid", ] +[[package]] +name = "delta_kernel" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bbaa9c975772c718e55ed1e502976a25d353a290b46b00ad1514751514be46d" +dependencies = [ + "arrow", + "bytes", + "chrono", + "crc", + "delta_kernel_derive", + "futures", + "indexmap 2.14.0", + "itertools 0.14.0", + "object_store", + "parquet", + "percent-encoding", + "rand 0.9.4", + "reqwest 0.13.4", + "roaring", + "rustc_version", + "serde", + "serde_json", + "strum", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", + "url", + "uuid", + "z85", +] + +[[package]] +name = "delta_kernel_derive" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "797369f8300adc409361bba541ed868a7cade2637691a22af521ad0909be2454" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "der" version = "0.7.10" @@ -2896,6 +2984,15 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equator" version = "0.4.2" @@ -3545,9 +3642,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -4331,6 +4430,12 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -4417,6 +4522,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num" version = "0.4.3" @@ -5318,7 +5432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5337,7 +5451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5787,8 +5901,10 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2", "http 1.4.2", "http-body 1.0.1", "http-body-util", @@ -5797,6 +5913,7 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", "quinn", @@ -5977,7 +6094,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni 0.22.4", "log", @@ -6097,7 +6214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags 2.13.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -6310,6 +6427,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -6554,6 +6680,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -6623,6 +6770,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "thrift" version = "0.17.0" @@ -6854,6 +7010,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -6877,6 +7034,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -7052,10 +7235,17 @@ checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "value-bag" version = "1.12.0" @@ -7345,6 +7535,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -7733,6 +7934,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "z85" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e61e59a957b7ccee15d2049f86e8bfd6f66968fcd88f018950662d9b86e675" + [[package]] name = "zerocopy" version = "0.8.52" diff --git a/native/Cargo.toml b/native/Cargo.toml index 29cc70e806..43272db27f 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -18,6 +18,10 @@ [workspace] default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"] members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "hdfs", "fs-hdfs"] +# The contrib crate at ../contrib/delta/native is intentionally NOT a workspace member +# (workspace members must live hierarchically under the workspace root). It's pulled in +# as a path dep by `core/Cargo.toml` when the `contrib-delta` feature is enabled. +exclude = ["../contrib"] resolver = "2" [workspace.package] diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 5cb970c762..802930e1eb 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -69,6 +69,9 @@ aws-config = { workspace = true } aws-credential-types = { workspace = true } parking_lot = "0.12.5" datafusion-comet-objectstore-hdfs = { path = "../hdfs", optional = true, default-features = false, features = ["hdfs"] } +# Optional Delta Lake contrib (enabled by the `contrib-delta` feature). Source lives +# under `contrib/delta/native/` so non-Delta committers can ignore it. +comet-contrib-delta = { path = "../../contrib/delta/native", optional = true } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2"] } object_store_opendal = { version = "0.57.0", optional = true } hdfs-sys = {version = "0.3", optional = true, features = ["hdfs_3_3"]} @@ -100,6 +103,10 @@ default = ["hdfs-opendal"] hdfs = ["datafusion-comet-objectstore-hdfs"] hdfs-opendal = ["opendal", "object_store_opendal", "hdfs-sys"] jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] +# Delta Lake integration. When enabled, links the `comet-contrib-delta` crate +# into `libcomet` and activates the `OpStruct::DeltaScan` dispatcher arm. +# Default builds carry zero Delta surface. +contrib-delta = ["dep:comet-contrib-delta"] # exclude optional packages from cargo machete verifications [package.metadata.cargo-machete] diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 4a7a21006d..420eb9c0b9 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -219,7 +219,13 @@ pub fn get_runtime() -> &'static Runtime { TOKIO_RUNTIME.get_or_init(|| build_runtime(None)) } -/// Returns a short name for an OpStruct variant. +/// Returns a short name for an OpStruct variant. Used for tracing event names; +/// no contrib-specific logic. The `OpStruct::DeltaScan` arm stays unconditional +/// even in non-`contrib-delta` builds because the proto enum is generated +/// regardless of cargo feature flags and Rust requires an exhaustive match -- +/// returning a name string here keeps default-build error messages (e.g. "Received +/// a DeltaScan operator but core was built without the `contrib-delta` Cargo +/// feature") able to identify the offender by name. fn op_name(op: &OpStruct) -> &'static str { match op { OpStruct::Scan(_) => "Scan", @@ -240,6 +246,7 @@ fn op_name(op: &OpStruct) -> &'static str { OpStruct::CsvScan(_) => "CsvScan", OpStruct::ShuffleScan(_) => "ShuffleScan", OpStruct::BroadcastNestedLoopJoin(_) => "BroadcastNestedLoopJoin", + OpStruct::DeltaScan(_) => "DeltaScan", } } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index c07a92d700..133db8066f 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -21,6 +21,15 @@ pub mod expression_registry; pub mod macros; pub mod operator_registry; +// Glue that wires the optional Delta integration into core's plan-tree builder. +// Compiled only under `--features contrib-delta`; default builds carry zero Delta +// surface. The bulk of the Delta logic lives in the `comet-contrib-delta` crate -- +// this module is just the bridge that reaches into core's `pub(crate)` planner +// helpers (`create_expr`, `init_datasource_exec`, `prepare_object_store_with_configs`) +// and calls into that crate. +#[cfg(feature = "contrib-delta")] +mod delta_scan; + use crate::execution::operators::init_csv_datasource_exec; use crate::execution::operators::AlignedArrowStreamReader; use crate::execution::operators::IcebergScanExec; @@ -1597,6 +1606,27 @@ impl PhysicalPlanner { )), )) } + OpStruct::DeltaScan(scan) => { + // Delta Lake scan -- handled by the optional `contrib/delta/` integration. + // The dispatcher arm exists unconditionally so a default build that receives + // a Delta-shaped plan from a misconfigured driver gets a clear error instead + // of a "no match" decode failure. + #[cfg(not(feature = "contrib-delta"))] + { + let _ = scan; + Err(GeneralError( + "Received a DeltaScan operator but core was built without the \ + `contrib-delta` Cargo feature. Rebuild with both \ + `-Pcontrib-delta` (Maven) and `--features contrib-delta` (Cargo) \ + to enable Delta Lake support." + .into(), + )) + } + #[cfg(feature = "contrib-delta")] + { + delta_scan::plan_delta_scan(self, spark_plan, scan) + } + } OpStruct::ShuffleWriter(writer) => { assert_eq!(children.len(), 1); let (scans, shuffle_scans, child) = @@ -3194,7 +3224,13 @@ pub fn from_protobuf_eval_mode(value: i32) -> Result SchemaRef { let arrow_fields = spark_types diff --git a/native/core/src/execution/planner/delta_scan.rs b/native/core/src/execution/planner/delta_scan.rs new file mode 100644 index 0000000000..3ef4f351ad --- /dev/null +++ b/native/core/src/execution/planner/delta_scan.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `OpStruct::DeltaScan` planner body, feature-gated behind `contrib-delta`. +//! +//! Thin bridge between core's plan-tree builder and the [`comet_contrib_delta`] crate. ALL +//! Delta-specific scan planning lives in `comet_contrib_delta::planner::plan_delta_scan`; this +//! shim only computes the requested/partition Arrow schemas (core owns the proto -> arrow type +//! converter, used across the planner) and wraps the contrib's `ExecutionPlan` in a `SparkPlan`. +//! It lives in core (not the contrib crate) because the converter is core's `pub(crate)` and a +//! `contrib -> core` dependency would cycle with core's optional `contrib-delta` dep. Compiled +//! only under `--features contrib-delta`; default builds carry zero Delta surface. + +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion_comet_proto::spark_operator::{DeltaScan, Operator}; + +use crate::execution::operators::ExecutionError::GeneralError; +use crate::execution::planner::convert_spark_types_to_arrow_schema; +use crate::execution::planner::PhysicalPlanner; +use crate::execution::planner::PlanCreationResult; +use crate::execution::spark_plan::SparkPlan; + +pub(crate) fn plan_delta_scan( + // The kernel-read path doesn't need the planner (the old ParquetSource path used it to build + // pushed-down filter exprs); kept in the signature for the dispatcher call site. + _planner: &PhysicalPlanner, + spark_plan: &Operator, + scan: &DeltaScan, +) -> PlanCreationResult { + let common = scan + .common + .as_ref() + .ok_or_else(|| GeneralError("DeltaScan missing common data".into()))?; + + // Core owns the proto -> arrow schema converter (`to_arrow_datatype`, used across the planner), + // so compute the requested + partition schemas here and hand them to the contrib, which owns all + // Delta-specific scan planning (#77). + let required_schema: SchemaRef = + convert_spark_types_to_arrow_schema(common.required_schema.as_slice()); + let partition_schema: SchemaRef = + convert_spark_types_to_arrow_schema(common.partition_schema.as_slice()); + + let exec = comet_contrib_delta::planner::plan_delta_scan( + scan, + common, + &required_schema, + &partition_schema, + ) + .map_err(|e| GeneralError(format!("{e}")))?; + + Ok(( + vec![], + vec![], + Arc::new(SparkPlan::new(spark_plan.plan_id, exec, vec![])), + )) +} diff --git a/native/core/src/execution/planner/operator_registry.rs b/native/core/src/execution/planner/operator_registry.rs index dc98a32fb0..5cb5c5acb2 100644 --- a/native/core/src/execution/planner/operator_registry.rs +++ b/native/core/src/execution/planner/operator_registry.rs @@ -152,5 +152,11 @@ fn get_operator_type(spark_operator: &Operator) -> Option { OpStruct::CsvScan(_) => Some(OperatorType::CsvScan), OpStruct::ShuffleScan(_) => None, // Not yet in OperatorType enum OpStruct::BroadcastNestedLoopJoin(_) => None, + // Optional contrib; not in OperatorType enum. The arm stays unconditional + // even in non-`contrib-delta` builds because the proto enum is generated + // regardless of cargo features and Rust requires an exhaustive match. No + // contrib-specific logic lives here -- we just signal "no OperatorType + // mapping" so the supports-mixed-codegen check skips it. + OpStruct::DeltaScan(_) => None, } } diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 15265f1d86..788c1834c1 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -54,6 +54,10 @@ message Operator { CsvScan csv_scan = 115; ShuffleScan shuffle_scan = 116; BroadcastNestedLoopJoin broadcast_nested_loop_join = 117; + // Delta Lake scan. Wire format used by `contrib/delta/`. Only decoded when + // core is built with `--features contrib-delta`; in default builds the + // dispatcher arm is `#[cfg]`-stubbed out so the contrib has zero runtime cost. + DeltaScan delta_scan = 118; } } @@ -276,6 +280,224 @@ message IcebergDeleteFile { uint64 file_size_in_bytes = 5; } +// ===================================================================================== +// Delta Lake scan messages -- consumed by the optional `contrib/delta/` integration. +// Field numbers must remain stable; older serialised plans encode them on the wire. +// ===================================================================================== + +// Per-scan invariants. Lives at the head of every Delta scan operator payload. +message DeltaScanCommon { + // Required (output) schema -- pure-logical at every nesting level. The kernel-read path + // physicalises it via `column_mappings`; partition columns are split out by name. + repeated SparkStructField required_schema = 1; + // Partition columns (stored in add actions, not in file paths). + repeated SparkStructField partition_schema = 3; + // Fields 2 (data_schema), 4 (data_filters), 5 (projection_vector) belonged to the removed + // legacy ParquetSource read path; the kernel-read path reads only `required_schema` + + // `column_mappings`. Reserved so the numbers aren't reused. + reserved 2, 4, 5; + reserved "data_schema", "data_filters", "projection_vector"; + string session_timezone = 6; + bool case_sensitive = 7; + // Cloud storage credentials / options, keyed by Hadoop-style names. + map object_store_options = 8; + // Root path of the Delta table (file:// or s3://...). + string table_root = 9; + // Optional snapshot version; unset = latest. + optional uint64 snapshot_version = 10; + // Per-file parallelism knob within a single Spark task. + uint32 data_file_concurrency_limit = 11; + // Informational label for debug output. + string source = 12; + // Field 13 was `column_mappings` (logical->physical): the executor's only consumer was the + // `physicalise_field` schema rebuild, removed once the kernel-read path ships kernel's own + // `scan.physical_schema()`/`logical_schema()`. Partition-value key translation happens + // driver-side (see `DeltaScanTaskList.column_mappings`), so nothing executor-side needs it. + reserved 13; + optional string materialized_row_id_column_name = 14; + optional string materialized_row_commit_version_column_name = 15; + // When true the DataSourceExec swallows object-store NotFound errors as empty + // streams; matches Spark's `spark.sql.files.ignoreMissingFiles=true` semantics. + bool ignore_missing_files = 16; + // When true the native parquet reader matches Spark schema fields to parquet + // file fields by `PARQUET:field_id` metadata rather than by name. Used by + // Delta tables with `delta.columnMapping.mode = id` (the contrib copies + // `delta.columnMapping.id` to `PARQUET:field_id` on each schema field). + bool use_field_id = 17; + // Delta synthetic columns to append to the scan output via + // `DeltaSyntheticColumnsExec`. The corresponding columns must NOT appear in + // `data_schema` or `required_schema` -- they're synthesised at exec time + // from row positions / DV bitmaps and inserted after the parquet read. + bool emit_row_index = 18; + bool emit_is_row_deleted = 19; + // Row-tracking synthesis (row_id = baseRowId + physical_row_index, + // row_commit_version = defaultRowCommitVersion). Per-task baseRowId / + // defaultRowCommitVersion travel on `DeltaScanTask` (fields 6 + 7). + bool emit_row_id = 20; + bool emit_row_commit_version = 21; + // When synthetic columns aren't a contiguous suffix of `required_schema`, this + // sequence tells the native dispatcher how to project the final output. Each + // entry is an index into [parquet_output, appended_synthetics] -- positive + // means a column from the (stripped) parquet read at that index; the synthetic + // tail comes after, in the canonical row_index, is_row_deleted, row_id, + // row_commit_version order. Empty = synthetics are already a suffix; no + // reorder needed. + repeated int32 final_output_indices = 22; + // Override for the row_index column name. Defaults to "__delta_internal_row_index" + // when empty. Delta plans that read `_metadata.row_index` from row-tracking-enabled + // tables expose the row index as `_tmp_metadata_row_index` (the Delta-internal + // physical name from `DeltaParquetFileFormat.TMP_METADATA_ROW_INDEX_COLUMN_NAME`) + // rather than the canonical `__delta_internal_row_index`. Setting this lets the + // contrib produce a column with the alternate name from the same native synthesis. + string row_index_column_alias = 23; + // Spark `_metadata.*` virtual columns the scan must surface. Delta's + // `PreprocessTableWithDVs` strategy and the row-tracking projection both inject + // these into the FileScan output: `file_path`, `file_name`, `file_size`, + // `file_block_start`, `file_block_length`, `file_modification_time`. Each + // appears as a flat top-level column in the scan's output schema. Native + // synthesizes them per-task from `DeltaScanTask` data (file_path, file_size, + // modification_time) and per-batch derivations. Empty list = no metadata + // columns to emit. Names are emitted in the order given, appended AFTER + // synthetic columns (row_index / is_row_deleted / row_id / row_commit_version) + // unless `final_output_indices` reorders them. + repeated string metadata_column_names = 24; + // Field 25 was `bool kernel_read`: a transitional toggle from when the legacy ParquetSource read + // path still co-existed with the kernel-read path. The legacy path is gone (#50) and kernel-read + // is the only read path, so the toggle (and its `spark.comet.delta.kernelRead.enabled` config) + // were removed -- the driver no longer sends it and the native planner no longer gates on it. + reserved 25; + reserved "kernel_read"; + // Kernel-built data-column schemas for the kernel-read path, produced by the driver from + // `delta-kernel-rs`'s `Scan` (`scan.physical_schema()` / `scan.logical_schema()`), serialized as + // Arrow IPC schema messages. The physical schema carries the correct physical names AND field-ids + // at EVERY nesting level (kernel's own column-mapping resolution); the logical schema is the + // matching pure-logical shape. The executor reads parquet with the physical schema and relabels + // physical->logical via the schema pair -- no Comet-side physicalisation. Empty when the driver + // didn't build them (e.g. the non-kernel-read add-files path), in which case the executor falls + // back to physicalising `required_schema` via `column_mappings`. + bytes kernel_physical_schema = 26; + bytes kernel_logical_schema = 27; + // Stage B/C: when true, DeltaKernelScanExec produces ALL synthetic columns itself (row_index / + // row_id / file_path via kernel metadata columns; is_row_deleted by inverting the decoded DV; + // row_commit_version + Spark `_metadata.*` as per-file constants) and assembles the full output by + // name -- so the standalone DeltaSyntheticColumnsExec is NOT stacked on top and no + // `final_output_indices` reorder is needed. Default false (legacy DeltaSyntheticColumnsExec path). + // When set, `required_schema` is the FULL output (synthetics NOT stripped) and the per-file + // synthesis inputs ride on DeltaScanTask (base_row_id / default_row_commit_version / + // modification_time / byte range / dv). + bool synthesize_in_worker = 28; + // Change Data Feed (CDF / readChangeFeed): when true, DeltaKernelScanExec reads table changes via + // delta-kernel-rs's `TableChanges` API (single partition reconstructs TableChanges(start, end) and + // calls `execute()`), emitting the data columns plus `_change_type` / `_commit_version` / + // `_commit_timestamp`. The version range is below; `required_schema` is the full CDF output + // (assembled by name). No per-file tasks -- the worker enumerates + reads the whole range itself + // (kernel's CDF per-file API is pub(crate); only execute() is public). Default false. + bool cdf_read = 29; + // Inclusive start version of the CDF range. + uint64 cdf_start_version = 30; + // Inclusive end version of the CDF range. `has_cdf_end_version` distinguishes "latest" (unset). + optional uint64 cdf_end_version = 31; + // Delta's test-only deletion-vector filename prefix + // (`spark.databricks.delta.testOnly.dvFileNamePrefix`, default `test%dv%prefix-` under + // `Utils.isTesting`, empty in production). Delta prepends it to the DV filename at write time + // (`deletion_vector_.bin`) but delta-kernel-rs resolves the un-prefixed name, so + // executor-side DV reads splice it back in for "u"-storage DVs. Empty = no-op (production). + string dv_file_name_prefix = 32; +} + +// Operator payload for a Delta scan. +message DeltaScan { + DeltaScanCommon common = 1; + repeated DeltaScanTask tasks = 2; + // Absolute URL of the Delta table root, used by the executor to resolve relative + // ("u" storage-type) DV paths via kernel's `absolute_path` -- kernel joins + // `_delta_log/deletion_vectors/` onto this, so it MUST end with `/` (the + // driver normalises before sending). Empty for tables with no DVs (executor + // simply never decodes), but populate it whenever known. + string table_root = 3; +} + +message DeltaScanTask { + string file_path = 1; + uint64 file_size = 2; + optional uint64 record_count = 3; + repeated DeltaPartitionValue partition_values = 4; + // Field 5 (repeated uint64 deleted_row_indexes) removed in the Iceberg-style DV refactor: + // the driver used to materialise the FULL deletion-vector bitmap to a Vec here, which + // could reach ~1 GB for a single DV with ~99 M deleted rows (DELETE/MERGE on huge tables). + // The executor now resolves the DV itself via the `dv` field below -- matches the contrib's + // own Iceberg integration (IcebergScanCommon.delete_files_pool + IcebergFileScanTask.delete_files_idx). + reserved 5; + reserved "deleted_row_indexes"; + optional int64 base_row_id = 6; + optional int64 default_row_commit_version = 7; + // File-splitting byte range. Both unset = read whole file. + optional uint64 byte_range_start = 8; + optional uint64 byte_range_end = 9; + // File modification time in epoch milliseconds. Surfaced through Spark's + // `_metadata.file_modification_time` column when emit_metadata_columns + // includes "file_modification_time". Unset = 0 (rare; AddFile usually + // carries one). + optional int64 modification_time = 10; + // Deletion-vector descriptor for this file (absent = no DV applies). Mirrors + // delta_kernel::actions::deletion_vector::DeletionVectorDescriptor. The executor reads + // the actual DV bitmap on-task via dv_reader::read_dv_indexes -- the driver only ships + // the path/offset/size, so the per-scan-exec heap stays KB-scale regardless of DV size. + optional DeltaDvDescriptor dv = 11; + // Kernel's own fully-resolved physical->logical transform for THIS file, serialized as JSON + // (serde of `delta_kernel::expressions::Expression`). The driver gets it from kernel's scan + // enumeration (`ScanMetadata.scan_file_transforms`, one per scan file), where partition values + // and `baseRowId` (for row_id `GenerateRowId`) are already baked in as literals. The executor + // deserializes it and applies `transform_to_logical` -- so partition injection, column-mapping + // relabel (incl. nested), and row-tracking come from kernel rather than Comet-side reconstruction. + // Empty when the driver has no kernel transform for this file (e.g. a plain table with no mapping + // / partitions / row-tracking -> identity), in which case the executor uses a pass-through read. + bytes transform_json = 12; +} + +// Mirrors delta_kernel::actions::deletion_vector::DeletionVectorDescriptor. Carries everything +// an executor needs to read + decode the DV bitmap, without materialising it on the driver. +message DeltaDvDescriptor { + // "u" (UUID-named, relative to /_delta_log/deletion_vectors/) + // "p" (absolute URI in path_or_inline_dv) + // "i" (inline -- bytes carried in inline_bytes) + string storage_type = 1; + // For "u": Delta-encoded UUID path component. For "p": absolute URI. For "i": empty. + string path_or_inline_dv = 2; + // Byte offset within the DV file ("u"/"p" only). Absent ≡ 0. + optional uint64 offset = 3; + // Size in bytes of the serialised RoaringBitmap entry. + uint64 size_in_bytes = 4; + // Number of deleted rows (descriptor metadata; the executor doesn't need to count). + uint64 cardinality = 5; + // Inline-DV bytes (storage_type="i" only). Empty for "u"/"p". + bytes inline_bytes = 6; +} + +message DeltaPartitionValue { + string name = 1; + optional string value = 2; +} + +// JNI return value from planDeltaScan -- the list of tasks for a given snapshot. +message DeltaScanTaskList { + uint64 snapshot_version = 1; + string table_root = 2; + repeated DeltaScanTask tasks = 3; + repeated string unsupported_features = 4; + // Field 5 was `column_mappings`: column mapping is now fully resolved by the kernel-built schemas + // shipped on `DeltaScanCommon` (kernel resolves physical names + field-ids at every nesting + // level), and partition-value key translation happens driver-side, so nothing on the wire needs + // a column-mapping tree. + reserved 5; + // Kernel-built data-column schemas (Arrow IPC), produced by the driver from the projected + // `Scan` (`scan.physical_schema()` / `scan.logical_schema()`). Carried back to Scala and copied + // onto `DeltaScanCommon.kernel_physical_schema` / `kernel_logical_schema`. Empty when the driver + // had no projection to build the scan against. + bytes physical_schema = 6; + bytes logical_schema = 7; +} + message Projection { repeated spark.spark_expression.Expr project_list = 1; } diff --git a/pom.xml b/pom.xml index f53b153d28..7ffaa350b6 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,14 @@ under the License. UTF-8 UTF-8 11 + + 4.1.0 ${java.version} ${java.version} 3.11.0 diff --git a/spark/pom.xml b/spark/pom.xml index 780ecb63aa..49a95d09d3 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -122,6 +122,18 @@ under the License. + + + com.google.guava + failureaccess + 1.0.2 + test + org.codehaus.jackson jackson-mapper-asl @@ -231,6 +243,13 @@ under the License. spark-3.5 + + + 3.3.2 + org.apache.iceberg @@ -256,6 +275,15 @@ under the License. spark-4.0 + + + 4.0.0 + org.apache.iceberg @@ -283,6 +311,10 @@ under the License. true + + + 4.1.0 + @@ -331,6 +363,69 @@ under the License. + + + contrib-delta + + + + io.delta + delta-spark_${scala.binary.version} + ${delta.version} + provided + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-contrib-delta-source + generate-sources + + add-source + + + + ${project.parent.basedir}/contrib/delta/src/main/scala + + + + + add-contrib-delta-test-source + generate-test-sources + + add-test-source + + + + ${project.parent.basedir}/contrib/delta/src/test/scala + + + + + + + + + generate-docs diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 50cd0927b4..64dda8a3b9 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -265,6 +265,16 @@ case class CometExecRule(session: SparkSession) // spotless:on private def transform(plan: SparkPlan): SparkPlan = { def convertNode(op: SparkPlan): SparkPlan = op match { + // Delta scan marker produced by the optional contrib/delta integration. Matched by type + // (no compile-time dependency on the contrib) -- present only when -Pcontrib-delta was + // activated. The marker wraps the original, link-bearing scan, so the produced exec's + // originalPlan keeps its logicalLink with no workaround. If conversion declines, the marker + // itself falls back to the vanilla Spark Delta scan, so leaving it in the plan is safe. + case scan if DeltaIntegration.isDeltaScanMarker(scan) => + DeltaIntegration.scanHandler + .flatMap(handler => convertToComet(scan, handler)) + .getOrElse(scan) + // Fully native scan for V1. CometScanExec must always convert to a native scan; the JVM // fallback path has been removed. If conversion fails, fall back to the original Spark scan. case scan: CometScanExec => diff --git a/spark/src/main/scala/org/apache/comet/rules/CometPlanAdaptiveDynamicPruningFilters.scala b/spark/src/main/scala/org/apache/comet/rules/CometPlanAdaptiveDynamicPruningFilters.scala index 217f8bc314..8876a28e05 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometPlanAdaptiveDynamicPruningFilters.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometPlanAdaptiveDynamicPruningFilters.scala @@ -83,6 +83,15 @@ case object CometPlanAdaptiveDynamicPruningFilters if icebergScan.runtimeFilters.exists(hasCometSAB) => logDebug("Converting AQE DPP for CometIcebergNativeScanExec") convertIcebergScanDPP(icebergScan, plan) + // Comet scans whose DPP filters live in a @transient field (the contrib's + // CometDeltaNativeScanExec). transformExpressions/makeCopy can't rewrite + // them, and a rewritten copy is orphaned when the enclosing native block + // is rebuilt (#3510). The scan's `withDynamicPruningFilters` installs the + // rewrite in place and returns `this`, so it lands on the executing + // instance. + case p: CometScanWithPlanData if p.dynamicPruningFilters.exists(hasCometSAB) => + logDebug(s"Converting AQE DPP for ${p.getClass.getSimpleName} in place") + p.withDynamicPruningFilters(p.dynamicPruningFilters.map(f => convertFilter(f, plan))) case p: SparkPlan if !p.isInstanceOf[CometNativeScanExec] && !p.isInstanceOf[CometIcebergNativeScanExec] diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index af68f9530a..1f42c4029c 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -88,14 +88,6 @@ case class CometScanRule(session: SparkSession) case _ => false } - def hasMetadataCol(plan: SparkPlan): Boolean = { - plan.expressions.exists(_.exists { - case a: Attribute => - a.isMetadataCol - case _ => false - }) - } - def isIcebergMetadataTable(scanExec: BatchScanExec): Boolean = { // List of Iceberg metadata tables: // https://iceberg.apache.org/docs/latest/spark-queries/#inspecting-tables @@ -127,12 +119,15 @@ case class CometScanRule(session: SparkSession) case scan if !CometConf.COMET_NATIVE_SCAN_ENABLED.get(conf) => withFallbackReason(scan, "Comet Scan is not enabled") - case scan if hasMetadataCol(scan) => - withFallbackReason(scan, "Metadata column is not supported") - - // data source V1 + // V1 scans go through `transformV1Scan` which itself first delegates to any + // available V1 contrib (today: Delta) and only then applies generic Comet + // bailouts like the metadata-column rejection. This keeps the metadata-col + // guard in place for V2 and non-contrib V1 paths without referencing any + // specific contrib class from this outer match. case scanExec: FileSourceScanExec => transformV1Scan(fullPlan, scanExec) + case scan if hasMetadataCol(scan) => + withFallbackReason(scan, "Metadata column is not supported") // data source V2 case scanExec: BatchScanExec => @@ -148,6 +143,14 @@ case class CometScanRule(session: SparkSession) } } + /** True when any expression in `plan` references a metadata column. */ + private def hasMetadataCol(plan: SparkPlan): Boolean = { + plan.expressions.exists(_.exists { + case a: Attribute => a.isMetadataCol + case _ => false + }) + } + private def transformV1Scan(plan: SparkPlan, scanExec: FileSourceScanExec): SparkPlan = { // On Spark 3.4, injectQueryStageOptimizerRule is unavailable, so @@ -165,9 +168,27 @@ case class CometScanRule(session: SparkSession) scanExec.relation match { case r: HadoopFsRelation => + // Try the optional Delta contrib first. When this build wasn't compiled with + // `-Pcontrib-delta`, the bridge returns None and we fall through to the + // vanilla scan path. When the Delta classes are on the classpath, the contrib + // either claims the scan (returning a CometScanExec marker) or declines via + // its own `withFallbackReason` fallback message. + DeltaIntegration.transformV1IfDelta(plan, session, scanExec, r) match { + case Some(handled) => return handled + case None => // proceed with vanilla logic + } + // Metadata-col bailout moved here so V1 contribs (Delta) get first crack + // at scans with synthetic metadata columns before generic Comet rejects + // them. For non-contrib V1 scans this is equivalent to the outer check. + if (hasMetadataCol(scanExec)) { + return withFallbackReason(scanExec, "Metadata column is not supported") + } if (!CometScanExec.isFileFormatSupported(r.fileFormat)) { return withFallbackReason(scanExec, s"Unsupported file format ${r.fileFormat}") } + // NOTE: the object_store scheme gate lives in `nativeScan` (below), shared with the + // non-contrib path. The Delta delegation above runs before it, so contrib scans are + // unaffected; vanilla V1 scans hit the gate when this method calls `nativeScan`. val hadoopConf = r.sparkSession.sessionState.newHadoopConfWithOptions(r.options) // TODO is this restriction valid for all native scan types? diff --git a/spark/src/main/scala/org/apache/comet/rules/DeltaIntegration.scala b/spark/src/main/scala/org/apache/comet/rules/DeltaIntegration.scala new file mode 100644 index 0000000000..12c4fce6b8 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/rules/DeltaIntegration.scala @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.rules + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} +import org.apache.spark.sql.execution.datasources.HadoopFsRelation + +import org.apache.comet.serde.CometOperatorSerde +import org.apache.comet.util.ClassLoaders + +/** + * Reflection-based bridge to the optional `contrib/delta/` integration. + * + * On default builds the contrib classes don't exist on the classpath, so the reflective class + * lookups fail and every method here returns the "not handled" sentinel. On builds compiled with + * `-Pcontrib-delta` (Maven) + `--features contrib-delta` (Cargo), the contrib classes are present + * and the lookups resolve, dispatching the call into the contrib helpers. + * + * Keeping this bridge as one small file in core lets the Delta detection block in `CometScanRule` + * and the serde dispatch in `CometExecRule` stay ~10 lines each -- exactly the shape Parth's + * review on #4339 asked for. + * + * No `SPI`, no `ServiceLoader`, no registry: the contrib provides its own static helper objects + * with stable names; this bridge just calls them. + */ +object DeltaIntegration extends org.apache.spark.internal.Logging { + + // Scala compiles `object Foo` into BOTH `Foo.class` (a static-forwarder class) AND + // `Foo$.class` (the actual module class). Only the latter has the `MODULE$` singleton + // field that the reflection bridge dereferences. Looking up the unqualified name + // returns the forwarder, where `getField("MODULE$")` throws -- and the surrounding + // try/catch silently turns that into `None`, making every Delta scan fall through to + // Spark's reader. (This bug shipped silently until the test suite caught it; the Delta + // regression suite was passing because Delta's own tests don't depend on Comet + // engaging.) The trailing `$` selects the module class explicitly. + private val ScanRuleClass = "org.apache.comet.contrib.delta.DeltaScanRule$" + private val SerdeClass = "org.apache.comet.contrib.delta.CometDeltaNativeScan$" + + // Fully-qualified name of the contrib's planning-time marker (CometDeltaScanMarker), matched by + // class name so core carries no compile-time dependency on the contrib. CometExecRule routes any + // node of this type to the Delta serde handler. Type-based dispatch survives node copies / AQE + // re-planning (a TreeNodeTag would not), and the marker wraps the original, link-bearing scan as + // the produced exec's originalPlan -- so no logicalLink workaround is needed (mirrors Iceberg's + // CometBatchScanExec + nativeIcebergScanMetadata field). + private val MarkerClass = "org.apache.spark.sql.comet.CometDeltaScanMarker" + + /** True when `plan` is the Delta contrib's scan marker. */ + def isDeltaScanMarker(plan: SparkPlan): Boolean = plan.getClass.getName == MarkerClass + + // Lazy class lookups -- single reflection cost per JVM, cached either as the + // class handle or as the empty option if the contrib wasn't bundled. + @volatile private var scanRuleLookup: Option[Option[Class[AnyRef]]] = None + @volatile private var serdeLookup: Option[Option[Class[AnyRef]]] = None + + private def scanRuleCls: Option[Class[AnyRef]] = + scanRuleLookup.getOrElse { + // ClassLoaders.loadClass resolves via Spark's thread-context ClassLoader first (where + // user/vendor jars are wired), matching IcebergReflection's convention, then caches the + // result so the lookup is a single reflection cost per JVM. + val cls = + try { + Some(ClassLoaders.loadClass(ScanRuleClass).asInstanceOf[Class[AnyRef]]) + } catch { case _: ClassNotFoundException => None } + scanRuleLookup = Some(cls) + cls + } + + private def serdeCls: Option[Class[AnyRef]] = + serdeLookup.getOrElse { + val cls = + try { + Some(ClassLoaders.loadClass(SerdeClass).asInstanceOf[Class[AnyRef]]) + } catch { case _: ClassNotFoundException => None } + serdeLookup = Some(cls) + cls + } + + /** True when the Delta contrib was bundled into this build. */ + def isAvailable: Boolean = scanRuleCls.isDefined + + /** + * Delegate the V1 scan transform to the Delta contrib when both (a) the contrib is on the + * classpath, AND (b) the relation's file format is `DeltaParquetFileFormat`. + * + * Returns `Some(plan)` if the contrib handled the scan (either with a transformed + * `CometScanExec` marker or by explicitly declining via the `withFallbackReason` path); `None` + * to indicate "not a Delta scan, proceed with the vanilla CometScanRule path". + */ + // Cached reflective binding: resolved once per JVM. The contrib's + // `transformV1IfDelta` is invoked for every V1 scan in every plan, even + // non-Delta ones; resolving the Method on each call would be a per-scan + // reflection round-trip just to find we don't apply. + @volatile private var transformV1IfDeltaBindingCache + : Option[Option[(AnyRef, java.lang.reflect.Method)]] = None + + private def transformV1IfDeltaBinding: Option[(AnyRef, java.lang.reflect.Method)] = + transformV1IfDeltaBindingCache.getOrElse { + val binding = scanRuleCls.flatMap { cls => + try { + val module = cls.getField("MODULE$").get(null) + val m = cls.getMethod( + "transformV1IfDelta", + classOf[SparkPlan], + classOf[SparkSession], + classOf[FileSourceScanExec], + classOf[HadoopFsRelation]) + Some((module, m)) + } catch { + // Only swallow true reflection-binding failures (signature/access drift). + // Other exceptions (linkage errors, init failures) should surface. + case _: NoSuchMethodException | _: NoSuchFieldException | _: IllegalAccessException => + None + } + } + transformV1IfDeltaBindingCache = Some(binding) + binding + } + + def transformV1IfDelta( + plan: SparkPlan, + session: SparkSession, + scanExec: FileSourceScanExec, + relation: HadoopFsRelation): Option[SparkPlan] = { + transformV1IfDeltaBinding.flatMap { case (module, m) => + try { + Option(m.invoke(module, plan, session, scanExec, relation)) + .map(_.asInstanceOf[Option[SparkPlan]]) + .flatten + } catch { + // Reflection invocation errors (signature drift / access) -> silent None. + case _: IllegalAccessException | _: IllegalArgumentException => + None + // The contrib's transform threw a real exception -- DON'T silently swallow; + // surface it as a log warning + decline. Without this, kernel-rs IO errors, + // CCE on a Delta version bump, NPE in the CM-id translator etc. would + // silently fall back to vanilla and the user would never know Comet declined. + case e: java.lang.reflect.InvocationTargetException => + logWarning( + "CometDeltaNativeScan.transformV1IfDelta threw, declining to vanilla Delta " + + "for this scan", + Option(e.getCause).getOrElse(e)) + None + } + } + } + + // Change Data Feed support (`isCdfRelation` / `transformCdf` + their reflective binding) lands + // with the CDF unit that also wires the `readChangeFeed` hook into CometExecRule -- shipping it + // here would be dead code with no caller. + + // Cached Delta scan handler, resolved once per JVM (mirrors the binding caches above): the + // contrib's serde companion is looked up reflectively and the (`None` vs handler) result is + // memoised so CometExecRule pays no reflective field round-trip per Delta marker node. + @volatile private var scanHandlerCache: Option[Option[CometOperatorSerde[_]]] = None + + /** + * The Delta scan handler, resolved via reflection from the contrib's `CometDeltaNativeScan` + * companion object. Returns `None` when the contrib isn't bundled into this build. + * `CometExecRule` calls this and passes the result through the standard `convertToComet(scan, + * handler)` path so the Delta scan flows through the same code as `CometNativeScan` etc. + */ + def scanHandler: Option[CometOperatorSerde[_]] = + scanHandlerCache.getOrElse { + // Annotate the result type explicitly: Scala 2.12 (Spark 3.4 builds) cannot infer the + // existential `CometOperatorSerde[_]` as `flatMap`'s type parameter without an expected + // type here (2.13 can). The original un-cached `def` got it from the method return type. + val handler: Option[CometOperatorSerde[_]] = serdeCls.flatMap { cls => + try { + val module = cls.getField("MODULE$").get(null) + Some(module.asInstanceOf[CometOperatorSerde[_]]) + } catch { + // Only swallow true reflection-binding failures. A ClassCastException here + // would mean the contrib's serde object doesn't implement the expected trait + // -- a real bug worth surfacing. + case _: NoSuchFieldException | _: IllegalAccessException => None + } + } + scanHandlerCache = Some(handler) + handler + } +} diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala index 0ce8547563..f620652214 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala @@ -71,7 +71,8 @@ case class CometNativeScanExec( sourceKey: String) // Key for PlanDataInjector to match common+partition data at runtime extends CometLeafExec with DataSourceScanExec - with ShimStreamSourceAwareSparkPlan { + with ShimStreamSourceAwareSparkPlan + with CometScanWithPlanData { override lazy val metadata: Map[String, String] = if (originalPlan != null) originalPlan.metadata else Map.empty diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index af9e1df8a3..f74617ed28 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -27,6 +27,7 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.{Partition, TaskContext} import org.apache.spark.broadcast.Broadcast +import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSet, Expression, ExpressionSet, Generator, NamedExpression, SortOrder} @@ -87,14 +88,42 @@ private[comet] trait PlanDataInjector { /** * Registry and utilities for injecting per-partition planning data into operator trees. */ -private[comet] object PlanDataInjector { - - // Registry of injectors for different operator types - private val injectors: Seq[PlanDataInjector] = Seq( - IcebergPlanDataInjector, - NativeScanPlanDataInjector - // Future: DeltaPlanDataInjector, HudiPlanDataInjector, etc. - ) +private[comet] object PlanDataInjector extends Logging { + + // Registry of injectors for different operator types. The contrib/delta integration's + // DeltaPlanDataInjector is appended via one reflective class lookup -- present only when + // the contrib was bundled (i.e. -Pcontrib-delta on the Maven build). Default builds get + // the empty Option and an unmodified injectors list, so there's zero contrib surface at + // runtime on default builds. + private val injectors: Seq[PlanDataInjector] = { + val builtin: Seq[PlanDataInjector] = Seq(IcebergPlanDataInjector, NativeScanPlanDataInjector) + val deltaOpt: Option[PlanDataInjector] = + try { + // Scala compiles `object Foo` into BOTH `Foo.class` (a static-forwarder + // class with no MODULE$ field) AND `Foo$.class` (the module class that + // does have MODULE$). The trailing `$` selects the module class. + // scalastyle:off classforname + val cls = Class.forName("org.apache.spark.sql.comet.DeltaPlanDataInjector$") + // scalastyle:on classforname + Some(cls.getField("MODULE$").get(null).asInstanceOf[PlanDataInjector]) + } catch { + // Default builds (no -Pcontrib-delta) won't have the class -> silent None. + // This is the only EXPECTED miss, so it's the only quiet one. + case _: ClassNotFoundException => None + // The class IS on the classpath but couldn't be bound: missing MODULE$, + // access drift (NoSuchField/IllegalAccess), an initializer/linkage error, + // or a CCE on the PlanDataInjector cast. That's a misbuilt contrib jar, not + // a default build -- warn so it's diagnosable, then still decline so the + // rest of the planner stays alive. + case e: Throwable => + logWarning( + "Found org.apache.spark.sql.comet.DeltaPlanDataInjector$ on classpath " + + "but failed to load it; skipping contrib-delta plan-data injection", + e) + None + } + builtin ++ deltaOpt + } // O(1) lookup by op kind: most operators in any tree don't match any injector, so the per-op // `for (injector <- injectors if injector.canInject(op))` walk was paying N*M canInject calls @@ -718,8 +747,16 @@ abstract class CometNativeExec extends CometExec { */ def foreachUntilCometInput(plan: SparkPlan)(func: SparkPlan => Unit): Unit = { plan match { - case _: CometNativeScanExec | _: CometScanExec | _: CometBatchScanExec | - _: CometIcebergNativeScanExec | _: CometCsvNativeScanExec | _: ShuffleQueryStageExec | + // Match `CometLeafExec` first so contrib leaf scans (e.g. the Delta + // contrib's `CometDeltaNativeScanExec`) are recognised as input boundaries + // without requiring a core compile-time reference to the contrib class. + // All built-in leaf scans (`CometNativeScanExec`, `CometIcebergNativeScanExec`, + // `CometCsvNativeScanExec`) also extend `CometLeafExec`, so this is a + // strict superset of the previous enumeration -- it just generalises the + // input-boundary concept from "this fixed list" to "any leaf Comet exec". + case _: CometLeafExec => + func(plan) + case _: CometScanExec | _: CometBatchScanExec | _: ShuffleQueryStageExec | _: AQEShuffleReadExec | _: CometShuffleExchangeExec | _: CometUnionExec | _: CometTakeOrderedAndProjectExec | _: CometCoalesceExec | _: ReusedExchangeExec | _: CometBroadcastExchangeExec | _: BroadcastQueryStageExec | @@ -786,11 +823,16 @@ abstract class CometNativeExec extends CometExec { (Map.empty, Map.empty) } - case nativeScan: CometNativeScanExec => - nativeScan.ensureSubqueriesResolved() - ( - Map(nativeScan.sourceKey -> nativeScan.commonData), - Map(nativeScan.sourceKey -> nativeScan.perPartitionData)) + // Generic path for leaf scans that surface planning data via the + // `CometScanWithPlanData` trait. Catches `CometNativeScanExec` and any contrib + // leaf scan (e.g. the Delta contrib's `CometDeltaNativeScanExec`) without + // requiring core to compile-time reference contrib classes. + case s: CometScanWithPlanData => + s match { + case leaf: CometLeafExec => leaf.ensureSubqueriesResolved() + case _ => // no DPP lifecycle to drive + } + (Map(s.sourceKey -> s.commonData), Map(s.sourceKey -> s.perPartitionData)) // Broadcast stages are boundaries - don't collect per-partition data from inside them. // After DPP filtering, broadcast scans may have different partition counts than the @@ -901,6 +943,43 @@ abstract class CometLeafExec extends CometNativeExec with LeafExecNode { } } +/** + * Marker trait for scan execs that surface planning data (a `commonData` block + per-partition + * task bytes keyed by `sourceKey`) so that a parent `CometNativeExec` can find and inject the + * data when the scan is fused into a larger native subtree. + * + * Implemented by `CometNativeScanExec` and the contrib's `CometDeltaNativeScanExec` -- without + * it, [[PlanDataInjector.findAllPlanData]] cannot collect the per-partition tasks and the + * parent's native execution receives an empty input. (`CometIcebergNativeScanExec` does NOT use + * this trait; it has a dedicated `findAllPlanData` case.) + * + * Each implementation also resolves its own DPP subqueries via `ensureSubqueriesResolved` + * (overridden from [[CometLeafExec]]) before `commonData`/`perPartitionData` are read. + */ +trait CometScanWithPlanData { + def sourceKey: String + def commonData: Array[Byte] + def perPartitionData: Array[Array[Byte]] + + // DPP / partition filters that may carry AQE SubqueryAdaptiveBroadcast + // subqueries needing rewrite by CometPlanAdaptiveDynamicPruningFilters. + // Default empty: scans with dedicated handling (CometNativeScanExec, + // CometIcebergNativeScanExec) don't use this path. + def dynamicPruningFilters: Seq[Expression] = Nil + + // Install rewritten DPP filters on this scan. Implementers whose filters live + // in a @transient field (which TreeNode.makeCopy can't carry, #3510) update + // them via a transient side-channel and return `this` -- so the optimizer + // rule's rewrite lands on the SAME instance that executes, instead of a copy + // that gets dropped when the enclosing native block is rebuilt. Only called + // when `dynamicPruningFilters` is non-empty, so the default is never reached + // for scans that leave it empty. + def withDynamicPruningFilters(filters: Seq[Expression]): SparkPlan = + throw new UnsupportedOperationException( + s"${getClass.getSimpleName} exposes dynamicPruningFilters but does not " + + "override withDynamicPruningFilters") +} + abstract class CometUnaryExec extends CometNativeExec with UnaryExecNode abstract class CometBinaryExec extends CometNativeExec with BinaryExecNode diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala new file mode 100644 index 0000000000..f845713713 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet + +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.spark.sql.catalyst.expressions.Expression + +/** + * Unit coverage for the core SPI that lets out-of-tree contrib leaf scans (e.g. the Delta + * contrib's `CometDeltaNativeScanExec`) participate in Comet native planning without core holding + * a compile-time reference to them: the [[CometScanWithPlanData]] trait contract and the + * reflective `DeltaPlanDataInjector` slot in [[PlanDataInjector]]'s registry. + * + * Deliberately does not exercise the per-op injector registry mechanics; that surface is owned by + * `PlanDataInjectorSuite`. + */ +class CometScanWithPlanDataSuite extends AnyFunSuite { + + /** Minimal trait implementation that opts into none of the optional DPP behaviour. */ + private class StubScan extends CometScanWithPlanData { + override def sourceKey: String = "stub-key" + override def commonData: Array[Byte] = Array.emptyByteArray + override def perPartitionData: Array[Array[Byte]] = Array.empty + } + + test( + "CometScanWithPlanData defaults: no DPP filters and withDynamicPruningFilters is unsupported") { + val scan = new StubScan + assert(scan.dynamicPruningFilters == Nil, "a scan must expose no DPP filters by default") + // The default exists only so the DPP rule never calls it for scans that leave + // dynamicPruningFilters empty; if a scan does expose filters it must override this. + val ex = intercept[UnsupportedOperationException] { + scan.withDynamicPruningFilters(Seq.empty[Expression]) + } + assert( + ex.getMessage.contains("withDynamicPruningFilters"), + s"default failure must name the un-overridden method, got: ${ex.getMessage}") + } + + test("contrib DeltaPlanDataInjector slot is absent in default builds") { + // PlanDataInjector appends a reflective `DeltaPlanDataInjector$` slot to its registry only + // when the contrib-delta build bundled it. A default build must not contain the class, so the + // reflective lookup degrades to a plain ClassNotFoundException (-> None) and the registry keeps + // only its built-in injectors. (That the registry still passes non-scan operator trees through + // unchanged is covered by `PlanDataInjectorSuite`, which exercises the post-registry behaviour + // directly; this suite asserts only the contract A.1 adds -- the slot's optional presence.) + val deltaSlotAbsent = + try { + // scalastyle:off classforname + Class.forName("org.apache.spark.sql.comet.DeltaPlanDataInjector$") + // scalastyle:on classforname + false + } catch { + case _: ClassNotFoundException => true + } + assert( + deltaSlotAbsent, + "default (non -Pcontrib-delta) build must not carry the contrib DeltaPlanDataInjector") + } +}