From 3ac899f8474b7f2a8fea20348e4ce813b18da63c Mon Sep 17 00:00:00 2001 From: Najib Date: Tue, 28 Oct 2025 13:39:36 +0100 Subject: [PATCH 1/6] Align dependencies with datafusion 50.0.0 --- Cargo.lock | 450 ++++++++++++++++----------------- Cargo.toml | 40 +-- bindings/python/Cargo.toml | 6 +- bindings/python/pyproject.toml | 2 +- crates/catalog/sql/Cargo.toml | 2 +- 5 files changed, 237 insertions(+), 263 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa30a87e89..99737c43a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,12 +17,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - [[package]] name = "aes" version = "0.8.4" @@ -155,35 +149,6 @@ version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" -[[package]] -name = "apache-avro" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" -dependencies = [ - "bigdecimal", - "bzip2 0.4.4", - "crc32fast", - "digest", - "libflate", - "log", - "num-bigint", - "quad-rand", - "rand 0.8.5", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum 0.26.3", - "strum_macros 0.26.4", - "thiserror 1.0.69", - "typed-builder 0.19.1", - "uuid", - "xz2", - "zstd", -] - [[package]] name = "apache-avro" version = "0.20.0" @@ -192,6 +157,8 @@ checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" dependencies = [ "bigdecimal", "bon", + "bzip2 0.6.1", + "crc32fast", "digest", "log", "miniz_oxide", @@ -202,10 +169,12 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "snap", "strum 0.27.2", "strum_macros 0.27.2", "thiserror 2.0.16", "uuid", + "xz2", "zstd", ] @@ -229,9 +198,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" +checksum = "c26b57282a08ae92f727497805122fec964c6245cfa0e13f0e75452eaf3bc41f" dependencies = [ "arrow-arith", "arrow-array", @@ -250,9 +219,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" +checksum = "cebf38ca279120ff522f4954b81a39527425b6e9f615e6b72842f4de1ffe02b8" dependencies = [ "arrow-array", "arrow-buffer", @@ -264,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" +checksum = "744109142cdf8e7b02795e240e20756c2a782ac9180d4992802954a8f871c0de" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -281,9 +250,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" +checksum = "601bb103c4c374bcd1f62c66bcea67b42a2ee91a690486c37d4c180236f11ccc" dependencies = [ "bytes", "half", @@ -292,9 +261,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" +checksum = "eed61d9d73eda8df9e3014843def37af3050b5080a9acbe108f045a316d5a0be" dependencies = [ "arrow-array", "arrow-buffer", @@ -313,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "012c9fef3f4a11573b2c74aec53712ff9fdae4a95f4ce452d1bbf088ee00f06b" +checksum = "fa95b96ce0c06b4d33ac958370db8c0d31e88e54f9d6e08b0353d18374d9f991" dependencies = [ "arrow-array", "arrow-cast", @@ -328,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" +checksum = "43407f2c6ba2367f64d85d4603d6fb9c4b92ed79d2ffd21021b37efa96523e12" dependencies = [ "arrow-buffer", "arrow-schema", @@ -340,23 +309,25 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" +checksum = "e4b0487c4d2ad121cbc42c4db204f1509f8618e589bc77e635e9c40b502e3b90" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", + "arrow-select", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] name = "arrow-json" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" +checksum = "26d747573390905905a2dc4c5a61a96163fe2750457f90a04ee2a88680758c79" dependencies = [ "arrow-array", "arrow-buffer", @@ -376,9 +347,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" +checksum = "c142a147dceb59d057bad82400f1693847c80dca870d008bf7b91caf902810ae" dependencies = [ "arrow-array", "arrow-buffer", @@ -389,9 +360,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" +checksum = "dac6620667fccdab4204689ca173bd84a15de6bb6b756c3a8764d4d7d0c2fc04" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,9 +373,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +checksum = "dfa93af9ff2bb80de539e6eb2c1c8764abd0f4b73ffb0d7c82bf1f9868785e66" dependencies = [ "serde", "serde_json", @@ -412,9 +383,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" +checksum = "be8b2e0052cd20d36d64f32640b68a5ab54d805d24a473baee5d52017c85536c" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -426,9 +397,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" +checksum = "c2155e26e17f053c8975c546fc70cf19c00542f9abf43c23a88a46ef7204204f" dependencies = [ "arrow-array", "arrow-buffer", @@ -1360,21 +1331,20 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" dependencies = [ "bzip2-sys", - "libc", ] [[package]] name = "bzip2" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ - "bzip2-sys", + "libbz2-rs-sys", ] [[package]] @@ -1571,11 +1541,12 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.2.0" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8e18d0dca9578507f13f9803add0df13362b02c501c1c17734f0dbb52eaf0b" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ - "unicode-segmentation", + "strum 0.26.3", + "strum_macros 0.26.4", "unicode-width 0.2.1", ] @@ -1590,15 +1561,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.11" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" dependencies = [ "encode_unicode", "libc", "once_cell", "unicode-width 0.2.1", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -1659,15 +1630,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -1870,12 +1832,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" - [[package]] name = "dashmap" version = "6.1.0" @@ -1892,16 +1848,16 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.1" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" +checksum = "481d0c1cad7606cee11233abcdff8eec46e43dd25abda007db6d5d26ae8483c4" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2 0.6.1", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1922,6 +1878,7 @@ dependencies = [ "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", @@ -1929,6 +1886,7 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -1947,9 +1905,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" +checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" dependencies = [ "arrow", "async-trait", @@ -1973,9 +1931,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" +checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" dependencies = [ "arrow", "async-trait", @@ -1996,9 +1954,9 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "48.0.1" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85591b54815b0c9d3fbf3b0425b75e9fc49ef73e6886d6c19d622986e2551b53" +checksum = "a56a7d57a2a5535516d4fa5b3ac494959531df562558e1a827dd60075364b53e" dependencies = [ "arrow", "async-trait", @@ -2009,6 +1967,7 @@ dependencies = [ "dirs", "env_logger", "futures", + "log", "mimalloc", "object_store", "parking_lot", @@ -2021,17 +1980,19 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" +checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" dependencies = [ "ahash 0.8.12", - "apache-avro 0.17.0", + "apache-avro", "arrow", "arrow-ipc", "base64 0.22.1", + "chrono", "half", "hashbrown 0.14.5", + "hex", "indexmap 2.11.0", "libc", "log", @@ -2046,9 +2007,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" +checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e" dependencies = [ "futures", "log", @@ -2057,21 +2018,22 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" +checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2 0.6.1", "chrono", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", @@ -2093,11 +2055,11 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de733d231abb0fba663ff60fd37bf7171fa8b2e46e8a99e41362001821d116e" +checksum = "10d40b6953ebc9099b37adfd12fde97eb73ff0cee44355c6dea64b8a4537d561" dependencies = [ - "apache-avro 0.17.0", + "apache-avro", "arrow", "async-trait", "bytes", @@ -2118,9 +2080,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" +checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" dependencies = [ "arrow", "async-trait", @@ -2143,9 +2105,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" +checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" dependencies = [ "arrow", "async-trait", @@ -2168,9 +2130,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" +checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" dependencies = [ "arrow", "async-trait", @@ -2183,11 +2145,14 @@ dependencies = [ "datafusion-expr", "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -2199,17 +2164,18 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" +checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" [[package]] name = "datafusion-execution" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" +checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" dependencies = [ "arrow", + "async-trait", "dashmap", "datafusion-common", "datafusion-expr", @@ -2217,6 +2183,7 @@ dependencies = [ "log", "object_store", "parking_lot", + "parquet", "rand 0.9.2", "tempfile", "url", @@ -2224,11 +2191,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" +checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" dependencies = [ "arrow", + "async-trait", "chrono", "datafusion-common", "datafusion-doc", @@ -2245,9 +2213,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" +checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" dependencies = [ "arrow", "datafusion-common", @@ -2258,9 +2226,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" +checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" dependencies = [ "arrow", "arrow-buffer", @@ -2287,9 +2255,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" +checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" dependencies = [ "ahash 0.8.12", "arrow", @@ -2308,9 +2276,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" +checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" dependencies = [ "ahash 0.8.12", "arrow", @@ -2321,9 +2289,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" +checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" dependencies = [ "arrow", "arrow-ord", @@ -2333,6 +2301,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", "itertools 0.14.0", @@ -2342,9 +2311,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" +checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" dependencies = [ "arrow", "async-trait", @@ -2358,9 +2327,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" +checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" dependencies = [ "arrow", "datafusion-common", @@ -2376,9 +2345,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" +checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2386,9 +2355,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" +checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" dependencies = [ "datafusion-expr", "quote", @@ -2397,14 +2366,15 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" +checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.11.0", "itertools 0.14.0", @@ -2416,9 +2386,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" +checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" dependencies = [ "ahash 0.8.12", "arrow", @@ -2432,15 +2402,31 @@ dependencies = [ "indexmap 2.11.0", "itertools 0.14.0", "log", + "parking_lot", "paste", "petgraph 0.8.2", ] +[[package]] +name = "datafusion-physical-expr-adapter" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", +] + [[package]] name = "datafusion-physical-expr-common" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" +checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" dependencies = [ "ahash 0.8.12", "arrow", @@ -2452,9 +2438,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" +checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" dependencies = [ "arrow", "datafusion-common", @@ -2464,6 +2450,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", "log", "recursive", @@ -2471,9 +2458,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" +checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" dependencies = [ "ahash 0.8.12", "arrow", @@ -2485,6 +2472,7 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -2499,11 +2487,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-pruning" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + [[package]] name = "datafusion-session" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" +checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" dependencies = [ "arrow", "async-trait", @@ -2525,11 +2531,13 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f469b1301a5a2dec8b944eacb7b1194ed8c0f8203ce460b7451bb6fa2e89060" +checksum = "613efb6666a7d42fcb922b90cd0daa2b25ea486d141350e5d3e86e46df28309a" dependencies = [ "arrow", + "chrono", + "crc32fast", "datafusion-catalog", "datafusion-common", "datafusion-execution", @@ -2537,13 +2545,16 @@ dependencies = [ "datafusion-functions", "datafusion-macros", "log", + "sha1", + "url", + "xxhash-rust", ] [[package]] name = "datafusion-sql" -version = "48.0.1" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" +checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" dependencies = [ "arrow", "bigdecimal", @@ -2558,9 +2569,9 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "48.0.1" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a05ab096d3dd4c234f4ecf3c9c08b71396ac1810d4d57d3a13049bbf8a1b30" +checksum = "b7b12d1b07ef098dcfb449e95565d44208a948f862b92dc53258a8178d3eb77f" dependencies = [ "arrow", "async-trait", @@ -2585,9 +2596,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "48.0.1" +version = "50.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6029c08002772fb2c38a191d21a8b03af78750f42d9b896222f6b0e16c84cf" +checksum = "fcee6783df42ea7e2e2567f4bc92a0e9ce96d395c7c2f3e68ddaf35630c7005c" dependencies = [ "async-recursion", "async-trait", @@ -2838,7 +2849,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3596,7 +3607,7 @@ name = "iceberg" version = "0.7.0" dependencies = [ "anyhow", - "apache-avro 0.20.0", + "apache-avro", "array-init", "arrow-arith", "arrow-array", @@ -3647,7 +3658,7 @@ dependencies = [ "tera", "thrift", "tokio", - "typed-builder 0.20.1", + "typed-builder", "url", "uuid", "zstd", @@ -3734,7 +3745,7 @@ dependencies = [ "serde_json", "tokio", "tracing", - "typed-builder 0.20.1", + "typed-builder", "uuid", ] @@ -3763,7 +3774,7 @@ dependencies = [ "sqlx", "tempfile", "tokio", - "typed-builder 0.20.1", + "typed-builder", ] [[package]] @@ -4010,14 +4021,14 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.11" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +checksum = "ade6dfcba0dfb62ad59e59e7241ec8912af34fd29e0e743e3db992bd278e8b65" dependencies = [ "console", - "number_prefix", "portable-atomic", "unicode-width 0.2.1", + "unit-prefix", "web-time", ] @@ -4252,34 +4263,16 @@ dependencies = [ ] [[package]] -name = "libc" -version = "0.2.175" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" - -[[package]] -name = "libflate" -version = "2.1.0" +name = "libbz2-rs-sys" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" -dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", -] +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] -name = "libflate_lz77" -version = "2.1.0" +name = "libc" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" -dependencies = [ - "core2", - "hashbrown 0.14.5", - "rle-decode-fast", -] +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libloading" @@ -4288,7 +4281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.53.3", ] [[package]] @@ -4838,12 +4831,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "object" version = "0.36.7" @@ -5015,9 +5002,9 @@ dependencies = [ [[package]] name = "parquet" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +checksum = "89b56b41d1bd36aae415e42f91cae70ee75cf6cba74416b14dce3e958d5990ec" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -5040,6 +5027,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -5698,7 +5686,7 @@ dependencies = [ "once_cell", "socket2 0.6.0", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6069,12 +6057,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "roaring" version = "0.11.2" @@ -6186,7 +6168,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6298,9 +6280,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rustyline" -version = "16.0.0" +version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fd9ca5ebc709e8535e8ef7c658eb51457987e48c98ead2be482172accc408d" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ "bitflags", "cfg-if", @@ -6315,7 +6297,7 @@ dependencies = [ "unicode-segmentation", "unicode-width 0.2.1", "utf8parse", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6868,9 +6850,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.55.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" +checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" dependencies = [ "log", "recursive", @@ -7169,9 +7151,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.56.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13de2e20128f2a018dab1cfa30be83ae069219a65968c6f89df66ad124de2397" +checksum = "de6d24c270c6c672a86c183c3a8439ba46c1936f93cf7296aa692de3b0ff0228" dependencies = [ "heck", "pbjson", @@ -7262,7 +7244,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.8", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -7663,33 +7645,13 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -[[package]] -name = "typed-builder" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" -dependencies = [ - "typed-builder-macro 0.19.1", -] - [[package]] name = "typed-builder" version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ - "typed-builder-macro 0.20.1", -] - -[[package]] -name = "typed-builder-macro" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "typed-builder-macro", ] [[package]] @@ -7857,6 +7819,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -8569,6 +8537,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "xz2" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index 999b911753..b2c59f76cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,16 +42,16 @@ rust-version = "1.87" anyhow = "1.0.72" apache-avro = { version = "0.20", features = ["zstandard"] } array-init = "2" -arrow-arith = { version = "55.1" } -arrow-array = { version = "55.1" } -arrow-buffer = { version = "55.1" } -arrow-cast = { version = "55.1" } -arrow-ord = { version = "55.1" } -arrow-schema = { version = "55.1" } -arrow-select = { version = "55.1" } -arrow-string = { version = "55.1" } +arrow-arith = { version = "=56.1.0" } +arrow-array = { version = "=56.1.0" } +arrow-buffer = { version = "=56.1.0" } +arrow-cast = { version = "=56.1.0" } +arrow-ord = { version = "=56.1.0" } +arrow-schema = { version = "=56.1.0" } +arrow-select = { version = "=56.1.0" } +arrow-string = { version = "=56.1.0" } as-any = "0.3.2" -async-trait = "0.1.88" +async-trait = "0.1.89" aws-config = "1.8.1" aws-sdk-glue = "1.39" aws-sdk-s3tables = "1.28.0" @@ -62,9 +62,9 @@ bytes = "1.10" chrono = "0.4.41" clap = { version = "4.5.41", features = ["derive", "cargo"] } ctor = "0.2.8" -datafusion = "48.0.1" -datafusion-cli = "48.0.1" -datafusion-sqllogictest = "48.0.1" +datafusion = "=50.0.0" +datafusion-cli = "=50.0.0" +datafusion-sqllogictest = "=50.0.0" derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" @@ -81,7 +81,7 @@ iceberg-catalog-glue = { version = "0.7.0", path = "./crates/catalog/glue" } iceberg-catalog-s3tables = { version = "0.7.0", path = "./crates/catalog/s3tables" } iceberg-catalog-hms = { version = "0.7.0", path = "./crates/catalog/hms" } iceberg-datafusion = { version = "0.7.0", path = "./crates/integrations/datafusion" } -indicatif = "0.17" +indicatif = "0.18" itertools = "0.13" linkedbytes = "0.1.8" metainfo = "0.7.14" @@ -94,15 +94,15 @@ num-bigint = "0.4.6" once_cell = "1.20" opendal = "0.54.0" ordered-float = "4" -parquet = "55.1" +parquet = "=56.1.0" pilota = "0.11.10" port_scanner = "0.1.5" pretty_assertions = "1.4" rand = "0.8.5" -regex = "1.10.5" +regex = "1.11.2" reqwest = { version = "0.12.12", default-features = false, features = ["json"] } roaring = { version = "0.11" } -rust_decimal = "1.37.1" +rust_decimal = "1.37.2" serde = { version = "1.0.219", features = ["rc"] } serde_bytes = "0.11.17" serde_derive = "1.0.219" @@ -116,12 +116,12 @@ strum = "0.27.2" tempfile = "3.18" tera = "1" thrift = "0.17.0" -tokio = { version = "1.46.1", default-features = false } +tokio = { version = "1.47.1", default-features = false } toml = "0.8" -tracing = "0.1.40" -tracing-subscriber = "0.3.8" +tracing = "0.1.41" +tracing-subscriber = "0.3.20" typed-builder = "0.20" -url = "2.5.4" +url = "2.5.7" uuid = { version = "1.18", features = ["v7"] } volo = "0.10.6" volo-thrift = "0.10.8" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 879a1450b7..d5e4bd8e5a 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -31,9 +31,9 @@ license = "Apache-2.0" crate-type = ["cdylib"] [dependencies] -arrow = { version = "55.1", features = ["pyarrow", "chrono-tz"] } +arrow = { version = "56.1.0", features = ["pyarrow", "chrono-tz"] } iceberg = { path = "../../crates/iceberg" } -pyo3 = { version = "0.24.1", features = ["extension-module", "abi3-py39"] } +pyo3 = { version = "0.25", features = ["extension-module", "abi3-py39"] } iceberg-datafusion = { path = "../../crates/integrations/datafusion" } -datafusion-ffi = { version = "48.0.1" } +datafusion-ffi = { version = "50.0.0" } tokio = { version = "1.46.1", default-features = false } diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 36b5834228..516424e0b0 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -49,7 +49,7 @@ include = [ ignore = ["F403", "F405"] [tool.hatch.envs.dev] -dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "datafusion==45.*", "pyiceberg[sql-sqlite,pyarrow]>=0.10.0", "fastavro>=1.11.1"] +dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "datafusion==50.*", "pyiceberg[sql-sqlite,pyarrow]>=0.10.0", "fastavro>=1.11.1"] [tool.hatch.envs.dev.scripts] build = "maturin build --out dist --sdist" diff --git a/crates/catalog/sql/Cargo.toml b/crates/catalog/sql/Cargo.toml index 33ca700bf7..8de241dc4d 100644 --- a/crates/catalog/sql/Cargo.toml +++ b/crates/catalog/sql/Cargo.toml @@ -37,7 +37,7 @@ typed-builder = { workspace = true } [dev-dependencies] iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } itertools = { workspace = true } -regex = "1.10.5" +regex = { workspace = true } sqlx = { version = "0.8.1", features = [ "tls-rustls", "runtime-tokio", From 3a1cdb67d23fcef139ec8e6cb7bcf6e7b339b6fa Mon Sep 17 00:00:00 2001 From: Najib Date: Tue, 28 Oct 2025 13:40:11 +0100 Subject: [PATCH 2/6] Update hatch version in workflows --- .github/workflows/bindings_python_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 5c904b319d..ae3a6d1a54 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -91,6 +91,6 @@ jobs: shell: bash run: | set -e - pip install hatch==1.12.0 + pip install hatch==1.14.2 hatch run dev:pip install dist/pyiceberg_core-*.whl --force-reinstall hatch run dev:test From 938872cac8d7f15dbd281c67c8df77f848782111 Mon Sep 17 00:00:00 2001 From: Najib Date: Tue, 28 Oct 2025 13:40:38 +0100 Subject: [PATCH 3/6] Resolve conflicts after deps upgrade --- crates/iceberg/src/arrow/reader.rs | 10 ++++++---- .../datafusion/src/physical_plan/commit.rs | 5 ++++- .../integrations/datafusion/src/physical_plan/write.rs | 5 ++++- .../datafusion/src/table/table_provider_factory.rs | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/iceberg/src/arrow/reader.rs b/crates/iceberg/src/arrow/reader.rs index 05aa6a4c9f..a05803c1da 100644 --- a/crates/iceberg/src/arrow/reader.rs +++ b/crates/iceberg/src/arrow/reader.rs @@ -39,7 +39,9 @@ use parquet::arrow::arrow_reader::{ }; use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder, ProjectionMask}; -use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData}; +use parquet::file::metadata::{ + PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData, +}; use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; @@ -1390,9 +1392,9 @@ impl AsyncFileReader for ArrowFileReader { async move { let reader = ParquetMetaDataReader::new() .with_prefetch_hint(self.metadata_size_hint) - .with_column_indexes(self.preload_column_index) - .with_page_indexes(self.preload_page_index) - .with_offset_indexes(self.preload_offset_index); + .with_column_index_policy(PageIndexPolicy::from(self.preload_column_index)) + .with_page_index_policy(PageIndexPolicy::from(self.preload_page_index)) + .with_offset_index_policy(PageIndexPolicy::from(self.preload_offset_index)); let size = self.meta.size; let meta = reader.load_and_finish(self, size).await?; diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index 0ee0504faa..65e66a91fa 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -84,7 +84,10 @@ impl IcebergCommitExec { let count_array = Arc::new(UInt64Array::from(vec![count])) as ArrayRef; RecordBatch::try_from_iter_with_nullable(vec![("count", count_array, false)]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make count batch!".to_string())) + DataFusionError::ArrowError( + Box::new(e), + Some("Failed to make count batch!".to_string()), + ) }) } diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 625405c95b..8f3613235c 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -100,7 +100,10 @@ impl IcebergWriteExec { let files_array = Arc::new(StringArray::from(data_files)) as ArrayRef; RecordBatch::try_new(Self::make_result_schema(), vec![files_array]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make result batch".to_string())) + DataFusionError::ArrowError( + Box::new(e), + Some("Failed to make result batch".to_string()), + ) }) } diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs index a6d3146e54..e8e87dd318 100644 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ b/crates/integrations/datafusion/src/table/table_provider_factory.rs @@ -241,7 +241,7 @@ mod tests { options: Default::default(), table_partition_cols: Default::default(), order_exprs: Default::default(), - constraints: Constraints::empty(), + constraints: Constraints::default(), column_defaults: Default::default(), if_not_exists: Default::default(), temporary: false, From cc49f6561f62f0ad3459ad9bb349077843cfb963 Mon Sep 17 00:00:00 2001 From: Najib Date: Tue, 28 Oct 2025 13:49:37 +0100 Subject: [PATCH 4/6] Use the datafusion fork to avoid inconsistencies and build issues --- Cargo.lock | 171 +++++++++++++++++++++-------------------------------- Cargo.toml | 27 +++++---- 2 files changed, 83 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99737c43a6..775ff0b08e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1849,8 +1849,7 @@ dependencies = [ [[package]] name = "datafusion" version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481d0c1cad7606cee11233abcdff8eec46e43dd25abda007db6d5d26ae8483c4" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "arrow-ipc", @@ -1905,9 +1904,8 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -1931,9 +1929,8 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -1955,8 +1952,7 @@ dependencies = [ [[package]] name = "datafusion-cli" version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56a7d57a2a5535516d4fa5b3ac494959531df562558e1a827dd60075364b53e" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -1980,9 +1976,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2007,9 +2002,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "futures", "log", @@ -2018,9 +2012,8 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-compression", @@ -2055,9 +2048,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d40b6953ebc9099b37adfd12fde97eb73ff0cee44355c6dea64b8a4537d561" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "apache-avro", "arrow", @@ -2080,9 +2072,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2105,9 +2096,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2130,9 +2120,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2164,15 +2153,13 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" [[package]] name = "datafusion-execution" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2191,9 +2178,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2213,9 +2199,8 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "datafusion-common", @@ -2226,9 +2211,8 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "arrow-buffer", @@ -2255,9 +2239,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "ahash 0.8.12", "arrow", @@ -2276,9 +2259,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "ahash 0.8.12", "arrow", @@ -2289,9 +2271,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "arrow-ord", @@ -2311,9 +2292,8 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2327,9 +2307,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "datafusion-common", @@ -2345,9 +2324,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2355,9 +2333,8 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "datafusion-expr", "quote", @@ -2366,9 +2343,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "chrono", @@ -2386,9 +2362,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "ahash 0.8.12", "arrow", @@ -2409,9 +2384,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "datafusion-common", @@ -2424,9 +2398,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "ahash 0.8.12", "arrow", @@ -2438,9 +2411,8 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "datafusion-common", @@ -2458,9 +2430,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "ahash 0.8.12", "arrow", @@ -2472,7 +2443,7 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate-common", + "datafusion-functions-aggregate", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -2489,9 +2460,8 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "arrow-schema", @@ -2507,9 +2477,8 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2531,9 +2500,8 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613efb6666a7d42fcb922b90cd0daa2b25ea486d141350e5d3e86e46df28309a" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "chrono", @@ -2552,9 +2520,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "bigdecimal", @@ -2570,8 +2537,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b12d1b07ef098dcfb449e95565d44208a948f862b92dc53258a8178d3eb77f" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "arrow", "async-trait", @@ -2596,9 +2562,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "50.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcee6783df42ea7e2e2567f4bc92a0e9ce96d395c7c2f3e68ddaf35630c7005c" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b2c59f76cd..12c06c2cff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,14 +42,15 @@ rust-version = "1.87" anyhow = "1.0.72" apache-avro = { version = "0.20", features = ["zstandard"] } array-init = "2" -arrow-arith = { version = "=56.1.0" } -arrow-array = { version = "=56.1.0" } -arrow-buffer = { version = "=56.1.0" } -arrow-cast = { version = "=56.1.0" } -arrow-ord = { version = "=56.1.0" } -arrow-schema = { version = "=56.1.0" } -arrow-select = { version = "=56.1.0" } -arrow-string = { version = "=56.1.0" } +# Align arrow version with the one used by datafusion 50.0.0 +arrow-arith = { version = "56.1.0" } +arrow-array = { version = "56.1.0" } +arrow-buffer = { version = "56.1.0" } +arrow-cast = { version = "56.1.0" } +arrow-ord = { version = "56.1.0" } +arrow-schema = { version = "56.1.0" } +arrow-select = { version = "56.1.0" } +arrow-string = { version = "56.1.0" } as-any = "0.3.2" async-trait = "0.1.89" aws-config = "1.8.1" @@ -62,9 +63,10 @@ bytes = "1.10" chrono = "0.4.41" clap = { version = "4.5.41", features = ["derive", "cargo"] } ctor = "0.2.8" -datafusion = "=50.0.0" -datafusion-cli = "=50.0.0" -datafusion-sqllogictest = "=50.0.0" +# Use datafusion fork to make sure to reuse the same versions of other packages and to avoid build conflicts caused by lzma +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "f209f98479cc6ba19c67fcbb0d69c441ef46ed3e", package = "datafusion" } +datafusion-cli = { git = "https://github.com/DataDog/datafusion", rev = "f209f98479cc6ba19c67fcbb0d69c441ef46ed3e", package = "datafusion-cli" } +datafusion-sqllogictest = { git = "https://github.com/DataDog/datafusion", rev = "f209f98479cc6ba19c67fcbb0d69c441ef46ed3e", package = "datafusion-sqllogictest" } derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" @@ -94,7 +96,8 @@ num-bigint = "0.4.6" once_cell = "1.20" opendal = "0.54.0" ordered-float = "4" -parquet = "=56.1.0" +# Align parquet version with the one used by datafusion 50.0.0 +parquet = "56.1.0" pilota = "0.11.10" port_scanner = "0.1.5" pretty_assertions = "1.4" From a7032fca75f466375ab7f859c1dbc2939e777f3d Mon Sep 17 00:00:00 2001 From: Najib Date: Wed, 19 Nov 2025 14:09:56 +0100 Subject: [PATCH 5/6] Update datafusio version to match latest of our fork --- Cargo.lock | 266 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 9 +- 2 files changed, 143 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 775ff0b08e..417203ae2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,15 +151,16 @@ checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "apache-avro" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", - "bzip2 0.6.1", + "bzip2", "crc32fast", "digest", + "liblzma", "log", "miniz_oxide", "num-bigint", @@ -172,9 +173,8 @@ dependencies = [ "snap", "strum 0.27.2", "strum_macros 0.27.2", - "thiserror 2.0.16", + "thiserror 2.0.17", "uuid", - "xz2", "zstd", ] @@ -454,19 +454,15 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.19" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +checksum = "93c1f86859c1af3d514fa19e8323147ff10ea98684e6c7b307912509f50e67b2" dependencies = [ - "bzip2 0.5.2", - "flate2", + "compression-codecs", + "compression-core", "futures-core", - "memchr", "pin-project-lite", "tokio", - "xz2", - "zstd", - "zstd-safe", ] [[package]] @@ -1076,9 +1072,9 @@ checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bigdecimal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" dependencies = [ "autocfg", "libm", @@ -1193,9 +1189,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.7.2" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2529c31017402be841eb45892278a6c21a000c0a17643af326c73a73f83f0fb" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" dependencies = [ "bon-macros", "rustversion", @@ -1203,9 +1199,9 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.7.2" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82020dadcb845a345591863adb65d74fa8dc5c18a0b6d408470e13b7adc7005" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" dependencies = [ "darling 0.21.3", "ident_case", @@ -1329,15 +1325,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - [[package]] name = "bzip2" version = "0.6.1" @@ -1347,16 +1334,6 @@ dependencies = [ "libbz2-rs-sys", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "cbc" version = "0.1.2" @@ -1550,6 +1527,27 @@ dependencies = [ "unicode-width 0.2.1", ] +[[package]] +name = "compression-codecs" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680dc087785c5230f8e8843e2e57ac7c1c90488b6a91b88caa265410568f441b" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9b614a5787ef0c8802a55766480563cb3a93b435898c422ed2a359cf811582" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1849,14 +1847,14 @@ dependencies = [ [[package]] name = "datafusion" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1887,6 +1885,7 @@ dependencies = [ "futures", "hex", "itertools 0.14.0", + "liblzma", "log", "object_store", "parking_lot", @@ -1898,14 +1897,13 @@ dependencies = [ "tokio", "url", "uuid", - "xz2", "zstd", ] [[package]] name = "datafusion-catalog" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -1930,7 +1928,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -1952,7 +1950,7 @@ dependencies = [ [[package]] name = "datafusion-cli" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -1977,7 +1975,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2003,7 +2001,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "futures", "log", @@ -2013,13 +2011,13 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-common", "datafusion-common-runtime", @@ -2034,6 +2032,7 @@ dependencies = [ "futures", "glob", "itertools 0.14.0", + "liblzma", "log", "object_store", "parquet", @@ -2042,14 +2041,13 @@ dependencies = [ "tokio", "tokio-util", "url", - "xz2", "zstd", ] [[package]] name = "datafusion-datasource-avro" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "apache-avro", "arrow", @@ -2073,7 +2071,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2097,7 +2095,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2121,7 +2119,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2154,12 +2152,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" [[package]] name = "datafusion-execution" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2179,7 +2177,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2200,7 +2198,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "datafusion-common", @@ -2212,7 +2210,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "arrow-buffer", @@ -2240,7 +2238,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "ahash 0.8.12", "arrow", @@ -2260,7 +2258,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "ahash 0.8.12", "arrow", @@ -2272,7 +2270,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "arrow-ord", @@ -2293,7 +2291,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2308,7 +2306,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "datafusion-common", @@ -2325,7 +2323,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2334,7 +2332,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "datafusion-expr", "quote", @@ -2344,7 +2342,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "chrono", @@ -2363,7 +2361,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "ahash 0.8.12", "arrow", @@ -2385,7 +2383,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "datafusion-common", @@ -2399,7 +2397,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "ahash 0.8.12", "arrow", @@ -2412,7 +2410,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "datafusion-common", @@ -2431,7 +2429,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "ahash 0.8.12", "arrow", @@ -2461,7 +2459,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "arrow-schema", @@ -2478,7 +2476,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2501,7 +2499,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "chrono", @@ -2521,7 +2519,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "bigdecimal", @@ -2537,7 +2535,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "arrow", "async-trait", @@ -2556,14 +2554,14 @@ dependencies = [ "sqllogictest", "sqlparser", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] [[package]] name = "datafusion-substrait" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=f209f98479cc6ba19c67fcbb0d69c441ef46ed3e#f209f98479cc6ba19c67fcbb0d69c441ef46ed3e" +source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" dependencies = [ "async-recursion", "async-trait", @@ -4249,6 +4247,26 @@ dependencies = [ "windows-targets 0.53.3", ] +[[package]] +name = "liblzma" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.15" @@ -4387,17 +4405,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "matchers" version = "0.2.0" @@ -4833,7 +4840,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "url", @@ -5096,7 +5103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.16", + "thiserror 2.0.17", "ucd-trie", ] @@ -5613,7 +5620,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.31", "socket2 0.6.0", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -5634,7 +5641,7 @@ dependencies = [ "rustls 0.23.31", "rustls-pki-types", "slab", - "thiserror 2.0.16", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -5792,7 +5799,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -5840,9 +5847,9 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" @@ -6438,27 +6445,38 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -6478,14 +6496,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -6653,7 +6672,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "time", ] @@ -6757,7 +6776,7 @@ dependencies = [ "simdutf8", "sonic-number", "sonic-simd", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -6809,7 +6828,7 @@ dependencies = [ "similar", "subst", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", ] @@ -6876,7 +6895,7 @@ dependencies = [ "serde_json", "sha2", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -6956,7 +6975,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "whoami", ] @@ -6993,7 +7012,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "whoami", ] @@ -7017,7 +7036,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "url", ] @@ -7251,11 +7270,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.16", + "thiserror-impl 2.0.17", ] [[package]] @@ -7271,9 +7290,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", @@ -7662,7 +7681,7 @@ dependencies = [ "serde", "serde_json", "syn 2.0.106", - "thiserror 2.0.16", + "thiserror 2.0.17", "unicode-ident", ] @@ -7881,7 +7900,7 @@ dependencies = [ "pin-project", "rand 0.9.2", "socket2 0.5.10", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tower", @@ -7912,7 +7931,7 @@ dependencies = [ "rustc-hash 2.1.1", "scopeguard", "sonic-rs", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "volo", @@ -8508,15 +8527,6 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 12c06c2cff..6db8e7e238 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,8 @@ rust-version = "1.87" [workspace.dependencies] anyhow = "1.0.72" -apache-avro = { version = "0.20", features = ["zstandard"] } +# Update to match our fork's version +apache-avro = { version = "0.21", features = ["zstandard"] } array-init = "2" # Align arrow version with the one used by datafusion 50.0.0 arrow-arith = { version = "56.1.0" } @@ -64,9 +65,9 @@ chrono = "0.4.41" clap = { version = "4.5.41", features = ["derive", "cargo"] } ctor = "0.2.8" # Use datafusion fork to make sure to reuse the same versions of other packages and to avoid build conflicts caused by lzma -datafusion = { git = "https://github.com/DataDog/datafusion", rev = "f209f98479cc6ba19c67fcbb0d69c441ef46ed3e", package = "datafusion" } -datafusion-cli = { git = "https://github.com/DataDog/datafusion", rev = "f209f98479cc6ba19c67fcbb0d69c441ef46ed3e", package = "datafusion-cli" } -datafusion-sqllogictest = { git = "https://github.com/DataDog/datafusion", rev = "f209f98479cc6ba19c67fcbb0d69c441ef46ed3e", package = "datafusion-sqllogictest" } +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d", package = "datafusion" } +datafusion-cli = { git = "https://github.com/DataDog/datafusion", rev = "34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d", package = "datafusion-cli" } +datafusion-sqllogictest = { git = "https://github.com/DataDog/datafusion", rev = "34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d", package = "datafusion-sqllogictest" } derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" From 35ff654b87f3eed1ca91936dfac4cdae0ced0493 Mon Sep 17 00:00:00 2001 From: Najib Date: Thu, 11 Dec 2025 19:51:19 +0100 Subject: [PATCH 6/6] Update datafusion fork version --- Cargo.lock | 77 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 6 ++--- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 417203ae2f..846086f7dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1847,7 +1847,7 @@ dependencies = [ [[package]] name = "datafusion" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "arrow-ipc", @@ -1903,7 +1903,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -1928,7 +1928,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -1950,7 +1950,7 @@ dependencies = [ [[package]] name = "datafusion-cli" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -1975,7 +1975,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2001,7 +2001,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "futures", "log", @@ -2011,7 +2011,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-compression", @@ -2047,7 +2047,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "apache-avro", "arrow", @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2095,7 +2095,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2119,7 +2119,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2152,12 +2152,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" [[package]] name = "datafusion-execution" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2177,7 +2177,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2198,7 +2198,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "datafusion-common", @@ -2210,7 +2210,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "arrow-buffer", @@ -2238,7 +2238,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2270,7 +2270,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "arrow-ord", @@ -2291,7 +2291,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2306,7 +2306,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "datafusion-common", @@ -2323,7 +2323,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2332,7 +2332,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "datafusion-expr", "quote", @@ -2342,7 +2342,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "chrono", @@ -2361,7 +2361,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2377,13 +2377,14 @@ dependencies = [ "log", "parking_lot", "paste", - "petgraph 0.8.2", + "petgraph 0.8.3", + "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "datafusion-common", @@ -2397,7 +2398,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2410,7 +2411,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "datafusion-common", @@ -2429,7 +2430,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2459,7 +2460,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "arrow-schema", @@ -2476,7 +2477,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2499,7 +2500,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "chrono", @@ -2519,7 +2520,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "bigdecimal", @@ -2535,7 +2536,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2561,7 +2562,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "50.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d#34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "async-recursion", "async-trait", @@ -5152,9 +5153,9 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", diff --git a/Cargo.toml b/Cargo.toml index 6db8e7e238..29b5ce9fda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,9 +65,9 @@ chrono = "0.4.41" clap = { version = "4.5.41", features = ["derive", "cargo"] } ctor = "0.2.8" # Use datafusion fork to make sure to reuse the same versions of other packages and to avoid build conflicts caused by lzma -datafusion = { git = "https://github.com/DataDog/datafusion", rev = "34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d", package = "datafusion" } -datafusion-cli = { git = "https://github.com/DataDog/datafusion", rev = "34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d", package = "datafusion-cli" } -datafusion-sqllogictest = { git = "https://github.com/DataDog/datafusion", rev = "34c8e49a36343092fc2a1dce2ceb503c2b5dfd3d", package = "datafusion-sqllogictest" } +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "4cc47d3418ee7dfe8f82394d356560401b950d63", package = "datafusion" } +datafusion-cli = { git = "https://github.com/DataDog/datafusion", rev = "4cc47d3418ee7dfe8f82394d356560401b950d63", package = "datafusion-cli" } +datafusion-sqllogictest = { git = "https://github.com/DataDog/datafusion", rev = "4cc47d3418ee7dfe8f82394d356560401b950d63", package = "datafusion-sqllogictest" } derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0"