From 901e04fc3bd33075cb1a285ea1234fbae0c65dbe Mon Sep 17 00:00:00 2001 From: Geoffrey Claude Date: Wed, 27 May 2026 12:31:29 +0200 Subject: [PATCH] Upgrade to DataFusion 54 --- CHANGELOG.md | 8 + Cargo.lock | 906 +++++++----------- Cargo.toml | 8 +- README.md | 8 +- .../src/exec_instrument_macros.rs | 31 +- .../src/exec_instrument_rule.rs | 14 +- datafusion-tracing/src/instrumented_exec.rs | 29 +- datafusion-tracing/src/lib.rs | 4 +- datafusion-tracing/src/utils.rs | 33 - tests/integration_tests.rs | 4 +- tests/snapshots/01_basic_trace.snap | 125 ++- tests/snapshots/02_basic_metrics_trace.snap | 126 ++- tests/snapshots/03_basic_preview_trace.snap | 125 ++- .../04_basic_compact_preview_trace.snap | 125 ++- .../snapshots/05_basic_all_options_trace.snap | 126 ++- .../06_object_store_all_options_trace.snap | 155 ++- ...crabble_all_options_01_ProjectionExec.snap | 26 +- ...rabble_all_options_03_RepartitionExec.snap | 26 +- ...rabble_all_options_05_RepartitionExec.snap | 4 +- ..._scrabble_all_options_10_HashJoinExec.snap | 26 +- ...crabble_all_options_11_AggregateExec.snap} | 0 ...crabble_all_options_11_ProjectionExec.snap | 17 - tests/snapshots/08_recursive_trace.snap | 199 +++- ...ecursive_all_options_01_WorkTableExec.snap | 10 +- ...ecursive_all_options_02_WorkTableExec.snap | 10 +- ...rsive_all_options_03_RepartitionExec.snap} | 0 ...rsive_all_options_04_RepartitionExec.snap} | 0 ...cursive_all_options_05_ProjectionExec.snap | 9 - ...rsive_all_options_05_RepartitionExec.snap} | 0 ..._recursive_all_options_06_FilterExec.snap} | 0 ...cursive_all_options_06_ProjectionExec.snap | 9 - ..._recursive_all_options_07_FilterExec.snap} | 0 ...cursive_all_options_08_ProjectionExec.snap | 9 + ...cursive_all_options_09_ProjectionExec.snap | 9 + ...all_options_10_CoalescePartitionsExec.snap | 9 - ...ve_all_options_10_PlaceholderRowExec.snap} | 0 ...all_options_11_CoalescePartitionsExec.snap | 10 +- ...all_options_12_CoalescePartitionsExec.snap | 9 + .../09_recursive_all_options_trace.snap | 274 +++++- tests/snapshots/10_topk_lineitem_trace.snap | 218 ++++- 40 files changed, 1864 insertions(+), 837 deletions(-) rename tests/snapshots/{07_scrabble_all_options_12_AggregateExec.snap => 07_scrabble_all_options_11_AggregateExec.snap} (100%) delete mode 100644 tests/snapshots/07_scrabble_all_options_11_ProjectionExec.snap rename tests/snapshots/{09_recursive_all_options_03_FilterExec.snap => 09_recursive_all_options_03_RepartitionExec.snap} (100%) rename tests/snapshots/{09_recursive_all_options_04_FilterExec.snap => 09_recursive_all_options_04_RepartitionExec.snap} (100%) delete mode 100644 tests/snapshots/09_recursive_all_options_05_ProjectionExec.snap rename tests/snapshots/{09_recursive_all_options_09_RepartitionExec.snap => 09_recursive_all_options_05_RepartitionExec.snap} (100%) rename tests/snapshots/{09_recursive_all_options_07_RepartitionExec.snap => 09_recursive_all_options_06_FilterExec.snap} (100%) delete mode 100644 tests/snapshots/09_recursive_all_options_06_ProjectionExec.snap rename tests/snapshots/{09_recursive_all_options_08_RepartitionExec.snap => 09_recursive_all_options_07_FilterExec.snap} (100%) create mode 100644 tests/snapshots/09_recursive_all_options_08_ProjectionExec.snap create mode 100644 tests/snapshots/09_recursive_all_options_09_ProjectionExec.snap delete mode 100644 tests/snapshots/09_recursive_all_options_10_CoalescePartitionsExec.snap rename tests/snapshots/{09_recursive_all_options_12_PlaceholderRowExec.snap => 09_recursive_all_options_10_PlaceholderRowExec.snap} (100%) create mode 100644 tests/snapshots/09_recursive_all_options_12_CoalescePartitionsExec.snap diff --git a/CHANGELOG.md b/CHANGELOG.md index bc2fcd3..56005a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Version numbers follow Apache DataFusion releases for compatibility alignment. +## [54.0.0] - 2026-06-08 + +### Changed + +- Update to Apache DataFusion 54.0.0 +- Align `InstrumentedExec` with DataFusion 54's `ExecutionPlan` downcast API + ## [53.0.2] - 2026-05-27 ### Fixed @@ -149,6 +156,7 @@ Initial public release of DataFusion Tracing. - Preview formatting utilities (`pretty_format_compact_batch`) - Integration with Jaeger, DataDog, and other OpenTelemetry-compatible collectors +[54.0.0]: https://github.com/datafusion-contrib/datafusion-tracing/compare/53.0.2...54.0.0 [53.0.2]: https://github.com/datafusion-contrib/datafusion-tracing/compare/53.0.1...53.0.2 [53.0.1]: https://github.com/datafusion-contrib/datafusion-tracing/compare/53.0.0...53.0.1 [53.0.0]: https://github.com/datafusion-contrib/datafusion-tracing/compare/52.0.0...53.0.0 diff --git a/Cargo.lock b/Cargo.lock index 5436e89..51c4f26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,15 +63,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arrow" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" dependencies = [ "arrow-arith", "arrow-array", @@ -90,9 +90,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" dependencies = [ "arrow-array", "arrow-buffer", @@ -104,9 +104,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash", "arrow-buffer", @@ -115,7 +115,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -123,9 +123,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" dependencies = [ "bytes", "half", @@ -135,9 +135,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" dependencies = [ "arrow-array", "arrow-buffer", @@ -157,9 +157,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" dependencies = [ "arrow-array", "arrow-cast", @@ -172,9 +172,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" dependencies = [ "arrow-array", "arrow-buffer", @@ -196,19 +196,21 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] name = "arrow-json" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", "indexmap", @@ -224,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -237,9 +239,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" dependencies = [ "arrow-array", "arrow-buffer", @@ -250,9 +252,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ "serde_core", "serde_json", @@ -260,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash", "arrow-array", @@ -274,9 +276,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ "arrow-array", "arrow-buffer", @@ -317,9 +319,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "base64" @@ -342,9 +344,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "brotli" @@ -369,9 +371,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "byteorder" @@ -381,15 +383,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.51" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -437,14 +439,13 @@ dependencies = [ [[package]] name = "console" -version = "0.15.11" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" dependencies = [ "encode_unicode", "libc", - "once_cell", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -462,7 +463,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] @@ -473,15 +474,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -540,9 +532,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -554,14 +546,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" +checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -589,13 +580,12 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", + "indexmap", "itertools", "log", "object_store", "parking_lot", "parquet", - "rand", - "regex", "sqlparser", "tempfile", "tokio", @@ -605,9 +595,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" +checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" dependencies = [ "arrow", "async-trait", @@ -630,9 +620,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" +checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" dependencies = [ "arrow", "async-trait", @@ -653,33 +643,34 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" +checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "libc", "log", "object_store", "parquet", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" +checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" dependencies = [ "futures", "log", @@ -688,9 +679,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" +checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" dependencies = [ "arrow", "async-trait", @@ -710,6 +701,7 @@ dependencies = [ "itertools", "log", "object_store", + "parking_lot", "rand", "tokio", "url", @@ -717,9 +709,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" +checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" dependencies = [ "arrow", "arrow-ipc", @@ -741,9 +733,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" +checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" dependencies = [ "arrow", "async-trait", @@ -764,9 +756,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" +checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" dependencies = [ "arrow", "async-trait", @@ -781,16 +773,15 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" +checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" dependencies = [ "arrow", "async-trait", @@ -800,6 +791,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -818,20 +810,19 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" +checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" [[package]] name = "datafusion-execution" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" +checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -847,11 +838,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" +checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -862,29 +854,27 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap", "itertools", - "paste", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" +checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" dependencies = [ "arrow", "datafusion-common", "indexmap", "itertools", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" +checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" dependencies = [ "arrow", "arrow-buffer", @@ -897,6 +887,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools", "log", @@ -904,17 +895,15 @@ dependencies = [ "num-traits", "rand", "regex", - "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" +checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -924,19 +913,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" +checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -945,9 +933,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" +checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" dependencies = [ "arrow", "arrow-ord", @@ -961,34 +949,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" +checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" +checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" dependencies = [ "arrow", "datafusion-common", @@ -999,14 +987,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" +checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1014,9 +1001,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" +checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" dependencies = [ "datafusion-doc", "quote", @@ -1025,9 +1012,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" +checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" dependencies = [ "arrow", "chrono", @@ -1044,11 +1031,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" +checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -1056,20 +1042,19 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" +checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" dependencies = [ "arrow", "datafusion-common", @@ -1082,26 +1067,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" +checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" +checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" dependencies = [ "arrow", "datafusion-common", @@ -1117,12 +1102,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" +checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -1137,7 +1123,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "log", @@ -1149,9 +1135,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" +checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" dependencies = [ "arrow", "datafusion-common", @@ -1160,15 +1146,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools", "log", ] [[package]] name = "datafusion-session" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" +checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" dependencies = [ "async-trait", "datafusion-common", @@ -1180,9 +1165,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" +checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" dependencies = [ "arrow", "bigdecimal", @@ -1198,7 +1183,7 @@ dependencies = [ [[package]] name = "datafusion-tracing" -version = "53.0.2" +version = "54.0.0" dependencies = [ "async-trait", "comfy-table", @@ -1217,7 +1202,7 @@ dependencies = [ [[package]] name = "datafusion-tracing-examples" -version = "53.0.2" +version = "54.0.0" dependencies = [ "datafusion", "integration-utils", @@ -1232,7 +1217,7 @@ dependencies = [ [[package]] name = "datafusion-tracing-tests" -version = "53.0.2" +version = "54.0.0" dependencies = [ "datafusion", "insta", @@ -1256,9 +1241,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -1276,9 +1261,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "encode_unicode" @@ -1299,20 +1284,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "find-msvc-tools" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fixedbitset" @@ -1332,13 +1317,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ - "crc32fast", - "libz-rs-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1370,9 +1354,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1385,9 +1369,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1395,15 +1379,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1412,15 +1396,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", @@ -1429,21 +1413,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -1453,15 +1437,14 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "libc", @@ -1501,9 +1484,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1547,9 +1530,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", @@ -1570,9 +1553,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -1615,9 +1598,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -1629,7 +1612,6 @@ dependencies = [ "httparse", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1650,14 +1632,13 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64", "bytes", "futures-channel", - "futures-core", "futures-util", "http", "http-body", @@ -1674,9 +1655,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1698,12 +1679,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1711,9 +1693,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1724,9 +1706,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1738,15 +1720,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1758,15 +1740,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1796,9 +1778,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1806,21 +1788,21 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] [[package]] name = "insta" -version = "1.46.0" +version = "1.47.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b66886d14d18d420ab5052cbff544fc5d34d0b2cdd35eb5976aaa10a4a472e5" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" dependencies = [ "console", "once_cell", @@ -1832,7 +1814,7 @@ dependencies = [ [[package]] name = "instrumented-object-store" -version = "53.0.2" +version = "54.0.0" dependencies = [ "async-trait", "bytes", @@ -1852,7 +1834,7 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "integration-utils" -version = "53.0.2" +version = "54.0.0" dependencies = [ "async-trait", "datafusion", @@ -1865,19 +1847,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.10" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" -dependencies = [ - "memchr", - "serde", -] +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "itertools" @@ -1890,9 +1862,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jobserver" @@ -1906,10 +1878,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1985,36 +1959,27 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.180" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libz-rs-sys" -version = "0.5.5" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" -dependencies = [ - "zlib-rs", -] +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "litrs" @@ -2033,15 +1998,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] @@ -2057,9 +2022,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "miniz_oxide" @@ -2073,13 +2038,13 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -2088,7 +2053,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -2157,9 +2122,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "opentelemetry" @@ -2190,9 +2155,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" +checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ "http", "opentelemetry", @@ -2271,9 +2236,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash", "arrow-array", @@ -2289,7 +2254,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "lz4_flex", "num-bigint", "num-integer", @@ -2349,18 +2314,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -2369,27 +2334,21 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2415,18 +2374,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -2434,9 +2393,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools", @@ -2468,9 +2427,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core", @@ -2488,9 +2447,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] @@ -2506,9 +2465,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -2518,9 +2477,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -2578,15 +2537,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -2597,9 +2556,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -2618,9 +2577,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "seq-macro" @@ -2635,7 +2594,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", - "serde_derive", ] [[package]] @@ -2660,9 +2618,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -2700,9 +2658,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simdutf8" @@ -2718,15 +2676,15 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -2742,19 +2700,19 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys", ] [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive", @@ -2810,31 +2768,31 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.24.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -2872,9 +2830,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -2882,9 +2840,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.49.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -2892,14 +2850,14 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -2933,9 +2891,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.2" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "base64", @@ -2959,9 +2917,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.2" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", @@ -2970,9 +2928,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -2989,20 +2947,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags", "bytes", "futures-util", "http", "http-body", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", + "url", ] [[package]] @@ -3100,9 +3058,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -3133,15 +3091,15 @@ checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-width" @@ -3175,9 +3133,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -3223,11 +3181,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen 0.46.0", + "wit-bindgen 0.57.1", ] [[package]] @@ -3241,9 +3199,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -3254,22 +3212,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3277,9 +3232,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -3290,9 +3245,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] @@ -3333,9 +3288,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -3373,7 +3328,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -3441,24 +3396,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -3468,141 +3405,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - -[[package]] -name = "wit-bindgen" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -3612,6 +3414,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -3693,15 +3501,15 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -3710,9 +3518,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -3722,18 +3530,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.33" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.33" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -3742,18 +3550,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -3763,9 +3571,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -3774,9 +3582,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -3785,9 +3593,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", @@ -3796,15 +3604,15 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" [[package]] name = "zmij" -version = "1.0.12" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index 6a34933..4b0b411 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ resolver = "2" [workspace.package] description = "DataFusion tracing of execution plans" -version = "53.0.2" +version = "54.0.0" edition = "2024" license = "Apache-2.0" repository = "https://github.com/datafusion-contrib/datafusion-tracing" @@ -38,10 +38,10 @@ rust-version = "1.88.0" [workspace.dependencies] async-trait = "0.1" -datafusion = { version = "53.0.0", default-features = false } -datafusion-tracing = { path = "datafusion-tracing", version = "53.0.2" } +datafusion = { version = "54.0.0", default-features = false } +datafusion-tracing = { path = "datafusion-tracing", version = "54.0.0" } futures = "0.3" -instrumented-object-store = { path = "instrumented-object-store", version = "53.0.2" } +instrumented-object-store = { path = "instrumented-object-store", version = "54.0.0" } tokio = { version = "1.48" } tracing = { version = "0.1" } tracing-futures = { version = "0.2", features = ["futures-03"] } diff --git a/README.md b/README.md index 2f6d49f..968b440 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ Include DataFusion Tracing in your project's `Cargo.toml`: ```toml [dependencies] -datafusion = "53.0.0" -datafusion-tracing = "53.0.0" +datafusion = "54.0.0" +datafusion-tracing = "54.0.0" ``` ### Quick Start Example @@ -182,11 +182,11 @@ Or pass a vector: builder.with_physical_optimizer_rules(vec![..., instrument_rule]) ``` -### InstrumentedExec visibility and `as_any` delegation +### InstrumentedExec visibility Instrumentation is designed to be mostly invisible: with the rule registered last, other optimizer rules typically never see `InstrumentedExec` at all. The wrapper itself is intentionally private so downstream code cannot depend on its internals; the supported surface is the optimizer rule and the standard `ExecutionPlan` trait. -For situations that absolutely need a read-only view of the underlying node type (for example, diagnostics or custom tooling), `InstrumentedExec` delegates `as_any()` to the inner plan. This lets callers inspect or downcast to the original node type without ever handling the wrapper directly, and any new nodes created via `ExecutionPlan` methods (such as `with_new_children()` or `repartitioned()`) are automatically re-wrapped so they remain instrumented. +Instrumented plans preserve the usual `ExecutionPlan` introspection behavior: callers can inspect wrapped node types without depending on `InstrumentedExec`. Any new nodes created via `ExecutionPlan` methods (such as `with_new_children()` or `repartitioned()`) are still automatically re-wrapped so they remain instrumented. ## Repository Structure diff --git a/datafusion-tracing/src/exec_instrument_macros.rs b/datafusion-tracing/src/exec_instrument_macros.rs index 7282fb4..e704aaf 100644 --- a/datafusion-tracing/src/exec_instrument_macros.rs +++ b/datafusion-tracing/src/exec_instrument_macros.rs @@ -90,9 +90,17 @@ /// The list of native datafusion metrics can be re-generated by running the following bash command at the root of the [datafusion](https://github.com/apache/datafusion) repository: /// ```bash /// ( -/// find . -type f -name '*.rs' ! -path '*/metrics/mod.rs' -exec grep -A2 'MetricBuilder::new' {} \; | grep -E '(counter|gauge|subset_time)' -/// grep -E -o 'Self::.*=>.*"' datafusion/physical-plan/src/metrics/value.rs -/// ) | cut -s -f 2 -d '"' | sort -u | sed 's/\(.*\)/datafusion.metrics.\1 = tracing::field::Empty,/g' +/// find datafusion -type f -name '*.rs' \ +/// ! -path '*/tests/*' \ +/// ! -path '*/benches/*' \ +/// ! -path '*/physical-expr-common/src/metrics/builder.rs' \ +/// -exec perl -0ne 'while (/\.(?:counter|global_counter|gauge|subset_time|pruning_metrics|ratio_metrics(?:_with_strategy)?)\(\s*"([A-Za-z0-9_]+)"/g) { print "$1\n" }' {} + +/// sed -n '/pub fn name/,/^ }/p' datafusion/physical-expr-common/src/metrics/value.rs \ +/// | grep -E '=> "[A-Za-z0-9_]+"' \ +/// | cut -d '"' -f 2 +/// ) | grep -Ev '^(test|my_counter|my_metric|the_counter|the_second_counter|the_third_counter|the_time)$' \ +/// | sort -u \ +/// | sed 's/\(.*\)/datafusion.metrics.\1 = tracing::field::Empty,/g' /// ``` #[macro_export] macro_rules! instrument_with_spans { @@ -114,6 +122,7 @@ macro_rules! instrument_with_spans { datafusion.preview_rows = tracing::field::Empty, datafusion.metrics.aggregate_arguments_time = tracing::field::Empty, datafusion.metrics.aggregation_time = tracing::field::Empty, + datafusion.metrics.avg_fanout = tracing::field::Empty, datafusion.metrics.batches_split = tracing::field::Empty, datafusion.metrics.bloom_filter_eval_time = tracing::field::Empty, datafusion.metrics.build_input_batches = tracing::field::Empty, @@ -121,44 +130,52 @@ macro_rules! instrument_with_spans { datafusion.metrics.build_mem_used = tracing::field::Empty, datafusion.metrics.build_time = tracing::field::Empty, datafusion.metrics.bytes_scanned = tracing::field::Empty, + datafusion.metrics.bytes_written = tracing::field::Empty, datafusion.metrics.elapsed_compute = tracing::field::Empty, datafusion.metrics.emitting_time = tracing::field::Empty, datafusion.metrics.end_timestamp = tracing::field::Empty, datafusion.metrics.fetch_time = tracing::field::Empty, datafusion.metrics.file_open_errors = tracing::field::Empty, datafusion.metrics.file_scan_errors = tracing::field::Empty, + datafusion.metrics.files_opened = tracing::field::Empty, + datafusion.metrics.files_processed = tracing::field::Empty, + datafusion.metrics.files_ranges_pruned_statistics = tracing::field::Empty, datafusion.metrics.input_batches = tracing::field::Empty, datafusion.metrics.input_rows = tracing::field::Empty, datafusion.metrics.join_time = tracing::field::Empty, datafusion.metrics.left_input_batches = tracing::field::Empty, datafusion.metrics.left_input_rows = tracing::field::Empty, + datafusion.metrics.limit_pruned_row_groups = tracing::field::Empty, datafusion.metrics.max_mem_used = tracing::field::Empty, datafusion.metrics.max_queued = tracing::field::Empty, datafusion.metrics.mem_used = tracing::field::Empty, datafusion.metrics.metadata_load_time = tracing::field::Empty, - datafusion.metrics.num_bytes = tracing::field::Empty, datafusion.metrics.num_predicate_creation_errors = tracing::field::Empty, datafusion.metrics.output_batches = tracing::field::Empty, datafusion.metrics.output_bytes = tracing::field::Empty, datafusion.metrics.output_rows = tracing::field::Empty, datafusion.metrics.page_index_eval_time = tracing::field::Empty, - datafusion.metrics.page_index_rows_matched = tracing::field::Empty, + datafusion.metrics.page_index_pages_pruned = tracing::field::Empty, + datafusion.metrics.page_index_pages_skipped_by_fully_matched = tracing::field::Empty, datafusion.metrics.page_index_rows_pruned = tracing::field::Empty, datafusion.metrics.peak_mem_used = tracing::field::Empty, datafusion.metrics.predicate_cache_inner_records = tracing::field::Empty, datafusion.metrics.predicate_cache_records = tracing::field::Empty, datafusion.metrics.predicate_evaluation_errors = tracing::field::Empty, + datafusion.metrics.probe_hit_rate = tracing::field::Empty, datafusion.metrics.pushdown_rows_matched = tracing::field::Empty, datafusion.metrics.pushdown_rows_pruned = tracing::field::Empty, + datafusion.metrics.reduction_factor = tracing::field::Empty, datafusion.metrics.repartition_time = tracing::field::Empty, datafusion.metrics.right_input_batches = tracing::field::Empty, datafusion.metrics.right_input_rows = tracing::field::Empty, - datafusion.metrics.row_groups_matched_bloom_filter = tracing::field::Empty, - datafusion.metrics.row_groups_matched_statistics = tracing::field::Empty, datafusion.metrics.row_groups_pruned_bloom_filter = tracing::field::Empty, datafusion.metrics.row_groups_pruned_statistics = tracing::field::Empty, datafusion.metrics.row_pushdown_eval_time = tracing::field::Empty, datafusion.metrics.row_replacements = tracing::field::Empty, + datafusion.metrics.rows_written = tracing::field::Empty, + datafusion.metrics.scan_efficiency_ratio = tracing::field::Empty, + datafusion.metrics.selectivity = tracing::field::Empty, datafusion.metrics.send_time = tracing::field::Empty, datafusion.metrics.skipped_aggregation_rows = tracing::field::Empty, datafusion.metrics.spill_count = tracing::field::Empty, diff --git a/datafusion-tracing/src/exec_instrument_rule.rs b/datafusion-tracing/src/exec_instrument_rule.rs index 8afd250..b07983b 100644 --- a/datafusion-tracing/src/exec_instrument_rule.rs +++ b/datafusion-tracing/src/exec_instrument_rule.rs @@ -19,7 +19,6 @@ use crate::instrumented_exec::{InstrumentedExec, SpanCreateFn}; use crate::options::InstrumentationOptions; -use crate::utils::InternalOptimizerGuard; use datafusion::common::runtime::{JoinSetTracer, set_join_set_tracer}; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion::{ @@ -68,13 +67,6 @@ impl PhysicalOptimizerRule for InstrumentRule { plan: Arc, _config: &ConfigOptions, ) -> datafusion::error::Result> { - // Activate the internal optimization context for the duration of this pass. - // This allows InstrumentedExec to reveal its type via as_any(). - // - // This guard is safe because PhysicalOptimizerRule::optimize is synchronous - // and won't be suspended or moved across threads during execution. - let _guard = InternalOptimizerGuard::new(); - // Iterate over the plan using transform_down to ensure all nodes are instrumented, // including any new nodes added by other optimizer rules. plan.transform_down(|plan| { @@ -150,6 +142,12 @@ mod tests { // First optimization pass let optimized_once = rule.optimize(plan, &ConfigOptions::default())?; + // Public downcasts see the wrapped plan, while the instrumentation + // rule can still identify its own wrapper internally. + assert!(optimized_once.is::()); + assert!(!optimized_once.is::()); + assert!(InstrumentedExec::is_instrumented(optimized_once.as_ref())); + // Second optimization pass let optimized_twice = rule.optimize(optimized_once.clone(), &ConfigOptions::default())?; diff --git a/datafusion-tracing/src/instrumented_exec.rs b/datafusion-tracing/src/instrumented_exec.rs index bb4788d..2cf3796 100644 --- a/datafusion-tracing/src/instrumented_exec.rs +++ b/datafusion-tracing/src/instrumented_exec.rs @@ -22,7 +22,6 @@ use crate::{ node::{NodeRecorder, NodeRecordingStream}, options::InstrumentationOptions, preview::{PreviewFn, PreviewRecorder, PreviewRecordingStream}, - utils::is_internal_optimizer_check, }; use datafusion::{ arrow::{array::RecordBatch, datatypes::SchemaRef}, @@ -242,9 +241,10 @@ impl InstrumentedExec { /// Returns true if the plan is an `InstrumentedExec` wrapper. /// - /// This relies on the internal optimization context being active in the current thread. + /// Uses a direct `Any` check because public `ExecutionPlan` downcasts + /// delegate to the wrapped plan instead. pub(crate) fn is_instrumented(plan: &dyn ExecutionPlan) -> bool { - plan.as_any().is::() + (plan as &dyn Any).is::() } } @@ -263,7 +263,7 @@ impl ExecutionPlan for InstrumentedExec { fn benefits_from_input_partitioning(&self) -> Vec; fn children(&self) -> Vec<&Arc>; fn metrics(&self) -> Option; - fn partition_statistics(&self, partition: Option) -> Result; + fn partition_statistics(&self, partition: Option) -> Result>; fn supports_limit_pushdown(&self) -> bool; fn fetch(&self) -> Option; fn cardinality_effect(&self) -> CardinalityEffect; @@ -388,19 +388,10 @@ impl ExecutionPlan for InstrumentedExec { Some(self.with_new_inner(new_inner)) } - /// Returns the plan as any to allow for downcasting. - /// - /// During optimization passes, this returns `self` (the `InstrumentedExec`) to - /// allow the optimizer to identify already-instrumented nodes. - /// - /// Otherwise, this delegates to the inner plan to provide "transparent downcasting", - /// allowing users to downcast an instrumented node to its original type. - fn as_any(&self) -> &dyn Any { - if is_internal_optimizer_check() { - self - } else { - self.inner.as_any() - } + /// Delegate public downcasts to the inner plan so instrumentation stays + /// transparent to normal plan inspection. + fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> { + Some(self.inner.as_ref()) } /// Executes the plan for a given partition and context, instrumented with tracing and metrics recording. @@ -923,10 +914,6 @@ mod tests { self.inner.name() } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { self.inner.properties() } diff --git a/datafusion-tracing/src/lib.rs b/datafusion-tracing/src/lib.rs index 63b9261..6882358 100644 --- a/datafusion-tracing/src/lib.rs +++ b/datafusion-tracing/src/lib.rs @@ -49,8 +49,8 @@ //! //! ```toml //! [dependencies] -//! datafusion = "53.0.0" -//! datafusion-tracing = "53.0.0" +//! datafusion = "54.0.0" +//! datafusion-tracing = "54.0.0" //! ``` //! //! ## Quick Start Example diff --git a/datafusion-tracing/src/utils.rs b/datafusion-tracing/src/utils.rs index 64fcba0..a9be0d6 100644 --- a/datafusion-tracing/src/utils.rs +++ b/datafusion-tracing/src/utils.rs @@ -18,41 +18,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. use datafusion::physical_plan::{DisplayFormatType, ExecutionPlan}; -use std::cell::Cell; use std::fmt; -thread_local! { - /// Flag to indicate that we are performing an internal check to identify - /// instrumentation nodes. This allows `InstrumentedExec` to reveal its - /// type during optimization passes while remaining transparent - /// to users. - static IS_INTERNAL_OPTIMIZER_CHECK: Cell = const { Cell::new(false) }; -} - -pub(crate) fn is_internal_optimizer_check() -> bool { - IS_INTERNAL_OPTIMIZER_CHECK.with(|c| c.get()) -} - -/// RAII guard to safely set and reset the internal optimizer check flag. -pub(crate) struct InternalOptimizerGuard(bool); - -impl InternalOptimizerGuard { - pub fn new() -> Self { - let prev = IS_INTERNAL_OPTIMIZER_CHECK.with(|c| { - let prev = c.get(); - c.set(true); - prev - }); - Self(prev) - } -} - -impl Drop for InternalOptimizerGuard { - fn drop(&mut self) { - IS_INTERNAL_OPTIMIZER_CHECK.with(|c| c.set(self.0)); - } -} - /// Helper struct for default display formatting of an `ExecutionPlan`. pub(crate) struct DefaultDisplay<'a>(pub &'a dyn ExecutionPlan); diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e25251f..d86ec4f 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -176,7 +176,9 @@ async fn test_scrabble_all_options() -> Result<()> { .with_compact_preview() // skip preview assertions for these spans as they depend on the partitioning // and are not deterministic - .ignore_preview_spans(&[13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]) + .ignore_preview_spans(&[ + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + ]) // ignore the full trace assertions as they are not deterministic: exact span ordering can vary for this complex query .ignore_full_trace(), ) diff --git a/tests/snapshots/01_basic_trace.snap b/tests/snapshots/01_basic_trace.snap index 971bef6..da0e583 100644 --- a/tests/snapshots/01_basic_trace.snap +++ b/tests/snapshots/01_basic_trace.snap @@ -1,6 +1,6 @@ --- source: tests/integration_tests.rs -expression: json_lines +expression: full_trace_lines --- [ { @@ -171,6 +171,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "01_basic" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1049,6 +1076,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "01_basic" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1165,6 +1221,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "01_basic" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1194,6 +1279,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "01_basic" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1437,6 +1551,15 @@ expression: json_lines { "name": "run_traced_query", "query_name": "select_one" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(1)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", diff --git a/tests/snapshots/02_basic_metrics_trace.snap b/tests/snapshots/02_basic_metrics_trace.snap index 875fb4f..55b37ef 100644 --- a/tests/snapshots/02_basic_metrics_trace.snap +++ b/tests/snapshots/02_basic_metrics_trace.snap @@ -1,7 +1,6 @@ --- source: tests/integration_tests.rs -assertion_line: 250 -expression: json_lines +expression: full_trace_lines --- [ { @@ -172,6 +171,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "02_basic_metrics" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1050,6 +1076,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "02_basic_metrics" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1166,6 +1221,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "02_basic_metrics" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1195,6 +1279,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "02_basic_metrics" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1438,6 +1551,15 @@ expression: json_lines { "name": "run_traced_query", "query_name": "select_one" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(1)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", diff --git a/tests/snapshots/03_basic_preview_trace.snap b/tests/snapshots/03_basic_preview_trace.snap index 271d088..38bb95e 100644 --- a/tests/snapshots/03_basic_preview_trace.snap +++ b/tests/snapshots/03_basic_preview_trace.snap @@ -1,6 +1,6 @@ --- source: tests/integration_tests.rs -expression: json_lines +expression: full_trace_lines --- [ { @@ -171,6 +171,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "03_basic_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1049,6 +1076,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "03_basic_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1165,6 +1221,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "03_basic_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1194,6 +1279,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "03_basic_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1438,6 +1552,15 @@ expression: json_lines { "name": "run_traced_query", "query_name": "select_one" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(1)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", diff --git a/tests/snapshots/04_basic_compact_preview_trace.snap b/tests/snapshots/04_basic_compact_preview_trace.snap index 0701190..42784dd 100644 --- a/tests/snapshots/04_basic_compact_preview_trace.snap +++ b/tests/snapshots/04_basic_compact_preview_trace.snap @@ -1,6 +1,6 @@ --- source: tests/integration_tests.rs -expression: json_lines +expression: full_trace_lines --- [ { @@ -171,6 +171,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "04_basic_compact_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1049,6 +1076,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "04_basic_compact_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1165,6 +1221,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "04_basic_compact_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1194,6 +1279,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "04_basic_compact_preview" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1438,6 +1552,15 @@ expression: json_lines { "name": "run_traced_query", "query_name": "select_one" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(1)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", diff --git a/tests/snapshots/05_basic_all_options_trace.snap b/tests/snapshots/05_basic_all_options_trace.snap index d1981fb..fe85d00 100644 --- a/tests/snapshots/05_basic_all_options_trace.snap +++ b/tests/snapshots/05_basic_all_options_trace.snap @@ -1,7 +1,6 @@ --- source: tests/integration_tests.rs -assertion_line: 250 -expression: json_lines +expression: full_trace_lines --- [ { @@ -172,6 +171,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "05_basic_all_options" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1050,6 +1076,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "05_basic_all_options" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1166,6 +1221,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "05_basic_all_options" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1195,6 +1279,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "05_basic_all_options" + }, + { + "name": "run_traced_query", + "query_name": "select_one" + }, + { + "logical_plan": "Projection: Int64(1) [Int64(1):Int64]\n EmptyRelation: rows=1 []", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1439,6 +1552,15 @@ expression: json_lines { "name": "run_traced_query", "query_name": "select_one" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(1)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", diff --git a/tests/snapshots/06_object_store_all_options_trace.snap b/tests/snapshots/06_object_store_all_options_trace.snap index 27b92c4..81e0937 100644 --- a/tests/snapshots/06_object_store_all_options_trace.snap +++ b/tests/snapshots/06_object_store_all_options_trace.snap @@ -171,6 +171,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "06_object_store_all_options" + }, + { + "name": "run_traced_query", + "query_name": "order_nations" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -843,6 +870,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "06_object_store_all_options" + }, + { + "name": "run_traced_query", + "query_name": "order_nations" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 2, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1750,6 +1804,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "06_object_store_all_options" + }, + { + "name": "run_traced_query", + "query_name": "order_nations" + }, + { + "logical_plan": "Sort: nation.n_name ASC NULLS LAST [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]\n TableScan: nation projection=[n_nationkey, n_name, n_regionkey, n_comment] [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1866,6 +1949,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "06_object_store_all_options" + }, + { + "name": "run_traced_query", + "query_name": "order_nations" + }, + { + "logical_plan": "Sort: nation.n_name ASC NULLS LAST [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]\n TableScan: nation projection=[n_nationkey, n_name, n_regionkey, n_comment] [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1895,6 +2007,35 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "06_object_store_all_options" + }, + { + "name": "run_traced_query", + "query_name": "order_nations" + }, + { + "logical_plan": "Sort: nation.n_name ASC NULLS LAST [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]\n TableScan: nation projection=[n_nationkey, n_name, n_regionkey, n_comment] [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1929,7 +2070,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "PushdownSort" + "otel.name": "PushdownSort (modified)" }, "spans": [ { @@ -2073,7 +2214,7 @@ expression: json_lines "level": "INFO", "message": "close", "span": { - "datafusion.effective_rules": "OutputRequirements, OutputRequirements", + "datafusion.effective_rules": "OutputRequirements, OutputRequirements, PushdownSort", "name": "Phase", "otel.name": "optimize_physical_plan" }, @@ -2101,7 +2242,7 @@ expression: json_lines "span": { "logical_plan": "Sort: nation.n_name ASC NULLS LAST [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]\n TableScan: nation projection=[n_nationkey, n_name, n_regionkey, n_comment] [n_nationkey:Int64, n_name:Utf8View, n_regionkey:Int64, n_comment:Utf8View]", "name": "create_physical_plan", - "physical_plan": "SortExec: expr=[n_name@1 ASC NULLS LAST], preserve_partitioning=[false]\n DataSourceExec: file_groups={1 group: [[/nation.parquet]]}, projection=[n_nationkey, n_name, n_regionkey, n_comment], file_type=parquet\n" + "physical_plan": "SortExec: expr=[n_name@1 ASC NULLS LAST], preserve_partitioning=[false]\n DataSourceExec: file_groups={1 group: [[/nation.parquet]]}, projection=[n_nationkey, n_name, n_regionkey, n_comment], file_type=parquet, sort_order_for_reorder=[n_name@1 ASC NULLS LAST]\n" }, "spans": [ { @@ -2172,12 +2313,17 @@ expression: json_lines "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.metrics.file_open_errors": "0", "datafusion.metrics.file_scan_errors": "0", + "datafusion.metrics.files_opened": "1", + "datafusion.metrics.files_processed": "1", + "datafusion.metrics.files_ranges_pruned_statistics": "1 total → 1 matched", + "datafusion.metrics.limit_pruned_row_groups": "0 total → 0 matched", "datafusion.metrics.metadata_load_time": "0.00ms", "datafusion.metrics.num_predicate_creation_errors": "0", "datafusion.metrics.output_batches": "1", "datafusion.metrics.output_bytes": "3.4 KB", "datafusion.metrics.output_rows": "25", "datafusion.metrics.page_index_eval_time": "0.00ms", + "datafusion.metrics.page_index_pages_pruned": "0 total → 0 matched", "datafusion.metrics.page_index_rows_pruned": "0 total → 0 matched", "datafusion.metrics.predicate_cache_inner_records": "0", "datafusion.metrics.predicate_cache_records": "0", @@ -2187,13 +2333,14 @@ expression: json_lines "datafusion.metrics.row_groups_pruned_bloom_filter": "1 total → 1 matched", "datafusion.metrics.row_groups_pruned_statistics": "1 total → 1 matched", "datafusion.metrics.row_pushdown_eval_time": "0.00ms", + "datafusion.metrics.scan_efficiency_ratio": "64.04% (1.71 K/2.67 K)", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.metrics.statistics_eval_time": "0.00ms", "datafusion.metrics.time_elapsed_opening": "0.00ms", "datafusion.metrics.time_elapsed_processing": "0.00ms", "datafusion.metrics.time_elapsed_scanning_total": "0.00ms", "datafusion.metrics.time_elapsed_scanning_until_data": "0.00ms", - "datafusion.node": "DataSourceExec: file_groups={1 group: [[/nation.parquet]]}, projection=[n_nationkey, n_name, n_regionkey, n_comment], file_type=parquet", + "datafusion.node": "DataSourceExec: file_groups={1 group: [[/nation.parquet]]}, projection=[n_nationkey, n_name, n_regionkey, n_comment], file_type=parquet, sort_order_for_reorder=[n_name@1 ASC NULLS LAST]", "datafusion.partitioning": "UnknownPartitioning(1)", "datafusion.preview": "", "env": "production", diff --git a/tests/snapshots/07_scrabble_all_options_01_ProjectionExec.snap b/tests/snapshots/07_scrabble_all_options_01_ProjectionExec.snap index 8ba4f7a..048aa37 100644 --- a/tests/snapshots/07_scrabble_all_options_01_ProjectionExec.snap +++ b/tests/snapshots/07_scrabble_all_options_01_ProjectionExec.snap @@ -2,16 +2,16 @@ source: tests/integration_tests.rs expression: preview --- -+--------+--------+ -| letter | points | -|========+========| -| A | 1 | -|--------+--------| -| B | 3 | -|--------+--------| -| C | 3 | -|--------+--------| -| D | 2 | -|--------+--------| -| E | 1 | -+--------+--------+ ++--------+--------+----------------------------+ +| letter | points | CAST(s.letter AS Utf8View) | +|========+========+============================| +| A | 1 | A | +|--------+--------+----------------------------| +| B | 3 | B | +|--------+--------+----------------------------| +| C | 3 | C | +|--------+--------+----------------------------| +| D | 2 | D | +|--------+--------+----------------------------| +| E | 1 | E | ++--------+--------+----------------------------+ diff --git a/tests/snapshots/07_scrabble_all_options_03_RepartitionExec.snap b/tests/snapshots/07_scrabble_all_options_03_RepartitionExec.snap index 50aaeda..f8b741e 100644 --- a/tests/snapshots/07_scrabble_all_options_03_RepartitionExec.snap +++ b/tests/snapshots/07_scrabble_all_options_03_RepartitionExec.snap @@ -2,16 +2,16 @@ source: tests/integration_tests.rs expression: preview --- -+-------------+---------------+-------------+ -| n_nationkey | n_name | n_regionkey | -|=============+===============+=============| -| 1 | ARGENTINA | 1 | -|-------------+---------------+-------------| -| 2 | BRAZIL | 1 | -|-------------+---------------+-------------| -| 3 | CANADA | 1 | -|-------------+---------------+-------------| -| 17 | PERU | 1 | -|-------------+---------------+-------------| -| 24 | UNITED STATES | 1 | -+-------------+---------------+-------------+ ++-------------+-----------+-------------+ +| n_nationkey | n_name | n_regionkey | +|=============+===========+=============| +| 1 | ARGENTINA | 1 | +|-------------+-----------+-------------| +| 2 | BRAZIL | 1 | +|-------------+-----------+-------------| +| 3 | CANADA | 1 | +|-------------+-----------+-------------| +| 6 | FRANCE | 3 | +|-------------+-----------+-------------| +| 7 | GERMANY | 3 | ++-------------+-----------+-------------+ diff --git a/tests/snapshots/07_scrabble_all_options_05_RepartitionExec.snap b/tests/snapshots/07_scrabble_all_options_05_RepartitionExec.snap index f0cae99..4f3323c 100644 --- a/tests/snapshots/07_scrabble_all_options_05_RepartitionExec.snap +++ b/tests/snapshots/07_scrabble_all_options_05_RepartitionExec.snap @@ -9,9 +9,9 @@ expression: preview |-------------+-------------| | 3 | EUROPE | |-------------+-------------| -| 0 | AFRICA | -|-------------+-------------| | 2 | ASIA | |-------------+-------------| | 4 | MIDDLE EAST | +|-------------+-------------| +| 0 | AFRICA | +-------------+-------------+ diff --git a/tests/snapshots/07_scrabble_all_options_10_HashJoinExec.snap b/tests/snapshots/07_scrabble_all_options_10_HashJoinExec.snap index ce6b631..cef5ff8 100644 --- a/tests/snapshots/07_scrabble_all_options_10_HashJoinExec.snap +++ b/tests/snapshots/07_scrabble_all_options_10_HashJoinExec.snap @@ -2,16 +2,16 @@ source: tests/integration_tests.rs expression: preview --- -+--------+-------------+-------------+-------------+ -| points | n_nationkey | nation_name | region_name | -|========+=============+=============+=============| -| 1 | 1 | ARGENTINA | AMERICA | -|--------+-------------+-------------+-------------| -| 1 | 1 | ARGENTINA | AMERICA | -|--------+-------------+-------------+-------------| -| 2 | 1 | ARGENTINA | AMERICA | -|--------+-------------+-------------+-------------| -| 1 | 1 | ARGENTINA | AMERICA | -|--------+-------------+-------------+-------------| -| 1 | 1 | ARGENTINA | AMERICA | -+--------+-------------+-------------+-------------+ ++-------------+-------------+-------------+--------+ +| n_nationkey | nation_name | region_name | points | +|=============+=============+=============+========| +| 1 | ARGENTINA | AMERICA | 1 | +|-------------+-------------+-------------+--------| +| 1 | ARGENTINA | AMERICA | 1 | +|-------------+-------------+-------------+--------| +| 1 | ARGENTINA | AMERICA | 2 | +|-------------+-------------+-------------+--------| +| 1 | ARGENTINA | AMERICA | 1 | +|-------------+-------------+-------------+--------| +| 1 | ARGENTINA | AMERICA | 1 | ++-------------+-------------+-------------+--------+ diff --git a/tests/snapshots/07_scrabble_all_options_12_AggregateExec.snap b/tests/snapshots/07_scrabble_all_options_11_AggregateExec.snap similarity index 100% rename from tests/snapshots/07_scrabble_all_options_12_AggregateExec.snap rename to tests/snapshots/07_scrabble_all_options_11_AggregateExec.snap diff --git a/tests/snapshots/07_scrabble_all_options_11_ProjectionExec.snap b/tests/snapshots/07_scrabble_all_options_11_ProjectionExec.snap deleted file mode 100644 index cef5ff8..0000000 --- a/tests/snapshots/07_scrabble_all_options_11_ProjectionExec.snap +++ /dev/null @@ -1,17 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+-------------+-------------+-------------+--------+ -| n_nationkey | nation_name | region_name | points | -|=============+=============+=============+========| -| 1 | ARGENTINA | AMERICA | 1 | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | 1 | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | 2 | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | 1 | -|-------------+-------------+-------------+--------| -| 1 | ARGENTINA | AMERICA | 1 | -+-------------+-------------+-------------+--------+ diff --git a/tests/snapshots/08_recursive_trace.snap b/tests/snapshots/08_recursive_trace.snap index f66fdc8..ccb5712 100644 --- a/tests/snapshots/08_recursive_trace.snap +++ b/tests/snapshots/08_recursive_trace.snap @@ -171,6 +171,33 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "08_recursive" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -843,6 +870,33 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "08_recursive" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 2, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1721,6 +1775,35 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "08_recursive" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1837,6 +1920,35 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "08_recursive" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1866,6 +1978,35 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "08_recursive" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -2072,7 +2213,7 @@ expression: full_trace_lines "span": { "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", "name": "create_physical_plan", - "physical_plan": "RecursiveQueryExec: name=numbers, is_distinct=false\n ProjectionExec: expr=[1 as n]\n PlaceholderRowExec\n CoalescePartitionsExec\n ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]\n FilterExec: n@0 < 3\n RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1\n WorkTableExec: name=numbers\n" + "physical_plan": "RecursiveQueryExec: name=numbers, is_distinct=false\n ProjectionExec: expr=[1 as n]\n PlaceholderRowExec\n CoalescePartitionsExec\n ProjectionExec: expr=[n@0 + 1 as n]\n FilterExec: n@0 < 3\n RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1\n WorkTableExec: name=numbers\n" }, "spans": [ { @@ -2166,11 +2307,11 @@ expression: full_trace_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "FilterExec: n@0 < 3", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", "datafusion.partitioning": "RoundRobinBatch(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "FilterExec", + "otel.name": "RepartitionExec", "region": "us-west" }, "spans": [ @@ -2199,6 +2340,24 @@ expression: full_trace_lines "name": "InstrumentedExec", "otel.name": "CoalescePartitionsExec", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "RoundRobinBatch(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "RoundRobinBatch(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "FilterExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2211,11 +2370,11 @@ expression: full_trace_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", + "datafusion.node": "FilterExec: n@0 < 3", "datafusion.partitioning": "RoundRobinBatch(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "ProjectionExec", + "otel.name": "FilterExec", "region": "us-west" }, "spans": [ @@ -2244,6 +2403,15 @@ expression: full_trace_lines "name": "InstrumentedExec", "otel.name": "CoalescePartitionsExec", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "RoundRobinBatch(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2256,11 +2424,11 @@ expression: full_trace_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", + "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as n]", "datafusion.partitioning": "RoundRobinBatch(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "RepartitionExec", + "otel.name": "ProjectionExec", "region": "us-west" }, "spans": [ @@ -2301,11 +2469,11 @@ expression: full_trace_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "CoalescePartitionsExec", + "datafusion.node": "PlaceholderRowExec", "datafusion.partitioning": "UnknownPartitioning(1)", "env": "production", "name": "InstrumentedExec", - "otel.name": "CoalescePartitionsExec", + "otel.name": "PlaceholderRowExec", "region": "us-west" }, "spans": [ @@ -2325,6 +2493,15 @@ expression: full_trace_lines "name": "InstrumentedExec", "otel.name": "RecursiveQueryExec", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(1)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2337,11 +2514,11 @@ expression: full_trace_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "PlaceholderRowExec", + "datafusion.node": "CoalescePartitionsExec", "datafusion.partitioning": "UnknownPartitioning(1)", "env": "production", "name": "InstrumentedExec", - "otel.name": "PlaceholderRowExec", + "otel.name": "CoalescePartitionsExec", "region": "us-west" }, "spans": [ diff --git a/tests/snapshots/09_recursive_all_options_01_WorkTableExec.snap b/tests/snapshots/09_recursive_all_options_01_WorkTableExec.snap index c3cfc8b..6b6948e 100644 --- a/tests/snapshots/09_recursive_all_options_01_WorkTableExec.snap +++ b/tests/snapshots/09_recursive_all_options_01_WorkTableExec.snap @@ -2,8 +2,8 @@ source: tests/integration_tests.rs expression: preview --- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 2 | -+----------------------+ ++---+ +| n | +|===| +| 2 | ++---+ diff --git a/tests/snapshots/09_recursive_all_options_02_WorkTableExec.snap b/tests/snapshots/09_recursive_all_options_02_WorkTableExec.snap index 7f8e94d..5bf8538 100644 --- a/tests/snapshots/09_recursive_all_options_02_WorkTableExec.snap +++ b/tests/snapshots/09_recursive_all_options_02_WorkTableExec.snap @@ -2,8 +2,8 @@ source: tests/integration_tests.rs expression: preview --- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 3 | -+----------------------+ ++---+ +| n | +|===| +| 3 | ++---+ diff --git a/tests/snapshots/09_recursive_all_options_03_FilterExec.snap b/tests/snapshots/09_recursive_all_options_03_RepartitionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_03_FilterExec.snap rename to tests/snapshots/09_recursive_all_options_03_RepartitionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_04_FilterExec.snap b/tests/snapshots/09_recursive_all_options_04_RepartitionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_04_FilterExec.snap rename to tests/snapshots/09_recursive_all_options_04_RepartitionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_05_ProjectionExec.snap b/tests/snapshots/09_recursive_all_options_05_ProjectionExec.snap deleted file mode 100644 index c3cfc8b..0000000 --- a/tests/snapshots/09_recursive_all_options_05_ProjectionExec.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 2 | -+----------------------+ diff --git a/tests/snapshots/09_recursive_all_options_09_RepartitionExec.snap b/tests/snapshots/09_recursive_all_options_05_RepartitionExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_09_RepartitionExec.snap rename to tests/snapshots/09_recursive_all_options_05_RepartitionExec.snap diff --git a/tests/snapshots/09_recursive_all_options_07_RepartitionExec.snap b/tests/snapshots/09_recursive_all_options_06_FilterExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_07_RepartitionExec.snap rename to tests/snapshots/09_recursive_all_options_06_FilterExec.snap diff --git a/tests/snapshots/09_recursive_all_options_06_ProjectionExec.snap b/tests/snapshots/09_recursive_all_options_06_ProjectionExec.snap deleted file mode 100644 index 7f8e94d..0000000 --- a/tests/snapshots/09_recursive_all_options_06_ProjectionExec.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 3 | -+----------------------+ diff --git a/tests/snapshots/09_recursive_all_options_08_RepartitionExec.snap b/tests/snapshots/09_recursive_all_options_07_FilterExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_08_RepartitionExec.snap rename to tests/snapshots/09_recursive_all_options_07_FilterExec.snap diff --git a/tests/snapshots/09_recursive_all_options_08_ProjectionExec.snap b/tests/snapshots/09_recursive_all_options_08_ProjectionExec.snap new file mode 100644 index 0000000..6b6948e --- /dev/null +++ b/tests/snapshots/09_recursive_all_options_08_ProjectionExec.snap @@ -0,0 +1,9 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---+ +| n | +|===| +| 2 | ++---+ diff --git a/tests/snapshots/09_recursive_all_options_09_ProjectionExec.snap b/tests/snapshots/09_recursive_all_options_09_ProjectionExec.snap new file mode 100644 index 0000000..5bf8538 --- /dev/null +++ b/tests/snapshots/09_recursive_all_options_09_ProjectionExec.snap @@ -0,0 +1,9 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---+ +| n | +|===| +| 3 | ++---+ diff --git a/tests/snapshots/09_recursive_all_options_10_CoalescePartitionsExec.snap b/tests/snapshots/09_recursive_all_options_10_CoalescePartitionsExec.snap deleted file mode 100644 index c3cfc8b..0000000 --- a/tests/snapshots/09_recursive_all_options_10_CoalescePartitionsExec.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: tests/integration_tests.rs -expression: preview ---- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 2 | -+----------------------+ diff --git a/tests/snapshots/09_recursive_all_options_12_PlaceholderRowExec.snap b/tests/snapshots/09_recursive_all_options_10_PlaceholderRowExec.snap similarity index 100% rename from tests/snapshots/09_recursive_all_options_12_PlaceholderRowExec.snap rename to tests/snapshots/09_recursive_all_options_10_PlaceholderRowExec.snap diff --git a/tests/snapshots/09_recursive_all_options_11_CoalescePartitionsExec.snap b/tests/snapshots/09_recursive_all_options_11_CoalescePartitionsExec.snap index 7f8e94d..6b6948e 100644 --- a/tests/snapshots/09_recursive_all_options_11_CoalescePartitionsExec.snap +++ b/tests/snapshots/09_recursive_all_options_11_CoalescePartitionsExec.snap @@ -2,8 +2,8 @@ source: tests/integration_tests.rs expression: preview --- -+----------------------+ -| numbers.n + Int64(1) | -|======================| -| 3 | -+----------------------+ ++---+ +| n | +|===| +| 2 | ++---+ diff --git a/tests/snapshots/09_recursive_all_options_12_CoalescePartitionsExec.snap b/tests/snapshots/09_recursive_all_options_12_CoalescePartitionsExec.snap new file mode 100644 index 0000000..5bf8538 --- /dev/null +++ b/tests/snapshots/09_recursive_all_options_12_CoalescePartitionsExec.snap @@ -0,0 +1,9 @@ +--- +source: tests/integration_tests.rs +expression: preview +--- ++---+ +| n | +|===| +| 3 | ++---+ diff --git a/tests/snapshots/09_recursive_all_options_trace.snap b/tests/snapshots/09_recursive_all_options_trace.snap index 7a51026..768ab67 100644 --- a/tests/snapshots/09_recursive_all_options_trace.snap +++ b/tests/snapshots/09_recursive_all_options_trace.snap @@ -171,6 +171,33 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "09_recursive_all_options" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -843,6 +870,33 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "09_recursive_all_options" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 2, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1721,6 +1775,35 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "09_recursive_all_options" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1837,6 +1920,35 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "HashJoinBuffering" + }, + "spans": [ + { + "name": "test", + "test_name": "09_recursive_all_options" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1866,6 +1978,35 @@ expression: full_trace_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "TopKRepartition" + }, + "spans": [ + { + "name": "test", + "test_name": "09_recursive_all_options" + }, + { + "name": "run_traced_query", + "query_name": "recursive" + }, + { + "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -2072,7 +2213,7 @@ expression: full_trace_lines "span": { "logical_plan": "SubqueryAlias: numbers [n:Int64]\n Projection: n AS n [n:Int64]\n RecursiveQuery: is_distinct=false [n:Int64]\n Projection: Int64(1) AS n [n:Int64]\n EmptyRelation: rows=1 []\n Projection: numbers.n + Int64(1) [numbers.n + Int64(1):Int64]\n Filter: numbers.n < Int64(3) [n:Int64]\n TableScan: numbers projection=[n] [n:Int64]", "name": "create_physical_plan", - "physical_plan": "RecursiveQueryExec: name=numbers, is_distinct=false\n ProjectionExec: expr=[1 as n]\n PlaceholderRowExec\n CoalescePartitionsExec\n ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]\n FilterExec: n@0 < 3\n RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1\n WorkTableExec: name=numbers\n" + "physical_plan": "RecursiveQueryExec: name=numbers, is_distinct=false\n ProjectionExec: expr=[1 as n]\n PlaceholderRowExec\n CoalescePartitionsExec\n ProjectionExec: expr=[n@0 + 1 as n]\n FilterExec: n@0 < 3\n RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1\n WorkTableExec: name=numbers\n" }, "spans": [ { @@ -2169,15 +2310,22 @@ expression: full_trace_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_batches": "0", - "datafusion.metrics.output_bytes": "0.0 B", - "datafusion.metrics.output_rows": "0", + "datafusion.metrics.fetch_time": "0.00ms", + "datafusion.metrics.output_batches": "1", + "datafusion.metrics.output_bytes": "64.0 KB", + "datafusion.metrics.output_rows": "1", + "datafusion.metrics.repartition_time": "0.00ms", + "datafusion.metrics.send_time": "0.00ms", + "datafusion.metrics.spill_count": "0", + "datafusion.metrics.spilled_bytes": "0.0 B", + "datafusion.metrics.spilled_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "FilterExec: n@0 < 3", + "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "FilterExec", + "otel.name": "RepartitionExec", "region": "us-west" }, "spans": [ @@ -2206,6 +2354,24 @@ expression: full_trace_lines "name": "InstrumentedExec", "otel.name": "CoalescePartitionsExec", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "RoundRobinBatch(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "RoundRobinBatch(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "FilterExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2220,13 +2386,13 @@ expression: full_trace_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_batches": "1", - "datafusion.metrics.output_bytes": "64.0 KB", - "datafusion.metrics.output_rows": "1", + "datafusion.metrics.output_batches": "0", + "datafusion.metrics.output_bytes": "0.0 B", + "datafusion.metrics.output_rows": "0", + "datafusion.metrics.selectivity": "0% (0/1)", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.node": "FilterExec: n@0 < 3", "datafusion.partitioning": "RoundRobinBatch(8)", - "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", "otel.name": "FilterExec", @@ -2258,6 +2424,15 @@ expression: full_trace_lines "name": "InstrumentedExec", "otel.name": "CoalescePartitionsExec", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "RoundRobinBatch(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2272,15 +2447,17 @@ expression: full_trace_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_batches": "0", - "datafusion.metrics.output_bytes": "0.0 B", - "datafusion.metrics.output_rows": "0", + "datafusion.metrics.output_batches": "1", + "datafusion.metrics.output_bytes": "64.0 KB", + "datafusion.metrics.output_rows": "1", + "datafusion.metrics.selectivity": "100% (1/1)", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", + "datafusion.node": "FilterExec: n@0 < 3", "datafusion.partitioning": "RoundRobinBatch(8)", + "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "ProjectionExec", + "otel.name": "FilterExec", "region": "us-west" }, "spans": [ @@ -2309,6 +2486,15 @@ expression: full_trace_lines "name": "InstrumentedExec", "otel.name": "CoalescePartitionsExec", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "RoundRobinBatch(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2323,13 +2509,12 @@ expression: full_trace_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_batches": "1", - "datafusion.metrics.output_bytes": "8.0 B", - "datafusion.metrics.output_rows": "1", + "datafusion.metrics.output_batches": "0", + "datafusion.metrics.output_bytes": "0.0 B", + "datafusion.metrics.output_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as numbers.n + Int64(1)]", + "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as n]", "datafusion.partitioning": "RoundRobinBatch(8)", - "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", "otel.name": "ProjectionExec", @@ -2375,22 +2560,16 @@ expression: full_trace_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.fetch_time": "0.00ms", "datafusion.metrics.output_batches": "1", - "datafusion.metrics.output_bytes": "64.0 KB", + "datafusion.metrics.output_bytes": "8.0 B", "datafusion.metrics.output_rows": "1", - "datafusion.metrics.repartition_time": "0.00ms", - "datafusion.metrics.send_time": "0.00ms", - "datafusion.metrics.spill_count": "0", - "datafusion.metrics.spilled_bytes": "0.0 B", - "datafusion.metrics.spilled_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", + "datafusion.node": "ProjectionExec: expr=[n@0 + 1 as n]", "datafusion.partitioning": "RoundRobinBatch(8)", "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "RepartitionExec", + "otel.name": "ProjectionExec", "region": "us-west" }, "spans": [ @@ -2431,17 +2610,12 @@ expression: full_trace_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.metrics.elapsed_compute": "0.00ms", - "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_batches": "0", - "datafusion.metrics.output_bytes": "0.0 B", - "datafusion.metrics.output_rows": "0", - "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.node": "CoalescePartitionsExec", + "datafusion.node": "PlaceholderRowExec", "datafusion.partitioning": "UnknownPartitioning(1)", + "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "CoalescePartitionsExec", + "otel.name": "PlaceholderRowExec", "region": "us-west" }, "spans": [ @@ -2461,6 +2635,15 @@ expression: full_trace_lines "name": "InstrumentedExec", "otel.name": "RecursiveQueryExec", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(1)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "ProjectionExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2475,13 +2658,12 @@ expression: full_trace_lines "datafusion.emission_type": "Incremental", "datafusion.metrics.elapsed_compute": "0.00ms", "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", - "datafusion.metrics.output_batches": "1", - "datafusion.metrics.output_bytes": "8.0 B", - "datafusion.metrics.output_rows": "1", + "datafusion.metrics.output_batches": "0", + "datafusion.metrics.output_bytes": "0.0 B", + "datafusion.metrics.output_rows": "0", "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", "datafusion.node": "CoalescePartitionsExec", "datafusion.partitioning": "UnknownPartitioning(1)", - "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", "otel.name": "CoalescePartitionsExec", @@ -2516,12 +2698,18 @@ expression: full_trace_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "PlaceholderRowExec", + "datafusion.metrics.elapsed_compute": "0.00ms", + "datafusion.metrics.end_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.metrics.output_batches": "1", + "datafusion.metrics.output_bytes": "8.0 B", + "datafusion.metrics.output_rows": "1", + "datafusion.metrics.start_timestamp": "1970-01-01 00:00:00 UTC", + "datafusion.node": "CoalescePartitionsExec", "datafusion.partitioning": "UnknownPartitioning(1)", "datafusion.preview": "", "env": "production", "name": "InstrumentedExec", - "otel.name": "PlaceholderRowExec", + "otel.name": "CoalescePartitionsExec", "region": "us-west" }, "spans": [ diff --git a/tests/snapshots/10_topk_lineitem_trace.snap b/tests/snapshots/10_topk_lineitem_trace.snap index 699d1d8..c7da7f7 100644 --- a/tests/snapshots/10_topk_lineitem_trace.snap +++ b/tests/snapshots/10_topk_lineitem_trace.snap @@ -1,6 +1,6 @@ --- source: tests/integration_tests.rs -expression: json_lines +expression: full_trace_lines --- [ { @@ -172,6 +172,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "10_topk_lineitem" + }, + { + "name": "run_traced_query", + "query_name": "topk_lineitem" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 1, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -844,6 +871,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "10_topk_lineitem" + }, + { + "name": "run_traced_query", + "query_name": "topk_lineitem" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 2, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -1516,6 +1570,33 @@ expression: json_lines "time.busy": "0.00ms", "time.idle": "0.00ms" }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "unions_to_filter" + }, + "spans": [ + { + "name": "test", + "test_name": "10_topk_lineitem" + }, + { + "name": "run_traced_query", + "query_name": "topk_lineitem" + }, + { + "datafusion.optimizer.max_passes": 3, + "datafusion.optimizer.pass": 3, + "name": "Phase", + "otel.name": "optimize_logical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, { "level": "INFO", "message": "close", @@ -2399,7 +2480,36 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "ProjectionPushdown" + "otel.name": "WindowTopN" + }, + "spans": [ + { + "name": "test", + "test_name": "10_topk_lineitem" + }, + { + "name": "run_traced_query", + "query_name": "topk_lineitem" + }, + { + "logical_plan": "Sort: lineitem.l_orderkey ASC NULLS LAST, lineitem.l_linenumber ASC NULLS LAST, fetch=3 [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Projection: lineitem.l_orderkey, lineitem.l_linenumber, lineitem.l_shipmode, lineitem.l_quantity [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Filter: lineitem.l_shipmode = Utf8View(\"SHIP\") [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]\n TableScan: lineitem projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View(\"SHIP\")] [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "name": "Rule", + "otel.name": "ProjectionPushdown (modified)" }, "spans": [ { @@ -2515,7 +2625,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "LimitPushdown" + "otel.name": "HashJoinBuffering" }, "spans": [ { @@ -2544,7 +2654,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "ProjectionPushdown" + "otel.name": "LimitPushdown" }, "spans": [ { @@ -2573,7 +2683,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "PushdownSort" + "otel.name": "TopKRepartition" }, "spans": [ { @@ -2602,7 +2712,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "EnsureCooperative" + "otel.name": "ProjectionPushdown" }, "spans": [ { @@ -2631,7 +2741,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "FilterPushdown(Post) (modified)" + "otel.name": "PushdownSort" }, "spans": [ { @@ -2660,7 +2770,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "SanityCheckPlan" + "otel.name": "EnsureCooperative" }, "spans": [ { @@ -2689,7 +2799,7 @@ expression: json_lines "message": "close", "span": { "name": "Rule", - "otel.name": "Instrument" + "otel.name": "FilterPushdown(Post) (modified)" }, "spans": [ { @@ -2717,9 +2827,8 @@ expression: json_lines "level": "INFO", "message": "close", "span": { - "datafusion.effective_rules": "OutputRequirements, FilterPushdown, EnforceDistribution, EnforceSorting, OutputRequirements, FilterPushdown(Post)", - "name": "Phase", - "otel.name": "optimize_physical_plan" + "name": "Rule", + "otel.name": "SanityCheckPlan" }, "spans": [ { @@ -2733,6 +2842,10 @@ expression: json_lines { "logical_plan": "Sort: lineitem.l_orderkey ASC NULLS LAST, lineitem.l_linenumber ASC NULLS LAST, fetch=3 [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Projection: lineitem.l_orderkey, lineitem.l_linenumber, lineitem.l_shipmode, lineitem.l_quantity [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Filter: lineitem.l_shipmode = Utf8View(\"SHIP\") [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]\n TableScan: lineitem projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View(\"SHIP\")] [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]", "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" } ], "target": "integration_utils", @@ -2743,9 +2856,8 @@ expression: json_lines "level": "INFO", "message": "close", "span": { - "logical_plan": "Sort: lineitem.l_orderkey ASC NULLS LAST, lineitem.l_linenumber ASC NULLS LAST, fetch=3 [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Projection: lineitem.l_orderkey, lineitem.l_linenumber, lineitem.l_shipmode, lineitem.l_quantity [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Filter: lineitem.l_shipmode = Utf8View(\"SHIP\") [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]\n TableScan: lineitem projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View(\"SHIP\")] [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]", - "name": "create_physical_plan", - "physical_plan": "SortPreservingMergeExec: [l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], fetch=3\n SortExec: TopK(fetch=3), expr=[l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], preserve_partitioning=[true]\n ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_linenumber@1 as l_linenumber, l_shipmode@3 as l_shipmode, l_quantity@2 as l_quantity]\n FilterExec: l_shipmode@3 = SHIP\n DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..2516294], [/lineitem.parquet:2516294..5032588], [/lineitem.parquet:5032588..7548882], [/lineitem.parquet:7548882..10065176], [/lineitem.parquet:10065176..12581470], [/lineitem.parquet:12581470..15097764], [/lineitem.parquet:15097764..17614058], [/lineitem.parquet:17614058..20130345]]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@14 = SHIP AND DynamicFilter [ empty ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1, required_guarantees=[l_shipmode in (SHIP)]\n" + "name": "Rule", + "otel.name": "Instrument" }, "spans": [ { @@ -2755,9 +2867,17 @@ expression: json_lines { "name": "run_traced_query", "query_name": "topk_lineitem" + }, + { + "logical_plan": "Sort: lineitem.l_orderkey ASC NULLS LAST, lineitem.l_linenumber ASC NULLS LAST, fetch=3 [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Projection: lineitem.l_orderkey, lineitem.l_linenumber, lineitem.l_shipmode, lineitem.l_quantity [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Filter: lineitem.l_shipmode = Utf8View(\"SHIP\") [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]\n TableScan: lineitem projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View(\"SHIP\")] [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]", + "name": "create_physical_plan" + }, + { + "name": "Phase", + "otel.name": "optimize_physical_plan" } ], - "target": "datafusion_tracing::planner", + "target": "integration_utils", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -2765,14 +2885,9 @@ expression: json_lines "level": "INFO", "message": "close", "span": { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Incremental", - "datafusion.node": "DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..2516294], [/lineitem.parquet:2516294..5032588], [/lineitem.parquet:5032588..7548882], [/lineitem.parquet:7548882..10065176], [/lineitem.parquet:10065176..12581470], ...]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@14 = SHIP AND DynamicFilter [ l_orderkey@0 < 35 OR l_orderkey@0 = 35 AND l_linenumber@3 < 4 ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1 AND (l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 < 35 OR l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 <= 35 AND 35 <= l_orderkey_max@6 AND l_linenumber_null_count@8 != row_count@3 AND l_linenumber_min@7 < 4), required_guarantees=[l_shipmode in (SHIP)]", - "datafusion.partitioning": "UnknownPartitioning(8)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "DataSourceExec", - "region": "us-west" + "datafusion.effective_rules": "OutputRequirements, FilterPushdown, EnforceDistribution, EnforceSorting, ProjectionPushdown, OutputRequirements, FilterPushdown(Post)", + "name": "Phase", + "otel.name": "optimize_physical_plan" }, "spans": [ { @@ -2784,25 +2899,33 @@ expression: json_lines "query_name": "topk_lineitem" }, { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Final", - "datafusion.partitioning": "UnknownPartitioning(1)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "SortPreservingMergeExec", - "region": "us-west" + "logical_plan": "Sort: lineitem.l_orderkey ASC NULLS LAST, lineitem.l_linenumber ASC NULLS LAST, fetch=3 [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Projection: lineitem.l_orderkey, lineitem.l_linenumber, lineitem.l_shipmode, lineitem.l_quantity [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Filter: lineitem.l_shipmode = Utf8View(\"SHIP\") [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]\n TableScan: lineitem projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View(\"SHIP\")] [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]", + "name": "create_physical_plan" + } + ], + "target": "integration_utils", + "time.busy": "0.00ms", + "time.idle": "0.00ms" + }, + { + "level": "INFO", + "message": "close", + "span": { + "logical_plan": "Sort: lineitem.l_orderkey ASC NULLS LAST, lineitem.l_linenumber ASC NULLS LAST, fetch=3 [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Projection: lineitem.l_orderkey, lineitem.l_linenumber, lineitem.l_shipmode, lineitem.l_quantity [l_orderkey:Int64, l_linenumber:Int32, l_shipmode:Utf8View, l_quantity:Decimal128(15, 2)]\n Filter: lineitem.l_shipmode = Utf8View(\"SHIP\") [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]\n TableScan: lineitem projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View(\"SHIP\")] [l_orderkey:Int64, l_linenumber:Int32, l_quantity:Decimal128(15, 2), l_shipmode:Utf8View]", + "name": "create_physical_plan", + "physical_plan": "SortPreservingMergeExec: [l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], fetch=3\n SortExec: TopK(fetch=3), expr=[l_orderkey@0 ASC NULLS LAST, l_linenumber@1 ASC NULLS LAST], preserve_partitioning=[true]\n FilterExec: l_shipmode@3 = SHIP, projection=[l_orderkey@0, l_linenumber@1, l_shipmode@3, l_quantity@2]\n DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..2516294], [/lineitem.parquet:2516294..5032588], [/lineitem.parquet:5032588..7548882], [/lineitem.parquet:7548882..10065176], [/lineitem.parquet:10065176..12581470], [/lineitem.parquet:12581470..15097764], [/lineitem.parquet:15097764..17614058], [/lineitem.parquet:17614058..20130345]]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@14 = SHIP AND DynamicFilter [ empty ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1, required_guarantees=[l_shipmode in (SHIP)]\n" + }, + "spans": [ + { + "name": "test", + "test_name": "10_topk_lineitem" }, { - "datafusion.boundedness": "Bounded", - "datafusion.emission_type": "Final", - "datafusion.partitioning": "UnknownPartitioning(8)", - "env": "production", - "name": "InstrumentedExec", - "otel.name": "SortExec(TopK)", - "region": "us-west" + "name": "run_traced_query", + "query_name": "topk_lineitem" } ], - "target": "integration_utils", + "target": "datafusion_tracing::planner", "time.busy": "0.00ms", "time.idle": "0.00ms" }, @@ -2812,11 +2935,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "FilterExec: l_shipmode@3 = SHIP", + "datafusion.node": "DataSourceExec: file_groups={8 groups: [[/lineitem.parquet:0..2516294], [/lineitem.parquet:2516294..5032588], [/lineitem.parquet:5032588..7548882], [/lineitem.parquet:7548882..10065176], [/lineitem.parquet:10065176..12581470], ...]}, projection=[l_orderkey, l_linenumber, l_quantity, l_shipmode], file_type=parquet, predicate=l_shipmode@14 = SHIP AND DynamicFilter [ l_orderkey@0 < 35 OR l_orderkey@0 = 35 AND l_linenumber@3 < 4 ], pruning_predicate=l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1 AND (l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 < 35 OR l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@4 <= 35 AND 35 <= l_orderkey_max@6 AND l_linenumber_null_count@8 != row_count@3 AND l_linenumber_min@7 < 4), required_guarantees=[l_shipmode in (SHIP)]", "datafusion.partitioning": "UnknownPartitioning(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "FilterExec", + "otel.name": "DataSourceExec", "region": "us-west" }, "spans": [ @@ -2845,6 +2968,15 @@ expression: json_lines "name": "InstrumentedExec", "otel.name": "SortExec(TopK)", "region": "us-west" + }, + { + "datafusion.boundedness": "Bounded", + "datafusion.emission_type": "Incremental", + "datafusion.partitioning": "UnknownPartitioning(8)", + "env": "production", + "name": "InstrumentedExec", + "otel.name": "FilterExec", + "region": "us-west" } ], "target": "integration_utils", @@ -2857,11 +2989,11 @@ expression: json_lines "span": { "datafusion.boundedness": "Bounded", "datafusion.emission_type": "Incremental", - "datafusion.node": "ProjectionExec: expr=[l_orderkey@0 as l_orderkey, l_linenumber@1 as l_linenumber, l_shipmode@3 as l_shipmode, l_quantity@2 as l_quantity]", + "datafusion.node": "FilterExec: l_shipmode@3 = SHIP, projection=[l_orderkey@0, l_linenumber@1, l_shipmode@3, l_quantity@2]", "datafusion.partitioning": "UnknownPartitioning(8)", "env": "production", "name": "InstrumentedExec", - "otel.name": "ProjectionExec", + "otel.name": "FilterExec", "region": "us-west" }, "spans": [