fmt

Author: adriangb

datafusion/datasource-parquet/src/opener.rs

@@ -1366,14 +1366,29 @@ impl RowGroupsPrunedParquetOpen {
         // projection and **every** predicate conjunct's columns, regardless
         // of whether each conjunct is currently row-level or post-scan.
         //
-        // Why all conjuncts (not just post-scan): a mid-stream
-        // `maybe_swap_strategy` call can demote a row-level filter to
-        // post-scan when its measured throughput drops below
-        // `min_bytes_per_sec`. The decoder's projection mask is fixed for
-        // the file (we don't grow it on swap), so any column that *might*
-        // be referenced by a post-scan filter at some point during the
-        // file must already be in the mask — otherwise the post-scan
-        // rebase fails with a schema-lookup error.
+        // The arrow-rs decoder *does* support swapping projection masks
+        // between row groups (`StrategySwap::with_projection`), so this
+        // could in principle be dynamic. We don't do that because:
+        //
+        // 1. Correctness — a mid-stream `maybe_swap_strategy` call can
+        //    demote a row-level filter to post-scan when its measured
+        //    throughput drops below `min_bytes_per_sec`. The post-scan
+        //    rebase resolves the filter against the decoder's output
+        //    schema, so any column that *might* be referenced by a
+        //    post-scan filter at some point during the file must already
+        //    be in the mask — otherwise the rebase fails with a
+        //    schema-lookup error.
+        //
+        // 2. Empirically operational, not defensive — the byte-ratio
+        //    placement heuristic (`filter_collecting_byte_ratio_threshold`)
+        //    routes filters with out-of-projection heavy columns to
+        //    post-scan from file-open, so those columns are needed at
+        //    every row group anyway. A `DF_MASK_PROBE` instrumentation
+        //    run on TPC-DS / TPC-H / ClickBench smoke measured
+        //    < 0.001 % per-RG mask waste under default config — i.e. the
+        //    static union mask matches the per-RG optimum at essentially
+        //    every swap point. A dynamic-shrink scheme would gain nothing
+        //    in these workloads.
         //
         // Filter-only columns are stripped when the projector runs after
         // post-scan filters, so the user-visible output schema is

datafusion/datasource-parquet/src/page_filter.rs

@@ -484,8 +484,7 @@ impl PagePruningAccessPlanFilter {
                     continue;
                 };
                 let total_pages = page_match_flags.len();
-                let matched_pages =
-                    page_match_flags.iter().filter(|m| **m).count();
+                let matched_pages = page_match_flags.iter().filter(|m| **m).count();
                 total_pages_select += matched_pages;
                 total_pages_skip += total_pages - matched_pages;