Rewrite ParquetOpener to use push-based ParquetPushDecoder

Replace the async pull-based ParquetRecordBatchStreamBuilder with
arrow-rs's SansIO ParquetPushDecoder for reading Parquet files.
The caller now controls IO explicitly via DecodeResult::NeedsData,
pushing byte ranges to the decoder and receiving decoded batches.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: Dandandan

.asf.yaml

@@ -41,6 +41,7 @@ github:
     - sql
   enabled_merge_buttons:
     squash: true
+    squash_commit_message: PR_TITLE_AND_DESC
     merge: false
     rebase: false
   features:
@@ -50,11 +51,27 @@ github:
     main:
       required_pull_request_reviews:
         required_approving_review_count: 1
+    # needs to be updated as part of the release process
+    # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names
+    # https://github.com/apache/infrastructure-asfyaml?tab=readme-ov-file#branch-protection
+    # these branches protection blocks autogenerated during release process which is described in
+    # https://github.com/apache/datafusion/tree/main/dev/release#2-add-a-protection-to-release-candidate-branch
+    branch-50:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-51:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+    branch-52:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
   pull_requests:
     # enable updating head branches of pull requests
     allow_update_branch: true
+    allow_auto_merge: true
 
 # publishes the content of the `asf-site` branch to
 # https://datafusion.apache.org/
 publish:
   whoami: asf-site
+

.devcontainer/Dockerfile

@@ -4,10 +4,12 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
     && apt-get purge -y imagemagick imagemagick-6-common
 
-# Add protoc
-# https://datafusion.apache.org/contributor-guide/getting_started.html#protoc-installation
-RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protoc-25.1-linux-x86_64.zip \
-    && unzip protoc-25.1-linux-x86_64.zip -d $HOME/.local \
-    && rm protoc-25.1-linux-x86_64.zip
+# setup the containers WORKDIR so npm install works
+# https://stackoverflow.com/questions/57534295/npm-err-tracker-idealtree-already-exists-while-creating-the-docker-image-for
+WORKDIR /root
 
-ENV PATH="$PATH:$HOME/.local/bin"
+# Add protoc, npm, prettier
+# https://datafusion.apache.org/contributor-guide/development_environment.html#protoc-installation
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends protobuf-compiler libprotobuf-dev npm nodejs\
+    && rm -rf /var/lib/apt/lists/*

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,5 +1,6 @@
 name: Bug report
 description: Create a report to help us improve
+type: Bug
 labels: bug
 body:
   - type: textarea

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,5 +1,6 @@
 name: Feature request
 description: Suggest an idea for this project
+type: Feature
 labels: enhancement
 body:
   - type: textarea

.github/actions/setup-builder/action.yaml

@@ -46,3 +46,17 @@ runs:
       # https://github.com/actions/checkout/issues/766
       shell: bash
       run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+    - name: Remove unnecessary preinstalled software
+      shell: bash
+      run: |
+        echo "Disk space before cleanup:"
+        df -h 
+        apt-get clean
+        # remove tool cache: about 8.5GB (github has host /opt/hostedtoolcache mounted as /__t)
+        rm -rf /__t/* || true
+        # remove Haskell runtime: about 6.3GB (host /usr/local/.ghcup)
+        rm -rf /host/usr/local/.ghcup || true
+        # remove Android library: about 7.8GB (host /usr/local/lib/android)
+        rm -rf /host/usr/local/lib/android || true
+        echo "Disk space after cleanup:"
+        df -h

.github/actions/setup-macos-aarch64-builder/action.yaml

@@ -44,6 +44,8 @@ runs:
         rustup default stable
         rustup component add rustfmt
     - name: Setup rust cache
-      uses: Swatinem/rust-cache@v2
+      uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1  # v2.8.1
+      with:
+           save-if: ${{ github.ref_name == 'main' }}
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime        

.github/actions/setup-macos-builder/action.yaml

@@ -20,10 +20,6 @@ description: 'Setup Rust Runtime Environment'
 runs:
   using: "composite"
   steps:
-    # https://github.com/apache/datafusion/issues/15535
-    # disabled because neither version nor git hash works with apache github policy
-    #- name: Run sccache-cache
-    #  uses: mozilla-actions/sccache-action@65101d47ea8028ed0c98a1cdea8dd9182e9b5133 # v0.0.8
     - name: Configure runtime env
       shell: bash
       # do not produce debug symbols to keep memory usage down
@@ -32,11 +28,6 @@ runs:
       # 
       # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
       # See for more details: https://github.com/rust-lang/rust/issues/119560
-      #
-      # readd the following to the run below once sccache-cache is re-enabled
-      # echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
-      # echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
       run: |
         echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
         echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
-     

.github/actions/setup-rust-runtime/action.yaml

@@ -20,9 +20,10 @@ updates:
   - package-ecosystem: cargo
     directory: "/"
     schedule:
-      interval: daily
+      interval: weekly
     target-branch: main
     labels: [auto-dependencies]
+    open-pull-requests-limit: 15
     ignore:
       # major version bumps of arrow* and parquet are handled manually
       - dependency-name: "arrow*"
@@ -44,9 +45,31 @@ updates:
         patterns:
           - "prost*"
           - "pbjson*"
+
+        # Catch-all: group only minor/patch into a single PR,
+        # excluding deps we want always separate (and excluding arrow/parquet which have their own group)
+      all-other-cargo-deps:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        exclude-patterns:
+          - "arrow*"
+          - "parquet"
+          - "object_store"
+          - "sqlparser"
+          - "prost*"
+          - "pbjson*"
+        update-types:
+          - "minor"
+          - "patch"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     open-pull-requests-limit: 10
     labels: [auto-dependencies]
+  - package-ecosystem: "pip"
+    directory: "/docs"
+    schedule:
+      interval: "weekly"
+    labels: [auto-dependencies]

.github/dependabot.yml

@@ -23,25 +23,29 @@ concurrency:
 
 on:
   push:
+    branches:
+      - main
     paths:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
-    branches:
-      - main
 
   pull_request:
     paths:
       - "**/Cargo.toml"
       - "**/Cargo.lock"
+  
+  merge_group:
 
 jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Install cargo-audit
-        run: cargo install cargo-audit
+        uses: taiki-e/install-action@d6e286fa45544157a02d45a43742857ebbc25d12  # v2.68.16
+        with:
+          tool: cargo-audit
       - name: Run audit check
-        # Ignored until https://github.com/apache/datafusion/issues/15571
-        # ignored py03 warning until arrow 55 upgrade
-        run: cargo audit --ignore RUSTSEC-2024-0370 --ignore RUSTSEC-2025-0020
+        # Note: you can ignore specific RUSTSEC issues using the `--ignore` flag ,for example:
+        # run: cargo audit --ignore RUSTSEC-2026-0001
+        run: cargo audit