feat: add HF Storage Bucket support via OpenDAL

.github/issue-labeler.yml

@@ -47,7 +47,7 @@ A-panic:
 A-plugin:
   - '/plugin/i'
 A-sql:
-  - '/\bsql\b|sqlcontext/i'
+  - '/\bsql\b|sql_expr|sqlcontext/i'
 A-selectors:
   - '/selector/i'
 A-streaming:

.github/workflows/build-hf-sink-wheels.yml

@@ -0,0 +1,173 @@
+name: Build HF Sink Wheels
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        description: Build with minimal optimization (faster, for testing)
+        type: boolean
+        default: false
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: '3.10'
+  CARGO_INCREMENTAL: 0
+  CARGO_NET_RETRY: 10
+  RUSTUP_MAX_RETRIES: 10
+  # CPU features for x86_64 (modern CPUs with AVX2)
+  NONCOMPAT_TUNE_CPU: 'skylake'
+  NONCOMPAT_FEATURES: '+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq,+movbe'
+  NONCOMPAT_CC_FEATURES: '-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mpclmul -mmovbe'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  # Build the base polars Python package (pure Python wrapper)
+  base-package:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Build base package
+        run: uv build py-polars --out-dir dist
+
+      - name: Upload base package
+        uses: actions/upload-artifact@v4
+        with:
+          name: base-package
+          path: dist
+
+  # Build Linux x64 wheel
+  build-wheel-linux-x64:
+    needs: [base-package]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download base package
+        uses: actions/download-artifact@v4
+        with:
+          name: base-package
+          path: dist
+
+      # Avoid potential out-of-memory errors during compilation
+      - name: Set swap space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install base polars package
+        run: |
+          pip install --force-reinstall --no-dependencies dist/polars-*.whl
+          rm -rf dist/*
+
+      - name: Set RUSTFLAGS for x86-64
+        run: |
+          echo "RUSTFLAGS=-C target-feature=$NONCOMPAT_FEATURES -Z tune-cpu=$NONCOMPAT_TUNE_CPU" >> $GITHUB_ENV
+          echo "CFLAGS=$NONCOMPAT_CC_FEATURES -mtune=$NONCOMPAT_TUNE_CPU" >> $GITHUB_ENV
+
+      - name: Build wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          target: x86_64-unknown-linux-gnu
+          args: >
+            --profile ${{ inputs.dry-run == true && 'mindebug-dev' || 'dist-release' }}
+            --manifest-path py-polars/runtime/polars-runtime-32/Cargo.toml
+            --features hf
+            --out dist
+          manylinux: auto
+          maturin-version: 1.8.3
+
+      - name: Test wheel
+        run: |
+          pip install --force-reinstall --verbose dist/*.whl
+          python -c 'import polars; polars.show_versions()'
+          # Quick sanity check for HF sink feature
+          python -c 'import polars as pl; df = pl.DataFrame({"a": [1,2,3]}); print("Polars loaded OK")'
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-linux-x64
+          path: dist/*.whl
+
+  # Build Linux ARM64 wheel
+  build-wheel-linux-arm64:
+    needs: [base-package]
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download base package
+        uses: actions/download-artifact@v4
+        with:
+          name: base-package
+          path: dist
+
+      # Avoid potential out-of-memory errors during compilation
+      - name: Set swap space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install base polars package
+        run: |
+          pip install --force-reinstall --no-dependencies dist/polars-*.whl
+          rm -rf dist/*
+
+      - name: Set jemalloc page size for aarch64
+        run: |
+          echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> $GITHUB_ENV
+
+      - name: Build wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          target: aarch64-unknown-linux-gnu
+          args: >
+            --profile ${{ inputs.dry-run == true && 'mindebug-dev' || 'dist-release' }}
+            --manifest-path py-polars/runtime/polars-runtime-32/Cargo.toml
+            --features hf
+            --out dist
+          manylinux: auto
+          maturin-version: 1.8.3
+
+      - name: Test wheel
+        run: |
+          pip install --force-reinstall --verbose dist/*.whl
+          python -c 'import polars; polars.show_versions()'
+          # Quick sanity check for HF sink feature
+          python -c 'import polars as pl; df = pl.DataFrame({"a": [1,2,3]}); print("Polars loaded OK")'
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-linux-arm64
+          path: dist/*.whl

.github/workflows/docs-python.yml

@@ -4,14 +4,14 @@ on:
   pull_request:
     paths:
       - py-polars/docs/**
-      - py-polars/polars/**
+      - py-polars/src/polars/**
       - .github/workflows/docs-python.yml
   push:
     branches:
       - main
     paths:
       - py-polars/docs/**
-      - py-polars/polars/**
+      - py-polars/src/polars/**
       - .github/workflows/docs-python.yml
   repository_dispatch:
     types:

.github/workflows/test-coverage.yml

@@ -157,7 +157,7 @@ jobs:
         run: >
           pytest
           -n auto
-          -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"
+          -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not benchmark and not docs"
           -k 'not test_polars_import'
           --cov --cov-report xml:auto-streaming.xml --cov-fail-under=0
 
@@ -170,7 +170,7 @@ jobs:
         run: >
           pytest
           -n auto
-          -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"
+          -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not benchmark and not docs"
           -k 'not test_polars_import'
           --cov --cov-report xml:small-morsel.xml --cov-fail-under=0
 

.github/workflows/test-python.yml

@@ -43,13 +43,28 @@ jobs:
         os: [ubuntu-latest]
         python-version: ['3.10', '3.12', '3.13', '3.14', '3.14t']
         ideal_morsel_size: [100000]
+        auto_new_streaming: [false]
         include:
           - os: windows-latest
             python-version: '3.14'
             ideal_morsel_size: 100000
+            auto_new_streaming: false
+          - os: windows-latest
+            python-version: '3.14'
+            ideal_morsel_size: 100000
+            auto_new_streaming: true
+          - os: ubuntu-latest
+            python-version: '3.14'
+            ideal_morsel_size: 4
+            auto_new_streaming: false
+          - os: ubuntu-latest
+            python-version: '3.14'
+            ideal_morsel_size: 100000
+            auto_new_streaming: true
           - os: ubuntu-latest
             python-version: '3.14'
             ideal_morsel_size: 4
+            auto_new_streaming: true
 
     steps:
       - uses: actions/checkout@v6
@@ -114,33 +129,33 @@ jobs:
           maturin develop --manifest-path runtime/polars-runtime-32/Cargo.toml
 
       - name: Run doctests
-        if: github.ref_name != 'main' && matrix.python-version == '3.14' && matrix.os == 'ubuntu-latest'
+        if: github.ref_name != 'main' && matrix.python-version == '3.14' && matrix.os == 'ubuntu-latest' && !matrix.auto_new_streaming
         run: |
           python tests/docs/run_doctest.py
           pytest tests/docs/test_user_guide.py -m docs
 
       - name: Run tests
-        if: github.ref_name != 'main' && matrix.python-version != '3.14t'
+        if: github.ref_name != 'main' && matrix.python-version != '3.14t' && !matrix.auto_new_streaming
         env:
           POLARS_TIMEOUT_MS: 60000
         run: pytest -n auto -m "not release and not benchmark and not docs"
 
       - name: Run tests with new streaming engine
-        if: github.ref_name != 'main' && matrix.python-version != '3.14t'
+        if: github.ref_name != 'main' && matrix.python-version != '3.14t' && matrix.auto_new_streaming
         env:
           POLARS_AUTO_NEW_STREAMING: 1
           POLARS_TIMEOUT_MS: 60000
-        run: pytest -n auto -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"
+        run: pytest -n auto -m "not may_fail_auto_streaming and not release and not benchmark and not docs"
 
       - name: Run tests async reader tests
-        if: github.ref_name != 'main' && matrix.os != 'windows-latest' && matrix.python-version != '3.14t'
+        if: github.ref_name != 'main' && matrix.os != 'windows-latest' && matrix.python-version != '3.14t' && !matrix.auto_new_streaming
         env:
           POLARS_FORCE_ASYNC: 1
           POLARS_TIMEOUT_MS: 60000
         run: pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/
 
       - name: Run tests multiscan force empty capabilities
-        if: github.ref_name != 'main' && matrix.python-version != '3.14t'
+        if: github.ref_name != 'main' && matrix.python-version != '3.14t' && !matrix.auto_new_streaming
         env:
           POLARS_FORCE_EMPTY_READER_CAPABILITIES: 1
           POLARS_TIMEOUT_MS: 60000

.gitignore

@@ -42,7 +42,14 @@ target/
 *.tbl
 
 # Project
-/docs/assets/data/
+/docs/assets/data/*
+!/docs/assets/data/alltypes_plain.parquet
+!/docs/assets/data/apple_stock.csv
+!/docs/assets/data/iris.csv
+!/docs/assets/data/monopoly_props_groups.csv
+!/docs/assets/data/monopoly_props_prices.csv
+!/docs/assets/data/pokemon.csv
+!/docs/assets/data/reddit.csv
 /docs/assets/people.md
 
 # User specific source setups