Merge branch 'pola-rs:main' into main

Author: davanstrien

.github/CODEOWNERS

@@ -5,6 +5,6 @@
 /crates/polars-parquet/  @ritchie46 @orlp @c-peters @coastalwhite
 /crates/polars-time/  @ritchie46 @orlp @c-peters @MarcoGorelli
 /crates/polars-python/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa
-/crates/polars-python/src/lazyframe/visit.rs  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
-/crates/polars-python/src/lazyframe/visitor/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence-
+/crates/polars-python/src/lazyframe/visit.rs  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence- @Matt711
+/crates/polars-python/src/lazyframe/visitor/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa @wence- @Matt711
 /py-polars/  @ritchie46 @c-peters @alexander-beedie @MarcoGorelli @reswqa

.github/issue-labeler.yml

@@ -3,7 +3,7 @@
 documentation:
   - '/\bdocs\b|documentation/i'
 performance:
-  - '/performance/i'
+  - '/performance|\bslow/i'
 regression:
   - '/regression/i'
 A-dtype-categorical:
@@ -48,5 +48,7 @@ A-plugin:
   - '/plugin/i'
 A-sql:
   - '/\bsql\b|sqlcontext/i'
+A-selectors:
+  - '/selector/i'
 A-streaming:
   - '/streaming/i'

.github/pull_request_template.md

@@ -2,6 +2,12 @@
 
 Please see the contribution guidelines: https://docs.pola.rs/development/contributing/#pull-requests.
 
+First-time contributors must adhere to the following rules, or your PR(s) will be closed:
+
+- You must post a screenshot of you successfully running the test suite (`make test`),
+  locally on your machine (not the CI).
+- You may not have more than one open PR at a time.
+
 Note that if you used AI to generate code there are additional requirements you
 must fulfill for your PR to be reviewed. See the contribution guidelines for
 details, afterwards you can use the following template if you wish:

.github/release-drafter.yml

@@ -117,6 +117,9 @@ autolabeler:
   - label: A-plugin
     title:
       - '/plugin/i'
+  - label: A-selectors
+    title:
+      - '/selector/i'
   - label: A-streaming
     title:
       - '/streaming/i'

.github/workflows/docs-global.yml

@@ -11,6 +11,9 @@ on:
       - python-release-docs
   workflow_dispatch:
 
+env:
+  RUSTFLAGS: -C debuginfo=0 # Do not produce debug symbols to keep memory usage down
+
 jobs:
   markdown-link-check:
     runs-on: ubuntu-latest

.github/workflows/lint-global.yml

@@ -16,3 +16,7 @@ jobs:
         uses: dprint/check@v2.3
       - name: Spell Check with Typos
         uses: crate-ci/typos@v1.42.0
+      - name: Install ripgrep
+        run: sudo apt-get update && sudo apt-get install -y ripgrep
+      - name: Check for FIXME comments
+        run: make check-fixme

.github/workflows/lint-rust.yml

@@ -48,7 +48,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy with all features enabled
-        run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings -D clippy::dbg_macro
+        run: python3 tools/cargo-fail-warning.py clippy --workspace --all-targets --all-features --locked -- -W clippy::dbg_macro
 
   # Default feature set should compile on the stable toolchain
   clippy-stable:
@@ -68,7 +68,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy
-        run: cargo clippy --all-targets --locked -- -D warnings -D clippy::dbg_macro
+        run: python3 tools/cargo-fail-warning.py clippy --all-targets --locked -- -W clippy::dbg_macro
 
   rustfmt:
     if: github.ref_name != 'main'

.github/workflows/pr-labeler.yml

@@ -12,14 +12,40 @@ jobs:
   labeler:
     runs-on: ubuntu-latest
     steps:
-      - name: Check pull request title
-        uses: thehanimo/pr-title-checker@v1.4.2
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Label pull request
         uses: release-drafter/release-drafter@v6
         with:
           disable-releaser: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Label first-time contributors
+        # First-time contributor check from https://github.com/llvm/llvm-project/blob/0af2d43e06415ce1e8a5d49e864c3881048dd08b/.github/workflows/new-prs.yml#L22.
+        # Only label PRs opened for the first time by someone new to this repo
+        # or to GitHub as a whole. Ideally we'd look for FIRST_TIMER or
+        # FIRST_TIME_CONTRIBUTOR, but this does not appear to work. Instead
+        # check that we do not have any of the other author associations.
+        # See https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=opened#pull_request
+        # for all the possible values.
+        if: >-
+          (github.event.action == 'opened') &&
+          (github.event.pull_request.author_association != 'COLLABORATOR') &&
+          (github.event.pull_request.author_association != 'CONTRIBUTOR') &&
+          (github.event.pull_request.author_association != 'MANNEQUIN') &&
+          (github.event.pull_request.author_association != 'MEMBER') &&
+          (github.event.pull_request.author_association != 'OWNER')
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.payload.pull_request.number,
+              labels: ['first-contribution'],
+            });
+
+      - name: Check pull request title
+        uses: thehanimo/pr-title-checker@v1.4.2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-coverage.yml

@@ -159,7 +159,20 @@ jobs:
           -n auto
           -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"
           -k 'not test_polars_import'
-          --cov --cov-report xml:main.xml --cov-fail-under=0
+          --cov --cov-report xml:auto-streaming.xml --cov-fail-under=0
+
+      - name: Run Python tests - streaming with morsel size 4
+        working-directory: py-polars
+        env:
+          POLARS_AUTO_NEW_STREAMING: 1
+          POLARS_IDEAL_MORSEL_SIZE: 4
+          POLARS_TIMEOUT_MS: 60000
+        run: >
+          pytest
+          -n auto
+          -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only"
+          -k 'not test_polars_import'
+          --cov --cov-report xml:small-morsel.xml --cov-fail-under=0
 
       - name: Run Python tests - async reader
         working-directory: py-polars
@@ -182,6 +195,8 @@ jobs:
           path: |
             coverage-python.lcov
             py-polars/main.xml
+            py-polars/auto-streaming.xml
+            py-polars/small-morsel.xml
             py-polars/async.xml
 
   upload-coverage:

.github/workflows/test-pyodide.yml

@@ -1,22 +1,9 @@
 name: Test Pyodide
 
 on:
-  pull_request:
-    paths:
-      - Cargo.lock
-      - py-polars/**
-      - docs/source/src/python/**
-      - crates/**
-      - .github/workflows/test-pyodide.yml
   push:
     branches:
-      - main
-    paths:
-      - Cargo.lock
-      - crates/**
-      - docs/source/src/python/**
-      - py-polars/**
-      - .github/workflows/test-pyodide.yml
+      - disabled-pyodide-tests  # disabled for now
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -43,7 +30,7 @@ jobs:
           FEATURES: csv|ipc|ipc_streaming|parquet|async|scan_lines|json|extract_jsonpath|catalog|cloud|polars_cloud|tokio|clipboard|decompress|new_streaming
         run: |
           sed -i 's/serde_json = { workspace = true, optional = true }/serde_json = { workspace = true }/' crates/polars-python/Cargo.toml
-          sed -i 's/"serde_json", //' crates/polars-python/Cargo.toml
+          sed -i 's/"serde_json",//' crates/polars-python/Cargo.toml
           sed -E -i "/^  \"(${FEATURES})\",$/d" crates/polars-python/Cargo.toml py-polars/runtime/polars-runtime-32/Cargo.toml
 
       - name: Setup emsdk