test-python.yml

name: Test Python

on:
  pull_request:
    paths:
      - Cargo.lock
      - py-polars/**
      - docs/source/src/python/**
      - crates/**
      - .github/workflows/test-python.yml
  push:
    branches:
      - main
    paths:
      - Cargo.lock
      - crates/**
      - docs/source/src/python/**
      - py-polars/**
      - .github/workflows/test-python.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  RUSTFLAGS: -C debuginfo=0 # Do not produce debug symbols to keep memory usage down
  RUST_BACKTRACE: 1
  PYTHONUTF8: 1

defaults:
  run:
    working-directory: py-polars
    shell: bash

jobs:
  test-python:
    runs-on: ${{ matrix.os }}
    env:
      POLARS_IDEAL_MORSEL_SIZE: ${{ matrix.ideal_morsel_size }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ['3.10', '3.12', '3.13', '3.14', '3.14t']
        ideal_morsel_size: [100000]
        auto_new_streaming: [false]
        include:
          - os: windows-latest
            python-version: '3.14'
            ideal_morsel_size: 100000
            auto_new_streaming: false
          - os: windows-latest
            python-version: '3.14'
            ideal_morsel_size: 100000
            auto_new_streaming: true
          - os: ubuntu-latest
            python-version: '3.14'
            ideal_morsel_size: 4
            auto_new_streaming: false
          - os: ubuntu-latest
            python-version: '3.14'
            ideal_morsel_size: 100000
            auto_new_streaming: true
          - os: ubuntu-latest
            python-version: '3.14'
            ideal_morsel_size: 4
            auto_new_streaming: true

    steps:
      - uses: actions/checkout@v6

      - name: Free some disk space (remove unused packages)
        uses: jlumbroso/free-disk-space@v1.3.1
        if: ${{ matrix.os != 'windows-latest' }}

      - name: Enable long paths on Windows
        if: matrix.os == 'windows-latest'
        shell:
          powershell
        run: |
          Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Set up Graphviz
        uses: ts-graphviz/setup-graphviz@v2

      - name: Create virtual environment
        env:
          BIN: ${{ matrix.os == 'windows-latest' && 'Scripts' || 'bin' }}
        run: |
          python -m venv .venv
          echo "$GITHUB_WORKSPACE/py-polars/.venv/$BIN" >> $GITHUB_PATH
          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/py-polars/.venv" >> $GITHUB_ENV
          echo "VENV=$GITHUB_WORKSPACE/py-polars/.venv" >> $GITHUB_ENV

      - name: Install maturin
        if: matrix.python-version == '3.14t'
        run: |
          pip install uv
          uv pip install maturin

      - name: Install Python dependencies
        if: matrix.python-version != '3.14t'
        run: |
          pip install uv
          # Install typing-extensions separately whilst the `--extra-index-url` in `requirements-ci.txt`
          # doesn't have an up-to-date typing-extensions, see
          # https://github.com/astral-sh/uv/issues/6028#issuecomment-2287232150
          uv pip install -U typing-extensions
          uv pip install --compile-bytecode -r requirements-dev.txt -r requirements-ci.txt --verbose --index-strategy=unsafe-best-match
      - name: Set up Rust
        run: rustup show

      - name: Cache Rust
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: py-polars
          save-if: ${{ github.ref_name == 'main' }}

      - name: Install Polars
        run: |
          # Make sure we use the Polars we're about to build.
          uv pip uninstall polars-runtime-32 polars-runtime-64 polars-runtime-compat
          pip install --no-deps -e .
          maturin develop --manifest-path runtime/polars-runtime-32/Cargo.toml

      - name: Run doctests
        if: github.ref_name != 'main' && matrix.python-version == '3.14' && matrix.os == 'ubuntu-latest' && !matrix.auto_new_streaming
        run: |
          python tests/docs/run_doctest.py
          pytest tests/docs/test_user_guide.py -m docs

      - name: Run tests
        if: github.ref_name != 'main' && matrix.python-version != '3.14t' && !matrix.auto_new_streaming
        env:
          POLARS_TIMEOUT_MS: 60000
        run: pytest -n auto -m "not release and not benchmark and not docs"

      - name: Run tests with new streaming engine
        if: github.ref_name != 'main' && matrix.python-version != '3.14t' && matrix.auto_new_streaming
        env:
          POLARS_AUTO_NEW_STREAMING: 1
          POLARS_TIMEOUT_MS: 60000
        run: pytest -n auto -m "not may_fail_auto_streaming and not release and not benchmark and not docs"

      - name: Run tests async reader tests
        if: github.ref_name != 'main' && matrix.os != 'windows-latest' && matrix.python-version != '3.14t' && !matrix.auto_new_streaming
        env:
          POLARS_FORCE_ASYNC: 1
          POLARS_TIMEOUT_MS: 60000
        run: pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/

      - name: Run tests multiscan force empty capabilities
        if: github.ref_name != 'main' && matrix.python-version != '3.14t' && !matrix.auto_new_streaming
        env:
          POLARS_FORCE_EMPTY_READER_CAPABILITIES: 1
          POLARS_TIMEOUT_MS: 60000
        run: |
          pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/test_multiscan.py
          pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/test_scan_row_deletion.py
          pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/test_iceberg.py

      - name: Check import without optional dependencies
        if: github.ref_name != 'main' && matrix.os == 'ubuntu-latest' && (matrix.python-version == '3.14' || matrix.python-version == '3.14t')
        run: |
          declare -a deps=("pandas"
          "pyarrow"
          "fsspec"
          "matplotlib"
          "backports.zoneinfo"
          "connectorx"
          "pyiceberg"
          "deltalake"
          "xlsx2csv"
          )
          for d in "${deps[@]}"
          do
            echo "uninstall $i and check imports..."
            pip uninstall "$d" -y
            python -c 'import polars'
          done