test-pywry.yml

name: Test pywry

on:
  pull_request:
    types: [opened, synchronize, reopened, labeled, unlabeled]
    branches: [main, develop]
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

defaults:
  run:
    working-directory: pywry

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-24.04
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v5

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.12'
          cache: 'pip'
          cache-dependency-path: pywry/pyproject.toml

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[notebook,auth,mcp,acp,deepagent,openai,anthropic,magentic]"
          pip install ruff ty pytauri-wheel plotly pandas cryptography

      - name: Run Ruff linter
        run: ruff check pywry/ tests/

      - name: Run Ruff formatter check
        run: ruff format --check pywry/ tests/

      - name: Run ty type checker
        run: ty check pywry/

  gen-matrix:
    name: Resolve test matrix
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: ${{ github.workspace }}
    outputs:
      full-matrix: ${{ steps.matrix.outputs.full-matrix }}
      pr-matrix: ${{ steps.matrix.outputs.pr-matrix }}
      pytest-targets: ${{ steps.targets.outputs.targets }}
      is-scoped: ${{ steps.targets.outputs.is-scoped }}
    steps:
      - id: matrix
        shell: bash
        run: |
          FULL_MATRIX='{"include":[{"os":"ubuntu-24.04","python-version":"3.11"},{"os":"ubuntu-24.04","python-version":"3.14"},{"os":"windows-2025","python-version":"3.11"},{"os":"windows-2025","python-version":"3.14"},{"os":"macos-15-intel","python-version":"3.11"},{"os":"macos-15-intel","python-version":"3.14"},{"os":"ubuntu-24.04-arm","python-version":"3.11"},{"os":"ubuntu-24.04-arm","python-version":"3.14"},{"os":"windows-11-arm","python-version":"3.12"},{"os":"windows-11-arm","python-version":"3.14"},{"os":"macos-latest","python-version":"3.11"},{"os":"macos-latest","python-version":"3.14"}]}'
          PR_MATRIX='{"include":[{"os":"ubuntu-24.04","python-version":"3.14"}]}'
          echo "full-matrix=$FULL_MATRIX" >> "$GITHUB_OUTPUT"
          echo "pr-matrix=$PR_MATRIX" >> "$GITHUB_OUTPUT"

      - name: Checkout for diff
        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full-test')
        uses: actions/checkout@v5
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - id: targets
        shell: bash
        env:
          IS_PR: ${{ github.event_name == 'pull_request' }}
          HAS_FULL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'full-test') }}
          BASE_REF: ${{ github.event.pull_request.base.ref }}
        run: |
          if [[ "$IS_PR" != "true" || "$HAS_FULL_LABEL" == "true" ]]; then
            echo "Full suite (dispatch or full-test label)"
            echo "targets=tests/" >> "$GITHUB_OUTPUT"
            echo "is-scoped=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          git fetch --no-tags --prune origin "$BASE_REF" 2>&1 || true
          CHANGED=$(git diff --name-only "origin/$BASE_REF"...HEAD)
          echo "Changed files in PR:"
          echo "$CHANGED" | sed 's/^/  /'

          # Source / workflow / shared-test-infra changes force the full suite.
          SOURCE_NON_TEST=$(echo "$CHANGED" | grep -E '^pywry/' | grep -vE '^pywry/(tests|docs)/' | grep -vE '^pywry/.*\.md$' || true)
          WORKFLOW_SELF=$(echo "$CHANGED" | grep -E '^\.github/workflows/test-pywry\.yml$' || true)
          INFRA_TEST=$(echo "$CHANGED" | grep -E '^pywry/tests/(conftest|constants|__init__)\.py$' || true)
          TEST_FILES=$(echo "$CHANGED" | grep -E '^pywry/tests/test_.*\.py$' || true)

          if [[ -n "$SOURCE_NON_TEST" || -n "$WORKFLOW_SELF" || -n "$INFRA_TEST" ]]; then
            echo "Full suite (source/workflow/test-infra changes)"
            echo "targets=tests/" >> "$GITHUB_OUTPUT"
            echo "is-scoped=false" >> "$GITHUB_OUTPUT"
          elif [[ -n "$TEST_FILES" ]]; then
            T=$(echo "$TEST_FILES" | sed 's|^pywry/||' | tr '\n' ' ' | sed 's/ *$//')
            echo "Scoped to changed test files: $T"
            echo "targets=$T" >> "$GITHUB_OUTPUT"
            echo "is-scoped=true" >> "$GITHUB_OUTPUT"
          else
            echo "No pywry source/test changes; skipping pytest selection"
            echo "targets=__skip__" >> "$GITHUB_OUTPUT"
            echo "is-scoped=false" >> "$GITHUB_OUTPUT"
          fi

  test:
    name: Test - ${{ matrix.os }} - Python ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    needs: gen-matrix
    if: needs.gen-matrix.outputs.pytest-targets != '__skip__'
    permissions:
      contents: read
    strategy:
      fail-fast: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full-test') }}
      # ``deepagents`` (pulled in by ``.[dev]`` → ``features``) hard-
      # requires Python >=3.11 — 3.10 can't resolve the dev extra.
      # ``sqlcipher3`` is a source build against the SQLCipher C
      # library installed in the ``Build SQLCipher`` step below, so
      # any Python ABI works (no binary-wheel coverage gaps).
      matrix: ${{ fromJSON(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'full-test') ? needs.gen-matrix.outputs.pr-matrix : needs.gen-matrix.outputs.full-matrix) }}

    steps:
      - uses: actions/checkout@v5

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}
          allow-prereleases: true
          cache: 'pip'
          cache-dependency-path: pywry/pyproject.toml

      - name: Set up Rust (Linux only)
        if: runner.os == 'Linux'
        uses: dtolnay/rust-toolchain@stable

      - name: Set up Rust (Windows ARM64)
        if: runner.os == 'Windows' && runner.arch == 'ARM64'
        uses: dtolnay/rust-toolchain@stable
        with:
          targets: aarch64-pc-windows-msvc

      - name: Install Linux dependencies
        if: runner.os == 'Linux'
        run: |
          for i in 1 2 3; do sudo apt-get update && break || sleep 15; done
          for i in 1 2 3; do sudo apt-get install -y --fix-missing xvfb dbus-x11 at-spi2-core libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 libxcb-shape0 libgl1 libegl1 libwebkit2gtk-4.1-dev libgtk-3-dev build-essential libssl-dev && break || { sleep 30; sudo apt-get update; }; done

      - name: Build SQLCipher from source (Linux)
        if: runner.os == 'Linux'
        shell: bash
        run: |
          # Build from the upstream source (https://github.com/sqlcipher/sqlcipher)
          # so the ``sqlcipher3`` Python package's setup.py links
          # against a known-good library on every Python ABI —
          # binary wheels stop at Python 3.13.  ``--disable-tcl``
          # skips the Tcl extension (not needed by the Python
          # binding, and modern Tcl bottles break its compile).
          git clone --depth=1 --branch=v4.6.1 https://github.com/sqlcipher/sqlcipher.git /tmp/sqlcipher
          cd /tmp/sqlcipher
          ./configure \
            --disable-tcl \
            --enable-tempstore=yes \
            CFLAGS="-DSQLITE_HAS_CODEC -DSQLCIPHER_CRYPTO_OPENSSL" \
            LDFLAGS="-lcrypto"
          make -j$(nproc) libsqlcipher.la || make -j$(nproc)
          sudo make install-libLTLIBRARIES install-includeHEADERS || sudo make install
          sudo ldconfig

      - name: Build SQLCipher from source (macOS)
        if: runner.os == 'macOS'
        shell: bash
        run: |
          # Intentionally NOT installing ``tcl-tk`` from Homebrew:
          # the current bottle is Tcl 9.0, whose headers drop the
          # ``CONST`` macro and ``TCL_CHANNEL_VERSION_2``, so
          # SQLCipher's ``tclsqlite.c`` (still Tcl-8 API) fails to
          # compile.  SQLCipher's configure auto-disables the Tcl
          # extension when no ``tclConfig.sh`` is on the usual
          # search paths — we only need the C library anyway; the
          # ``sqlcipher3`` Python binding doesn't touch Tcl.
          brew install openssl@3
          OPENSSL_PREFIX="$(brew --prefix openssl@3)"
          git clone --depth=1 --branch=v4.6.1 https://github.com/sqlcipher/sqlcipher.git /tmp/sqlcipher
          cd /tmp/sqlcipher
          ./configure \
            --disable-tcl \
            --enable-tempstore=yes \
            CFLAGS="-DSQLITE_HAS_CODEC -DSQLCIPHER_CRYPTO_OPENSSL -I$OPENSSL_PREFIX/include" \
            LDFLAGS="-L$OPENSSL_PREFIX/lib -lcrypto"
          # Build only the library target — the default ``make`` tries
          # to build the ``sqlcipher`` shell which links against Tcl.
          make -j$(sysctl -n hw.logicalcpu) libsqlcipher.la || make -j$(sysctl -n hw.logicalcpu)
          sudo make install-libLTLIBRARIES install-includeHEADERS || sudo make install
          echo "LDFLAGS=-L/usr/local/lib -L$OPENSSL_PREFIX/lib -lsqlcipher -lcrypto" >> $GITHUB_ENV
          echo "CPPFLAGS=-I/usr/local/include -I/usr/local/include/sqlcipher -I$OPENSSL_PREFIX/include" >> $GITHUB_ENV

      - name: Install SQLCipher via vcpkg (Windows x64)
        if: runner.os == 'Windows' && runner.arch != 'ARM64'
        shell: pwsh
        run: |
          # vcpkg builds SQLCipher from source using the same upstream
          # repo (https://github.com/sqlcipher/sqlcipher), just
          # packaged as an MSVC-friendly port so we don't have to wire
          # nmake + OpenSSL manually.  ``SQLCIPHER_PATH`` points
          # ``sqlcipher3``'s setup.py at the install root.  Skipped on
          # arm64-windows: the transitive ``tcl`` port doesn't build
          # there even with ``--allow-unsupported``, and the
          # ``SqliteStateBackend`` handles the missing binding by
          # falling back to plain SQLite.
          $env:VCPKG_ROOT = "$env:VCPKG_INSTALLATION_ROOT"
          & "$env:VCPKG_ROOT\vcpkg" install "sqlcipher:x64-windows"
          $sqlcipherDir = "$env:VCPKG_ROOT\installed\x64-windows"
          echo "SQLCIPHER_PATH=$sqlcipherDir" >> $env:GITHUB_ENV
          echo "INCLUDE=$sqlcipherDir\include;$sqlcipherDir\include\sqlcipher;$env:INCLUDE" >> $env:GITHUB_ENV
          echo "LIB=$sqlcipherDir\lib;$env:LIB" >> $env:GITHUB_ENV
          echo "PATH=$sqlcipherDir\bin;$env:PATH" >> $env:GITHUB_ENV

      - name: Cache vcpkg packages (Windows ARM)
        if: runner.os == 'Windows' && runner.arch == 'ARM64'
        uses: actions/cache@v5
        with:
          path: C:/vcpkg/installed
          key: vcpkg-arm64-windows-openssl-static-md-lua-${{ hashFiles('.github/workflows/test-pywry.yml') }}
          restore-keys: |
            vcpkg-arm64-windows-openssl-static-md-lua-

      - name: Install OpenSSL via vcpkg (Windows ARM)
        if: runner.os == 'Windows' && runner.arch == 'ARM64'
        shell: pwsh
        run: |
          $env:VCPKG_ROOT = "$env:VCPKG_INSTALLATION_ROOT"
          & "$env:VCPKG_ROOT\vcpkg" install openssl:arm64-windows-static-md
          $opensslDir = "$env:VCPKG_ROOT\installed\arm64-windows-static-md"
          echo "OPENSSL_DIR=$opensslDir" >> $env:GITHUB_ENV
          echo "OPENSSL_LIB_DIR=$opensslDir\lib" >> $env:GITHUB_ENV
          echo "OPENSSL_INCLUDE_DIR=$opensslDir\include" >> $env:GITHUB_ENV

      - name: Setup build environment (Windows ARM)
        if: runner.os == 'Windows' && runner.arch == 'ARM64'
        shell: pwsh
        run: |
          $buildTemp = "C:\buildtmp"
          New-Item -ItemType Directory -Force -Path $buildTemp | Out-Null
          echo "TMP=$buildTemp" >> $env:GITHUB_ENV
          echo "TEMP=$buildTemp" >> $env:GITHUB_ENV
          $env:VCPKG_ROOT = "$env:VCPKG_INSTALLATION_ROOT"
          & "$env:VCPKG_ROOT\vcpkg" install lua:arm64-windows
          $luaDir = "$env:VCPKG_ROOT\installed\arm64-windows"
          echo "LUA_DIR=$luaDir" >> $env:GITHUB_ENV
          echo "LUA_LIB=$luaDir\lib" >> $env:GITHUB_ENV
          echo "LUA_INC=$luaDir\include" >> $env:GITHUB_ENV

      - name: Build cryptography from source (Windows ARM)
        if: runner.os == 'Windows' && runner.arch == 'ARM64'
        shell: pwsh
        run: |
          pip install --no-cache-dir --no-binary=cryptography cryptography
          python -c "from cryptography.hazmat.bindings._rust import openssl; print('cryptography OK')"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e ".[dev]" pytest-xdist

      - name: Verify Docker (Linux)
        if: runner.os == 'Linux'
        run: docker version

      - name: Configure Docker (macOS Intel)
        if: runner.os == 'macOS' && runner.arch == 'X64'
        shell: bash
        run: |
          # ``docker/setup-docker-action@v4`` doesn't reliably bring up
          # a Docker daemon on ``macos-15-intel`` runners — testcontainers
          # then fails with ``FileNotFoundError: /var/run/docker.sock``
          # during the Redis fixture setup.  Mirror the resilient
          # Docker-Desktop → Colima fallback used in publish-pywry.yml.
          set -euo pipefail

          if docker info >/dev/null 2>&1; then
            echo "Docker daemon already available"
          else
            if [ -d "/Applications/Docker.app" ]; then
              echo "Starting Docker Desktop"
              open -a Docker
              for _ in {1..30}; do
                if docker info >/dev/null 2>&1; then
                  break
                fi
                sleep 5
              done
            fi
          fi

          if ! docker info >/dev/null 2>&1; then
            echo "Falling back to Colima"
            brew install docker colima
            colima start --runtime docker --arch x86_64 --cpu 2 --memory 4 --disk 20
          fi

          if [ -S "$HOME/.colima/default/docker.sock" ]; then
            sudo mkdir -p /var/run
            sudo ln -sf "$HOME/.colima/default/docker.sock" /var/run/docker.sock
          fi

          echo "DOCKER_HOST=unix:///var/run/docker.sock" >> "$GITHUB_ENV"
          docker info

      - name: Run tests (Linux)
        if: runner.os == 'Linux'
        env:
          PYWRY_HEADLESS: "1"
          NO_AT_BRIDGE: "1"
          PYTEST_TARGETS: ${{ needs.gen-matrix.outputs.pytest-targets }}
        run: |
          dbus-run-session -- xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" python -m pytest -c pytest.ini $PYTEST_TARGETS -n auto -v --tb=short
          rc=$?
          # Exit 5 = no tests collected after platform filtering on a scoped run; treat as success.
          if [[ $rc -eq 5 && "$PYTEST_TARGETS" != "tests/" ]]; then rc=0; fi
          exit $rc

      - name: Run tests (Windows x64)
        if: runner.os == 'Windows' && runner.arch == 'X64'
        env:
          PYWRY_HEADLESS: "1"
          PYWRY_DEPLOY__STATE_BACKEND: "memory"
          PYTHONUTF8: "1"
          PYTEST_TARGETS: ${{ needs.gen-matrix.outputs.pytest-targets }}
        shell: pwsh
        run: |
          python -m pytest -c pytest.ini $env:PYTEST_TARGETS.Split(' ') -n auto -v --tb=short --ignore=tests/test_state_redis_integration.py --ignore=tests/test_auth_rbac_integration.py --ignore=tests/test_deploy_mode_integration.py --ignore=tests/test_e2e_deploy_mode.py --ignore=tests/test_e2e_rbac_widgets.py -m "not redis and not container"
          if ($LASTEXITCODE -eq 5 -and $env:PYTEST_TARGETS -ne 'tests/') { exit 0 }
          exit $LASTEXITCODE

      - name: Run tests (Windows ARM)
        if: runner.os == 'Windows' && runner.arch == 'ARM64'
        env:
          PYWRY_HEADLESS: "1"
          PYWRY_DEPLOY__STATE_BACKEND: "memory"
          PYTHONUTF8: "1"
          PYTEST_TARGETS: ${{ needs.gen-matrix.outputs.pytest-targets }}
        shell: pwsh
        run: |
          python -m pytest -c pytest.ini $env:PYTEST_TARGETS.Split(' ') -n auto -v --tb=short --ignore=tests/test_state_redis_integration.py --ignore=tests/test_auth_rbac_integration.py --ignore=tests/test_deploy_mode_integration.py --ignore=tests/test_e2e_deploy_mode.py --ignore=tests/test_e2e_rbac_widgets.py --ignore=tests/test_inline_ssl.py -m "not redis and not container"
          if ($LASTEXITCODE -eq 5 -and $env:PYTEST_TARGETS -ne 'tests/') { exit 0 }
          exit $LASTEXITCODE

      - name: Run tests (macOS Intel)
        if: runner.os == 'macOS' && runner.arch == 'X64'
        env:
          PYWRY_HEADLESS: "1"
          PYTEST_TARGETS: ${{ needs.gen-matrix.outputs.pytest-targets }}
        run: |
          python -m pytest -c pytest.ini $PYTEST_TARGETS -n auto -v --tb=short
          rc=$?
          if [[ $rc -eq 5 && "$PYTEST_TARGETS" != "tests/" ]]; then rc=0; fi
          exit $rc

      - name: Run tests (macOS ARM)
        if: runner.os == 'macOS' && runner.arch == 'ARM64'
        env:
          PYWRY_HEADLESS: "1"
          PYWRY_DEPLOY__STATE_BACKEND: "memory"
          PYTEST_TARGETS: ${{ needs.gen-matrix.outputs.pytest-targets }}
        run: |
          python -m pytest -c pytest.ini $PYTEST_TARGETS -n auto -v --tb=short --ignore=tests/test_state_redis_integration.py --ignore=tests/test_auth_rbac_integration.py --ignore=tests/test_deploy_mode_integration.py --ignore=tests/test_e2e_deploy_mode.py --ignore=tests/test_e2e_rbac_widgets.py -m "not redis and not container"
          rc=$?
          if [[ $rc -eq 5 && "$PYTEST_TARGETS" != "tests/" ]]; then rc=0; fi
          exit $rc

  ci-required:
    name: CI Required
    runs-on: ubuntu-latest
    if: always()
    needs:
      - lint
      - gen-matrix
      - test
    permissions:
      contents: read
    defaults:
      run:
        working-directory: ${{ github.workspace }}
    steps:
      - name: Enforce required CI outcomes
        shell: bash
        env:
          LINT_RESULT: ${{ needs.lint.result }}
          TEST_RESULT: ${{ needs.test.result }}
          PYTEST_TARGETS: ${{ needs.gen-matrix.outputs.pytest-targets }}
        run: |
          set -euo pipefail

          if [[ "$LINT_RESULT" != "success" ]]; then
            echo "Lint job failed with result: $LINT_RESULT"
            exit 1
          fi

          if [[ "$PYTEST_TARGETS" == "__skip__" ]]; then
            echo "No pywry source/test changes detected; test matrix intentionally skipped."
            exit 0
          fi

          if [[ "$TEST_RESULT" != "success" ]]; then
            echo "Test matrix did not succeed (result: $TEST_RESULT)"
            exit 1
          fi

          echo "All required CI checks passed."