test.yml

name: Python CI

on:
  pull_request:
    paths-ignore:
      - "**.md"
      - "LICENSE"
  push:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  lint:
    name: Lint and Format
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Lint with Ruff
        run: uv run ruff check src/

      - name: Check formatting
        run: uv run ruff format --check src/

  type-check:
    name: Type Check
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Type check with mypy
        run: uv run mypy src/promptfoo/

  test:
    name: Test (py${{ matrix.python-version }}, ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 15
    strategy:
      matrix:
        # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
        # causing BlockingIOError [Errno 35] when spawning subprocess
        os: [ubuntu-latest, windows-latest]
        # Test only min and max supported Python versions for efficiency
        python-version: ["3.9", "3.13"]
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Configure npm cache on Windows
        if: matrix.os == 'windows-latest'
        run: |
          "NPM_CONFIG_CACHE=$env:RUNNER_TEMP\\npm-cache" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          npm cache clean --force

      - name: Install promptfoo globally
        run: npm install -g promptfoo@latest
        env:
          NODE_OPTIONS: --max-old-space-size=4096

      - name: Add npm global bin to PATH (Windows)
        if: matrix.os == 'windows-latest'
        run: |
          $globalPrefix = (npm config get prefix).Trim()
          if (-not $globalPrefix) {
            $globalPrefix = Join-Path $env:APPDATA "npm"
          }
          "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          "npm_config_prefix=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          $binPaths = @($globalPrefix, (Join-Path $globalPrefix "bin")) | Where-Object { Test-Path $_ }
          foreach ($binPath in $binPaths) {
            $binPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          }

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package
        run: uv sync

      - name: Test CLI can be invoked
        run: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  test-npx-fallback:
    name: Test npx fallback (py${{ matrix.python-version }}, ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 15
    strategy:
      matrix:
        # Test npx fallback (without global install)
        # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
        os: [ubuntu-latest, windows-latest]
        # Use middle-version Python for this test
        python-version: ["3.12"]
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Configure npm cache on Windows
        if: matrix.os == 'windows-latest'
        run: |
          "NPM_CONFIG_CACHE=$env:RUNNER_TEMP\\npm-cache" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
          npm cache clean --force

      # Intentionally skip installing promptfoo globally
      # This tests the npx fallback path

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package
        run: uv sync

      - name: Test CLI fallback to npx (no global install)
        run: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  build:
    name: Build Package
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Build package
        run: uv build

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          name: dist
          path: dist/

  ci-success:
    name: CI Success
    needs: [lint, type-check, test, test-npx-fallback, build]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check if all jobs succeeded
        run: |
          LINT_RESULT="${{ needs.lint.result }}"
          TYPE_CHECK_RESULT="${{ needs.type-check.result }}"
          TEST_RESULT="${{ needs.test.result }}"
          TEST_NPX_FALLBACK_RESULT="${{ needs.test-npx-fallback.result }}"
          BUILD_RESULT="${{ needs.build.result }}"

          echo "Job results:"
          echo "  lint: $LINT_RESULT"
          echo "  type-check: $TYPE_CHECK_RESULT"
          echo "  test: $TEST_RESULT"
          echo "  test-npx-fallback: $TEST_NPX_FALLBACK_RESULT"
          echo "  build: $BUILD_RESULT"

          if [[ "$LINT_RESULT" == "failure" || "$LINT_RESULT" == "cancelled" ||
                "$TYPE_CHECK_RESULT" == "failure" || "$TYPE_CHECK_RESULT" == "cancelled" ||
                "$TEST_RESULT" == "failure" || "$TEST_RESULT" == "cancelled" ||
                "$TEST_NPX_FALLBACK_RESULT" == "failure" || "$TEST_NPX_FALLBACK_RESULT" == "cancelled" ||
                "$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" ]]; then
            echo "Some CI checks failed!"
            exit 1
          else
            echo "All CI checks passed!"
            exit 0
          fi