promptfoo-python/.github/workflows/test.yml at 7d8d3708d784111abbd88c7b5abf3810de2db5eb · promptfoo/promptfoo-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
name: Python CI

on:
  pull_request:
    paths-ignore:
      - "**.md"
      - "LICENSE"
  push:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  lint:
    name: Lint and Format
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Lint with Ruff
        run: uv run ruff check src/

      - name: Check formatting
        run: uv run ruff format --check src/

  type-check:
    name: Type Check
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Type check with mypy
        run: uv run mypy src/promptfoo/

  test:
    name: Test Python ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    timeout-minutes: 15
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Clear npm cache on Windows
        if: runner.os == 'Windows'
        run: npm cache clean --force
        continue-on-error: true

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package
        run: uv sync

      - name: Test CLI can be invoked
        run: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  build:
    name: Build Package
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Build package
        run: uv build

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          name: dist
          path: dist/

  ci-success:
    name: CI Success
    needs: [lint, type-check, test, build]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check if all jobs succeeded
        run: |
          LINT_RESULT="${{ needs.lint.result }}"
          TYPE_CHECK_RESULT="${{ needs.type-check.result }}"
          TEST_RESULT="${{ needs.test.result }}"
          BUILD_RESULT="${{ needs.build.result }}"

          echo "Job results:"
          echo "  lint: $LINT_RESULT"
          echo "  type-check: $TYPE_CHECK_RESULT"
          echo "  test: $TEST_RESULT"
          echo "  build: $BUILD_RESULT"

          if [[ "$LINT_RESULT" == "failure" || "$LINT_RESULT" == "cancelled" ||
                "$TYPE_CHECK_RESULT" == "failure" || "$TYPE_CHECK_RESULT" == "cancelled" ||
                "$TEST_RESULT" == "failure" || "$TEST_RESULT" == "cancelled" ||
                "$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" ]]; then
            echo "Some CI checks failed!"
            exit 1
          else
            echo "All CI checks passed!"
            exit 0
          fi