promptfoo-python/.github/workflows/test.yml at 5e5b59efd23dba6947b91cb270809687151f48fb · promptfoo/promptfoo-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: Python CI

on:
  pull_request:
    paths-ignore:
      - "**.md"
      - "LICENSE"
  push:
    branches:
      - main
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  lint:
    name: Lint and Format
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Lint with Ruff
        run: uv run ruff check src/

      - name: Check formatting
        run: uv run ruff format --check src/

  type-check:
    name: Type Check
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Install dependencies
        run: uv sync --extra dev

      - name: Type check with mypy
        run: uv run mypy src/promptfoo/

  test:
    name: Test Python ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    timeout-minutes: 15
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-node@v4
        with:
          node-version: "20"

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin ${{ matrix.python-version }}

      - name: Install package
        run: uv sync

      - name: Clear npm cache on Windows
        if: runner.os == 'Windows'
        run: npm cache clean --force
        continue-on-error: true

      - name: Test CLI can be invoked
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 3
          max_attempts: 3
          retry_on: error
          command: uv run promptfoo --version

      - name: Test Node.js detection
        run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"

  build:
    name: Build Package
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Pin Python version
        run: uv python pin 3.12

      - name: Build package
        run: uv build

      - name: Upload artifacts
        uses: actions/upload-artifact@v6
        with:
          name: dist
          path: dist/

  ci-success:
    name: CI Success
    needs: [lint, type-check, test, build]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check if all jobs succeeded
        run: |
          LINT_RESULT="${{ needs.lint.result }}"
          TYPE_CHECK_RESULT="${{ needs.type-check.result }}"
          TEST_RESULT="${{ needs.test.result }}"
          BUILD_RESULT="${{ needs.build.result }}"

          echo "Job results:"
          echo "  lint: $LINT_RESULT"
          echo "  type-check: $TYPE_CHECK_RESULT"
          echo "  test: $TEST_RESULT"
          echo "  build: $BUILD_RESULT"

          if [[ "$LINT_RESULT" == "failure" || "$LINT_RESULT" == "cancelled" ||
                "$TYPE_CHECK_RESULT" == "failure" || "$TYPE_CHECK_RESULT" == "cancelled" ||
                "$TEST_RESULT" == "failure" || "$TEST_RESULT" == "cancelled" ||
                "$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" ]]; then
            echo "Some CI checks failed!"
            exit 1
          else
            echo "All CI checks passed!"
            exit 0
          fi