|
27 | 27 | COMMENT: ${{ github.event.comment.body }} |
28 | 28 | run: | |
29 | 29 | python3 ci/orchestrate_evaluations.py "$PR_NUMBER" --matrix-only --filter-provider all > /tmp/matrix.json |
30 | | - MATRIX=$(cat /tmp/matrix.json) |
31 | | - echo "result=$MATRIX" >> $GITHUB_OUTPUT |
| 30 | + echo "result<<EOF" >> $GITHUB_OUTPUT |
| 31 | + cat /tmp/matrix.json >> $GITHUB_OUTPUT |
| 32 | + echo "EOF" >> $GITHUB_OUTPUT |
32 | 33 |
|
33 | 34 | evaluate: |
34 | 35 | needs: prepare |
@@ -59,12 +60,23 @@ jobs: |
59 | 60 | COPILOT_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
60 | 61 | OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} |
61 | 62 | run: | |
62 | | - uv run --project tests --frozen tests/evaluator.py \ |
63 | | - --provider "${{ matrix.provider }}" \ |
64 | | - --model "${{ matrix.model }}" \ |
65 | | - --judge --verbose --report --threshold 50 \ |
66 | | - ${{ matrix.extra_args }} \ |
67 | | - ${{ matrix.skill && format('--skill {0}', matrix.skill) || '--all' }} |
| 63 | + CMD="uv run --project tests --frozen tests/evaluator.py" |
| 64 | + CMD="$CMD --provider ${{ matrix.provider }}" |
| 65 | + CMD="$CMD --model ${{ matrix.model }}" |
| 66 | + CMD="$CMD --judge --verbose --report --threshold 50" |
| 67 | + |
| 68 | + if [ -n "${{ matrix.extra_args }}" ]; then |
| 69 | + CMD="$CMD ${{ matrix.extra_args }}" |
| 70 | + fi |
| 71 | + |
| 72 | + if [ -n "${{ matrix.skill }}" ]; then |
| 73 | + CMD="$CMD --skill ${{ matrix.skill }}" |
| 74 | + else |
| 75 | + CMD="$CMD --all" |
| 76 | + fi |
| 77 | + |
| 78 | + echo "Running: $CMD" |
| 79 | + eval "$CMD" |
68 | 80 |
|
69 | 81 | - name: Upload results |
70 | 82 | if: always() |
|
0 commit comments