|
| 1 | +name: Flaky Test Analysis |
| 2 | +permissions: |
| 3 | + contents: read |
| 4 | + actions: read |
| 5 | +on: |
| 6 | + workflow_dispatch: |
| 7 | + inputs: |
| 8 | + start_date: |
| 9 | + description: 'Start date (YYYY-MM-DD). Defaults to 7 days before yesterday.' |
| 10 | + required: false |
| 11 | + end_date: |
| 12 | + description: 'End date (YYYY-MM-DD). Defaults to yesterday.' |
| 13 | + required: false |
| 14 | + top_n: |
| 15 | + description: 'Number of flakiest tests to report' |
| 16 | + required: false |
| 17 | + default: '20' |
| 18 | + window_size: |
| 19 | + description: 'Window size in days for flip rate calculation' |
| 20 | + required: false |
| 21 | + default: '1' |
| 22 | + branch: |
| 23 | + description: 'Only include runs from this branch (e.g. develop). All branches if empty.' |
| 24 | + required: false |
| 25 | +jobs: |
| 26 | + analyze: |
| 27 | + runs-on: ubuntu-latest |
| 28 | + steps: |
| 29 | + - uses: actions/checkout@v4 |
| 30 | + |
| 31 | + - uses: actions/setup-python@v5 |
| 32 | + with: |
| 33 | + python-version: '3.12' |
| 34 | + |
| 35 | + - name: Install dependencies |
| 36 | + run: pip install flaky-tests-detection |
| 37 | + |
| 38 | + - name: Run flaky test analysis |
| 39 | + id: run |
| 40 | + env: |
| 41 | + GH_TOKEN: ${{ github.token }} |
| 42 | + START_DATE: ${{ inputs.start_date }} |
| 43 | + END_DATE: ${{ inputs.end_date }} |
| 44 | + TOP_N: ${{ inputs.top_n }} |
| 45 | + WINDOW_SIZE: ${{ inputs.window_size }} |
| 46 | + BRANCH: ${{ inputs.branch }} |
| 47 | + RESULTS_FILE: flaky-results.txt |
| 48 | + run: | |
| 49 | + ./tools/flaky-test-analysis.sh |
| 50 | + echo "start=${START_DATE:-$(date -u -d '7 days ago' +%Y-%m-%d)}" >> "$GITHUB_OUTPUT" |
| 51 | + echo "end=${END_DATE:-$(date -u -d 'yesterday' +%Y-%m-%d)}" >> "$GITHUB_OUTPUT" |
| 52 | +
|
| 53 | + - name: Generate job summary |
| 54 | + if: always() |
| 55 | + run: | |
| 56 | + { |
| 57 | + echo "## Flaky Test Analysis" |
| 58 | + echo "" |
| 59 | + echo "**Date range:** \`${{ steps.run.outputs.start }}\` .. \`${{ steps.run.outputs.end }}\`" |
| 60 | + BRANCH_INFO="${{ inputs.branch }}" |
| 61 | + if [[ -n "$BRANCH_INFO" ]]; then |
| 62 | + echo "**Branch:** \`${BRANCH_INFO}\` | **Top N:** ${{ inputs.top_n || '20' }} | **Window size:** ${{ inputs.window_size || '1' }} day(s)" |
| 63 | + else |
| 64 | + echo "**Top N:** ${{ inputs.top_n || '20' }} | **Window size:** ${{ inputs.window_size || '1' }} day(s)" |
| 65 | + fi |
| 66 | + echo "" |
| 67 | +
|
| 68 | + if [[ ! -f flaky-results.txt ]]; then |
| 69 | + echo "> No results produced. Check the workflow logs." |
| 70 | + exit 0 |
| 71 | + fi |
| 72 | +
|
| 73 | + # Extract the score lines (format: suite::[It] description --- score: N) |
| 74 | + flaky_lines=$(grep -E ' --- score: ' flaky-results.txt || true) |
| 75 | +
|
| 76 | + if [[ -z "$flaky_lines" ]]; then |
| 77 | + echo "> No flaky tests detected in this period." |
| 78 | + else |
| 79 | + echo "| Score | Suite | Test |" |
| 80 | + echo "|------:|-------|------|" |
| 81 | + echo "$flaky_lines" | while IFS= read -r line; do |
| 82 | + score=$(echo "$line" | sed -E 's/.* --- score: (.+)/\1/') |
| 83 | + suite=$(echo "$line" | sed -E 's/^([^:]+)::.*/\1/') |
| 84 | + test_name=$(echo "$line" | sed -E 's/^[^:]+::\[It\] (.*) --- score:.*/\1/') |
| 85 | + echo "| ${score} | ${suite} | ${test_name} |" |
| 86 | + done |
| 87 | + fi |
| 88 | +
|
| 89 | + } >> "$GITHUB_STEP_SUMMARY" |
| 90 | +
|
| 91 | + - name: Generate failure stats summary |
| 92 | + if: always() |
| 93 | + run: python3 tools/test-failure-stats.py ./junit-reports >> "$GITHUB_STEP_SUMMARY" |
| 94 | + |
| 95 | + - name: Upload results |
| 96 | + if: always() |
| 97 | + uses: actions/upload-artifact@v4 |
| 98 | + with: |
| 99 | + name: flaky-test-results-${{ steps.run.outputs.start }}-to-${{ steps.run.outputs.end }} |
| 100 | + path: | |
| 101 | + flaky-results.txt |
| 102 | + *_flip_rate_*.png |
| 103 | + if-no-files-found: ignore |
| 104 | + retention-days: 30 |
0 commit comments