[NoQA] Investigate Reassure flakiness v2 #469
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Reassure Performance Tests | |
| on: | |
| pull_request: | |
| types: [opened, synchronize] | |
| branches-ignore: [staging, production] | |
| paths-ignore: ['**.md', '**.sh'] | |
| jobs: | |
| # Note: We run baseline and delta performance checks in the same runner to reduce hardware variance across machines | |
| perf-tests: | |
| if: ${{ github.actor != 'OSBotify' }} | |
| runs-on: ubuntu-24.04-v4 | |
| steps: | |
| # v4 | |
| - name: Checkout | |
| uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 | |
| with: | |
| ref: ${{ github.event.pull_request.base.ref }} | |
| # fetch-depth: 0 | |
| # v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e | |
| with: | |
| node-version-file: '.nvmrc' | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Reassure stability check | |
| run: npx reassure check-stability --verbose | |
| - name: Checkout PR head SHA | |
| run: | | |
| git fetch origin ${{ github.event.pull_request.head.sha }} --no-tags --depth=1 | |
| git switch --force --detach ${{ github.event.pull_request.head.sha }} | |
| - name: Reinstall dependencies | |
| run: npm ci | |
| - name: Run Reassure delta tests | |
| run: npx reassure --branch --verbose | |
| # - name: Reassure tests | |
| # run: | | |
| # git log --oneline --graph --decorate --all -n 10 | |
| # npm ci | |
| # npx reassure --baseline --verbose | |
| # git fetch origin ${{ github.event.pull_request.head.sha }} --no-tags --depth=1 | |
| # git switch --force --detach ${{ github.event.pull_request.head.sha }} | |
| # git log --oneline --graph --decorate --all -n 10 | |
| # npm ci | |
| # npx reassure --branch --verbose | |
| # - name: Set dummy git credentials | |
| # run: | | |
| # git config --global user.email "test@test.com" | |
| # git config --global user.name "Test" | |
| # - name: Get common ancestor commit | |
| # run: | | |
| # git fetch origin main | |
| # common_ancestor=$(git merge-base "${{ github.sha }}" origin/main) | |
| # echo "COMMIT_HASH=$common_ancestor" >> "$GITHUB_ENV" | |
| # - name: Clean up deleted files | |
| # run: | | |
| # DELETED_FILES=$(git diff --name-only --diff-filter=D "$COMMIT_HASH" "${{ github.sha }}") | |
| # for file in $DELETED_FILES; do | |
| # if [ -n "$file" ]; then | |
| # rm -f "$file" | |
| # echo "Deleted file: $file" | |
| # fi | |
| # done | |
| # - name: Run performance testing script | |
| # shell: bash | |
| # run: | | |
| # set -e | |
| # BASELINE_BRANCH=${BASELINE_BRANCH:="main"} | |
| # git fetch origin "$BASELINE_BRANCH" --no-tags --depth=1 | |
| # git switch "$BASELINE_BRANCH" | |
| # npm install --force || (rm -rf node_modules && npm install --force) | |
| # NODE_OPTIONS=--experimental-vm-modules npx reassure --baseline | |
| # git switch --force --detach - | |
| # git merge --no-commit --allow-unrelated-histories "$BASELINE_BRANCH" -X ours | |
| # git checkout --ours . | |
| # npm install --force || (rm -rf node_modules && npm install --force) | |
| # NODE_OPTIONS=--experimental-vm-modules npx reassure --branch | |
| - name: Upload Reassure results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results | |
| path: .reassure/ | |
| if-no-files-found: ignore | |
| include-hidden-files: true | |
| - name: Validate output.json | |
| id: validateReassureOutput | |
| uses: ./.github/actions/javascript/validateReassureOutput | |
| with: | |
| ALLOWED_DURATION_DEVIATION: 10 | |
| ALLOWED_RELATIVE_DURATION_DEVIATION: 20 |