Skip to content

Commit 1a3b903

Browse files
committed
fix(ci): retry only failed snap test cases instead of skipping flaky tests
Revert the ignoredPlatforms skip on create-next-command-library and instead add a shared retry script that re-runs only the specific snap test cases whose snap.txt changed, up to 2 times. Closes #1209
1 parent 7feb2e7 commit 1a3b903

5 files changed

Lines changed: 42 additions & 15 deletions

File tree

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env bash
2+
# Retry only the snap test cases whose snap.txt changed, up to max_retries times.
3+
# Usage: retry-failed-snap-tests.sh [max_retries]
4+
set -euo pipefail
5+
6+
max_retries=${1:-2}
7+
8+
for retry in $(seq 1 "$max_retries"); do
9+
changed=$(git diff --name-only -- 'packages/cli/snap-tests*/*/snap.txt')
10+
if [ -z "$changed" ]; then
11+
exit 0
12+
fi
13+
14+
echo "::warning::Snapshot diff detected, retry $retry/$max_retries for failed cases..."
15+
git diff --stat -- 'packages/cli/snap-tests*/*/snap.txt'
16+
17+
failed_local=$(echo "$changed" | grep -v 'snap-tests-global/' | sed -E 's|packages/cli/snap-tests/([^/]+)/.*|\1|' | sort -u || true)
18+
failed_global=$(echo "$changed" | grep 'snap-tests-global/' | sed -E 's|packages/cli/snap-tests-global/([^/]+)/.*|\1|' | sort -u || true)
19+
20+
echo "$changed" | xargs git checkout --
21+
22+
for name in $failed_local; do
23+
echo "Retrying local snap test: $name"
24+
RUST_BACKTRACE=1 pnpm -F ./packages/cli snap-test-local "$name"
25+
done
26+
for name in $failed_global; do
27+
echo "Retrying global snap test: $name"
28+
RUST_BACKTRACE=1 pnpm -F ./packages/cli snap-test-global "$name"
29+
done
30+
done
31+
32+
# Final check after all retries
33+
if ! git diff --quiet -- 'packages/cli/snap-tests*/*/snap.txt'; then
34+
echo "::error::Snapshot diff detected after $max_retries retries. Run 'pnpm -F vite-plus snap-test' locally and commit the updated snap.txt files."
35+
git diff --stat
36+
git diff
37+
exit 1
38+
fi

.github/workflows/ci.yml

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -656,12 +656,7 @@ jobs:
656656
run: |
657657
RUST_BACKTRACE=1 pnpm -F ./packages/cli snap-test-local --shard=${{ matrix.shard }}/${{ matrix.shardTotal }}
658658
RUST_BACKTRACE=1 pnpm -F ./packages/cli snap-test-global --shard=${{ matrix.shard }}/${{ matrix.shardTotal }}
659-
if ! git diff --quiet; then
660-
echo "::error::Snapshot diff detected. Run 'pnpm -F vite-plus snap-test' locally and commit the updated snap.txt files."
661-
git diff --stat
662-
git diff
663-
exit 1
664-
fi
659+
bash .github/scripts/retry-failed-snap-tests.sh
665660
env:
666661
RUST_MIN_STACK: 8388608
667662

@@ -735,15 +730,9 @@ jobs:
735730
tsc --version
736731
vp uninstall -g typescript
737732
738-
# Run snap tests
739733
git config --global --add safe.directory /workspace
740734
RUST_BACKTRACE=1 pnpm test
741-
if ! git diff --quiet; then
742-
echo '::error::Snapshot diff detected. Run pnpm -F vite-plus snap-test locally and commit the updated snap.txt files.'
743-
git diff --stat
744-
git diff
745-
exit 1
746-
fi
735+
bash .github/scripts/retry-failed-snap-tests.sh
747736
"
748737
749738
install-e2e-test:

packages/cli/snap-tests-global/create-next-command-library/steps.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
{
2-
"ignoredPlatforms": ["win32", "darwin"],
32
"commands": [
43
{
54
"command": "vp create vite:library --no-interactive # verify next command suggests vp run",

packages/cli/snap-tests/check-fix-missing-stderr/snap.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,5 @@ Error: The `fmt` field in the default export must be an object.
1515
Ensure the file has a valid default export of a JSON-serializable configuration object.
1616

1717
Formatting failed before analysis started
18+
19+
test error, will be removed after test.

packages/cli/snap-tests/check-fix-missing-stderr/steps.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
{
2-
"ignoredPlatforms": ["win32"],
32
"commands": [
43
"vp check --fix",
54
"vp check"

0 commit comments

Comments
 (0)