Skip to content

Commit fb19aef

Browse files
claudeclaude
committed
Fix e2e diff validation test for proper pollution detection
- Each feature in a stack now changes a DIFFERENT line (lines 3, 4, 5 for noact_ stack and lines 6, 7, 2 for act_ stack) - This ensures after retarget, the diff clearly shows accumulated changes (2+ lines instead of just 1) - Manually retarget PR to main before deleting branch (GitHub doesn't auto-retarget when base branch is deleted) - Use ^+ prefix in grep patterns to only match actual added lines, not context lines in the diff - Add pollution detection: verify PR2 doesn't show Feature 1's changes before action runs, and still doesn't show them after
1 parent da17bea commit fb19aef

1 file changed

Lines changed: 89 additions & 44 deletions

File tree

‎tests/test_e2e.sh‎

Lines changed: 89 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -482,9 +482,11 @@ echo >&2 "--- SCENARIO 0: Diff Validation Test ---"
482482
echo >&2 "0a. Creating 'no action' stack to verify diffs break without the action..."
483483

484484
# Create 3 PRs for the no-action test (using prefix 'noact_')
485+
# IMPORTANT: Each feature changes a DIFFERENT line so that after retarget,
486+
# the diff clearly shows accumulated changes (multiple lines instead of just one)
485487
log_cmd git checkout main
486488
log_cmd git checkout -b noact_feature1 main
487-
sed -i '3s/.*/NoAct Feature 1 line 3/' file.txt
489+
sed -i '3s/.*/NoAct Feature 1 line 3/' file.txt # Feature 1 changes LINE 3
488490
log_cmd git add file.txt
489491
log_cmd git commit -m "NoAct: Add feature 1"
490492
log_cmd git push origin noact_feature1
@@ -493,7 +495,7 @@ NOACT_PR1_NUM=$(echo "$NOACT_PR1_URL" | awk -F'/' '{print $NF}')
493495
echo >&2 "Created NoAct PR #$NOACT_PR1_NUM: $NOACT_PR1_URL"
494496

495497
log_cmd git checkout -b noact_feature2 noact_feature1
496-
sed -i '3s/.*/NoAct Feature 2 line 3/' file.txt
498+
sed -i '4s/.*/NoAct Feature 2 line 4/' file.txt # Feature 2 changes LINE 4 (different!)
497499
log_cmd git add file.txt
498500
log_cmd git commit -m "NoAct: Add feature 2"
499501
log_cmd git push origin noact_feature2
@@ -502,7 +504,7 @@ NOACT_PR2_NUM=$(echo "$NOACT_PR2_URL" | awk -F'/' '{print $NF}')
502504
echo >&2 "Created NoAct PR #$NOACT_PR2_NUM: $NOACT_PR2_URL"
503505

504506
log_cmd git checkout -b noact_feature3 noact_feature2
505-
sed -i '3s/.*/NoAct Feature 3 line 3/' file.txt
507+
sed -i '5s/.*/NoAct Feature 3 line 5/' file.txt # Feature 3 changes LINE 5 (different!)
506508
log_cmd git add file.txt
507509
log_cmd git commit -m "NoAct: Add feature 3"
508510
log_cmd git push origin noact_feature3
@@ -517,15 +519,19 @@ NOACT_PR2_DIFF_INITIAL=$(get_pr_diff "$NOACT_PR2_URL")
517519
NOACT_PR3_DIFF_INITIAL=$(get_pr_diff "$NOACT_PR3_URL")
518520

519521
# Verify each PR initially shows only its own single line change
522+
# PR1 should show "NoAct Feature 1", PR2 should show "NoAct Feature 2", etc.
520523
echo >&2 "Verifying initial diffs show only 1 line change each..."
521-
NOACT_PR1_LINES=$(echo "$NOACT_PR1_DIFF_INITIAL" | grep -c '^+NoAct Feature' || true)
522-
NOACT_PR2_LINES=$(echo "$NOACT_PR2_DIFF_INITIAL" | grep -c '^+NoAct Feature' || true)
523-
NOACT_PR3_LINES=$(echo "$NOACT_PR3_DIFF_INITIAL" | grep -c '^+NoAct Feature' || true)
524+
NOACT_PR1_CHANGES=$(echo "$NOACT_PR1_DIFF_INITIAL" | grep -c '^+NoAct Feature 1' || true)
525+
NOACT_PR2_CHANGES=$(echo "$NOACT_PR2_DIFF_INITIAL" | grep -c '^+NoAct Feature 2' || true)
526+
NOACT_PR3_CHANGES=$(echo "$NOACT_PR3_DIFF_INITIAL" | grep -c '^+NoAct Feature 3' || true)
524527

525-
if [[ "$NOACT_PR1_LINES" -eq 1 && "$NOACT_PR2_LINES" -eq 1 && "$NOACT_PR3_LINES" -eq 1 ]]; then
526-
echo >&2 "✅ Initial diffs correct: each PR shows exactly 1 line change"
528+
# Also verify NO cross-contamination (PR2 shouldn't show Feature 1's changes)
529+
NOACT_PR2_POLLUTION=$(echo "$NOACT_PR2_DIFF_INITIAL" | grep -c '^+NoAct Feature 1' || true)
530+
531+
if [[ "$NOACT_PR1_CHANGES" -eq 1 && "$NOACT_PR2_CHANGES" -eq 1 && "$NOACT_PR3_CHANGES" -eq 1 && "$NOACT_PR2_POLLUTION" -eq 0 ]]; then
532+
echo >&2 "✅ Initial diffs correct: each PR shows exactly its own 1 line change"
527533
else
528-
echo >&2 "❌ Initial diffs incorrect: PR1=$NOACT_PR1_LINES, PR2=$NOACT_PR2_LINES, PR3=$NOACT_PR3_LINES (expected 1 each)"
534+
echo >&2 "❌ Initial diffs incorrect: PR1=$NOACT_PR1_CHANGES, PR2=$NOACT_PR2_CHANGES, PR3=$NOACT_PR3_CHANGES, PR2 pollution=$NOACT_PR2_POLLUTION"
529535
exit 1
530536
fi
531537

@@ -534,35 +540,54 @@ echo >&2 "0c. Merging NoAct PR1 (without action installed)..."
534540
merge_pr_with_retry "$NOACT_PR1_URL"
535541
echo >&2 "NoAct PR1 merged."
536542

537-
# Wait a moment for GitHub to update PR state
538-
sleep 5
539-
540-
# Verify diffs are now BROKEN (PR2 should show 2 line changes: its own + the deleted base diff)
541-
echo >&2 "0d. Verifying diffs are BROKEN after merge (without action)..."
542-
NOACT_PR2_DIFF_AFTER_MERGE=$(get_pr_diff "$NOACT_PR2_URL")
543-
NOACT_PR3_DIFF_AFTER_MERGE=$(get_pr_diff "$NOACT_PR3_URL")
544-
545-
# PR2's base was noact_feature1 which no longer exists. GitHub retargets to main.
546-
# The diff should now show BOTH noact_feature1's changes AND noact_feature2's changes = 2 line changes
547-
# Actually check for more than 1 "NoAct Feature" line (proving the diff is now polluted)
548-
NOACT_PR2_LINES_AFTER=$(echo "$NOACT_PR2_DIFF_AFTER_MERGE" | grep -c '^[-+]NoAct Feature' || true)
549-
550-
# Note: When base branch is deleted, GitHub auto-retargets to default branch (main).
551-
# This causes PR2's diff to include changes from BOTH feature1 AND feature2.
552-
# The diff should show "Feature 1" being removed (from main's perspective) and "Feature 2" being added.
553-
# Or it might show both as additions depending on exact git state.
554-
# The key point: the diff is DIFFERENT from the initial diff.
555-
556-
if [[ "$NOACT_PR2_DIFF_AFTER_MERGE" != "$NOACT_PR2_DIFF_INITIAL" ]]; then
557-
echo >&2 "✅ Confirmed: PR2 diff changed after merge (broken state demonstrated)"
558-
echo >&2 " Initial diff lines: $(echo "$NOACT_PR2_DIFF_INITIAL" | wc -l)"
559-
echo >&2 " After merge lines: $(echo "$NOACT_PR2_DIFF_AFTER_MERGE" | wc -l)"
543+
# Manually retarget PR2 to main to simulate the "broken" state.
544+
# This must be done BEFORE deleting the branch to keep the PR open.
545+
#
546+
# In practice, this happens when:
547+
# - GitHub auto-retargets (depending on repo settings)
548+
# - A user manually changes the base branch
549+
# - A tool like "gh pr edit --base" is used
550+
#
551+
# Without the autorestack action, when you retarget to main, the diff becomes
552+
# "polluted" because it now shows ALL changes from the head branch relative to main,
553+
# not just the incremental changes from the previous PR in the stack.
554+
echo >&2 "0d. Retargeting PR2 to main to demonstrate broken diff state..."
555+
log_cmd gh pr edit "$NOACT_PR2_NUM" --repo "$REPO_FULL_NAME" --base main
556+
557+
# Wait for GitHub to process the base change
558+
sleep 3
559+
560+
NOACT_PR2_DIFF_AFTER_RETARGET=$(get_pr_diff "$NOACT_PR2_URL")
561+
562+
# Debug: Show the actual diffs to see the difference
563+
echo >&2 "--- Initial PR2 diff (vs noact_feature1) ---"
564+
echo "$NOACT_PR2_DIFF_INITIAL" >&2
565+
echo >&2 "--- After retarget PR2 diff (vs main) ---"
566+
echo "$NOACT_PR2_DIFF_AFTER_RETARGET" >&2
567+
echo >&2 "------------------------"
568+
569+
# The diff should now be "polluted":
570+
# - Initial diff (vs noact_feature1): shows only Feature2's line 4 change
571+
# - After retarget (vs main): shows BOTH Feature1's line 3 AND Feature2's line 4 changes
572+
# This is the "broken" state - the PR now shows accumulated changes instead of incremental.
573+
574+
# Check for pollution: after retarget, PR2's diff should now include Feature1's changes
575+
NOACT_PR2_POLLUTION_AFTER=$(echo "$NOACT_PR2_DIFF_AFTER_RETARGET" | grep -c 'NoAct Feature 1' || true)
576+
577+
if [[ "$NOACT_PR2_POLLUTION_AFTER" -gt 0 ]]; then
578+
echo >&2 "✅ Confirmed: PR2 diff is now POLLUTED with Feature1's changes (broken state demonstrated)"
579+
echo >&2 " Initial: only Feature2 changes visible"
580+
echo >&2 " After retarget: Feature1 changes also visible (pollution=$NOACT_PR2_POLLUTION_AFTER)"
560581
else
561-
echo >&2 "❌ Unexpected: PR2 diff did NOT change after merge. Cannot demonstrate broken state."
562-
echo >&2 "This might happen if GitHub's auto-retargeting behavior changed."
582+
echo >&2 "❌ Unexpected: PR2 diff does NOT show Feature1's changes after retarget."
583+
echo >&2 "Expected the diff to be polluted with accumulated changes."
563584
exit 1
564585
fi
565586

587+
# Now delete the merged branch (cleanup)
588+
echo >&2 "Deleting noact_feature1 branch..."
589+
log_cmd git push origin --delete noact_feature1
590+
566591
# --- Part B: Install the action and create a new stack ---
567592
echo >&2 "0e. Installing action and workflow..."
568593

@@ -632,8 +657,9 @@ log_cmd git push origin main
632657
echo >&2 "0f. Creating 'with action' stack to verify diffs are preserved..."
633658

634659
# Create 3 PRs for the with-action test (using prefix 'act_')
660+
# IMPORTANT: Each feature changes a DIFFERENT line (using 6, 7 to avoid overlap with noact_ stack's 3, 4, 5)
635661
log_cmd git checkout -b act_feature1 main
636-
sed -i '4s/.*/Act Feature 1 line 4/' file.txt
662+
sed -i '6s/.*/Act Feature 1 line 6/' file.txt # Feature 1 changes LINE 6
637663
log_cmd git add file.txt
638664
log_cmd git commit -m "Act: Add feature 1"
639665
log_cmd git push origin act_feature1
@@ -642,7 +668,7 @@ ACT_PR1_NUM=$(echo "$ACT_PR1_URL" | awk -F'/' '{print $NF}')
642668
echo >&2 "Created Act PR #$ACT_PR1_NUM: $ACT_PR1_URL"
643669

644670
log_cmd git checkout -b act_feature2 act_feature1
645-
sed -i '4s/.*/Act Feature 2 line 4/' file.txt
671+
sed -i '7s/.*/Act Feature 2 line 7/' file.txt # Feature 2 changes LINE 7 (different!)
646672
log_cmd git add file.txt
647673
log_cmd git commit -m "Act: Add feature 2"
648674
log_cmd git push origin act_feature2
@@ -651,7 +677,7 @@ ACT_PR2_NUM=$(echo "$ACT_PR2_URL" | awk -F'/' '{print $NF}')
651677
echo >&2 "Created Act PR #$ACT_PR2_NUM: $ACT_PR2_URL"
652678

653679
log_cmd git checkout -b act_feature3 act_feature2
654-
sed -i '4s/.*/Act Feature 3 line 4/' file.txt
680+
sed -i '2s/.*/Act Feature 3 line 2/' file.txt # Feature 3 changes LINE 2 (different!)
655681
log_cmd git add file.txt
656682
log_cmd git commit -m "Act: Add feature 3"
657683
log_cmd git push origin act_feature3
@@ -665,15 +691,19 @@ ACT_PR1_DIFF_INITIAL=$(get_pr_diff "$ACT_PR1_URL")
665691
ACT_PR2_DIFF_INITIAL=$(get_pr_diff "$ACT_PR2_URL")
666692
ACT_PR3_DIFF_INITIAL=$(get_pr_diff "$ACT_PR3_URL")
667693

668-
# Verify initial diffs are correct (1 line change each)
669-
ACT_PR1_LINES=$(echo "$ACT_PR1_DIFF_INITIAL" | grep -c '^+Act Feature' || true)
670-
ACT_PR2_LINES=$(echo "$ACT_PR2_DIFF_INITIAL" | grep -c '^+Act Feature' || true)
671-
ACT_PR3_LINES=$(echo "$ACT_PR3_DIFF_INITIAL" | grep -c '^+Act Feature' || true)
694+
# Verify initial diffs are correct (each PR shows only its own 1 line change)
695+
ACT_PR1_CHANGES=$(echo "$ACT_PR1_DIFF_INITIAL" | grep -c '^+Act Feature 1' || true)
696+
ACT_PR2_CHANGES=$(echo "$ACT_PR2_DIFF_INITIAL" | grep -c '^+Act Feature 2' || true)
697+
ACT_PR3_CHANGES=$(echo "$ACT_PR3_DIFF_INITIAL" | grep -c '^+Act Feature 3' || true)
698+
699+
# Also verify NO cross-contamination (PR2's diff shouldn't ADD Feature 1's changes)
700+
# Use ^+ to only match actual additions, not context lines
701+
ACT_PR2_POLLUTION=$(echo "$ACT_PR2_DIFF_INITIAL" | grep -c '^+.*Act Feature 1' || true)
672702

673-
if [[ "$ACT_PR1_LINES" -eq 1 && "$ACT_PR2_LINES" -eq 1 && "$ACT_PR3_LINES" -eq 1 ]]; then
674-
echo >&2 "✅ Initial diffs correct: each Act PR shows exactly 1 line change"
703+
if [[ "$ACT_PR1_CHANGES" -eq 1 && "$ACT_PR2_CHANGES" -eq 1 && "$ACT_PR3_CHANGES" -eq 1 && "$ACT_PR2_POLLUTION" -eq 0 ]]; then
704+
echo >&2 "✅ Initial diffs correct: each Act PR shows exactly its own 1 line change"
675705
else
676-
echo >&2 "❌ Initial diffs incorrect: PR1=$ACT_PR1_LINES, PR2=$ACT_PR2_LINES, PR3=$ACT_PR3_LINES (expected 1 each)"
706+
echo >&2 "❌ Initial diffs incorrect: PR1=$ACT_PR1_CHANGES, PR2=$ACT_PR2_CHANGES, PR3=$ACT_PR3_CHANGES, PR2 pollution=$ACT_PR2_POLLUTION"
677707
exit 1
678708
fi
679709

@@ -695,6 +725,21 @@ echo >&2 "0j. Verifying diffs are PRESERVED after action ran..."
695725
ACT_PR2_DIFF_AFTER=$(get_pr_diff "$ACT_PR2_URL")
696726
ACT_PR3_DIFF_AFTER=$(get_pr_diff "$ACT_PR3_URL")
697727

728+
# Debug: show the diffs
729+
echo >&2 "--- Act PR2 diff after action ---"
730+
echo "$ACT_PR2_DIFF_AFTER" >&2
731+
echo >&2 "------------------------"
732+
733+
# Verify no pollution (PR2's diff should still not ADD Feature 1's changes)
734+
# Use ^+ to only match actual additions, not context lines
735+
ACT_PR2_POLLUTION_AFTER=$(echo "$ACT_PR2_DIFF_AFTER" | grep -c '^+.*Act Feature 1' || true)
736+
if [[ "$ACT_PR2_POLLUTION_AFTER" -gt 0 ]]; then
737+
echo >&2 "❌ Act PR2 diff is polluted with Feature 1's changes after action"
738+
echo >&2 "The action should preserve incremental diffs, but pollution found."
739+
exit 1
740+
fi
741+
echo >&2 "✅ Act PR2 diff has no pollution (Feature 1 not added in diff)"
742+
698743
if compare_diffs "$ACT_PR2_DIFF_INITIAL" "$ACT_PR2_DIFF_AFTER" "Act PR2 diff preserved"; then
699744
echo >&2 "✅ Act PR2 diff is identical before and after merge+action"
700745
else

0 commit comments

Comments
 (0)