VCR: Catch compilation error

.ci/magician/cmd/test_terraform_vcr.go

@@ -237,6 +237,12 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 		return nil
 	}
 
+	if hasBuildFailures, err := handleBuildFailures(prNumber, buildID, buildStatusTargetURL, mmCommitSha, replayingResult, vcr.Replaying, gh); err != nil {
+		return fmt.Errorf("error handling build failures: %w", err)
+	} else if hasBuildFailures {
+		return nil
+	}
+
 	var servicesArr []string
 	for s := range services {
 		servicesArr = append(servicesArr, s)
@@ -297,6 +303,12 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 			return nil
 		}
 
+		if hasBuildFailures, err := handleBuildFailures(prNumber, buildID, buildStatusTargetURL, mmCommitSha, recordingResult, vcr.Recording, gh); err != nil {
+			return fmt.Errorf("error handling build failures: %w", err)
+		} else if hasBuildFailures {
+			return nil
+		}
+
 		replayingAfterRecordingResult := vcr.Result{}
 		var replayingAfterRecordingErr error
 		if len(recordingResult.PassedTests) > 0 {
@@ -397,10 +409,11 @@ func notRunTests(gaDiff, betaDiff string, result vcr.Result) ([]string, []string
 
 func subtestResult(original vcr.Result) vcr.Result {
 	return vcr.Result{
-		PassedTests:  excludeCompoundTests(original.PassedTests, original.PassedSubtests),
-		FailedTests:  excludeCompoundTests(original.FailedTests, original.FailedSubtests),
-		SkippedTests: excludeCompoundTests(original.SkippedTests, original.SkippedSubtests),
-		Panics:       original.Panics,
+		PassedTests:   excludeCompoundTests(original.PassedTests, original.PassedSubtests),
+		FailedTests:   excludeCompoundTests(original.FailedTests, original.FailedSubtests),
+		SkippedTests:  excludeCompoundTests(original.SkippedTests, original.SkippedSubtests),
+		Panics:        original.Panics,
+		BuildFailures: original.BuildFailures,
 	}
 }
 
@@ -485,6 +498,7 @@ func runReplaying(runFullVCR bool, version provider.Version, services map[string
 		result.SkippedTests = append(result.SkippedTests, serviceResult.SkippedTests...)
 		result.FailedTests = append(result.FailedTests, serviceResult.FailedTests...)
 		result.Panics = append(result.Panics, serviceResult.Panics...)
+		result.BuildFailures = append(result.BuildFailures, serviceResult.BuildFailures...)
 	} else {
 		fmt.Println("runReplaying: no impacted services")
 	}
@@ -508,6 +522,28 @@ View the [build log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/head
 	return false, nil
 }
 
+func handleBuildFailures(prNumber, buildID, buildStatusTargetURL, mmCommitSha string, result vcr.Result, mode vcr.Mode, gh GithubClient) (bool, error) {
+	if len(result.BuildFailures) > 0 {
+		comment := color("red", fmt.Sprintf("The provider failed to build during VCR tests in %s mode\n", mode.Upper()))
+		comment += "\nThe following packages failed to build:\n"
+		for _, pkg := range result.BuildFailures {
+			comment += fmt.Sprintf("- `%s`\n", pkg)
+		}
+		comment += fmt.Sprintf(`
+Please fix the compilation errors to complete your PR.
+
+View the [build log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-%s/artifacts/%s/build-log/%s_test.log)`, prNumber, buildID, mode.Lower())
+		if err := gh.PostComment(prNumber, comment); err != nil {
+			return true, fmt.Errorf("error posting comment: %v", err)
+		}
+		if err := gh.PostBuildStatus(prNumber, "VCR-test", "failure", buildStatusTargetURL, mmCommitSha); err != nil {
+			return true, fmt.Errorf("error posting failure status: %v", err)
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
 func init() {
 	rootCmd.AddCommand(testTerraformVCRCmd)
 }

.ci/magician/cmd/test_terraform_vcr_test.go

@@ -579,3 +579,42 @@ func TestRecordReplay(t *testing.T) {
 		})
 	}
 }
+
+func TestHandleBuildFailures(t *testing.T) {
+	gh := &mockGithub{
+		calledMethods: make(map[string][][]any),
+	}
+	result := vcr.Result{
+		BuildFailures: []string{"package1", "package2"},
+	}
+
+	handled, err := handleBuildFailures("123", "build-456", "http://target", "sha789", result, vcr.Replaying, gh)
+
+	assert.NoError(t, err)
+	assert.True(t, handled)
+
+	assert.Len(t, gh.calledMethods["PostComment"], 1)
+	assert.Equal(t, "123", gh.calledMethods["PostComment"][0][0])
+	comment := gh.calledMethods["PostComment"][0][1].(string)
+	assert.Contains(t, comment, "package1")
+	assert.Contains(t, comment, "package2")
+
+	assert.Len(t, gh.calledMethods["PostBuildStatus"], 1)
+	assert.Equal(t, "123", gh.calledMethods["PostBuildStatus"][0][0])
+	assert.Equal(t, "VCR-test", gh.calledMethods["PostBuildStatus"][0][1])
+	assert.Equal(t, "failure", gh.calledMethods["PostBuildStatus"][0][2])
+}
+
+func TestHandleBuildFailures_NoFailures(t *testing.T) {
+	gh := &mockGithub{
+		calledMethods: make(map[string][][]any),
+	}
+	result := vcr.Result{}
+
+	handled, err := handleBuildFailures("123", "build-456", "http://target", "sha789", result, vcr.Replaying, gh)
+
+	assert.NoError(t, err)
+	assert.False(t, handled)
+	assert.Len(t, gh.calledMethods["PostComment"], 0)
+	assert.Len(t, gh.calledMethods["PostBuildStatus"], 0)
+}

.ci/magician/cmd/vcr_cassette_update.go

@@ -169,6 +169,10 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
 		return fmt.Errorf("provider crashed while running the VCR tests in REPLAYING mode: %v", replayingResult.Panics)
 	}
 
+	if len(replayingResult.BuildFailures) != 0 {
+		return fmt.Errorf("provider failed to build during VCR tests in REPLAYING mode: %v", replayingResult.BuildFailures)
+	}
+
 	if len(replayingResult.FailedTests) != 0 {
 		fmt.Println("running tests in RECORDING mode now")
 
@@ -217,6 +221,10 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
 		if len(recordingResult.Panics) != 0 {
 			return fmt.Errorf("provider crashed while running the VCR tests in RECORDING mode: %v", recordingResult.Panics)
 		}
+
+		if len(recordingResult.BuildFailures) != 0 {
+			return fmt.Errorf("provider failed to build during VCR tests in RECORDING mode: %v", recordingResult.BuildFailures)
+		}
 	}
 	return nil
 }

.ci/magician/cmd/vcr_cassette_update_test.go

@@ -433,3 +433,31 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 		})
 	}
 }
+
+func TestExecVCRCassetteUpdate_BuildFailure(t *testing.T) {
+	rnr := &mockRunner{
+		calledMethods: make(map[string][]ParameterList),
+		cwd:           "/mock/dir/magic-modules/.ci/magician",
+		dirStack:      list.New(),
+		cmdResults: map[string]string{
+			"gopath/src/github.com/hashicorp/terraform-provider-google-beta go [test  -p 16 -parallel 32 -v -run=TestAcc -timeout 360m -ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc -vet=off] map[ACCTEST_PARALLELISM:32 GOOGLE_APPLICATION_CREDENTIALS:/mock/dir/magic-modules/.ci/magician/sa_key.json GOOGLE_CREDENTIALS:sa_key GOOGLE_TEST_DIRECTORY: SA_KEY:sa_key TF_ACC:1 TF_ACC_REFRESH_AFTER_APPLY:1 TF_LOG:DEBUG TF_LOG_CORE:WARN TF_LOG_PATH_MASK:/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/%s.log TF_LOG_SDK_FRAMEWORK:INFO TF_SCHEMA_PANIC_ON_ERROR:1 VCR_MODE:REPLAYING VCR_PATH:/mock/dir/magic-modules/.ci/magician/cassettes/beta]": "FAIL\tgithub.com/hashicorp/terraform-provider-google-beta/google-beta/services/corebilling [build failed]",
+		},
+	}
+
+	ctlr := source.NewController("gopath", "hashicorp", "token", rnr)
+	vt, err := vcr.NewTester(map[string]string{
+		"SA_KEY": "sa_key",
+	}, "ci-vcr-cassettes", "", rnr, false)
+	if err != nil {
+		t.Fatalf("Failed to create new tester: %v", err)
+	}
+
+	err = execVCRCassetteUpdate("buildID", "2024-07-08", rnr, ctlr, vt)
+	if err == nil {
+		t.Fatalf("execVCRCassetteUpdate expected to return error on build failure, got nil")
+	}
+
+	if !strings.Contains(err.Error(), "provider failed to build during VCR tests in REPLAYING mode") {
+		t.Errorf("Unexpected error message: %v", err)
+	}
+}

.ci/magician/vcr/tester.go

@@ -24,6 +24,7 @@ type Result struct {
 	SkippedSubtests []string
 	FailedSubtests  []string
 	Panics          []string
+	BuildFailures   []string
 }
 
 type Mode int
@@ -82,6 +83,8 @@ var subtestResultsExpression = regexp.MustCompile(`(?m:^    --- (PASS|FAIL|SKIP)
 
 var testPanicExpression = regexp.MustCompile(`(?m:^panic: .*)`)
 
+var buildFailedExpression = regexp.MustCompile(`FAIL\s+(\S+)\s+\[build failed\]`)
+
 var safeToLog = map[string]bool{
 	"ACCTEST_PARALLELISM":                        true,
 	"COMMIT_SHA":                                 true,
@@ -723,9 +726,18 @@ func collectResult(output string) Result {
 		}
 		subtestResultSets[submatches[1]][fmt.Sprintf("%s__%s", submatches[2], submatches[3])] = struct{}{}
 	}
-	results := make(map[string][]string, 4)
+	results := make(map[string][]string, 5)
 	results["PANIC"] = testPanicExpression.FindAllString(output, -1)
 	sort.Strings(results["PANIC"])
+
+	buildFailuresMatches := buildFailedExpression.FindAllStringSubmatch(output, -1)
+	for _, submatches := range buildFailuresMatches {
+		if len(submatches) == 2 {
+			results["BUILD_FAILURE"] = append(results["BUILD_FAILURE"], filepath.Base(submatches[1]))
+		}
+	}
+	sort.Strings(results["BUILD_FAILURE"])
+
 	subtestResults := make(map[string][]string, 3)
 	for _, kind := range []string{"FAIL", "PASS", "SKIP"} {
 		for test := range resultSets[kind] {
@@ -745,5 +757,6 @@ func collectResult(output string) Result {
 		PassedSubtests:  subtestResults["PASS"],
 		SkippedSubtests: subtestResults["SKIP"],
 		Panics:          results["PANIC"],
+		BuildFailures:   results["BUILD_FAILURE"],
 	}
 }

.ci/magician/vcr/tester_test.go

@@ -49,6 +49,30 @@ func TestCollectResults(t *testing.T) {
 				FailedSubtests: []string{"TestAccServiceOneResourceTwo__test_two"},
 			},
 		},
+		{
+			name: "build failure",
+			output: `FAIL	github.com/hashicorp/terraform-provider-google-beta/google-beta/services/corebilling [build failed]
+--- PASS: TestAccServiceTwoResourceOne (100.00s)
+`,
+			expected: Result{
+				PassedTests:   []string{"TestAccServiceTwoResourceOne"},
+				BuildFailures: []string{"corebilling"},
+			},
+		},
+		{
+			name: "build failure in middle",
+			output: `Error replaying tests:
+error running go: exit status 1
+stdout:
+FAIL	github.com/hashicorp/terraform-provider-google-beta/google-beta/services/corebilling [build failed]
+FAIL
+stderr:
+go: downloading ...
+`,
+			expected: Result{
+				BuildFailures: []string{"corebilling"},
+			},
+		},
 	} {
 		if diff := cmp.Diff(test.expected, collectResult(test.output)); diff != "" {
 			t.Errorf("collectResult(%q) got unexpected diff (-want +got):\n%s", test.output, diff)