Best-effort attempt to preserve log files in e2e tests

- Tests are tagged with "supports-graceful-termination" so bazel sends a
  SIGTERM. Bazel still sends a SIGKILL after 5 seconds in this case
- TestContext.SetUp installs a signal handler that cancels the test on
  receiving a SIGINT or SIGTERM
- TestContext instances are reused between test runs to preserve the
  cancelled state
- TearDown() is called in a `defer` statement which still runs when the
  test panics due to Go's `-timeout` flag which is used by bazel.
- Commands in TestContext.TearDown are run using context.TODO to escape
  the possibly-cancelled test context

Bug: b/493728561

Author: Databean

e2etests/cvd/bugreport_tests/BUILD.bazel

@@ -41,5 +41,6 @@ go_test(
         "exclusive",
         "external",
         "no-sandbox",
+        "supports-graceful-termination",
     ],
 )

e2etests/cvd/bugreport_tests/main_test.go

@@ -23,6 +23,7 @@ import (
 func TestTakeBugreport(t *testing.T) {
 	c := e2etests.TestContext{}
 	c.SetUp(t)
+	defer c.TearDown()
 
 	if err := c.CVDFetch(e2etests.FetchArgs{
 		DefaultBuildBranch: "aosp-android-latest-release",
@@ -37,6 +38,4 @@ func TestTakeBugreport(t *testing.T) {
 	if _, err := c.RunCmd("cvd", "host_bugreport"); err != nil {
 		t.Fatal(err)
 	}
-
-	c.TearDown()
 }

e2etests/cvd/common/common.go

@@ -18,6 +18,7 @@ package e2etests
 
 import (
 	"bytes"
+	"context"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -26,9 +27,11 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"os/signal"
 	"path"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"testing"
 
 	"github.com/bazelbuild/rules_go/go/runfiles"
@@ -68,11 +71,13 @@ type TestContext struct {
 	// registered cleanup functions are invoked.
 	cleanups []func()
 	teardownCalled bool
+
+	context context.Context
+	cancelled bool
 }
 
-// Runs the given command with the given set of envvars overrided.
-func (tc *TestContext) RunCmdWithEnv(command []string, envvars map[string]string) (string, error) {
-	cmd := exec.CommandContext(tc.t.Context(), command[0], command[1:]...)
+func runCmdWithContextEnv(ctx context.Context, command []string, envvars map[string]string) (string, error) {
+	cmd := exec.CommandContext(ctx, command[0], command[1:]...)
 
 	cmdOutputBuf := bytes.Buffer{}
 	cmdWriter := io.MultiWriter(&cmdOutputBuf, log.Writer())
@@ -84,7 +89,7 @@ func (tc *TestContext) RunCmdWithEnv(command []string, envvars map[string]string
 		envvarPairs = append(envvarPairs, fmt.Sprintf("%s=%s", k, v))
 	}
 	cmd.Env = os.Environ()
-    cmd.Env = append(cmd.Env, envvarPairs...)
+	cmd.Env = append(cmd.Env, envvarPairs...)
 
 	log.Printf("Running `%s %s`\n", strings.Join(envvarPairs, " "), strings.Join(command, " "))
 	err := cmd.Run()
@@ -95,6 +100,11 @@ func (tc *TestContext) RunCmdWithEnv(command []string, envvars map[string]string
 	return cmdOutputBuf.String(), nil
 }
 
+// Runs the given command with the given set of envvars overrided.
+func (tc *TestContext) RunCmdWithEnv(command []string, envvars map[string]string) (string, error) {
+	return runCmdWithContextEnv(tc.context, command, envvars)
+}
+
 // Waits for a device to be available via adb.
 func (tc *TestContext) RunAdbWaitForDevice() error {
 	adbCommand := []string{
@@ -283,6 +293,8 @@ func  (tc *TestContext) CVDLoad(load LoadArgs) error {
 func (tc *TestContext) SetUp(t *testing.T) {
 	tc.t = t
 	tc.teardownCalled = false
+	cancellableContext, cancel := context.WithCancel(t.Context())
+	tc.context = cancellableContext
 
 	log.Printf("Initializing %s test...", tc.t.Name())
 
@@ -314,6 +326,21 @@ func (tc *TestContext) SetUp(t *testing.T) {
 
 	log.Printf("Initialized %s test!", tc.t.Name())
 
+	sigChan := make(chan os.Signal, 10)
+	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		for sig := range sigChan {
+			t.Errorf("Received signal '%s'", sig.String())
+			tc.cancelled = true
+			cancel()
+		}
+	}()
+
+	if tc.cancelled {
+		t.Fatalf("Previous test was cancelled")
+	}
+
 	tc.t.Cleanup(func() {
 		if !tc.teardownCalled {
 			tc.t.Fatalf("Test %s forgot to call TearDown().", tc.t.Name())
@@ -373,7 +400,7 @@ func (tc *TestContext) TearDown() {
 					err := os.MkdirAll(outinstancedir, os.ModePerm)
 					if err == nil {
 						logdir := path.Join(instancedir, "logs")
-						_, err := tc.RunCmd("cp", "-r", "--dereference", logdir, outinstancedir)
+						_, err := runCmdWithContextEnv(context.TODO(), []string{"cp", "-r", "--dereference", logdir, outinstancedir}, map[string]string{})
 						if err != nil {
 							log.Printf("failed to copy %s to %s: %w", logdir, outinstancedir, err)
 						}
@@ -387,7 +414,7 @@ func (tc *TestContext) TearDown() {
 		}
 	}
 
-	tc.RunCmd("cvd", "reset", "-y")
+	runCmdWithContextEnv(context.TODO(), []string{"cvd", "reset", "-y"}, map[string]string{})
 
 	log.Printf("Finished cleaning up after test!")
 }
@@ -459,6 +486,7 @@ type XtsArgs struct {
 func RunXts(t *testing.T, cuttlefishArgs FetchAndCreateArgs, xtsArgs XtsArgs) {
 	tc := TestContext{}
 	tc.SetUp(t)
+	defer tc.TearDown()
 
 	localXtsDir := findLocalXTS(cuttlefishArgs, xtsArgs)
 	if localXtsDir != "" {
@@ -511,9 +539,9 @@ func RunXts(t *testing.T, cuttlefishArgs FetchAndCreateArgs, xtsArgs XtsArgs) {
 	}
 
 	var xtsResult xtsResult
-    if err := xml.Unmarshal([]byte(xtsResultsBytes), &xtsResult); err != nil {
+	if err := xml.Unmarshal([]byte(xtsResultsBytes), &xtsResult); err != nil {
 		t.Fatalf("failed to parse XTS XML results from %s: %w", xtsResultsPath, err)
-    }
+	}
 
 	for _, xtsModule := range(xtsResult.Modules) {
 		for _, xtsTestCase := range(xtsModule.TestCases) {
@@ -529,6 +557,4 @@ func RunXts(t *testing.T, cuttlefishArgs FetchAndCreateArgs, xtsArgs XtsArgs) {
 		}
 	}
 	log.Printf("Finished parsing XTS results!", err)
-
-	tc.TearDown()
 }

e2etests/cvd/cvd_create_tests/BUILD.bazel

@@ -25,5 +25,6 @@ go_test(
         "exclusive",
         "external",
         "no-sandbox",
+        "supports-graceful-termination",
     ],
 )

e2etests/cvd/cvd_create_tests/main_test.go

@@ -35,10 +35,11 @@ func TestCvdCreate(t *testing.T) {
 			target: "aosp_cf_x86_64_only_phone-trunk_staging-userdebug",
 		},
 	}
+	c := e2etests.TestContext{}
 	for _, tc := range testcases {
 		t.Run(fmt.Sprintf("BUILD=%s/%s", tc.branch, tc.target), func(t *testing.T) {
-			c := e2etests.TestContext{}
 			c.SetUp(t)
+			defer c.TearDown()
 
 			if err := c.CVDFetch(e2etests.FetchArgs{
 				DefaultBuildBranch: tc.branch,
@@ -51,7 +52,6 @@ func TestCvdCreate(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			c.TearDown()
 		})
 	}
 }

e2etests/cvd/cvd_load_tests/BUILD.bazel

@@ -25,5 +25,6 @@ go_test(
         "exclusive",
         "external",
         "no-sandbox",
+        "supports-graceful-termination",
     ],
 )

e2etests/cvd/cvd_load_tests/main_test.go

@@ -109,9 +109,9 @@ func TestCvdLoad(t *testing.T) {
   }
 }`,
 		},
-    {
-      name: "AospMainX64PhoneX2",
-      loadconfig: `
+		{
+			name: "AospMainX64PhoneX2",
+			loadconfig: `
 {
   "instances": [
     {
@@ -134,20 +134,20 @@ func TestCvdLoad(t *testing.T) {
     "host_package": "@ab\/aosp-android-latest-release\/aosp_cf_x86_64_only_phone-userdebug"
   }
 }`,
-    },
+		},
 	}
+	c := e2etests.TestContext{}
 	for _, tc := range testcases {
 		t.Run(fmt.Sprintf("BUILD=%s", tc.name), func(t *testing.T) {
-			c := e2etests.TestContext{}
-      c.SetUp(t)
-
-			if err := c.CVDLoad(e2etests.LoadArgs{
-        LoadConfig: tc.loadconfig,
-      }); err != nil {
-        t.Fatal(err)
-      }
+			c.SetUp(t)
+			defer c.TearDown()
 
-      c.TearDown()
+			err := c.CVDLoad(e2etests.LoadArgs{
+				LoadConfig: tc.loadconfig,
+			});
+			if err != nil {
+				t.Fatal(err)
+			}
 		})
 	}
 }

e2etests/cvd/display_tests/BUILD.bazel

@@ -25,6 +25,7 @@ go_test(
         "exclusive",
         "external",
         "no-sandbox",
+        "supports-graceful-termination",
     ],
 )
 
@@ -41,5 +42,6 @@ go_test(
         "exclusive",
         "external",
         "no-sandbox",
+        "supports-graceful-termination",
     ],
 )

e2etests/cvd/display_tests/main_test.go

@@ -20,9 +20,9 @@ import (
 	"github.com/google/android-cuttlefish/e2etests/cvd/common"
 )
 
-func TestAddDisplay(t *testing.T) {
-	c := e2etests.TestContext{}
+func addDisplay(c e2etests.TestContext, t *testing.T) {
 	c.SetUp(t)
+	defer c.TearDown()
 
 	if err := c.CVDFetch(e2etests.FetchArgs{
 		DefaultBuildBranch: "aosp-android-latest-release",
@@ -38,13 +38,11 @@ func TestAddDisplay(t *testing.T) {
 	if _, err := c.RunCmd("cvd", "display", "add", "--display=width=500,height=500",); err != nil {
 		t.Fatal(err)
 	}
-
-	c.TearDown()
 }
 
-func TestListDisplays(t *testing.T) {
-	c := e2etests.TestContext{}
+func listDisplays(c e2etests.TestContext, t *testing.T) {
 	c.SetUp(t)
+	defer c.TearDown()
 
 	if err := c.CVDFetch(e2etests.FetchArgs{
 		DefaultBuildBranch: "aosp-android-latest-release",
@@ -60,6 +58,10 @@ func TestListDisplays(t *testing.T) {
 	if _, err := c.RunCmd("cvd", "display", "list",); err != nil {
 		t.Fatal(err)
 	}
+}
 
-	c.TearDown()
+func TestRunAll(t *testing.T) {
+	c := e2etests.TestContext{}
+	t.Run("AddDisplay", func(t *testing.T) { addDisplay(c, t) })
+	t.Run("ListDisplays", func(t *testing.T) { listDisplays(c, t) })
 }

e2etests/cvd/env_tests/BUILD.bazel

@@ -25,6 +25,7 @@ go_test(
         "exclusive",
         "external",
         "no-sandbox",
+        "supports-graceful-termination",
     ],
 )
 
@@ -41,5 +42,6 @@ go_test(
         "exclusive",
         "external",
         "no-sandbox",
+        "supports-graceful-termination",
     ],
 )