allocRunner: add restore test when allocDir is gone

Author: rodrigol-chan

client/allocrunner/alloc_runner_unix_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client/allocdir"
 	regMock "github.com/hashicorp/nomad/client/serviceregistration/mock"
 	"github.com/hashicorp/nomad/client/state"
 	"github.com/hashicorp/nomad/client/taskenv"
@@ -108,6 +109,8 @@ func TestAllocRunner_Restore_RunningTerminal(t *testing.T) {
 
 	// Use original statedb to maintain hook state
 	conf2.StateDB = conf.StateDB
+	conf2.ClientConfig.AllocDir = conf.ClientConfig.AllocDir
+	conf2.ClientConfig.AllocMountsDir = conf.ClientConfig.AllocMountsDir
 
 	// Restore, start, and wait for task to be killed
 	ar2Iface, err := NewAllocRunner(conf2)
@@ -203,6 +206,8 @@ func TestAllocRunner_Restore_CompletedBatch(t *testing.T) {
 
 	// Use original statedb to maintain hook state
 	conf2.StateDB = conf.StateDB
+	conf2.ClientConfig.AllocDir = conf.ClientConfig.AllocDir
+	conf2.ClientConfig.AllocMountsDir = conf.ClientConfig.AllocMountsDir
 
 	// Restore, start, and wait for task to be killed
 	ar2Iface, err := NewAllocRunner(conf2)
@@ -307,3 +312,80 @@ func (*allocFailingPrestartHook) Name() string { return "failing_prestart" }
 func (*allocFailingPrestartHook) Prerun(_ *taskenv.TaskEnv) error {
 	return fmt.Errorf("failing prestart hooks")
 }
+
+// TestAllocRunner_Restore_MissingAllocDir asserts that restoring a running
+// system alloc fails if client.alloc_dir was wiped.
+func TestAllocRunner_Restore_MissingAllocDir(t *testing.T) {
+	ci.Parallel(t)
+
+	alloc := mock.Alloc()
+	alloc.Job.Type = structs.JobTypeSystem
+	taskName := alloc.Job.TaskGroups[0].Tasks[0].Name
+	alloc.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver"
+	alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{
+		"run_for": "1h",
+	}
+
+	conf, cleanup := testAllocRunnerConfig(t, alloc.Copy())
+	defer cleanup()
+
+	// Persist task hook state across runner restarts to simulate data_dir.
+	conf.StateDB = state.NewMemDB(conf.Logger)
+
+	arIface, err := NewAllocRunner(conf)
+	must.NoError(t, err)
+	ar := arIface.(*allocRunner)
+
+	go ar.Run()
+	defer destroy(ar)
+
+	testutil.WaitForResult(func() (bool, error) {
+		state := ar.AllocState()
+		ts, ok := state.TaskStates[taskName]
+		if !ok || ts == nil {
+			return false, fmt.Errorf("expected task to be running")
+		}
+		if ts.State != structs.TaskStateRunning {
+			return false, fmt.Errorf("expected task to be running")
+		}
+		return true, nil
+	}, func(err error) {
+		require.NoError(t, err)
+	})
+
+	allocDirPath := ar.GetAllocDir().AllocDirPath()
+
+	ar.Shutdown()
+	select {
+	case <-ar.ShutdownCh():
+	case <-time.After(30 * time.Second):
+		require.Fail(t, "AR took too long to exit")
+	}
+
+	// Unmount dangling mounts to allow folder deletion in this test process suite
+	allocDirImpl, _ := ar.GetAllocDir().(*allocdir.AllocDir)
+	if allocDirImpl != nil {
+		require.NoError(t, allocDirImpl.UnmountAll())
+	}
+	require.NoError(t, os.RemoveAll(allocDirPath))
+
+	conf2, cleanup2 := testAllocRunnerConfig(t, alloc.Copy())
+	defer cleanup2()
+
+	// Reuse persistent state and the same alloc storage path after restart.
+	conf2.StateDB = conf.StateDB
+	conf2.ClientConfig.AllocDir = conf.ClientConfig.AllocDir
+	conf2.ClientConfig.AllocMountsDir = conf.ClientConfig.AllocMountsDir
+
+	ar2Iface, err := NewAllocRunner(conf2)
+	must.NoError(t, err)
+	ar2 := ar2Iface.(*allocRunner)
+	defer destroy(ar2)
+
+	err = ar2.Restore()
+	must.ErrorContains(t, err, "allocation directory is inacessible")
+
+	// Usually set by client.go restoreState handle code
+	ar2.SetClientStatus(structs.AllocClientStatusFailed)
+	must.Eq(t, structs.AllocClientStatusFailed, ar2.AllocState().ClientStatus)
+}