fix: data races and test fixes exposed by ARM64 testing

ARM64's weaker memory model reliably triggers races that x86 papers over:

- clean-nfs-cache: fd use-after-close between Scanner and Statter
  goroutines — pass directory path string instead of *os.File
- nbd/path_direct: loop variable capture in goroutine closure
- envd conversion_test: shared connect.Response across parallel subtests
  — use RunAndReturn to create fresh response per call
- errorcollector_test: plain bool and ctx variable reuse in concurrent
  test — use atomic.Bool and distinct context variables
- db/testutils: goose v3.26 SetDialect() races on package globals when
  parallel tests run migrations — serialize with sync.Mutex
- uffd/page_mmap: graceful skip on hugepage ENOMEM in CI
- async_wp_test: skip UFFD write-protection test on ARM64 (unsupported)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 2 people

packages/db/pkg/testutils/db.go

@@ -5,6 +5,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -106,6 +107,12 @@ func SetupDatabase(t *testing.T) *Database {
 	}
 }
 
+// gooseMu serializes goose operations across parallel tests.
+// goose.OpenDBWithDriver calls goose.SetDialect which writes to package-level
+// globals (dialect, store) without synchronization. Concurrent test goroutines
+// race on these globals, triggering the race detector on ARM64.
+var gooseMu sync.Mutex
+
 // runDatabaseMigrations executes all required database migrations
 func runDatabaseMigrations(t *testing.T, connStr string) {
 	t.Helper()
@@ -115,6 +122,9 @@ func runDatabaseMigrations(t *testing.T, connStr string) {
 	require.NoError(t, err, "Failed to find git root")
 	repoRoot := strings.TrimSpace(string(output))
 
+	gooseMu.Lock()
+	defer gooseMu.Unlock()
+
 	db, err := goose.OpenDBWithDriver("pgx", connStr)
 	require.NoError(t, err)
 	t.Cleanup(func() {

packages/envd/internal/services/legacy/conversion_test.go

@@ -2,6 +2,7 @@ package legacy
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -22,12 +23,19 @@ import (
 func TestFilesystemClient_FieldFormatter(t *testing.T) {
 	t.Parallel()
 	fsh := filesystemconnectmocks.NewMockFilesystemHandler(t)
-	fsh.EXPECT().Move(mock.Anything, mock.Anything).Return(connect.NewResponse(&filesystem.MoveResponse{
-		Entry: &filesystem.EntryInfo{
-			Name:  "test-name",
-			Owner: "new-extra-field",
+	// Use RunAndReturn to create a fresh response per call. Using Return()
+	// shares one Response across parallel subtests, causing a data race on
+	// the lazily-initialized header/trailer maps inside connect.Response.
+	fsh.EXPECT().Move(mock.Anything, mock.Anything).RunAndReturn(
+		func(_ context.Context, _ *connect.Request[filesystem.MoveRequest]) (*connect.Response[filesystem.MoveResponse], error) {
+			return connect.NewResponse(&filesystem.MoveResponse{
+				Entry: &filesystem.EntryInfo{
+					Name:  "test-name",
+					Owner: "new-extra-field",
+				},
+			}), nil
 		},
-	}), nil)
+	)
 
 	_, handler := filesystemconnect.NewFilesystemHandler(fsh,
 		connect.WithInterceptors(

packages/orchestrator/cmd/clean-nfs-cache/cleaner/clean.go

@@ -99,7 +99,7 @@ type Candidate struct {
 }
 
 type statReq struct {
-	df       *os.File
+	dirPath  string
 	name     string
 	response chan *statReq
 	f        *File

packages/orchestrator/cmd/clean-nfs-cache/cleaner/scan.go

@@ -60,7 +60,7 @@ func (c *Cleaner) Statter(ctx context.Context, done *sync.WaitGroup) {
 		case <-ctx.Done():
 			return
 		case req := <-c.statRequestCh:
-			f, err := c.statInDir(req.df, req.name)
+			f, err := c.statInDir(req.dirPath, req.name)
 			req.f = f
 			req.err = err
 			req.response <- req
@@ -201,13 +201,16 @@ func (c *Cleaner) scanDir(ctx context.Context, path []*Dir) (out *Dir, err error
 		}
 	}
 
-	// submit all stat requests
+	// Submit stat requests using the directory path (not the *os.File).
+	// The file descriptor df is closed when scanDir returns (defer above),
+	// but Statter goroutines may still be processing requests concurrently.
+	// Passing the path avoids a race between df.Close() and df.Fd().
 	responseCh := make(chan *statReq, len(filenames))
 	for _, name := range filenames {
 		select {
 		case <-ctx.Done():
 			return nil, ctx.Err()
-		case c.statRequestCh <- &statReq{df: df, name: name, response: responseCh}:
+		case c.statRequestCh <- &statReq{dirPath: absPath, name: name, response: responseCh}:
 			// submitted
 		}
 	}

packages/orchestrator/cmd/clean-nfs-cache/cleaner/stat_linux.go

@@ -4,7 +4,7 @@ package cleaner
 
 import (
 	"fmt"
-	"os"
+	"path/filepath"
 
 	"golang.org/x/sys/unix"
 )
@@ -30,11 +30,11 @@ func (c *Cleaner) stat(fullPath string) (*Candidate, error) {
 	}, nil
 }
 
-func (c *Cleaner) statInDir(df *os.File, filename string) (*File, error) {
+func (c *Cleaner) statInDir(dirPath string, filename string) (*File, error) {
 	c.StatxC.Add(1)
 	c.StatxInDirC.Add(1)
 	var statx unix.Statx_t
-	err := unix.Statx(int(df.Fd()), filename,
+	err := unix.Statx(unix.AT_FDCWD, filepath.Join(dirPath, filename),
 		unix.AT_STATX_DONT_SYNC|unix.AT_SYMLINK_NOFOLLOW|unix.AT_NO_AUTOMOUNT,
 		unix.STATX_ATIME|unix.STATX_SIZE,
 		&statx,

packages/orchestrator/cmd/clean-nfs-cache/cleaner/stat_osx.go

@@ -31,10 +31,10 @@ func (c *Cleaner) stat(path string) (*Candidate, error) {
 	}, nil
 }
 
-func (c *Cleaner) statInDir(df *os.File, filename string) (*File, error) {
+func (c *Cleaner) statInDir(dirPath string, filename string) (*File, error) {
 	c.StatxInDirC.Add(1)
-	// performance on OS X doeas not matter, so just use the full stat
-	cand, err := c.stat(filepath.Join(df.Name(), filename))
+	// performance on OS X does not matter, so just use the full stat
+	cand, err := c.stat(filepath.Join(dirPath, filename))
 	if err != nil {
 		return nil, err
 	}

packages/orchestrator/pkg/sandbox/nbd/path_direct.go

@@ -78,7 +78,7 @@ func (d *DirectPathMount) Open(ctx context.Context) (retDeviceIndex uint32, err
 
 	telemetry.ReportEvent(ctx, "got backend size")
 
-	deviceIndex := uint32(math.MaxUint32)
+	var deviceIndex uint32
 
 	for {
 		deviceIndex, err = d.devicePool.GetDevice(ctx)
@@ -119,14 +119,19 @@ func (d *DirectPathMount) Open(ctx context.Context) (retDeviceIndex uint32, err
 			server.Close()
 
 			dispatch := NewDispatch(serverc, d.Backend)
+			// Capture loop variables for the goroutine closure to avoid a data
+			// race: deviceIndex is reassigned on each retry iteration of the
+			// outer for-loop while the goroutine may still read it.
+			devIdx := deviceIndex
+			sockIdx := i
 			// Start reading commands on the socket and dispatching them to our provider
 			d.handlersWg.Go(func() {
 				handleErr := dispatch.Handle(ctx)
 				// The error is expected to happen if the nbd (socket connection) is closed
 				logger.L().Info(ctx, "closing handler for NBD commands",
 					zap.Error(handleErr),
-					zap.Uint32("device_index", deviceIndex),
-					zap.Int("socket_index", i),
+					zap.Uint32("device_index", devIdx),
+					zap.Int("socket_index", sockIdx),
 				)
 			})
 

packages/orchestrator/pkg/sandbox/uffd/testutils/page_mmap.go

@@ -1,6 +1,7 @@
 package testutils
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"syscall"
@@ -20,7 +21,16 @@ func NewPageMmap(t *testing.T, size, pagesize uint64) ([]byte, uintptr, error) {
 	}
 
 	if pagesize == header.HugepageSize {
-		return newMmap(t, size, header.HugepageSize, unix.MAP_HUGETLB|unix.MAP_HUGE_2MB)
+		b, addr, err := newMmap(t, size, header.HugepageSize, unix.MAP_HUGETLB|unix.MAP_HUGE_2MB)
+		// Hugepage allocation can fail with ENOMEM on CI runners that don't
+		// have enough (or any) hugepages pre-allocated in /proc/sys/vm/nr_hugepages.
+		// Skip gracefully rather than failing the test.
+		if err != nil && errors.Is(err, syscall.ENOMEM) {
+			pages := int(math.Ceil(float64(size) / float64(header.HugepageSize)))
+			t.Skipf("skipping: hugepage mmap failed (need %d hugepages): %v", pages, err)
+		}
+
+		return b, addr, err
 	}
 
 	return nil, 0, fmt.Errorf("unsupported page size: %d", pagesize)

packages/orchestrator/pkg/sandbox/uffd/userfaultfd/async_wp_test.go

@@ -2,6 +2,7 @@ package userfaultfd
 
 import (
 	"os"
+	"runtime"
 	"testing"
 	"unsafe"
 
@@ -33,6 +34,9 @@ func TestAsyncWriteProtection(t *testing.T) {
 	if os.Geteuid() != 0 {
 		t.Skip("this test requires root privileges")
 	}
+	if runtime.GOARCH == "arm64" {
+		t.Skip("ARM64 kernels do not support UFFD write protection")
+	}
 
 	tests := []struct {
 		name          string

packages/shared/pkg/utils/errorcollector_test.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"context"
 	"errors"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -49,23 +50,31 @@ func TestErrorCollector(t *testing.T) {
 
 		ec := NewErrorCollector(1)
 
-		// Block the collector's only slot
+		// Block the collector's only slot.
+		// ctx1 and ctx2 must be distinct variables: the closure passed to ec.Go
+		// captures the context variable by reference. If we reused a single "ctx"
+		// variable, the first closure's <-ctx.Done() would race with the main
+		// goroutine's reassignment of ctx on the second WithCancel call.
 		started := make(chan struct{})
-		ctx, cancel1 := context.WithCancel(t.Context())
-		ec.Go(ctx, func() error {
+		ctx1, cancel1 := context.WithCancel(t.Context())
+		ec.Go(ctx1, func() error {
 			close(started)
-			<-ctx.Done()
+			<-ctx1.Done()
 
 			return nil
 		})
 
 		<-started
 
-		// This Go call should block on the semaphore
-		var wasCalled bool
-		ctx, cancel2 := context.WithCancel(t.Context())
-		ec.Go(ctx, func() error {
-			wasCalled = true
+		// This Go call should block on the semaphore.
+		// wasCalled must be atomic: the goroutine spawned by ec.Go may write it
+		// concurrently with the main goroutine's read in assert.False below.
+		// A plain bool causes a data race that the -race detector catches on ARM64
+		// (weaker memory model) even though it appears safe on x86.
+		var wasCalled atomic.Bool
+		ctx2, cancel2 := context.WithCancel(t.Context())
+		ec.Go(ctx2, func() error {
+			wasCalled.Store(true)
 
 			return nil
 		})
@@ -78,6 +87,6 @@ func TestErrorCollector(t *testing.T) {
 
 		err := ec.Wait()
 		require.ErrorIs(t, err, context.Canceled)
-		assert.False(t, wasCalled)
+		assert.False(t, wasCalled.Load())
 	})
 }