Release v0.1.26

Makefile

@@ -9,7 +9,7 @@ endif
 export PATH := $(HOME)/go/bin:$(PATH)
 
 # Base command to start the tests with ginkgo
-GINKGO_RUN?=CGO_ENABLED=0 ginkgo run ${PARALLEL_OPT} --v
+GINKGO_RUN?=CGO_ENABLED=0 ginkgo run ${PARALLEL_OPT} --v --fail-fast
 
 all: fmt vet
 
@@ -22,9 +22,19 @@ vet:
 	go vet ./...
 
 # TODO: would be nice to tie this to a script that also deletes all the workflows and makes sure nothing is left over
+# Clean up resources that may be left behind by interrupted test runs
 .PHONY: clean
 clean:
+	@echo "Removing triage namespace..."
 	kubectl delete namespace nnf-system-needs-triage --ignore-not-found
+	@echo "Removing leftover UID verification pods..."
+	kubectl delete pods -n default --ignore-not-found $$(kubectl get pods -n default --no-headers -o custom-columns=':metadata.name' 2>/dev/null | grep '^verify-uid-') 2>/dev/null; true
+	@echo "Checking for persistent storage instances..."
+	@psi=$$(kubectl get persistentstorageinstances -A --no-headers 2>/dev/null); \
+	if [ -n "$$psi" ]; then \
+		echo "WARNING: Persistent storage instances still exist. These require a destroy_persistent workflow to remove:" >&2; \
+		echo "$$psi" >&2; \
+	fi
 
 .PHONY: init
 init:
@@ -33,12 +43,22 @@ init:
 # Run all the tests
 .PHONY: test
 test:
-	${GINKGO_RUN} .
+	@if kubectl config current-context 2>/dev/null | grep -q 'kind-'; then \
+		echo "Detected kind cluster. Switching to 'make kind' (excludes global-lustre tests)." >&2; \
+		$(MAKE) kind; \
+	else \
+		${GINKGO_RUN} . ; \
+	fi
 
 # Alias for all the tests
 .PHONY: full
 full: test
 
+# Run tests compatible with a kind environment (excludes tests requiring real Lustre mounts)
+.PHONY: kind
+kind:
+	${GINKGO_RUN} --label-filter='!global-lustre && !multi-storage && !lustre-csimount && !high-capacity' .
+
 # Run one test to ensure system is in working order
 .PHONY: sanity
 sanity:

go.mod

@@ -3,10 +3,10 @@ module github.com/NearNodeFlash/nnf-integration-test
 go 1.25.7
 
 require (
-	github.com/DataWorkflowServices/dws v0.0.1-0.20260316202252-4e24ce7ae372
-	github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260409140943-c59379bba6df
+	github.com/DataWorkflowServices/dws v0.0.1-0.20260506165801-7e73c2416704
+	github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260506210158-1c2551dfbf1c
 	github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 // indirect
-	github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260409160114-bcb00b928378
+	github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260507014732-f747aca10125
 	github.com/onsi/ginkgo/v2 v2.22.2
 	github.com/onsi/gomega v1.36.2
 	go.openly.dev/pointy v1.3.0
@@ -19,25 +19,25 @@ require (
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -63,7 +63,7 @@ require (
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.41.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/protobuf v1.36.1 // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -1,18 +1,18 @@
-github.com/DataWorkflowServices/dws v0.0.1-0.20260316202252-4e24ce7ae372 h1:XHGgQFKImeAzM2CNNnCshrpf8NoDDplh6V0u7keVMbU=
-github.com/DataWorkflowServices/dws v0.0.1-0.20260316202252-4e24ce7ae372/go.mod h1:4zp8Ar95lcBjmk/GavkALeIGP9btG176rFQ08XQtVXE=
-github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260409140943-c59379bba6df h1:5S/AIw9QwGvceNvSYnFdeH0pRyb8EEdya9exLwj1j+Y=
-github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260409140943-c59379bba6df/go.mod h1:TZFQc3lV1bX3ufKsKUzP6pQx+ujLJTA2Z/zkPjAun5U=
+github.com/DataWorkflowServices/dws v0.0.1-0.20260506165801-7e73c2416704 h1:/bbdHCdc+M0tIsO+Q/VOob+P56v4P9BNpGeWmtKDgB0=
+github.com/DataWorkflowServices/dws v0.0.1-0.20260506165801-7e73c2416704/go.mod h1:4zp8Ar95lcBjmk/GavkALeIGP9btG176rFQ08XQtVXE=
+github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260506210158-1c2551dfbf1c h1:7uPTARsFc5vAVOuTNAsKzeFVwrGun+THd27UvyedQdE=
+github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260506210158-1c2551dfbf1c/go.mod h1:pYIaXNmF3wOZ/RS+VctciIjPymE2n5oVOTslkYywvnw=
 github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 h1:kZHhYVMa2KdBpdXbAtnE5DSu6pN/9VEVU2t8wPUAqaY=
 github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379/go.mod h1:vJXM5DUx6FZtK8yuYNMSv2O1mfUmrpRVuRZp/o/NDRs=
-github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260409160114-bcb00b928378 h1:RKTFWSDkZ42YOkTpzYndzHy7bYyWQ/PW4QosxQoQDDA=
-github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260409160114-bcb00b928378/go.mod h1:3qSSwBwOSfEJMFJ7W3pkAmemwZiy4qT41ne41HZjt2Y=
+github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260507014732-f747aca10125 h1:5DbVTbtwY35+pTVtbC2YgcwHFHyNhWQSXnfNS1lWxOs=
+github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260507014732-f747aca10125/go.mod h1:QX2lh2QCBZjorvqomgrG4tbHg2fvSUpiEPlaD1d+rqI=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -27,8 +27,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
 github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA=
 github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
@@ -44,21 +44,19 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
 github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -191,10 +189,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
-google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

int_test.go

@@ -87,6 +87,7 @@ var tests = []*T{
 
 	// GFS2 Fence
 	MakeTest("GFS2 Fence", "#DW jobdw type=gfs2 name=gfs2-fence capacity=50GB").WithLabels(GFS2Fence).
+		RequiresHardware().
 		DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).  // start pacemaker
 		DelayInState(dwsv1alpha7.StatePreRun, 60*time.Second).  // fence node(s)
 		DelayInState(dwsv1alpha7.StateDataOut, 15*time.Second), // stop pacemaker on surviving node(s)
@@ -102,7 +103,8 @@ var tests = []*T{
 	// WithStorageProfile().DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).StopAfter(dwsv1alpha7.StatePreRun).Focused(),
 	MakeTest("XFS with Storage Profile and LV Create",
 		"#DW jobdw type=xfs name=xfs-storage-profile capacity=14TB profile=my-xfs-storage-profile").
-		WithStorageProfileLvCreate("--zero n --activate y --type raid5 --nosync --extents $PERCENT_VG --stripes $DEVICE_NUM-1 --stripesize=64KiB --name $LV_NAME $VG_NAME"),
+		WithStorageProfileLvCreate("--zero n --activate y --type raid5 --nosync --extents $PERCENT_VG --stripes $DEVICE_NUM-1 --stripesize=64KiB --name $LV_NAME $VG_NAME").
+		WithLabels("high-capacity"),
 
 	// Persistent
 	MakeTest("Persistent Lustre",
@@ -150,7 +152,7 @@ var tests = []*T{
 		"#DW jobdw type=lustre name=lustre-with-containers-mpi capacity=100GB",
 		"#DW container name=lustre-with-containers-mpi profile=example-mpi "+
 			"DW_JOB_foo_local_storage=lustre-with-containers-mpi").
-		WithPermissions(userID, groupID).WithLabels("mpi"),
+		WithPermissions(userID, groupID).WithLabels("mpi", "lustre-csimount"),
 	MakeTest("GFS2 and Global Lustre with MPI Containers",
 		"#DW jobdw type=gfs2 name=gfs2-and-global-with-containers-mpi capacity=100GB",
 		"#DW container name=gfs2-and-global-with-containers-mpi profile=example-mpi "+
@@ -217,15 +219,15 @@ var tests = []*T{
 		WithContainerProfile("example-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}).
 		ExpectError(dwsv1alpha7.StatePostRun),
 
-	// Containers - Unsupported Filesystems. These should fail as xfs/raw filesystems are not supported for containers.
+	// Containers - Unsupported Filesystems. XFS is not supported for containers.
 	MakeTest("XFS with Containers",
 		"#DW jobdw type=xfs name=xfs-with-containers capacity=100GB",
 		"#DW container name=xfs-with-containers profile=example-success DW_JOB_foo_local_storage=xfs-with-containers").
 		ExpectError(dwsv1alpha7.StateProposal).WithLabels("unsupported-fs"),
 	MakeTest("Raw with Containers",
 		"#DW jobdw type=raw name=raw-with-containers capacity=100GB",
 		"#DW container name=raw-with-containers profile=example-success DW_JOB_foo_local_storage=raw-with-containers").
-		ExpectError(dwsv1alpha7.StateProposal).WithLabels("unsupported-fs"),
+		WithPermissions(userID, groupID).WithLabels("non-mpi"),
 
 	// Containers - Multiple Storages
 	MakeTest("GFS2 and Lustre with Containers",

internal/options.go

@@ -89,6 +89,12 @@ func (t *T) DelayInState(state dwsv1alpha7.WorkflowState, duration time.Duration
 	return t
 }
 
+// RequiresHardware marks a test as requiring real hardware (skipped in kind environments).
+func (t *T) RequiresHardware() *T {
+	t.options.hardwareRequired = true
+	return t
+}
+
 type TExpectError struct {
 	state dwsv1alpha7.WorkflowState
 }
@@ -294,11 +300,12 @@ func (t *T) WithGlobalLustreFromPersistentLustre(name string, namespaces []strin
 			fsType = args["type"]
 		}
 
-		if args["command"] == "copy_in" {
+		switch args["command"] {
+		case "copy_in":
 			if path, found := args["source"]; found {
 				t.options.globalLustre.in = path
 			}
-		} else if args["command"] == "copy_out" {
+		case "copy_out":
 			if path, found := args["destination"]; found {
 				// Account for index mount directories in the path. This only works for file-file
 				// data movement to assume where the '*' goes for the index mount directories.
@@ -433,9 +440,11 @@ func (t *T) Prepare(ctx context.Context, k8sClient client.Client) error {
 		capacity := o.persistentLustre.capacity
 
 		o.persistentLustre.create = MakeTest(name+"-create",
-			fmt.Sprintf("#DW create_persistent type=lustre name=%s capacity=%s", name, capacity))
+			fmt.Sprintf("#DW create_persistent type=lustre name=%s capacity=%s", name, capacity)).
+			WithPermissions(t.workflow.Spec.UserID, t.workflow.Spec.GroupID)
 		o.persistentLustre.destroy = MakeTest(name+"-destroy",
-			fmt.Sprintf("#DW destroy_persistent name=%s", name))
+			fmt.Sprintf("#DW destroy_persistent name=%s", name)).
+			WithPermissions(t.workflow.Spec.UserID, t.workflow.Spec.GroupID)
 
 		// Create the persistent lustre instance
 		By(fmt.Sprintf("Creating persistent lustre instance '%s'", name))
@@ -590,7 +599,8 @@ func (t *T) Cleanup(ctx context.Context, k8sClient client.Client) error {
 		name := o.cleanupPersistent.name
 		By(fmt.Sprintf("Destroying persistent filesystem '%s'", name))
 
-		test := MakeTest(name+"-destroy", fmt.Sprintf("#DW destroy_persistent name=%s", name))
+		test := MakeTest(name+"-destroy", fmt.Sprintf("#DW destroy_persistent name=%s", name)).
+			WithPermissions(t.workflow.Spec.UserID, t.workflow.Spec.GroupID)
 		Expect(k8sClient.Create(ctx, test.Workflow())).To(Succeed())
 		test.Execute(ctx, k8sClient)
 		Expect(k8sClient.Delete(ctx, test.Workflow())).To(Succeed())

internal/utils.go

@@ -254,11 +254,12 @@ func runHelperPod(ctx context.Context, k8sClient client.Client, t *T, name, comm
 	Expect(k8sClient.Create(ctx, pod)).To(Succeed())
 	t.helperPods = append(t.helperPods, pod)
 
-	// Wait for successful completion
+	// Wait for successful completion. Use a generous timeout to account for
+	// image pulls which can take several minutes on a cold cache.
 	Eventually(func(g Gomega) bool {
 		g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pod), pod)).To(Succeed())
 		return pod.Status.Phase == corev1.PodSucceeded
-	}).WithTimeout(time.Minute).WithPolling(time.Second).Should(BeTrue())
+	}).WithTimeout(5 * time.Minute).WithPolling(time.Second).Should(BeTrue())
 }
 
 // HasContainerDirective returns true if this test includes a #DW container directive.