diff --git a/Makefile b/Makefile index f8d944a..f955a26 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ endif export PATH := $(HOME)/go/bin:$(PATH) # Base command to start the tests with ginkgo -GINKGO_RUN?=CGO_ENABLED=0 ginkgo run ${PARALLEL_OPT} --v +GINKGO_RUN?=CGO_ENABLED=0 ginkgo run ${PARALLEL_OPT} --v --fail-fast all: fmt vet @@ -22,9 +22,19 @@ vet: go vet ./... # TODO: would be nice to tie this to a script that also deletes all the workflows and makes sure nothing is left over +# Clean up resources that may be left behind by interrupted test runs .PHONY: clean clean: + @echo "Removing triage namespace..." kubectl delete namespace nnf-system-needs-triage --ignore-not-found + @echo "Removing leftover UID verification pods..." + kubectl delete pods -n default --ignore-not-found $$(kubectl get pods -n default --no-headers -o custom-columns=':metadata.name' 2>/dev/null | grep '^verify-uid-') 2>/dev/null; true + @echo "Checking for persistent storage instances..." + @psi=$$(kubectl get persistentstorageinstances -A --no-headers 2>/dev/null); \ + if [ -n "$$psi" ]; then \ + echo "WARNING: Persistent storage instances still exist. These require a destroy_persistent workflow to remove:" >&2; \ + echo "$$psi" >&2; \ + fi .PHONY: init init: @@ -33,12 +43,22 @@ init: # Run all the tests .PHONY: test test: - ${GINKGO_RUN} . + @if kubectl config current-context 2>/dev/null | grep -q 'kind-'; then \ + echo "Detected kind cluster. Switching to 'make kind' (excludes global-lustre tests)." >&2; \ + $(MAKE) kind; \ + else \ + ${GINKGO_RUN} . ; \ + fi # Alias for all the tests .PHONY: full full: test +# Run tests compatible with a kind environment (excludes tests requiring real Lustre mounts) +.PHONY: kind +kind: + ${GINKGO_RUN} --label-filter='!global-lustre && !multi-storage && !lustre-csimount && !high-capacity' . + # Run one test to ensure system is in working order .PHONY: sanity sanity: diff --git a/go.mod b/go.mod index c26883e..f938e78 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/NearNodeFlash/nnf-integration-test go 1.25.7 require ( - github.com/DataWorkflowServices/dws v0.0.1-0.20260316202252-4e24ce7ae372 - github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260409140943-c59379bba6df + github.com/DataWorkflowServices/dws v0.0.1-0.20260506165801-7e73c2416704 + github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260506210158-1c2551dfbf1c github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 // indirect - github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260409160114-bcb00b928378 + github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260507014732-f747aca10125 github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 go.openly.dev/pointy v1.3.0 @@ -19,12 +19,12 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -32,12 +32,12 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -63,7 +63,7 @@ require ( golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.41.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.36.1 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index ca9d951..9f11789 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,18 @@ -github.com/DataWorkflowServices/dws v0.0.1-0.20260316202252-4e24ce7ae372 h1:XHGgQFKImeAzM2CNNnCshrpf8NoDDplh6V0u7keVMbU= -github.com/DataWorkflowServices/dws v0.0.1-0.20260316202252-4e24ce7ae372/go.mod h1:4zp8Ar95lcBjmk/GavkALeIGP9btG176rFQ08XQtVXE= -github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260409140943-c59379bba6df h1:5S/AIw9QwGvceNvSYnFdeH0pRyb8EEdya9exLwj1j+Y= -github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260409140943-c59379bba6df/go.mod h1:TZFQc3lV1bX3ufKsKUzP6pQx+ujLJTA2Z/zkPjAun5U= +github.com/DataWorkflowServices/dws v0.0.1-0.20260506165801-7e73c2416704 h1:/bbdHCdc+M0tIsO+Q/VOob+P56v4P9BNpGeWmtKDgB0= +github.com/DataWorkflowServices/dws v0.0.1-0.20260506165801-7e73c2416704/go.mod h1:4zp8Ar95lcBjmk/GavkALeIGP9btG176rFQ08XQtVXE= +github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260506210158-1c2551dfbf1c h1:7uPTARsFc5vAVOuTNAsKzeFVwrGun+THd27UvyedQdE= +github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260506210158-1c2551dfbf1c/go.mod h1:pYIaXNmF3wOZ/RS+VctciIjPymE2n5oVOTslkYywvnw= github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 h1:kZHhYVMa2KdBpdXbAtnE5DSu6pN/9VEVU2t8wPUAqaY= github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379/go.mod h1:vJXM5DUx6FZtK8yuYNMSv2O1mfUmrpRVuRZp/o/NDRs= -github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260409160114-bcb00b928378 h1:RKTFWSDkZ42YOkTpzYndzHy7bYyWQ/PW4QosxQoQDDA= -github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260409160114-bcb00b928378/go.mod h1:3qSSwBwOSfEJMFJ7W3pkAmemwZiy4qT41ne41HZjt2Y= +github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260507014732-f747aca10125 h1:5DbVTbtwY35+pTVtbC2YgcwHFHyNhWQSXnfNS1lWxOs= +github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260507014732-f747aca10125/go.mod h1:QX2lh2QCBZjorvqomgrG4tbHg2fvSUpiEPlaD1d+rqI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -27,8 +27,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -44,21 +44,19 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -191,10 +189,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/int_test.go b/int_test.go index b875600..ac06d94 100644 --- a/int_test.go +++ b/int_test.go @@ -87,6 +87,7 @@ var tests = []*T{ // GFS2 Fence MakeTest("GFS2 Fence", "#DW jobdw type=gfs2 name=gfs2-fence capacity=50GB").WithLabels(GFS2Fence). + RequiresHardware(). DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second). // start pacemaker DelayInState(dwsv1alpha7.StatePreRun, 60*time.Second). // fence node(s) DelayInState(dwsv1alpha7.StateDataOut, 15*time.Second), // stop pacemaker on surviving node(s) @@ -102,7 +103,8 @@ var tests = []*T{ // WithStorageProfile().DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).StopAfter(dwsv1alpha7.StatePreRun).Focused(), MakeTest("XFS with Storage Profile and LV Create", "#DW jobdw type=xfs name=xfs-storage-profile capacity=14TB profile=my-xfs-storage-profile"). - WithStorageProfileLvCreate("--zero n --activate y --type raid5 --nosync --extents $PERCENT_VG --stripes $DEVICE_NUM-1 --stripesize=64KiB --name $LV_NAME $VG_NAME"), + WithStorageProfileLvCreate("--zero n --activate y --type raid5 --nosync --extents $PERCENT_VG --stripes $DEVICE_NUM-1 --stripesize=64KiB --name $LV_NAME $VG_NAME"). + WithLabels("high-capacity"), // Persistent MakeTest("Persistent Lustre", @@ -150,7 +152,7 @@ var tests = []*T{ "#DW jobdw type=lustre name=lustre-with-containers-mpi capacity=100GB", "#DW container name=lustre-with-containers-mpi profile=example-mpi "+ "DW_JOB_foo_local_storage=lustre-with-containers-mpi"). - WithPermissions(userID, groupID).WithLabels("mpi"), + WithPermissions(userID, groupID).WithLabels("mpi", "lustre-csimount"), MakeTest("GFS2 and Global Lustre with MPI Containers", "#DW jobdw type=gfs2 name=gfs2-and-global-with-containers-mpi capacity=100GB", "#DW container name=gfs2-and-global-with-containers-mpi profile=example-mpi "+ @@ -217,7 +219,7 @@ var tests = []*T{ WithContainerProfile("example-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}). ExpectError(dwsv1alpha7.StatePostRun), - // Containers - Unsupported Filesystems. These should fail as xfs/raw filesystems are not supported for containers. + // Containers - Unsupported Filesystems. XFS is not supported for containers. MakeTest("XFS with Containers", "#DW jobdw type=xfs name=xfs-with-containers capacity=100GB", "#DW container name=xfs-with-containers profile=example-success DW_JOB_foo_local_storage=xfs-with-containers"). @@ -225,7 +227,7 @@ var tests = []*T{ MakeTest("Raw with Containers", "#DW jobdw type=raw name=raw-with-containers capacity=100GB", "#DW container name=raw-with-containers profile=example-success DW_JOB_foo_local_storage=raw-with-containers"). - ExpectError(dwsv1alpha7.StateProposal).WithLabels("unsupported-fs"), + WithPermissions(userID, groupID).WithLabels("non-mpi"), // Containers - Multiple Storages MakeTest("GFS2 and Lustre with Containers", diff --git a/internal/options.go b/internal/options.go index 5329f4f..e845012 100644 --- a/internal/options.go +++ b/internal/options.go @@ -89,6 +89,12 @@ func (t *T) DelayInState(state dwsv1alpha7.WorkflowState, duration time.Duration return t } +// RequiresHardware marks a test as requiring real hardware (skipped in kind environments). +func (t *T) RequiresHardware() *T { + t.options.hardwareRequired = true + return t +} + type TExpectError struct { state dwsv1alpha7.WorkflowState } @@ -294,11 +300,12 @@ func (t *T) WithGlobalLustreFromPersistentLustre(name string, namespaces []strin fsType = args["type"] } - if args["command"] == "copy_in" { + switch args["command"] { + case "copy_in": if path, found := args["source"]; found { t.options.globalLustre.in = path } - } else if args["command"] == "copy_out" { + case "copy_out": if path, found := args["destination"]; found { // Account for index mount directories in the path. This only works for file-file // data movement to assume where the '*' goes for the index mount directories. @@ -433,9 +440,11 @@ func (t *T) Prepare(ctx context.Context, k8sClient client.Client) error { capacity := o.persistentLustre.capacity o.persistentLustre.create = MakeTest(name+"-create", - fmt.Sprintf("#DW create_persistent type=lustre name=%s capacity=%s", name, capacity)) + fmt.Sprintf("#DW create_persistent type=lustre name=%s capacity=%s", name, capacity)). + WithPermissions(t.workflow.Spec.UserID, t.workflow.Spec.GroupID) o.persistentLustre.destroy = MakeTest(name+"-destroy", - fmt.Sprintf("#DW destroy_persistent name=%s", name)) + fmt.Sprintf("#DW destroy_persistent name=%s", name)). + WithPermissions(t.workflow.Spec.UserID, t.workflow.Spec.GroupID) // Create the persistent lustre instance By(fmt.Sprintf("Creating persistent lustre instance '%s'", name)) @@ -590,7 +599,8 @@ func (t *T) Cleanup(ctx context.Context, k8sClient client.Client) error { name := o.cleanupPersistent.name By(fmt.Sprintf("Destroying persistent filesystem '%s'", name)) - test := MakeTest(name+"-destroy", fmt.Sprintf("#DW destroy_persistent name=%s", name)) + test := MakeTest(name+"-destroy", fmt.Sprintf("#DW destroy_persistent name=%s", name)). + WithPermissions(t.workflow.Spec.UserID, t.workflow.Spec.GroupID) Expect(k8sClient.Create(ctx, test.Workflow())).To(Succeed()) test.Execute(ctx, k8sClient) Expect(k8sClient.Delete(ctx, test.Workflow())).To(Succeed()) diff --git a/internal/utils.go b/internal/utils.go index 406ff6f..3a4a6fd 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -254,11 +254,12 @@ func runHelperPod(ctx context.Context, k8sClient client.Client, t *T, name, comm Expect(k8sClient.Create(ctx, pod)).To(Succeed()) t.helperPods = append(t.helperPods, pod) - // Wait for successful completion + // Wait for successful completion. Use a generous timeout to account for + // image pulls which can take several minutes on a cold cache. Eventually(func(g Gomega) bool { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) return pod.Status.Phase == corev1.PodSucceeded - }).WithTimeout(time.Minute).WithPolling(time.Second).Should(BeTrue()) + }).WithTimeout(5 * time.Minute).WithPolling(time.Second).Should(BeTrue()) } // HasContainerDirective returns true if this test includes a #DW container directive.