Merge release v0.1.21

Release v0.1.21

Author: matthew-richerson

Makefile

@@ -59,6 +59,11 @@ dm:
 container:
 	${GINKGO_RUN} --label-filter='container' .
 
+# Run gfs2 fence test
+.PHONY: gfs2_fence
+gfs2_fence:
+	${GINKGO_RUN} --label-filter='gfs2_fence' .
+
 .PHONY: .version
 .version: ## Uses the git-version-gen script to generate a tag version
 	./git-version-gen --fallback `git rev-parse HEAD` > .version

go.mod

@@ -3,10 +3,10 @@ module github.com/NearNodeFlash/nnf-integration-test
 go 1.23.9
 
 require (
-	github.com/DataWorkflowServices/dws v0.0.1-0.20250826235427-3ba3755ecbb4
-	github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20250827135601-4d150db05d08
-	github.com/NearNodeFlash/nnf-ec v0.0.1-0.20250902204910-828ccfe450c3 // indirect
-	github.com/NearNodeFlash/nnf-sos v0.0.1-0.20250910191707-694eda76a3d0
+	github.com/DataWorkflowServices/dws v0.0.1-0.20251201192617-99a03aa3170f
+	github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20251210162744-b3f481b47161
+	github.com/NearNodeFlash/nnf-ec v0.0.1-0.20251021182141-d22098d70c26 // indirect
+	github.com/NearNodeFlash/nnf-sos v0.0.1-0.20251210164542-0745ce4c3473
 	github.com/onsi/ginkgo/v2 v2.22.2
 	github.com/onsi/gomega v1.36.2
 	go.openly.dev/pointy v1.3.0

go.sum

@@ -1,11 +1,11 @@
-github.com/DataWorkflowServices/dws v0.0.1-0.20250826235427-3ba3755ecbb4 h1:c5bKT/hZUHzgx4HxiFMGoxEFDXPv/qlFmLwDTbJAA00=
-github.com/DataWorkflowServices/dws v0.0.1-0.20250826235427-3ba3755ecbb4/go.mod h1:NbNOIG2ybY2XHVSJn0KF7edPlX3I1ji7je6MGzuPeaA=
-github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20250827135601-4d150db05d08 h1:1IYv2gDH/+hxBo6TZenIRs30PSpzO49USDkj/aBo/g4=
-github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20250827135601-4d150db05d08/go.mod h1:1lpNL1ZYtVwDU6Fuju5u/xiza6Z2Ua8KWZeaGBKpn5o=
-github.com/NearNodeFlash/nnf-ec v0.0.1-0.20250902204910-828ccfe450c3 h1:CAJW97TU2fzzECwbwgmYkYe0kzvkZccXg6f8Ukom8Ws=
-github.com/NearNodeFlash/nnf-ec v0.0.1-0.20250902204910-828ccfe450c3/go.mod h1:slDsqUJtN2uZ5Z7z4J6wWrtKksQAaH4JqRzUonDXNKo=
-github.com/NearNodeFlash/nnf-sos v0.0.1-0.20250910191707-694eda76a3d0 h1:IPvOz9ZYP4XTmofIlSiuw3L0d6UxFbrFZ51fLNEkf5g=
-github.com/NearNodeFlash/nnf-sos v0.0.1-0.20250910191707-694eda76a3d0/go.mod h1:wT5lUEvqLjpPGGbuyWmJYhThvTDkJM4Rqe22DrT7zdw=
+github.com/DataWorkflowServices/dws v0.0.1-0.20251201192617-99a03aa3170f h1:ezj0+9HY6VvPnJbDoIXr/IzwBFe1X2pLLEZpHodOulU=
+github.com/DataWorkflowServices/dws v0.0.1-0.20251201192617-99a03aa3170f/go.mod h1:NbNOIG2ybY2XHVSJn0KF7edPlX3I1ji7je6MGzuPeaA=
+github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20251210162744-b3f481b47161 h1:7NCIXYOp8jW/OS3KQg27+xw5DikxoHJRYRLWiT6fQ8I=
+github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20251210162744-b3f481b47161/go.mod h1:4QNBl5vMm4dFW49oqXa/1NzIjgt8EJKuAixmeoIVLCU=
+github.com/NearNodeFlash/nnf-ec v0.0.1-0.20251021182141-d22098d70c26 h1:CHEvLPUEZDj0Z1hn0kXB//mHKh2eQ2EP/mMjy4zB9+g=
+github.com/NearNodeFlash/nnf-ec v0.0.1-0.20251021182141-d22098d70c26/go.mod h1:slDsqUJtN2uZ5Z7z4J6wWrtKksQAaH4JqRzUonDXNKo=
+github.com/NearNodeFlash/nnf-sos v0.0.1-0.20251210164542-0745ce4c3473 h1:VgFA+7Tklv0TNjkdcNtCRJspheNaP3wxMxjCMgKtOX4=
+github.com/NearNodeFlash/nnf-sos v0.0.1-0.20251210164542-0745ce4c3473/go.mod h1:6EQR4F5Wscxpy4ekZOdu9+xfkfuvrD1ZeM6Z+zi/gdA=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

int_test.go

@@ -21,14 +21,15 @@ package test
 
 import (
 	"fmt"
+	"time"
 
 	. "github.com/NearNodeFlash/nnf-integration-test/internal"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"go.openly.dev/pointy"
 
-	dwsv1alpha6 "github.com/DataWorkflowServices/dws/api/v1alpha6"
+	dwsv1alpha7 "github.com/DataWorkflowServices/dws/api/v1alpha7"
 )
 
 var (
@@ -41,9 +42,9 @@ var (
 	highTimeout = "5m"
 
 	// Which states use the high timeout
-	highTimeoutStates = []dwsv1alpha6.WorkflowState{
-		dwsv1alpha6.StateSetup,
-		dwsv1alpha6.StateTeardown,
+	highTimeoutStates = []dwsv1alpha7.WorkflowState{
+		dwsv1alpha7.StateSetup,
+		dwsv1alpha7.StateTeardown,
 	}
 )
 
@@ -56,8 +57,12 @@ var tests = []*T{
 	// Mark a test case as Pending(). Ginkgo will not run any tests that have the Pending decorator
 	//   MakeTest("Pending", "#DW ...").Pending()
 	//
-	// Mark a test case so it will stop after the workflow achieves the desired state of PreRun
-	//   MakeTest("Stop After", "#DW ...").StopAfter(wsv1alpha1.StatePreRun),
+	// Mark a test case, so it will stop after the workflow achieves the desired state of PreRun
+	//   MakeTest("Stop After", "#DW ...").StopAfter(dwsv1alpha7.StatePreRun),
+	//
+	// Mark a test case, so it delays for the specified period after a workflow achieves the desired state
+	// Multiple delays can be added by calling DelayInState multiple times.
+	//   MakeTest("Delay In State", "#DW ...").DelayInState(dwsv1alpha7.StateDataIn, 2*time.Minute).DelayInState(dwsv1alpha7.StateDataOut, 2*time.Minute),
 	//
 	// Duplicate a test case 20 times.
 	//   DuplicateTest(
@@ -73,13 +78,24 @@ var tests = []*T{
 	// External Computes
 	MakeTest("Lustre External", "#DW jobdw type=lustre name=lustre capacity=50GB").WithExternalComputes().WithLabels(ExternalLustre),
 
+	// GFS2 Fence
+	MakeTest("GFS2 Fence", "#DW jobdw type=gfs2 name=gfs2-fence capacity=50GB").WithLabels(GFS2Fence).
+		DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).  // start pacemaker
+		DelayInState(dwsv1alpha7.StatePreRun, 60*time.Second).  // fence node(s)
+		DelayInState(dwsv1alpha7.StateDataOut, 15*time.Second), // stop pacemaker on surviving node(s)
+
 	// Storage Profiles
 	MakeTest("XFS with Storage Profile",
 		"#DW jobdw type=xfs name=xfs-storage-profile capacity=50GB profile=my-xfs-storage-profile").
 		WithStorageProfile(),
 	MakeTest("GFS2 with Storage Profile",
 		"#DW jobdw type=gfs2 name=gfs2-storage-profile capacity=50GB profile=my-gfs2-storage-profile").
 		WithStorageProfile(),
+	// WithStorageProfile().DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).DelayInState(dwsv1alpha7.StateDataOut, 15*time.Second).Focused(),    // Useful for debugging
+	// WithStorageProfile().DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).StopAfter(dwsv1alpha7.StatePreRun).Focused(),
+	MakeTest("XFS with Storage Profile and LV Create",
+		"#DW jobdw type=xfs name=xfs-storage-profile capacity=14TB profile=my-xfs-storage-profile").
+		WithStorageProfileLvCreate("--zero n --activate y --type raid5 --nosync --extents $PERCENT_VG --stripes $DEVICE_NUM-1 --stripesize=64KiB --name $LV_NAME $VG_NAME"),
 
 	// Persistent
 	MakeTest("Persistent Lustre",
@@ -143,17 +159,17 @@ var tests = []*T{
 		"#DW container name=prerun-timeout-mpi profile=example-mpi-prerun-timeout").
 		WithPermissions(1050, 1051).WithLabels("mpi", "timeout").
 		WithContainerProfile("example-mpi", &ContainerProfileOptions{PrerunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
-		ExpectError(dwsv1alpha6.StatePreRun),
+		ExpectError(dwsv1alpha7.StatePreRun),
 	MakeTest("PostRun timeout on MPI containers",
 		"#DW container name=postrun-timeout-mpi profile=example-mpi-postrun-timeout").
 		WithPermissions(1050, 1051).WithLabels("mpi", "timeout").
 		WithContainerProfile("example-mpi-webserver", &ContainerProfileOptions{PostrunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
-		ExpectError(dwsv1alpha6.StatePostRun),
+		ExpectError(dwsv1alpha7.StatePostRun),
 	MakeTest("Non-zero exit on MPI containers",
 		"#DW container name=mpi-container-fail profile=example-mpi-fail-noretry").
 		WithPermissions(1050, 1051).WithLabels("mpi", "fail").
 		WithContainerProfile("example-mpi-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}).
-		ExpectError(dwsv1alpha6.StatePostRun),
+		ExpectError(dwsv1alpha7.StatePostRun),
 
 	// Containers - Non-MPI
 	MakeTest("GFS2 with Containers",
@@ -175,27 +191,27 @@ var tests = []*T{
 		"#DW container name=prerun-timeout profile=example-prerun-timeout").
 		WithPermissions(1050, 1051).WithLabels("non-mpi", "timeout").
 		WithContainerProfile("example-forever", &ContainerProfileOptions{PrerunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
-		ExpectError(dwsv1alpha6.StatePreRun),
+		ExpectError(dwsv1alpha7.StatePreRun),
 	MakeTest("PostRun timeout on non-MPI containers",
 		"#DW container name=postrun-timeout profile=example-postrun-timeout").
 		WithPermissions(1050, 1051).WithLabels("non-mpi", "timeout").
 		WithContainerProfile("example-forever", &ContainerProfileOptions{PostrunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
-		ExpectError(dwsv1alpha6.StatePostRun),
+		ExpectError(dwsv1alpha7.StatePostRun),
 	MakeTest("Non-zero exit on non-MPI containers",
 		"#DW container name=container-fail profile=example-fail-noretry").
 		WithPermissions(1050, 1051).WithLabels("non-mpi", "fail").
 		WithContainerProfile("example-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}).
-		ExpectError(dwsv1alpha6.StatePostRun),
+		ExpectError(dwsv1alpha7.StatePostRun),
 
 	// Containers - Unsupported Filesystems. These should fail as xfs/raw filesystems are not supported for containers.
 	MakeTest("XFS with Containers",
 		"#DW jobdw type=xfs name=xfs-with-containers capacity=100GB",
 		"#DW container name=xfs-with-containers profile=example-success DW_JOB_foo_local_storage=xfs-with-containers").
-		ExpectError(dwsv1alpha6.StateProposal).WithLabels("unsupported-fs"),
+		ExpectError(dwsv1alpha7.StateProposal).WithLabels("unsupported-fs"),
 	MakeTest("Raw with Containers",
 		"#DW jobdw type=raw name=raw-with-containers capacity=100GB",
 		"#DW container name=raw-with-containers profile=example-success DW_JOB_foo_local_storage=raw-with-containers").
-		ExpectError(dwsv1alpha6.StateProposal).WithLabels("unsupported-fs"),
+		ExpectError(dwsv1alpha7.StateProposal).WithLabels("unsupported-fs"),
 
 	// Containers - Multiple Storages
 	MakeTest("GFS2 and Lustre with Containers",
@@ -250,7 +266,7 @@ var _ = Describe("NNF Integration Test", func() {
 						// TODO: Ginkgo's `--fail-fast` option still seems to execute DeferCleanup() calls
 						//       See if this is by design or if we might need to move this to an AfterEach()
 						if !context.SpecReport().Failed() {
-							t.AdvanceStateAndWaitForReady(ctx, k8sClient, workflow, dwsv1alpha6.StateTeardown)
+							t.AdvanceStateAndWaitForReady(ctx, k8sClient, workflow, dwsv1alpha7.StateTeardown)
 
 							Expect(k8sClient.Delete(ctx, workflow)).To(Succeed())
 						}

internal/internal.go

@@ -29,7 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
-	dwsv1alpha6 "github.com/DataWorkflowServices/dws/api/v1alpha6"
+	dwsv1alpha7 "github.com/DataWorkflowServices/dws/api/v1alpha7"
 	"github.com/DataWorkflowServices/dws/utils/dwdparse"
 )
 
@@ -68,7 +68,7 @@ type T struct {
 	decorators []interface{}
 
 	// Workflow defines the DWS Workflow resource that is the target of the test.
-	workflow *dwsv1alpha6.Workflow
+	workflow *dwsv1alpha7.Workflow
 
 	// User Id to use for the workflow
 	userId int
@@ -84,7 +84,7 @@ type T struct {
 	helperPods []*corev1.Pod
 
 	// Compute nodes that were assigned to the test. This is determined at test runtime.
-	computes *dwsv1alpha6.Computes
+	computes *dwsv1alpha7.Computes
 }
 
 func MakeTest(name string, directives ...string) *T {
@@ -118,13 +118,13 @@ func MakeTest(name string, directives ...string) *T {
 		t.directives = []string{}
 	}
 
-	t.workflow = &dwsv1alpha6.Workflow{
+	t.workflow = &dwsv1alpha7.Workflow{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      t.WorkflowName(),
 			Namespace: corev1.NamespaceDefault,
 		},
-		Spec: dwsv1alpha6.WorkflowSpec{
-			DesiredState: dwsv1alpha6.StateProposal,
+		Spec: dwsv1alpha7.WorkflowSpec{
+			DesiredState: dwsv1alpha7.StateProposal,
 			DWDirectives: t.WorkflowDirectives(),
 			JobID:        intstr.FromInt(GinkgoParallelProcess()),
 			WLMID:        strconv.Itoa(GinkgoParallelProcess()),
@@ -145,14 +145,15 @@ func (t *T) WorkflowDirectives() []string {
 	return t.directives
 }
 
-func (t *T) Workflow() *dwsv1alpha6.Workflow {
+func (t *T) Workflow() *dwsv1alpha7.Workflow {
 	return t.workflow
 }
 
 // To apply a set of labels for a particular test, use the withLables() method. Labels
 const (
 	Simple         = "simple"
 	ExternalLustre = "external_lustre"
+	GFS2Fence      = "gfs2_fence"
 )
 
 func (t *T) WithLabels(labels ...string) *T { t.labels = append(t.labels, labels...); return t }