Skip to content

Commit fadb5f4

Browse files
ajfloederajfloeder
authored
Merge release v0.1.24
Release v0.1.24
2 parents efbfbd0 + 4a303f7 commit fadb5f4

6 files changed

Lines changed: 290 additions & 19 deletions

File tree

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/DataWorkflowServices/dws v0.0.1-0.20260309165511-cf9255cfc6a7
77
github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260309172025-4b4ec57b9127
88
github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 // indirect
9-
github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260312170612-fb412e3dde6f
9+
github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260313175029-02a4b3a23e5d
1010
github.com/onsi/ginkgo/v2 v2.22.2
1111
github.com/onsi/gomega v1.36.2
1212
go.openly.dev/pointy v1.3.0

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260309172025-4b4ec57b9127
44
github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260309172025-4b4ec57b9127/go.mod h1:zVG05Nu1CB1qzpCPoSODieC/bYmSPn5Vd8zsbWq3uHw=
55
github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 h1:kZHhYVMa2KdBpdXbAtnE5DSu6pN/9VEVU2t8wPUAqaY=
66
github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379/go.mod h1:vJXM5DUx6FZtK8yuYNMSv2O1mfUmrpRVuRZp/o/NDRs=
7-
github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260312170612-fb412e3dde6f h1:qnyX+B9zx18qtkdXUCHbB+FwvgLerfT4Cn8rvp+uBM0=
8-
github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260312170612-fb412e3dde6f/go.mod h1:ki5lg1qIY2tHmCaMkAR5rj87XUBb39QHsdGSj9svpBE=
7+
github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260313175029-02a4b3a23e5d h1:48l8A1owxg5Br2Q6oJ30YGDPCo4re6NtyPuSJAjDCs0=
8+
github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260313175029-02a4b3a23e5d/go.mod h1:ki5lg1qIY2tHmCaMkAR5rj87XUBb39QHsdGSj9svpBE=
99
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
1010
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
1111
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

‎int_test.go‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,13 @@ var (
4646
dwsv1alpha7.StateSetup,
4747
dwsv1alpha7.StateTeardown,
4848
}
49+
50+
// User/Group IDs for tests that require permissions. These reference the
51+
// configurable values in internal/utils.go which can be overridden via
52+
// NNF_USER_ID and NNF_GROUP_ID environment variables. They must correspond
53+
// to a real user on the system's Rabbit/compute nodes.
54+
userID = TestUserID
55+
groupID = TestGroupID
4956
)
5057

5158
var tests = []*T{
@@ -110,7 +117,7 @@ var tests = []*T{
110117
"#DW copy_out source=$DW_JOB_xfs-data-movement/test.in destination=/lus/zenith/testuser/test.out").
111118
WithPersistentLustre("xfs-data-movement-lustre-instance").
112119
WithGlobalLustreFromPersistentLustre("zenith", []string{"default"}).
113-
WithPermissions(1050, 1051).
120+
WithPermissions(userID, groupID).
114121
WithLabels("dm").
115122
HardwareRequired(),
116123
MakeTest("GFS2 with Data Movement",
@@ -119,7 +126,7 @@ var tests = []*T{
119126
"#DW copy_out profile=no-xattr source=$DW_JOB_gfs2-data-movement/test.in destination=/lus/kelso/testuser/test.out").
120127
WithPersistentLustre("gfs2-data-movement-lustre-instance").
121128
WithGlobalLustreFromPersistentLustre("kelso", []string{"default"}).
122-
WithPermissions(1050, 1051).
129+
WithPermissions(userID, groupID).
123130
WithLabels("dm").
124131
HardwareRequired(),
125132
MakeTest("Lustre with Data Movement",
@@ -129,7 +136,7 @@ var tests = []*T{
129136
WithPersistentLustre("lustre-data-movement-lustre-instance").
130137
WithGlobalLustreFromPersistentLustre("flame", []string{"default"}).
131138
WithStorageProfileExternalMGSFromPersistentLustre().
132-
WithPermissions(1050, 1051).
139+
WithPermissions(userID, groupID).
133140
WithLabels("dm").
134141
HardwareRequired(),
135142

@@ -138,68 +145,68 @@ var tests = []*T{
138145
"#DW jobdw type=gfs2 name=gfs2-with-containers-mpi capacity=100GB",
139146
"#DW container name=gfs2-with-containers-mpi profile=example-mpi "+
140147
"DW_JOB_foo_local_storage=gfs2-with-containers-mpi").
141-
WithPermissions(1050, 1051).WithLabels("mpi"),
148+
WithPermissions(userID, groupID).WithLabels("mpi"),
142149
MakeTest("Lustre with MPI Containers",
143150
"#DW jobdw type=lustre name=lustre-with-containers-mpi capacity=100GB",
144151
"#DW container name=lustre-with-containers-mpi profile=example-mpi "+
145152
"DW_JOB_foo_local_storage=lustre-with-containers-mpi").
146-
WithPermissions(1050, 1051).WithLabels("mpi"),
153+
WithPermissions(userID, groupID).WithLabels("mpi"),
147154
MakeTest("GFS2 and Global Lustre with MPI Containers",
148155
"#DW jobdw type=gfs2 name=gfs2-and-global-with-containers-mpi capacity=100GB",
149156
"#DW container name=gfs2-and-global-with-containers-mpi profile=example-mpi "+
150157
"DW_JOB_foo_local_storage=gfs2-and-global-with-containers-mpi "+
151158
"DW_GLOBAL_foo_global_lustre=/lus/polly").
152-
WithPermissions(1050, 1051).
159+
WithPermissions(userID, groupID).
153160
WithPersistentLustre("gfs2-and-global-with-containers-polly").
154161
WithGlobalLustreFromPersistentLustre("polly", []string{"default"}).
155162
WithLabels("mpi", "global-lustre"),
156163

157164
// Containers - MPI failures
158165
MakeTest("PreRun timeout on MPI containers",
159166
"#DW container name=prerun-timeout-mpi profile=example-mpi-prerun-timeout").
160-
WithPermissions(1050, 1051).WithLabels("mpi", "timeout").
167+
WithPermissions(userID, groupID).WithLabels("mpi", "timeout").
161168
WithContainerProfile("example-mpi", &ContainerProfileOptions{PrerunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
162169
ExpectError(dwsv1alpha7.StatePreRun),
163170
MakeTest("PostRun timeout on MPI containers",
164171
"#DW container name=postrun-timeout-mpi profile=example-mpi-postrun-timeout").
165-
WithPermissions(1050, 1051).WithLabels("mpi", "timeout").
172+
WithPermissions(userID, groupID).WithLabels("mpi", "timeout").
166173
WithContainerProfile("example-mpi-webserver", &ContainerProfileOptions{PostrunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
167174
ExpectError(dwsv1alpha7.StatePostRun),
168175
MakeTest("Non-zero exit on MPI containers",
169176
"#DW container name=mpi-container-fail profile=example-mpi-fail-noretry").
170-
WithPermissions(1050, 1051).WithLabels("mpi", "fail").
177+
WithPermissions(userID, groupID).WithLabels("mpi", "fail").
171178
WithContainerProfile("example-mpi-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}).
172179
ExpectError(dwsv1alpha7.StatePostRun),
173180

174181
// Containers - Non-MPI
175182
MakeTest("GFS2 with Containers",
176183
"#DW jobdw type=gfs2 name=gfs2-with-containers capacity=100GB",
177184
"#DW container name=gfs2-with-containers profile=example-success DW_JOB_foo_local_storage=gfs2-with-containers").
178-
WithPermissions(1050, 1051).WithLabels("non-mpi"),
185+
WithPermissions(userID, groupID).WithLabels("non-mpi"),
179186
MakeTest("GFS2 and Global Lustre with Containers",
180187
"#DW jobdw type=gfs2 name=gfs2-and-global-with-containers capacity=100GB",
181188
"#DW container name=gfs2-and-global-with-containers profile=example-success "+
182189
"DW_JOB_foo_local_storage=gfs2-and-global-with-containers "+
183190
"DW_GLOBAL_foo_global_lustre=/lus/cherokee").
184-
WithPermissions(1050, 1051).
191+
WithPermissions(userID, groupID).
185192
WithPersistentLustre("gfs2-and-global-with-containers-cherokee").
186193
WithGlobalLustreFromPersistentLustre("cherokee", []string{"default"}).
187194
WithLabels("non-mpi", "global-lustre"),
188195

189196
// Containers - Non-MPI failures
190197
MakeTest("PreRun timeout on non-MPI containers",
191198
"#DW container name=prerun-timeout profile=example-prerun-timeout").
192-
WithPermissions(1050, 1051).WithLabels("non-mpi", "timeout").
199+
WithPermissions(userID, groupID).WithLabels("non-mpi", "timeout").
193200
WithContainerProfile("example-forever", &ContainerProfileOptions{PrerunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
194201
ExpectError(dwsv1alpha7.StatePreRun),
195202
MakeTest("PostRun timeout on non-MPI containers",
196203
"#DW container name=postrun-timeout profile=example-postrun-timeout").
197-
WithPermissions(1050, 1051).WithLabels("non-mpi", "timeout").
204+
WithPermissions(userID, groupID).WithLabels("non-mpi", "timeout").
198205
WithContainerProfile("example-forever", &ContainerProfileOptions{PostrunTimeoutSeconds: pointy.Int(1), NoStorage: true}).
199206
ExpectError(dwsv1alpha7.StatePostRun),
200207
MakeTest("Non-zero exit on non-MPI containers",
201208
"#DW container name=container-fail profile=example-fail-noretry").
202-
WithPermissions(1050, 1051).WithLabels("non-mpi", "fail").
209+
WithPermissions(userID, groupID).WithLabels("non-mpi", "fail").
203210
WithContainerProfile("example-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}).
204211
ExpectError(dwsv1alpha7.StatePostRun),
205212

@@ -219,14 +226,14 @@ var tests = []*T{
219226
"#DW persistentdw name=containers-persistent-storage",
220227
"#DW container name=gfs2-lustre-with-containers profile=example-success DW_JOB_foo_local_storage=containers-local-storage DW_PERSISTENT_foo_persistent_storage=containers-persistent-storage").
221228
WithPersistentLustre("containers-persistent-storage").
222-
WithPermissions(1050, 1051).
229+
WithPermissions(userID, groupID).
223230
WithLabels("multi-storage"),
224231
MakeTest("GFS2 and Lustre with Containers MPI",
225232
"#DW jobdw name=containers-local-storage-mpi type=gfs2 capacity=100GB",
226233
"#DW persistentdw name=containers-persistent-storage-mpi",
227234
"#DW container name=gfs2-lustre-with-containers-mpi profile=example-mpi DW_JOB_foo_local_storage=containers-local-storage-mpi DW_PERSISTENT_foo_persistent_storage=containers-persistent-storage-mpi").
228235
WithPersistentLustre("containers-persistent-storage-mpi").
229-
WithPermissions(1050, 1051).
236+
WithPermissions(userID, groupID).
230237
WithLabels("multi-storage"),
231238

232239
// External MGS
@@ -279,6 +286,12 @@ var _ = Describe("NNF Integration Test", func() {
279286
if report.Failed() {
280287
workflow := t.Workflow()
281288
AddReportEntry(fmt.Sprintf("Workflow '%s' Failed", workflow.Name), workflow.Status)
289+
290+
// Include container pod logs for container tests to aid diagnosis
291+
if t.HasContainerDirective() {
292+
AddReportEntry(fmt.Sprintf("Container Pod Logs for '%s'", workflow.Name),
293+
ReportContainerPodLogs(ctx, k8sClient, workflow))
294+
}
282295
}
283296
})
284297

‎internal/states.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,14 @@ func (t *T) preRun(ctx context.Context, k8sClient client.Client, workflow *dwsv1
199199

200200
func (t *T) postRun(ctx context.Context, k8sClient client.Client, workflow *dwsv1alpha7.Workflow) {
201201
t.AdvanceStateAndWaitForReady(ctx, k8sClient, workflow, dwsv1alpha7.StatePostRun)
202+
203+
// After successful PostRun, verify MPI container pod logs for SSH errors.
204+
// Only MPI workflows use SSH between launcher and workers. Skip for tests
205+
// that expect errors since those containers are designed to fail.
206+
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(workflow), workflow)).To(Succeed())
207+
if _, isMPI := workflow.Status.Env["NNF_CONTAINER_LAUNCHER"]; isMPI && t.options.expectError == nil {
208+
VerifyContainerPodLogs(ctx, k8sClient, workflow)
209+
}
202210
}
203211

204212
func (t *T) dataOut(ctx context.Context, k8sClient client.Client, workflow *dwsv1alpha7.Workflow) {

0 commit comments

Comments
 (0)