diff --git a/go.mod b/go.mod index 3f36929..1a0e9f4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/DataWorkflowServices/dws v0.0.1-0.20260309165511-cf9255cfc6a7 github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260309172025-4b4ec57b9127 github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 // indirect - github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260312170612-fb412e3dde6f + github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260313175029-02a4b3a23e5d github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 go.openly.dev/pointy v1.3.0 diff --git a/go.sum b/go.sum index 497857c..75a1fcc 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260309172025-4b4ec57b9127 github.com/NearNodeFlash/lustre-fs-operator v0.0.1-0.20260309172025-4b4ec57b9127/go.mod h1:zVG05Nu1CB1qzpCPoSODieC/bYmSPn5Vd8zsbWq3uHw= github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379 h1:kZHhYVMa2KdBpdXbAtnE5DSu6pN/9VEVU2t8wPUAqaY= github.com/NearNodeFlash/nnf-ec v0.0.1-0.20260310162051-7797cd568379/go.mod h1:vJXM5DUx6FZtK8yuYNMSv2O1mfUmrpRVuRZp/o/NDRs= -github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260312170612-fb412e3dde6f h1:qnyX+B9zx18qtkdXUCHbB+FwvgLerfT4Cn8rvp+uBM0= -github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260312170612-fb412e3dde6f/go.mod h1:ki5lg1qIY2tHmCaMkAR5rj87XUBb39QHsdGSj9svpBE= +github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260313175029-02a4b3a23e5d h1:48l8A1owxg5Br2Q6oJ30YGDPCo4re6NtyPuSJAjDCs0= +github.com/NearNodeFlash/nnf-sos v0.0.1-0.20260313175029-02a4b3a23e5d/go.mod h1:ki5lg1qIY2tHmCaMkAR5rj87XUBb39QHsdGSj9svpBE= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= diff --git a/int_test.go b/int_test.go index 791a8e9..467c053 100644 --- a/int_test.go +++ b/int_test.go @@ -46,6 +46,13 @@ var ( dwsv1alpha7.StateSetup, dwsv1alpha7.StateTeardown, } + + // User/Group IDs for tests that require permissions. These reference the + // configurable values in internal/utils.go which can be overridden via + // NNF_USER_ID and NNF_GROUP_ID environment variables. They must correspond + // to a real user on the system's Rabbit/compute nodes. + userID = TestUserID + groupID = TestGroupID ) var tests = []*T{ @@ -110,7 +117,7 @@ var tests = []*T{ "#DW copy_out source=$DW_JOB_xfs-data-movement/test.in destination=/lus/zenith/testuser/test.out"). WithPersistentLustre("xfs-data-movement-lustre-instance"). WithGlobalLustreFromPersistentLustre("zenith", []string{"default"}). - WithPermissions(1050, 1051). + WithPermissions(userID, groupID). WithLabels("dm"). HardwareRequired(), MakeTest("GFS2 with Data Movement", @@ -119,7 +126,7 @@ var tests = []*T{ "#DW copy_out profile=no-xattr source=$DW_JOB_gfs2-data-movement/test.in destination=/lus/kelso/testuser/test.out"). WithPersistentLustre("gfs2-data-movement-lustre-instance"). WithGlobalLustreFromPersistentLustre("kelso", []string{"default"}). - WithPermissions(1050, 1051). + WithPermissions(userID, groupID). WithLabels("dm"). HardwareRequired(), MakeTest("Lustre with Data Movement", @@ -129,7 +136,7 @@ var tests = []*T{ WithPersistentLustre("lustre-data-movement-lustre-instance"). WithGlobalLustreFromPersistentLustre("flame", []string{"default"}). WithStorageProfileExternalMGSFromPersistentLustre(). - WithPermissions(1050, 1051). + WithPermissions(userID, groupID). WithLabels("dm"). HardwareRequired(), @@ -138,18 +145,18 @@ var tests = []*T{ "#DW jobdw type=gfs2 name=gfs2-with-containers-mpi capacity=100GB", "#DW container name=gfs2-with-containers-mpi profile=example-mpi "+ "DW_JOB_foo_local_storage=gfs2-with-containers-mpi"). - WithPermissions(1050, 1051).WithLabels("mpi"), + WithPermissions(userID, groupID).WithLabels("mpi"), MakeTest("Lustre with MPI Containers", "#DW jobdw type=lustre name=lustre-with-containers-mpi capacity=100GB", "#DW container name=lustre-with-containers-mpi profile=example-mpi "+ "DW_JOB_foo_local_storage=lustre-with-containers-mpi"). - WithPermissions(1050, 1051).WithLabels("mpi"), + WithPermissions(userID, groupID).WithLabels("mpi"), MakeTest("GFS2 and Global Lustre with MPI Containers", "#DW jobdw type=gfs2 name=gfs2-and-global-with-containers-mpi capacity=100GB", "#DW container name=gfs2-and-global-with-containers-mpi profile=example-mpi "+ "DW_JOB_foo_local_storage=gfs2-and-global-with-containers-mpi "+ "DW_GLOBAL_foo_global_lustre=/lus/polly"). - WithPermissions(1050, 1051). + WithPermissions(userID, groupID). WithPersistentLustre("gfs2-and-global-with-containers-polly"). WithGlobalLustreFromPersistentLustre("polly", []string{"default"}). WithLabels("mpi", "global-lustre"), @@ -157,17 +164,17 @@ var tests = []*T{ // Containers - MPI failures MakeTest("PreRun timeout on MPI containers", "#DW container name=prerun-timeout-mpi profile=example-mpi-prerun-timeout"). - WithPermissions(1050, 1051).WithLabels("mpi", "timeout"). + WithPermissions(userID, groupID).WithLabels("mpi", "timeout"). WithContainerProfile("example-mpi", &ContainerProfileOptions{PrerunTimeoutSeconds: pointy.Int(1), NoStorage: true}). ExpectError(dwsv1alpha7.StatePreRun), MakeTest("PostRun timeout on MPI containers", "#DW container name=postrun-timeout-mpi profile=example-mpi-postrun-timeout"). - WithPermissions(1050, 1051).WithLabels("mpi", "timeout"). + WithPermissions(userID, groupID).WithLabels("mpi", "timeout"). WithContainerProfile("example-mpi-webserver", &ContainerProfileOptions{PostrunTimeoutSeconds: pointy.Int(1), NoStorage: true}). ExpectError(dwsv1alpha7.StatePostRun), MakeTest("Non-zero exit on MPI containers", "#DW container name=mpi-container-fail profile=example-mpi-fail-noretry"). - WithPermissions(1050, 1051).WithLabels("mpi", "fail"). + WithPermissions(userID, groupID).WithLabels("mpi", "fail"). WithContainerProfile("example-mpi-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}). ExpectError(dwsv1alpha7.StatePostRun), @@ -175,13 +182,13 @@ var tests = []*T{ MakeTest("GFS2 with Containers", "#DW jobdw type=gfs2 name=gfs2-with-containers capacity=100GB", "#DW container name=gfs2-with-containers profile=example-success DW_JOB_foo_local_storage=gfs2-with-containers"). - WithPermissions(1050, 1051).WithLabels("non-mpi"), + WithPermissions(userID, groupID).WithLabels("non-mpi"), MakeTest("GFS2 and Global Lustre with Containers", "#DW jobdw type=gfs2 name=gfs2-and-global-with-containers capacity=100GB", "#DW container name=gfs2-and-global-with-containers profile=example-success "+ "DW_JOB_foo_local_storage=gfs2-and-global-with-containers "+ "DW_GLOBAL_foo_global_lustre=/lus/cherokee"). - WithPermissions(1050, 1051). + WithPermissions(userID, groupID). WithPersistentLustre("gfs2-and-global-with-containers-cherokee"). WithGlobalLustreFromPersistentLustre("cherokee", []string{"default"}). WithLabels("non-mpi", "global-lustre"), @@ -189,17 +196,17 @@ var tests = []*T{ // Containers - Non-MPI failures MakeTest("PreRun timeout on non-MPI containers", "#DW container name=prerun-timeout profile=example-prerun-timeout"). - WithPermissions(1050, 1051).WithLabels("non-mpi", "timeout"). + WithPermissions(userID, groupID).WithLabels("non-mpi", "timeout"). WithContainerProfile("example-forever", &ContainerProfileOptions{PrerunTimeoutSeconds: pointy.Int(1), NoStorage: true}). ExpectError(dwsv1alpha7.StatePreRun), MakeTest("PostRun timeout on non-MPI containers", "#DW container name=postrun-timeout profile=example-postrun-timeout"). - WithPermissions(1050, 1051).WithLabels("non-mpi", "timeout"). + WithPermissions(userID, groupID).WithLabels("non-mpi", "timeout"). WithContainerProfile("example-forever", &ContainerProfileOptions{PostrunTimeoutSeconds: pointy.Int(1), NoStorage: true}). ExpectError(dwsv1alpha7.StatePostRun), MakeTest("Non-zero exit on non-MPI containers", "#DW container name=container-fail profile=example-fail-noretry"). - WithPermissions(1050, 1051).WithLabels("non-mpi", "fail"). + WithPermissions(userID, groupID).WithLabels("non-mpi", "fail"). WithContainerProfile("example-fail", &ContainerProfileOptions{RetryLimit: pointy.Int(0)}). ExpectError(dwsv1alpha7.StatePostRun), @@ -219,14 +226,14 @@ var tests = []*T{ "#DW persistentdw name=containers-persistent-storage", "#DW container name=gfs2-lustre-with-containers profile=example-success DW_JOB_foo_local_storage=containers-local-storage DW_PERSISTENT_foo_persistent_storage=containers-persistent-storage"). WithPersistentLustre("containers-persistent-storage"). - WithPermissions(1050, 1051). + WithPermissions(userID, groupID). WithLabels("multi-storage"), MakeTest("GFS2 and Lustre with Containers MPI", "#DW jobdw name=containers-local-storage-mpi type=gfs2 capacity=100GB", "#DW persistentdw name=containers-persistent-storage-mpi", "#DW container name=gfs2-lustre-with-containers-mpi profile=example-mpi DW_JOB_foo_local_storage=containers-local-storage-mpi DW_PERSISTENT_foo_persistent_storage=containers-persistent-storage-mpi"). WithPersistentLustre("containers-persistent-storage-mpi"). - WithPermissions(1050, 1051). + WithPermissions(userID, groupID). WithLabels("multi-storage"), // External MGS @@ -279,6 +286,12 @@ var _ = Describe("NNF Integration Test", func() { if report.Failed() { workflow := t.Workflow() AddReportEntry(fmt.Sprintf("Workflow '%s' Failed", workflow.Name), workflow.Status) + + // Include container pod logs for container tests to aid diagnosis + if t.HasContainerDirective() { + AddReportEntry(fmt.Sprintf("Container Pod Logs for '%s'", workflow.Name), + ReportContainerPodLogs(ctx, k8sClient, workflow)) + } } }) diff --git a/internal/states.go b/internal/states.go index 8236730..4087a3c 100644 --- a/internal/states.go +++ b/internal/states.go @@ -199,6 +199,14 @@ func (t *T) preRun(ctx context.Context, k8sClient client.Client, workflow *dwsv1 func (t *T) postRun(ctx context.Context, k8sClient client.Client, workflow *dwsv1alpha7.Workflow) { t.AdvanceStateAndWaitForReady(ctx, k8sClient, workflow, dwsv1alpha7.StatePostRun) + + // After successful PostRun, verify MPI container pod logs for SSH errors. + // Only MPI workflows use SSH between launcher and workers. Skip for tests + // that expect errors since those containers are designed to fail. + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(workflow), workflow)).To(Succeed()) + if _, isMPI := workflow.Status.Env["NNF_CONTAINER_LAUNCHER"]; isMPI && t.options.expectError == nil { + VerifyContainerPodLogs(ctx, k8sClient, workflow) + } } func (t *T) dataOut(ctx context.Context, k8sClient client.Client, workflow *dwsv1alpha7.Workflow) { diff --git a/internal/utils.go b/internal/utils.go index a45c08c..406ff6f 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -22,6 +22,7 @@ package internal import ( "context" "fmt" + "os" "os/exec" "strconv" "strings" @@ -31,16 +32,115 @@ import ( . "github.com/onsi/gomega" dwsv1alpha7 "github.com/DataWorkflowServices/dws/api/v1alpha7" + nnfv1alpha11 "github.com/NearNodeFlash/nnf-sos/api/v1alpha11" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/DataWorkflowServices/dws/utils/dwdparse" ) const ( helperImage = "ghcr.io/nearnodeflash/nnf-integration-test-helper" ) +var ( + // TestUserID and TestGroupID are the UID/GID used for workflow permissions in + // container and data movement tests. These must correspond to a real user on the + // system's compute/Rabbit nodes so that MPI SSH authentication is properly exercised. + // Defaults to the flux user (1051/1052) since that is the WLM user that submits + // MPI jobs in production. Using the mpiuser account (1050) would bypass the SSH key + // setup code path where issue #310 occurs. + // Override via NNF_USER_ID and NNF_GROUP_ID environment variables. + TestUserID uint32 = 1051 + TestGroupID uint32 = 1052 +) + +func init() { + if s := os.Getenv("NNF_USER_ID"); s != "" { + v, err := strconv.ParseUint(s, 10, 32) + if err != nil { + panic(fmt.Sprintf("invalid NNF_USER_ID '%s': %v", s, err)) + } + TestUserID = uint32(v) + } + if s := os.Getenv("NNF_GROUP_ID"); s != "" { + v, err := strconv.ParseUint(s, 10, 32) + if err != nil { + panic(fmt.Sprintf("invalid NNF_GROUP_ID '%s': %v", s, err)) + } + TestGroupID = uint32(v) + } +} + +// VerifyUserOnRabbit creates a pod on a Rabbit node to verify that the given UID +// corresponds to a real user. This is important for MPI container tests: if the user +// doesn't exist on the system, SSH key setup follows a different code path and won't +// exercise the authentication flow that caused issue #310. +// Checks all Rabbit nodes in the system configuration. +func VerifyUserOnRabbit(ctx context.Context, k8sClient client.Client, uid uint32) { + By(fmt.Sprintf("Verifying UID %d exists on the system", uid)) + + systemConfig := GetSystemConfiguraton(ctx, k8sClient) + + var missing []string + for _, storageNode := range systemConfig.Spec.StorageNodes { + rabbitName := storageNode.Name + podName := fmt.Sprintf("verify-uid-%d-%s", uid, rabbitName) + privileged := true + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: corev1.NamespaceDefault, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + NodeName: rabbitName, + HostPID: true, + Containers: []corev1.Container{{ + Name: "verify-uid", + Image: "alpine:latest", + // Use nsenter to run 'id' in the host's mount namespace so we + // see the host's /etc/passwd rather than the container's. + Command: []string{"nsenter", "-t", "1", "-m", "--", "id", fmt.Sprintf("%d", uid)}, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + }}, + }, + } + + Expect(k8sClient.Create(ctx, pod)).To(Succeed()) + + // Wait for pod to terminate + Eventually(func(g Gomega) corev1.PodPhase { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + return pod.Status.Phase + }).WithTimeout(time.Minute).WithPolling(time.Second).Should( + SatisfyAny(Equal(corev1.PodSucceeded), Equal(corev1.PodFailed)), + ) + + if pod.Status.Phase != corev1.PodSucceeded { + missing = append(missing, rabbitName) + } + + // Clean up the verification pod + Expect(k8sClient.Delete(ctx, pod)).To(Succeed()) + WaitForDeletion(ctx, k8sClient, pod) + + By(fmt.Sprintf("Checked UID %d on Rabbit node '%s': found=%t", uid, rabbitName, pod.Status.Phase == corev1.PodSucceeded)) + } + + Expect(missing).To(BeEmpty(), + fmt.Sprintf("UID %d does not exist on Rabbit node(s): %s. "+ + "MPI container tests require a real system user to properly exercise SSH authentication. "+ + "Set NNF_USER_ID and NNF_GROUP_ID environment variables to a valid user on the system.", + uid, strings.Join(missing, ", "))) + + By(fmt.Sprintf("Verified UID %d exists on all %d Rabbit node(s)", uid, len(systemConfig.Spec.StorageNodes))) +} + func GetSystemConfiguraton(ctx context.Context, k8sClient client.Client) *dwsv1alpha7.SystemConfiguration { // TODO: Move this to a global variable and initialized in the test suite. // Note that putting the GET in Prepare will not work for things like @@ -161,6 +261,151 @@ func runHelperPod(ctx context.Context, k8sClient client.Client, t *T, name, comm }).WithTimeout(time.Minute).WithPolling(time.Second).Should(BeTrue()) } +// HasContainerDirective returns true if this test includes a #DW container directive. +func (t *T) HasContainerDirective() bool { + for _, directive := range t.directives { + args, _ := dwdparse.BuildArgsMap(directive) + if args["command"] == "container" { + return true + } + } + return false +} + +// getPodLogs retrieves logs from a specific pod container using kubectl. +func getPodLogs(namespace, podName, containerName string) (string, error) { + args := []string{"logs", podName, "-n", namespace} + if containerName != "" { + args = append(args, "-c", containerName) + } + out, err := exec.Command("kubectl", args...).CombinedOutput() + return string(out), err +} + +// sshErrorPatterns are log patterns indicating MPI SSH communication failures. +// See NearNodeFlash/NearNodeFlash.github.io#310. +var sshErrorPatterns = []string{ + "Permission denied", + "ORTE was unable to reliably start one or more daemons", +} + +// findContainerPods discovers container pods for a workflow. For MPI workflows, +// NNF_CONTAINER_LAUNCHER is set and NNF_CONTAINER_HOSTNAMES lists the launcher and +// worker hostnames. The MPI operator names worker pods exactly as the hostname, but +// the launcher pod has a random suffix (e.g. "workflow-launcher-xxxxx"), so we use +// prefix matching. For non-MPI containers, pods are found using owner labels. +func findContainerPods(ctx context.Context, k8sClient client.Client, workflow *dwsv1alpha7.Workflow) []*corev1.Pod { + var result []*corev1.Pod + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(workflow), workflow)).To(Succeed()) + + // Only MPI workflows have NNF_CONTAINER_LAUNCHER set. Non-MPI containers also + // set NNF_CONTAINER_HOSTNAMES (with node names), so we must check LAUNCHER first. + if _, isMPI := workflow.Status.Env["NNF_CONTAINER_LAUNCHER"]; isMPI { + hostnames := workflow.Status.Env["NNF_CONTAINER_HOSTNAMES"] + By(fmt.Sprintf("Found MPI container hostnames from workflow status: '%s'", hostnames)) + + pods := &corev1.PodList{} + Expect(k8sClient.List(ctx, pods, client.InNamespace(workflow.Namespace))).To(Succeed()) + + // Build a set of hostnames for exact and prefix matching + hostnameList := strings.Split(hostnames, ",") + + for i := range pods.Items { + pod := &pods.Items[i] + for _, hostname := range hostnameList { + // Worker pods match exactly; launcher pods use the hostname as a prefix + if pod.Name == hostname || strings.HasPrefix(pod.Name, hostname+"-") { + result = append(result, pod) + break + } + } + } + + return result + } + + // Fallback: find pods using owner labels (for non-MPI containers or if env isn't set) + pods := &corev1.PodList{} + Expect(k8sClient.List(ctx, pods, client.InNamespace(workflow.Namespace))).To(Succeed()) + + for i := range pods.Items { + pod := &pods.Items[i] + // Check for NNF container label + if _, ok := pod.Labels[nnfv1alpha11.ContainerLabel]; ok { + if pod.Labels[dwsv1alpha7.WorkflowNameLabel] == workflow.Name || + pod.Labels[dwsv1alpha7.OwnerNameLabel] == workflow.Name { + result = append(result, pod) + } + } + } + + return result +} + +// VerifyContainerPodLogs finds container pods for the given workflow and verifies +// their logs don't contain SSH authentication errors that indicate MPI communication +// failures. This catches issues like NearNodeFlash/NearNodeFlash.github.io#310 where +// mpirun fails with "Permission denied" when SSHing to worker nodes. +func VerifyContainerPodLogs(ctx context.Context, k8sClient client.Client, workflow *dwsv1alpha7.Workflow) { + By("Verifying container pod logs for SSH errors") + + containerPods := findContainerPods(ctx, k8sClient, workflow) + Expect(containerPods).NotTo(BeEmpty(), "expected to find container pods for workflow '%s' but found none", workflow.Name) + + for _, pod := range containerPods { + By(fmt.Sprintf("Checking container pod '%s' (phase: %s)", pod.Name, pod.Status.Phase)) + + // Get and check logs from all containers in the pod + for _, container := range pod.Spec.Containers { + logs, err := getPodLogs(pod.Namespace, pod.Name, container.Name) + if err != nil { + By(fmt.Sprintf("Warning: could not retrieve logs for pod '%s' container '%s': %v", + pod.Name, container.Name, err)) + continue + } + + for _, pattern := range sshErrorPatterns { + Expect(logs).NotTo(ContainSubstring(pattern), + fmt.Sprintf("SSH error detected in pod '%s' container '%s': found '%s' in logs.\n"+ + "This indicates an MPI communication failure (see NearNodeFlash/NearNodeFlash.github.io#310).\n"+ + "Logs:\n%s", + pod.Name, container.Name, pattern, logs)) + } + } + } + + By(fmt.Sprintf("Verified %d container pod(s): no SSH errors found", len(containerPods))) +} + +// ReportContainerPodLogs collects container pod logs for a workflow and returns +// them as a formatted string for inclusion in test failure reports. +func ReportContainerPodLogs(ctx context.Context, k8sClient client.Client, workflow *dwsv1alpha7.Workflow) string { + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(workflow), workflow); err != nil { + return fmt.Sprintf("Failed to get workflow: %v", err) + } + + containerPods := findContainerPods(ctx, k8sClient, workflow) + + var sb strings.Builder + for _, pod := range containerPods { + sb.WriteString(fmt.Sprintf("\n--- Pod: %s (phase: %s) ---\n", pod.Name, pod.Status.Phase)) + for _, container := range pod.Spec.Containers { + logs, err := getPodLogs(pod.Namespace, pod.Name, container.Name) + if err != nil { + sb.WriteString(fmt.Sprintf(" [%s] Error getting logs: %v\n", container.Name, err)) + continue + } + sb.WriteString(fmt.Sprintf(" [%s] Logs:\n%s\n", container.Name, logs)) + } + } + + if sb.Len() == 0 { + return "No container pods found" + } + return sb.String() +} + func CleanupHelperPods(ctx context.Context, k8sClient client.Client, t *T) { for _, p := range t.helperPods { By(fmt.Sprintf("Deleting helper pod %s", p.Name)) diff --git a/suite_test.go b/suite_test.go index 31183d1..d5f7140 100644 --- a/suite_test.go +++ b/suite_test.go @@ -137,6 +137,11 @@ var _ = BeforeSuite(func() { } } + // Verify the test user exists on the system. MPI container tests require a real + // system user so that SSH key setup exercises the correct code paths. + // See NearNodeFlash/NearNodeFlash.github.io#310. + By(fmt.Sprintf("Verifying test user UID=%d GID=%d exists on the system", TestUserID, TestGroupID)) + VerifyUserOnRabbit(ctx, k8sClient, TestUserID) }) var _ = AfterSuite(func() {