Skip to content

Commit bc0eae4

Browse files
danilrwxdanilrwx
authored
test(usb): fix broken NodeUSBDevices e2e test (#2297)
Stabilize the VirtualMachineUSB e2e scenario and simplify the USB handling flow in the guest. Changes include: - check NodeUSBDevice detached state before creating the VM; - wait for the VM guest agent and USB readiness explicitly; - extract repeated USB test actions into helper methods; - rescan SCSI hosts before mount attempts in the guest; - detect the guest USB disk through lsblk while validating that the expected USB serial is present; - remount the USB filesystem after migration before reading the test data; - flush data with sync and unmount the filesystem before migration; - increase retries/timeouts around guest-side USB disk discovery after reconnect. --------- Signed-off-by: Daniil Antoshin <daniil.antoshin@flant.com>
1 parent 6ff05e7 commit bc0eae4

2 files changed

Lines changed: 150 additions & 61 deletions

File tree

‎test/e2e/internal/util/vm.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"regexp"
23+
"strings"
2324
"time"
2425

2526
. "github.com/onsi/ginkgo/v2"
@@ -184,6 +185,32 @@ func UntilSSHReady(f *framework.Framework, vm *v1alpha2.VirtualMachine, timeout
184185
}).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed())
185186
}
186187

188+
func UntilGuestCommandsReady(f *framework.Framework, vm *v1alpha2.VirtualMachine, commands []string, timeout time.Duration) {
189+
GinkgoHelper()
190+
191+
cmd := fmt.Sprintf(`
192+
missing=""
193+
for command in %s; do
194+
command -v "$command" >/dev/null 2>&1 || missing="$missing $command"
195+
done
196+
[ -z "$missing" ] || { echo "missing commands:$missing"; exit 1; }
197+
`, shellArgs(commands))
198+
199+
Eventually(func() error {
200+
_, err := f.SSHCommand(vm.Name, vm.Namespace, cmd, framework.WithSSHTimeout(5*time.Second))
201+
return err
202+
}).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed())
203+
}
204+
205+
func shellArgs(args []string) string {
206+
quoted := make([]string, 0, len(args))
207+
for _, arg := range args {
208+
quoted = append(quoted, fmt.Sprintf("%q", arg))
209+
}
210+
211+
return strings.Join(quoted, " ")
212+
}
213+
187214
func UntilVMMigrationSucceeded(key client.ObjectKey, timeout time.Duration) {
188215
GinkgoHelper()
189216

‎test/e2e/vm/usb.go‎

Lines changed: 123 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -62,58 +62,39 @@ var _ = Describe("VirtualMachineUSB", func() {
6262
}
6363

6464
t.GenerateEnvironmentResources()
65-
err := f.CreateWithDeferredDeletion(context.Background(), t.VD, t.VM)
65+
err := f.CreateWithDeferredDeletion(context.Background(), t.VD)
6666
Expect(err).NotTo(HaveOccurred())
6767

6868
t.assignNodeUSB()
69-
70-
util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, t.VM)
71-
util.UntilSSHReady(f, t.VM, framework.MiddleTimeout)
7269
})
7370

7471
By("Verifying NodeUSBDevice is not attached before VM attachment", func() {
75-
Eventually(func(g Gomega) {
76-
nodeUSBDevice, err := t.Framework.VirtClient().NodeUSBDevices().Get(t.ctx, t.NodeUSBDevice.Name, metav1.GetOptions{})
77-
g.Expect(err).NotTo(HaveOccurred())
78-
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice)).NotTo(BeNil())
79-
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice).Status).To(Equal(metav1.ConditionFalse))
80-
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
72+
t.waitForNodeUSBAttached(metav1.ConditionFalse)
73+
})
74+
75+
By("Creating VM with USB device", func() {
76+
err := f.CreateWithDeferredDeletion(context.Background(), t.VM)
77+
Expect(err).NotTo(HaveOccurred())
78+
79+
util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, t.VM)
80+
util.UntilSSHReady(f, t.VM, framework.MiddleTimeout)
81+
util.UntilGuestCommandsReady(f, t.VM, []string{"sudo", "tee", "udevadm"}, framework.LongTimeout)
8182
})
8283

8384
By("Waiting for USB device to be attached and ready", func() {
84-
Eventually(func() error {
85-
vm, err := t.Framework.VirtClient().VirtualMachines(t.VM.Namespace).Get(t.ctx, t.VM.Name, metav1.GetOptions{})
86-
Expect(err).NotTo(HaveOccurred())
87-
88-
for _, dev := range vm.Status.USBDevices {
89-
if dev.Name == t.NodeUSBDevice.Name && dev.Attached && dev.Ready {
90-
t.DevicePath = fmt.Sprintf("/dev/bus/usb/%d/%d", dev.Address.Bus, dev.Address.Port)
91-
return nil
92-
}
93-
}
94-
95-
return fmt.Errorf("USB device %s not attached or not ready", t.NodeUSBDevice.Name)
96-
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
85+
t.waitForVMUSBReady("USB device %s not attached or not ready")
9786
})
9887

9988
By("Verifying NodeUSBDevice is attached", func() {
100-
Eventually(func(g Gomega) {
101-
nodeUSBDevice, err := t.Framework.VirtClient().NodeUSBDevices().Get(t.ctx, t.NodeUSBDevice.Name, metav1.GetOptions{})
102-
g.Expect(err).NotTo(HaveOccurred())
103-
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice)).NotTo(BeNil())
104-
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice).Status).To(Equal(metav1.ConditionTrue))
105-
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
89+
t.waitForNodeUSBAttached(metav1.ConditionTrue)
10690
})
10791

10892
By("Mounting USB device", func() {
10993
t.mountUSB()
11094
})
11195

11296
By("Writing data to USB device", func() {
113-
result, err := t.Framework.SSHCommand(t.VM.Name, t.VM.Namespace, fmt.Sprintf("echo \"%s\" | sudo tee %s", t.testContent, t.testFile))
114-
115-
Expect(err).NotTo(HaveOccurred())
116-
Expect(result).To(ContainSubstring(t.testContent))
97+
t.writeUSBTestData()
11798
})
11899

119100
By("Migrating VM", func() {
@@ -125,37 +106,19 @@ var _ = Describe("VirtualMachineUSB", func() {
125106
})
126107

127108
By("Waiting for USB device to be ready after migration", func() {
128-
Eventually(func() error {
129-
vm, err := t.Framework.VirtClient().VirtualMachines(t.VM.Namespace).Get(t.ctx, t.VM.Name, metav1.GetOptions{})
130-
Expect(err).NotTo(HaveOccurred())
131-
132-
for _, dev := range vm.Status.USBDevices {
133-
if dev.Name == t.NodeUSBDevice.Name && dev.Attached && dev.Ready {
134-
return nil
135-
}
136-
}
137-
138-
return fmt.Errorf("USB device %s not ready after migration", t.NodeUSBDevice.Name)
139-
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
109+
t.waitForVMUSBReady("USB device %s not ready after migration")
140110
})
141111

142112
By("Verifying NodeUSBDevice is attached after migration", func() {
143-
Eventually(func(g Gomega) {
144-
nodeUSBDevice, err := t.Framework.VirtClient().NodeUSBDevices().Get(t.ctx, t.NodeUSBDevice.Name, metav1.GetOptions{})
145-
g.Expect(err).NotTo(HaveOccurred())
146-
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice)).NotTo(BeNil())
147-
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice).Status).To(Equal(metav1.ConditionTrue))
148-
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
113+
t.waitForNodeUSBAttached(metav1.ConditionTrue)
149114
})
150115

151116
By("Remounting USB device after migration", func() {
152117
t.mountUSB()
153118
})
154119

155120
By("Verifying data persists after migration", func() {
156-
result, err := t.Framework.SSHCommand(t.VM.Name, t.VM.Namespace, fmt.Sprintf("cat %s", t.testFile))
157-
Expect(err).NotTo(HaveOccurred())
158-
Expect(result).To(ContainSubstring(t.testContent))
121+
t.verifyUSBTestData()
159122
})
160123
})
161124
})
@@ -167,7 +130,6 @@ type VMUSBTest struct {
167130
VM *v1alpha2.VirtualMachine
168131
VD *v1alpha2.VirtualDisk
169132
NodeUSBDevice *v1alpha2.NodeUSBDevice
170-
DevicePath string
171133

172134
testFile string
173135
testContent string
@@ -256,15 +218,115 @@ func (t *VMUSBTest) assignNodeUSB() {
256218
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
257219
}
258220

221+
func (t *VMUSBTest) waitForNodeUSBAttached(status metav1.ConditionStatus) {
222+
Eventually(func(g Gomega) {
223+
nodeUSBDevice, err := t.Framework.VirtClient().NodeUSBDevices().Get(t.ctx, t.NodeUSBDevice.Name, metav1.GetOptions{})
224+
g.Expect(err).NotTo(HaveOccurred())
225+
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice)).NotTo(BeNil())
226+
g.Expect(nodeUSBAttachedCondition(nodeUSBDevice).Status).To(Equal(status))
227+
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
228+
}
229+
230+
func (t *VMUSBTest) waitForVMUSBReady(message string) {
231+
Eventually(func() error {
232+
vm, err := t.Framework.VirtClient().VirtualMachines(t.VM.Namespace).Get(t.ctx, t.VM.Name, metav1.GetOptions{})
233+
Expect(err).NotTo(HaveOccurred())
234+
235+
for _, dev := range vm.Status.USBDevices {
236+
if dev.Name == t.NodeUSBDevice.Name && dev.Attached && dev.Ready {
237+
return nil
238+
}
239+
}
240+
241+
return fmt.Errorf(message, t.NodeUSBDevice.Name)
242+
}).WithTimeout(framework.MaxTimeout).WithPolling(time.Second).Should(Succeed())
243+
}
244+
245+
func (t *VMUSBTest) writeUSBTestData() {
246+
result, err := t.Framework.SSHCommand(t.VM.Name, t.VM.Namespace, fmt.Sprintf("echo \"%s\" | sudo tee %s && sudo sync && sudo umount /mnt/usb", t.testContent, t.testFile))
247+
Expect(err).NotTo(HaveOccurred())
248+
Expect(result).To(ContainSubstring(t.testContent))
249+
}
250+
251+
func (t *VMUSBTest) verifyUSBTestData() {
252+
result, err := t.Framework.SSHCommand(t.VM.Name, t.VM.Namespace, fmt.Sprintf("cat %s", t.testFile))
253+
Expect(err).NotTo(HaveOccurred())
254+
Expect(result).To(ContainSubstring(t.testContent))
255+
}
256+
259257
func (t *VMUSBTest) mountUSB() {
258+
serial := t.NodeUSBDevice.Status.Attributes.Serial
259+
Expect(serial).NotTo(BeEmpty(), "USB device serial must be set")
260+
260261
mountCmd := fmt.Sprintf(`
261-
sudo mkdir -p /mnt/usb || true && \
262-
sudo mount %s /mnt/usb 2>/dev/null || sudo mount -o rw %s /mnt/usb || true && \
263-
ls -la /mnt/usb || true
264-
`, t.DevicePath, t.DevicePath)
262+
usb_serial=%q
263+
: > /tmp/usb-mount.err
264+
for serial_file in /sys/bus/usb/devices/*/serial; do
265+
if [ -f "$serial_file" ] && [ "$(cat "$serial_file")" = "$usb_serial" ]; then
266+
usb_present=1
267+
break
268+
fi
269+
done
270+
[ -n "$usb_present" ] || { echo "USB device with serial $usb_serial not found" >/tmp/usb-mount.err; exit 1; }
271+
272+
sudo modprobe usb-storage
273+
sudo modprobe uas || true
274+
275+
for host in /sys/class/scsi_host/host*; do
276+
echo "- - -" | sudo tee "$host/scan" >/dev/null || true
277+
done
278+
279+
sudo udevadm trigger
280+
sudo udevadm settle
281+
282+
for dev in /dev/sd*; do
283+
[ -b "$dev" ] || continue
284+
if lsblk -dno TRAN,RM "$dev" 2>/dev/null | grep -Eq '^usb[[:space:]]+1$'; then
285+
mount_device="$dev"
286+
break
287+
fi
288+
done
289+
[ -n "$mount_device" ] || {
290+
echo "USB block device not found for serial $usb_serial" >>/tmp/usb-mount.err
291+
lsblk -a -o NAME,PATH,TYPE,TRAN,RM,SERIAL,MODEL >>/tmp/usb-mount.err 2>&1 || true
292+
exit 1
293+
}
265294
266-
_, err := t.Framework.SSHCommand(t.VM.Name, t.VM.Namespace, mountCmd)
267-
Expect(err).NotTo(HaveOccurred())
295+
sudo mkdir -p /mnt/usb
296+
if sudo mountpoint -q /mnt/usb; then
297+
sudo umount /mnt/usb || true
298+
fi
299+
sudo mount -t auto "$mount_device" /mnt/usb 2>>/tmp/usb-mount.err || \
300+
sudo mount -t vfat -o rw "$mount_device" /mnt/usb 2>>/tmp/usb-mount.err || \
301+
sudo mount -o rw "$mount_device" /mnt/usb 2>>/tmp/usb-mount.err
302+
ls -la /mnt/usb
303+
`, serial)
304+
305+
Eventually(func() error {
306+
_, err := t.Framework.SSHCommand(t.VM.Name, t.VM.Namespace, mountCmd, framework.WithSSHTimeout(framework.MiddleTimeout))
307+
return err
308+
}).WithTimeout(framework.MiddleTimeout).WithPolling(time.Second).Should(Succeed(), t.usbDiagnostics())
309+
}
310+
311+
func (t *VMUSBTest) usbDiagnostics() string {
312+
diagnosticsCmd := `
313+
echo "mount error:" && cat /tmp/usb-mount.err 2>/dev/null || true
314+
echo "mount:" && mount || true
315+
echo "usb serials:" && for serial_file in /sys/bus/usb/devices/*/serial; do [ -f "$serial_file" ] && echo "$serial_file=$(cat "$serial_file")"; done || true
316+
echo "usb sysfs:" && find /sys/bus/usb/devices -maxdepth 3 -print || true
317+
echo "lsblk:" && lsblk -a -o NAME,PATH,TYPE,TRAN,RM,SERIAL,MODEL || true
318+
echo "disks:" && for dev in /dev/sd*; do [ -b "$dev" ] && echo "== $dev ==" && lsblk -dno NAME,PATH,TRAN,RM,SERIAL,MODEL "$dev"; done || true
319+
echo "lsusb:" && lsusb || true
320+
echo "fstype:" && blkid /dev/sd* || true
321+
echo "dmesg:" && sudo dmesg | tail -n 100 || true
322+
`
323+
324+
result, err := t.Framework.SSHCommand(t.VM.Name, t.VM.Namespace, diagnosticsCmd, framework.WithSSHTimeout(framework.MiddleTimeout))
325+
if err != nil {
326+
return fmt.Sprintf("failed to collect USB diagnostics: %v", err)
327+
}
328+
329+
return result
268330
}
269331

270332
func nodeUSBAttachedCondition(nodeUSBDevice *v1alpha2.NodeUSBDevice) *metav1.Condition {

0 commit comments

Comments
 (0)