Skip to content

Commit c2ae268

Browse files
sradcoAI Assistantcursoragent
committed
eval(kubevirt): add vm_troubleshoot tool evals for
missing StorageClass and cloud-init crashloop Adds two eval tasks that exercise the vm_troubleshoot tool's heuristic issue detection: 1. troubleshoot-vm-missing-storageclass: Creates a VM referencing a non-existent StorageClass. Verifies the agent calls vm_troubleshoot and reports the CRITICAL missing SC finding with alternatives. 2. troubleshoot-vm-cloudinit-shutdown: Creates a VM with "shutdown -h now" in cloud-init runcmd that causes CrashLoopBackOff. Verifies the agent identifies the dangerous command as root cause. Both scenarios are deterministic and non-flaky on any cluster with KubeVirt installed. Signed-off-by: Shirly Radco <sradco@redhat.com> Co-authored-by: AI Assistant <noreply@cursor.com> Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent d90a1ca commit c2ae268

5 files changed

Lines changed: 353 additions & 9 deletions

File tree

‎evals/tasks/kubevirt/troubleshoot-vm-cloudinit-shutdown/task.yaml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
kind: Task
2+
apiVersion: mcpchecker/v1alpha2
3+
metadata:
4+
labels:
5+
suite: kubevirt
6+
requires: kubevirt
7+
name: "troubleshoot-vm-cloudinit-shutdown"
8+
difficulty: medium
9+
description: "Diagnose a VM crashlooping due to a shutdown command in cloud-init"
10+
spec:
11+
requires:
12+
- extension: kubernetes
13+
as: k8s
14+
setup:
15+
- k8s.delete:
16+
apiVersion: v1
17+
kind: Namespace
18+
metadata:
19+
name: vm-test-troubleshoot-ci
20+
ignoreNotFound: true
21+
- k8s.create:
22+
apiVersion: v1
23+
kind: Namespace
24+
metadata:
25+
name: vm-test-troubleshoot-ci
26+
- k8s.create:
27+
apiVersion: kubevirt.io/v1
28+
kind: VirtualMachine
29+
metadata:
30+
name: crashloop-vm
31+
namespace: vm-test-troubleshoot-ci
32+
spec:
33+
runStrategy: Always
34+
template:
35+
spec:
36+
domain:
37+
devices:
38+
disks:
39+
- name: containerdisk
40+
disk:
41+
bus: virtio
42+
- name: cloudinit
43+
disk:
44+
bus: virtio
45+
resources:
46+
requests:
47+
memory: 2Gi
48+
terminationGracePeriodSeconds: 0
49+
volumes:
50+
- name: containerdisk
51+
containerDisk:
52+
image: quay.io/containerdisks/fedora:latest
53+
- name: cloudinit
54+
cloudInitNoCloud:
55+
userData: |
56+
#cloud-config
57+
runcmd:
58+
- shutdown -h now
59+
- script:
60+
inline: |-
61+
#!/usr/bin/env bash
62+
echo "Waiting for VM to attempt boot and crash..."
63+
sleep 60
64+
verify:
65+
- script:
66+
inline: |-
67+
#!/usr/bin/env bash
68+
NS="vm-test-troubleshoot-ci"
69+
70+
echo "=== Verification: Agent identified cloud-init issue ==="
71+
72+
if ! kubectl get virtualmachine crashloop-vm -n "$NS" > /dev/null 2>&1; then
73+
echo "✗ VirtualMachine crashloop-vm no longer exists"
74+
exit 1
75+
fi
76+
echo "✓ VirtualMachine crashloop-vm still exists"
77+
exit 0
78+
- llmJudge:
79+
contains: "shutdown -h now"
80+
cleanup:
81+
- k8s.delete:
82+
apiVersion: kubevirt.io/v1
83+
kind: VirtualMachine
84+
metadata:
85+
name: crashloop-vm
86+
namespace: vm-test-troubleshoot-ci
87+
ignoreNotFound: true
88+
- k8s.delete:
89+
apiVersion: v1
90+
kind: Namespace
91+
metadata:
92+
name: vm-test-troubleshoot-ci
93+
ignoreNotFound: true
94+
prompt:
95+
inline: |-
96+
A VirtualMachine named "crashloop-vm" in the vm-test-troubleshoot-ci namespace keeps restarting.
97+
Diagnose why this VM is crashlooping. Report:
98+
- The root cause of the crashloop
99+
- Which cloud-init command is causing the problem
100+
- How to fix it

‎evals/tasks/kubevirt/troubleshoot-vm-missing-storageclass/task.yaml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
kind: Task
2+
apiVersion: mcpchecker/v1alpha2
3+
metadata:
4+
labels:
5+
suite: kubevirt
6+
requires: kubevirt
7+
name: "troubleshoot-vm-missing-storageclass"
8+
difficulty: medium
9+
description: "Diagnose a VM stuck in Provisioning due to a non-existent StorageClass"
10+
spec:
11+
requires:
12+
- extension: kubernetes
13+
as: k8s
14+
setup:
15+
- k8s.delete:
16+
apiVersion: v1
17+
kind: Namespace
18+
metadata:
19+
name: vm-test-troubleshoot-sc
20+
ignoreNotFound: true
21+
- k8s.create:
22+
apiVersion: v1
23+
kind: Namespace
24+
metadata:
25+
name: vm-test-troubleshoot-sc
26+
- k8s.create:
27+
apiVersion: kubevirt.io/v1
28+
kind: VirtualMachine
29+
metadata:
30+
name: broken-sc-vm
31+
namespace: vm-test-troubleshoot-sc
32+
spec:
33+
runStrategy: Always
34+
dataVolumeTemplates:
35+
- metadata:
36+
name: broken-sc-vm-rootdisk
37+
spec:
38+
storage:
39+
storageClassName: non-existent-sc-xyz
40+
accessModes:
41+
- ReadWriteOnce
42+
resources:
43+
requests:
44+
storage: 10Gi
45+
source:
46+
registry:
47+
url: docker://quay.io/containerdisks/fedora:latest
48+
template:
49+
spec:
50+
domain:
51+
devices:
52+
disks:
53+
- name: rootdisk
54+
disk:
55+
bus: virtio
56+
resources:
57+
requests:
58+
memory: 2Gi
59+
terminationGracePeriodSeconds: 0
60+
volumes:
61+
- name: rootdisk
62+
dataVolume:
63+
name: broken-sc-vm-rootdisk
64+
- script:
65+
inline: |-
66+
#!/usr/bin/env bash
67+
echo "Waiting for VM to enter Provisioning state..."
68+
sleep 15
69+
verify:
70+
- script:
71+
inline: |-
72+
#!/usr/bin/env bash
73+
NS="vm-test-troubleshoot-sc"
74+
75+
echo "=== Verification: Agent identified missing StorageClass ==="
76+
77+
if ! kubectl get virtualmachine broken-sc-vm -n "$NS" > /dev/null 2>&1; then
78+
echo "✗ VirtualMachine broken-sc-vm no longer exists"
79+
exit 1
80+
fi
81+
echo "✓ VirtualMachine broken-sc-vm still exists"
82+
exit 0
83+
- llmJudge:
84+
contains: "non-existent-sc-xyz"
85+
cleanup:
86+
- k8s.delete:
87+
apiVersion: kubevirt.io/v1
88+
kind: VirtualMachine
89+
metadata:
90+
name: broken-sc-vm
91+
namespace: vm-test-troubleshoot-sc
92+
ignoreNotFound: true
93+
- k8s.delete:
94+
apiVersion: v1
95+
kind: Namespace
96+
metadata:
97+
name: vm-test-troubleshoot-sc
98+
ignoreNotFound: true
99+
prompt:
100+
inline: |-
101+
A VirtualMachine named "broken-sc-vm" in the vm-test-troubleshoot-sc namespace is not starting.
102+
Diagnose why this VM is stuck. Report:
103+
- The root cause of the issue
104+
- What StorageClass is missing
105+
- What alternative StorageClasses are available on the cluster

‎pkg/mcp/kubevirt_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -784,6 +784,97 @@ func (s *KubevirtSuite) TestVMTroubleshootPrompt() {
784784
})
785785
}
786786

787+
func (s *KubevirtSuite) TestVMTroubleshoot() {
788+
s.Run("vm_troubleshoot missing required params", func() {
789+
testCases := []string{"namespace", "name"}
790+
for _, param := range testCases {
791+
s.Run("missing "+param, func() {
792+
params := map[string]interface{}{
793+
"namespace": "default",
794+
"name": "test-vm",
795+
}
796+
delete(params, param)
797+
toolResult, err := s.CallTool("vm_troubleshoot", params)
798+
s.Require().Nilf(err, "call tool failed %v", err)
799+
s.Truef(toolResult.IsError, "expected call tool to fail due to missing %s", param)
800+
s.Equal(toolResult.Content[0].(*mcp.TextContent).Text, param+" parameter required")
801+
})
802+
}
803+
})
804+
805+
s.Run("vm_troubleshoot returns diagnostic report for existing VM", func() {
806+
dynamicClient := dynamic.NewForConfigOrDie(envTestRestConfig)
807+
vm := &unstructured.Unstructured{}
808+
vm.SetUnstructuredContent(map[string]interface{}{
809+
"apiVersion": "kubevirt.io/v1",
810+
"kind": "VirtualMachine",
811+
"metadata": map[string]interface{}{
812+
"name": "troubleshoot-vm",
813+
"namespace": "default",
814+
},
815+
"spec": map[string]interface{}{
816+
"runStrategy": "Always",
817+
"template": map[string]interface{}{
818+
"spec": map[string]interface{}{
819+
"volumes": []interface{}{
820+
map[string]interface{}{
821+
"name": "rootdisk",
822+
"containerDisk": map[string]interface{}{
823+
"image": "quay.io/containerdisks/fedora:latest",
824+
},
825+
},
826+
},
827+
},
828+
},
829+
},
830+
})
831+
_, err := dynamicClient.Resource(schema.GroupVersionResource{
832+
Group: "kubevirt.io",
833+
Version: "v1",
834+
Resource: "virtualmachines",
835+
}).Namespace("default").Create(s.T().Context(), vm, metav1.CreateOptions{})
836+
s.Require().NoError(err, "failed to create test VM")
837+
838+
toolResult, err := s.CallTool("vm_troubleshoot", map[string]interface{}{
839+
"namespace": "default",
840+
"name": "troubleshoot-vm",
841+
})
842+
s.Run("no error", func() {
843+
s.Nilf(err, "call tool failed %v", err)
844+
s.Falsef(toolResult.IsError, "call tool failed: %v", toolResult.Content)
845+
})
846+
847+
s.Run("returns diagnostic report with correct VM details", func() {
848+
text := toolResult.Content[0].(*mcp.TextContent).Text
849+
s.Truef(strings.HasPrefix(text, "# VirtualMachine Diagnostic Report: default/troubleshoot-vm"),
850+
"Expected diagnostic report header, got %v", text)
851+
s.Contains(text, "## VirtualMachine Status")
852+
s.Contains(text, "## VirtualMachineInstance")
853+
s.Contains(text, "troubleshoot-vm")
854+
})
855+
856+
// Cleanup
857+
_ = dynamicClient.Resource(schema.GroupVersionResource{
858+
Group: "kubevirt.io",
859+
Version: "v1",
860+
Resource: "virtualmachines",
861+
}).Namespace("default").Delete(s.T().Context(), "troubleshoot-vm", metav1.DeleteOptions{})
862+
})
863+
864+
s.Run("vm_troubleshoot handles non-existent VM gracefully", func() {
865+
toolResult, err := s.CallTool("vm_troubleshoot", map[string]interface{}{
866+
"namespace": "default",
867+
"name": "non-existent-vm",
868+
})
869+
s.Nilf(err, "call tool failed %v", err)
870+
s.Falsef(toolResult.IsError, "tool should not return error for non-existent VM")
871+
text := toolResult.Content[0].(*mcp.TextContent).Text
872+
s.Truef(strings.HasPrefix(text, "# VirtualMachine Diagnostic Report: default/non-existent-vm"),
873+
"Expected diagnostic report header, got %v", text)
874+
s.Contains(text, "not found")
875+
})
876+
}
877+
787878
func (s *KubevirtSuite) TestVMGuestInfo() {
788879
s.Run("vm_guest_info missing required params", func() {
789880
testCases := []string{"namespace", "name"}

‎pkg/toolsets/kubevirt/vm/troubleshoot/analysis.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ func isConfigKeyLine(line string) bool {
351351
if strings.Contains(lower, ":") && !strings.HasPrefix(lower, "-") && !strings.HasPrefix(lower, "[") {
352352
parts := strings.SplitN(lower, ":", 2)
353353
key := strings.TrimSpace(parts[0])
354-
if strings.Contains(key, "_") || strings.Contains(key, "-") {
354+
if len(key) > 0 && !strings.Contains(key, " ") {
355355
return true
356356
}
357357
}
@@ -368,7 +368,9 @@ func matchDangerousCommand(line string) string {
368368
switch {
369369
case stripped == "shutdown" || stripped == "shutdown -h now" ||
370370
stripped == "/sbin/shutdown" || strings.HasPrefix(stripped, "/sbin/shutdown ") ||
371-
strings.HasPrefix(stripped, "shutdown -"):
371+
strings.HasPrefix(stripped, "shutdown -h") ||
372+
strings.HasPrefix(stripped, "shutdown -P") || strings.HasPrefix(stripped, "shutdown -p") ||
373+
strings.HasPrefix(stripped, "shutdown -r"):
372374
return "shutdown"
373375
case stripped == "poweroff" || stripped == "/sbin/poweroff" ||
374376
stripped == "systemctl poweroff":
@@ -539,7 +541,7 @@ func checkMigrationStatus(ctx context.Context, dynamicClient dynamic.Interface,
539541
}
540542

541543
vmimList, err := dynamicClient.Resource(kubevirt.VirtualMachineInstanceMigrationGVR).Namespace(namespace).List(ctx, metav1.ListOptions{
542-
Limit: 50,
544+
LabelSelector: fmt.Sprintf("kubevirt.io/vmi-name=%s", name),
543545
})
544546
if err != nil || len(vmimList.Items) == 0 {
545547
return nil

0 commit comments

Comments
 (0)