Skip to content

Commit c863a2e

Browse files
codingbencodingben
committed
test(kubevirt): add basic building-block evals for vm_guest_info
Add five simple evaluation tasks that test basic vm_guest_info functionality: - get-vm-filesystems: Get filesystem information from inside a VM - get-vm-ip-address: Get IP address from inside the guest OS - get-vm-os-info: Get OS name and version from a VM - list-vm-users: List currently logged-in users in a VM - vm-guest-info: General test for retrieving guest agent information These evals focus on testing the basic building blocks rather than high-level use cases, making them more reliable and model-independent. Assisted-By: Claude <noreply@anthropic.com> Signed-off-by: Ben Oukhanov <boukhanov@redhat.com>
1 parent 5ac04d6 commit c863a2e

5 files changed

Lines changed: 491 additions & 0 deletions

File tree

‎evals/tasks/kubevirt/get-vm-filesystems/task.yaml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
kind: Task
2+
apiVersion: mcpchecker/v1alpha2
3+
metadata:
4+
labels:
5+
suite: kubevirt
6+
requires: kubevirt
7+
name: "get-vm-filesystems"
8+
difficulty: easy
9+
spec:
10+
requires:
11+
- extension: kubernetes
12+
as: k8s
13+
setup:
14+
- k8s.delete:
15+
apiVersion: v1
16+
kind: Namespace
17+
metadata:
18+
name: vm-test-filesystems
19+
ignoreNotFound: true
20+
- k8s.create:
21+
apiVersion: v1
22+
kind: Namespace
23+
metadata:
24+
name: vm-test-filesystems
25+
- k8s.create:
26+
apiVersion: kubevirt.io/v1
27+
kind: VirtualMachine
28+
metadata:
29+
name: test-vm
30+
namespace: vm-test-filesystems
31+
spec:
32+
runStrategy: Always
33+
template:
34+
spec:
35+
domain:
36+
devices:
37+
disks:
38+
- name: containerdisk
39+
disk:
40+
bus: virtio
41+
- name: cloudinitdisk
42+
disk:
43+
bus: virtio
44+
resources:
45+
requests:
46+
memory: 2Gi
47+
volumes:
48+
- name: containerdisk
49+
containerDisk:
50+
image: quay.io/containerdisks/fedora:latest
51+
- name: cloudinitdisk
52+
cloudInitNoCloud:
53+
userData: |
54+
#cloud-config
55+
- script:
56+
inline: |-
57+
#!/usr/bin/env bash
58+
kubectl wait --for=create vm/test-vm -n vm-test-filesystems --timeout=30s
59+
kubectl wait --for=condition=Ready vm/test-vm -n vm-test-filesystems --timeout=30s
60+
kubectl wait --for=condition=AgentConnected --timeout=30s vmi/test-vm -n vm-test-filesystems
61+
echo "VM created with guest agent connected."
62+
verify:
63+
- script:
64+
inline: |-
65+
#!/usr/bin/env bash
66+
set -e
67+
if ! kubectl get vm test-vm -n vm-test-filesystems &>/dev/null; then
68+
echo "ERROR: VirtualMachine 'test-vm' not found"
69+
exit 1
70+
fi
71+
echo "Verification passed: VM exists"
72+
exit 0
73+
- llmJudge:
74+
contains: "/"
75+
cleanup:
76+
- k8s.delete:
77+
apiVersion: kubevirt.io/v1
78+
kind: VirtualMachine
79+
metadata:
80+
name: test-vm
81+
namespace: vm-test-filesystems
82+
ignoreNotFound: true
83+
- k8s.delete:
84+
apiVersion: v1
85+
kind: Namespace
86+
metadata:
87+
name: vm-test-filesystems
88+
ignoreNotFound: true
89+
prompt:
90+
inline: |
91+
Get the filesystem information from the VirtualMachine "test-vm" in the vm-test-filesystems namespace.
92+
93+
Report the mounted filesystems you find.

‎evals/tasks/kubevirt/get-vm-ip-address/task.yaml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
kind: Task
2+
apiVersion: mcpchecker/v1alpha2
3+
metadata:
4+
labels:
5+
suite: kubevirt
6+
requires: kubevirt
7+
name: "get-vm-ip-address"
8+
difficulty: easy
9+
spec:
10+
requires:
11+
- extension: kubernetes
12+
as: k8s
13+
setup:
14+
- k8s.delete:
15+
apiVersion: v1
16+
kind: Namespace
17+
metadata:
18+
name: vm-test-ip-address
19+
ignoreNotFound: true
20+
- k8s.create:
21+
apiVersion: v1
22+
kind: Namespace
23+
metadata:
24+
name: vm-test-ip-address
25+
- k8s.create:
26+
apiVersion: kubevirt.io/v1
27+
kind: VirtualMachine
28+
metadata:
29+
name: test-vm
30+
namespace: vm-test-ip-address
31+
spec:
32+
runStrategy: Always
33+
template:
34+
spec:
35+
domain:
36+
devices:
37+
disks:
38+
- name: containerdisk
39+
disk:
40+
bus: virtio
41+
- name: cloudinitdisk
42+
disk:
43+
bus: virtio
44+
resources:
45+
requests:
46+
memory: 2Gi
47+
volumes:
48+
- name: containerdisk
49+
containerDisk:
50+
image: quay.io/containerdisks/fedora:latest
51+
- name: cloudinitdisk
52+
cloudInitNoCloud:
53+
userData: |
54+
#cloud-config
55+
password: fedora
56+
chpasswd: { expire: False }
57+
- script:
58+
inline: |-
59+
#!/usr/bin/env bash
60+
kubectl wait --for=create vm/test-vm -n vm-test-ip-address --timeout=30s
61+
kubectl wait --for=condition=Ready vm/test-vm -n vm-test-ip-address --timeout=30s
62+
kubectl wait --for=condition=AgentConnected --timeout=30s vmi/test-vm -n vm-test-ip-address
63+
echo "VM created with guest agent connected."
64+
verify:
65+
- script:
66+
inline: |-
67+
#!/usr/bin/env bash
68+
set -e
69+
if ! kubectl get vm test-vm -n vm-test-ip-address &>/dev/null; then
70+
echo "ERROR: VirtualMachine 'test-vm' not found"
71+
exit 1
72+
fi
73+
echo "Verification passed: VM exists"
74+
exit 0
75+
- llmJudge:
76+
contains: "10."
77+
cleanup:
78+
- k8s.delete:
79+
apiVersion: kubevirt.io/v1
80+
kind: VirtualMachine
81+
metadata:
82+
name: test-vm
83+
namespace: vm-test-ip-address
84+
ignoreNotFound: true
85+
- k8s.delete:
86+
apiVersion: v1
87+
kind: Namespace
88+
metadata:
89+
name: vm-test-ip-address
90+
ignoreNotFound: true
91+
prompt:
92+
inline: |
93+
Get the IP address of the VirtualMachine "test-vm" in the vm-test-ip-address namespace from inside the guest OS.
94+
95+
Report the IP address you find.

‎evals/tasks/kubevirt/get-vm-os-info/task.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
kind: Task
2+
apiVersion: mcpchecker/v1alpha2
3+
metadata:
4+
labels:
5+
suite: kubevirt
6+
requires: kubevirt
7+
name: "get-vm-os-info"
8+
difficulty: easy
9+
spec:
10+
requires:
11+
- extension: kubernetes
12+
as: k8s
13+
setup:
14+
- k8s.delete:
15+
apiVersion: v1
16+
kind: Namespace
17+
metadata:
18+
name: vm-test-os-info
19+
ignoreNotFound: true
20+
- k8s.create:
21+
apiVersion: v1
22+
kind: Namespace
23+
metadata:
24+
name: vm-test-os-info
25+
- k8s.create:
26+
apiVersion: kubevirt.io/v1
27+
kind: VirtualMachine
28+
metadata:
29+
name: test-vm
30+
namespace: vm-test-os-info
31+
spec:
32+
runStrategy: Always
33+
template:
34+
spec:
35+
domain:
36+
devices:
37+
disks:
38+
- name: containerdisk
39+
disk:
40+
bus: virtio
41+
- name: cloudinitdisk
42+
disk:
43+
bus: virtio
44+
resources:
45+
requests:
46+
memory: 2Gi
47+
volumes:
48+
- name: containerdisk
49+
containerDisk:
50+
image: quay.io/containerdisks/fedora:latest
51+
- name: cloudinitdisk
52+
cloudInitNoCloud:
53+
userData: |
54+
#cloud-config
55+
hostname: test-vm
56+
- script:
57+
inline: |-
58+
#!/usr/bin/env bash
59+
kubectl wait --for=create vm/test-vm -n vm-test-os-info --timeout=30s
60+
kubectl wait --for=condition=Ready vm/test-vm -n vm-test-os-info --timeout=30s
61+
kubectl wait --for=condition=AgentConnected --timeout=30s vmi/test-vm -n vm-test-os-info
62+
echo "VM created with guest agent connected."
63+
verify:
64+
- script:
65+
inline: |-
66+
#!/usr/bin/env bash
67+
set -e
68+
if ! kubectl get vm test-vm -n vm-test-os-info &>/dev/null; then
69+
echo "ERROR: VirtualMachine 'test-vm' not found"
70+
exit 1
71+
fi
72+
echo "Verification passed: VM exists"
73+
exit 0
74+
- llmJudge:
75+
contains: "Fedora"
76+
cleanup:
77+
- k8s.delete:
78+
apiVersion: kubevirt.io/v1
79+
kind: VirtualMachine
80+
metadata:
81+
name: test-vm
82+
namespace: vm-test-os-info
83+
ignoreNotFound: true
84+
- k8s.delete:
85+
apiVersion: v1
86+
kind: Namespace
87+
metadata:
88+
name: vm-test-os-info
89+
ignoreNotFound: true
90+
prompt:
91+
inline: |
92+
Get the OS information from the VirtualMachine "test-vm" in the vm-test-os-info namespace.
93+
94+
Report the OS name and version you find.

‎evals/tasks/kubevirt/list-vm-users/task.yaml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
kind: Task
2+
apiVersion: mcpchecker/v1alpha2
3+
metadata:
4+
labels:
5+
suite: kubevirt
6+
requires: kubevirt
7+
name: "list-vm-users"
8+
difficulty: easy
9+
spec:
10+
requires:
11+
- extension: kubernetes
12+
as: k8s
13+
setup:
14+
- k8s.delete:
15+
apiVersion: v1
16+
kind: Namespace
17+
metadata:
18+
name: vm-test-users
19+
ignoreNotFound: true
20+
- k8s.create:
21+
apiVersion: v1
22+
kind: Namespace
23+
metadata:
24+
name: vm-test-users
25+
- k8s.create:
26+
apiVersion: kubevirt.io/v1
27+
kind: VirtualMachine
28+
metadata:
29+
name: test-vm
30+
namespace: vm-test-users
31+
spec:
32+
runStrategy: Always
33+
template:
34+
spec:
35+
domain:
36+
devices:
37+
disks:
38+
- name: containerdisk
39+
disk:
40+
bus: virtio
41+
- name: cloudinitdisk
42+
disk:
43+
bus: virtio
44+
resources:
45+
requests:
46+
memory: 2Gi
47+
volumes:
48+
- name: containerdisk
49+
containerDisk:
50+
image: quay.io/containerdisks/fedora:latest
51+
- name: cloudinitdisk
52+
cloudInitNoCloud:
53+
userData: |
54+
#cloud-config
55+
users:
56+
- name: admin
57+
sudo: ALL=(ALL) NOPASSWD:ALL
58+
- name: developer
59+
groups: developers
60+
- script:
61+
inline: |-
62+
#!/usr/bin/env bash
63+
kubectl wait --for=create vm/test-vm -n vm-test-users --timeout=30s
64+
kubectl wait --for=condition=Ready vm/test-vm -n vm-test-users --timeout=30s
65+
kubectl wait --for=condition=AgentConnected --timeout=30s vmi/test-vm -n vm-test-users
66+
echo "VM created with guest agent connected."
67+
verify:
68+
- script:
69+
inline: |-
70+
#!/usr/bin/env bash
71+
set -e
72+
if ! kubectl get vm test-vm -n vm-test-users &>/dev/null; then
73+
echo "ERROR: VirtualMachine 'test-vm' not found"
74+
exit 1
75+
fi
76+
echo "Verification passed: VM exists"
77+
exit 0
78+
- llmJudge:
79+
contains: "admin"
80+
cleanup:
81+
- k8s.delete:
82+
apiVersion: kubevirt.io/v1
83+
kind: VirtualMachine
84+
metadata:
85+
name: test-vm
86+
namespace: vm-test-users
87+
ignoreNotFound: true
88+
- k8s.delete:
89+
apiVersion: v1
90+
kind: Namespace
91+
metadata:
92+
name: vm-test-users
93+
ignoreNotFound: true
94+
prompt:
95+
inline: |
96+
Get the list of currently logged-in users from the VirtualMachine "test-vm" in the vm-test-users namespace.
97+
98+
Report the usernames you find.

0 commit comments

Comments
 (0)