Skip to content

test(vdsnapshot): disable filesystem frozen check #981

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
eofff wants to merge 3 commits into from
Closed

test(vdsnapshot): disable filesystem frozen check #981

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
deaa7aa
omit
May 14, 2025
48bdf48
refactoring
May 22, 2025
956a495
wait snapshot phase
May 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions tests/e2e/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ toolchain go1.23.7

require (
github.com/deckhouse/sds-replicated-volume/api v0.0.0-20241109122839-a1ae840eb5db
github.com/deckhouse/virtualization-controller v0.0.0-00010101000000-000000000000
github.com/deckhouse/virtualization/api v0.0.0-20240923080356-bb5809dba578
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
github.com/onsi/ginkgo/v2 v2.21.0
Expand Down Expand Up @@ -62,13 +63,12 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/term v0.25.0 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/tools v0.26.0 // indirect
google.golang.org/protobuf v1.35.1 // indirect
Expand All @@ -85,4 +85,7 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
)

replace github.com/deckhouse/virtualization/api => ./../../api
replace (
github.com/deckhouse/virtualization-controller => ./../../images/virtualization-artifact
github.com/deckhouse/virtualization/api => ./../../api
)
16 changes: 8 additions & 8 deletions tests/e2e/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,8 +274,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand All @@ -300,20 +300,20 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand Down
120 changes: 77 additions & 43 deletions tests/e2e/vd_snapshots_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package e2e

import (
"context"
"errors"
"fmt"
"strings"
Expand All @@ -28,9 +29,11 @@ import (
. "github.com/onsi/gomega"
storagev1 "k8s.io/api/storage/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

sdsrepvolv1 "github.com/deckhouse/sds-replicated-volume/api/v1alpha1"

"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
"github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition"
"github.com/deckhouse/virtualization/tests/e2e/config"
Expand Down Expand Up @@ -222,6 +225,53 @@ func CheckFileSystemFrozen(vmName string) (bool, error) {
return false, nil
}

func WaitForVMsUnfrozen(timeoutSeconds int64) (bool, error) {
vms, err := virtClient.VirtualMachines(conf.Namespace).List(context.Background(), v1.ListOptions{})
if err != nil {
return false, err
}

vmFSFrozenByName := make(map[string]bool)

for _, vm := range vms.Items {
frozenCondition, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, vm.Status.Conditions)
vmFSFrozenByName[vm.Name] = frozenCondition.Status == v1.ConditionTrue
}

watcher, err := virtClient.VirtualMachines(conf.Namespace).Watch(context.Background(), v1.ListOptions{TimeoutSeconds: ptr.To(timeoutSeconds)})
if err != nil {
return false, err
}

var hasFrozen bool
for {
hasFrozen = false
for _, frozen := range vmFSFrozenByName {
hasFrozen = hasFrozen || frozen
Copy link
Copy Markdown
Contributor

@Isteb4k Isteb4k May 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hasFrozen is always false here; you unconditionally set it to false above

Copy link
Copy Markdown
Contributor

@Isteb4k Isteb4k May 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do you need to iterate all the virtual machines if you've already found the frozen one? Why not just use a break for the for loop at line 249?

}

if !hasFrozen {
watcher.Stop()
return true, nil
}

event, ok := <-watcher.ResultChan()
if !ok {
break
}

vm, ok := event.Object.(*virtv2.VirtualMachine)
if !ok {
continue
Copy link
Copy Markdown
Contributor

@Isteb4k Isteb4k May 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will result in an infinite loop in case of an error. If something goes wrong, throw an error in the test

}

frozenCondition, _ := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, vm.Status.Conditions)
vmFSFrozenByName[vm.Name] = frozenCondition.Status == v1.ConditionTrue
}

return false, nil
}

var _ = Describe("Virtual disk snapshots", ginkgoutil.CommonE2ETestDecorators(), func() {
BeforeEach(func() {
if config.IsReusable() {
Expand Down Expand Up @@ -490,52 +540,36 @@ var _ = Describe("Virtual disk snapshots", ginkgoutil.CommonE2ETestDecorators(),
}
})

// TODO: It is a known issue that disk snapshots are not always created consistently. To prevent this error from causing noise during testing, we disabled this check. It will need to be re-enabled once the consistency issue is fixed.
// It("checks snapshots of attached VDs", func() {
// By(fmt.Sprintf("Snapshots should be in %s phase", PhaseReady))
// WaitPhaseByLabel(kc.ResourceVDSnapshot, PhaseReady, kc.WaitOptions{
// Labels: attachedVirtualDiskLabel,
// Namespace: conf.Namespace,
// Timeout: MaxWaitTimeout,
// })
// By("Snapshots should be consistent", func() {
// vdSnapshots := virtv2.VirtualDiskSnapshotList{}
// err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{
// ExcludedLabels: []string{"hasNoConsumer"},
// Namespace: conf.Namespace,
// Labels: attachedVirtualDiskLabel,
// })
// Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err)
//
// for _, snapshot := range vdSnapshots.Items {
// Expect(snapshot.Status.Consistent).ToNot(BeNil())
// Expect(*snapshot.Status.Consistent).To(BeTrue(), "consistent field should be `true`: %s", snapshot.Name)
// }
// })
// })
It("checks snapshots of attached VDs", func() {
By(fmt.Sprintf("Snapshots should be in %s phase", PhaseReady))
WaitPhaseByLabel(kc.ResourceVDSnapshot, PhaseReady, kc.WaitOptions{
Labels: attachedVirtualDiskLabel,
Namespace: conf.Namespace,
Timeout: MaxWaitTimeout,
})
// TODO: It is a known issue that disk snapshots are not always created consistently. To prevent this error from causing noise during testing, we disabled this check. It will need to be re-enabled once the consistency issue is fixed.
// By("Snapshots should be consistent", func() {
// vdSnapshots := virtv2.VirtualDiskSnapshotList{}
// err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{
// ExcludedLabels: []string{"hasNoConsumer"},
// Namespace: conf.Namespace,
// Labels: attachedVirtualDiskLabel,
// })
// Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err)
//
// for _, snapshot := range vdSnapshots.Items {
// Expect(snapshot.Status.Consistent).ToNot(BeNil())
// Expect(*snapshot.Status.Consistent).To(BeTrue(), "consistent field should be `true`: %s", snapshot.Name)
// }
// })
})

It("checks `FileSystemFrozen` status of VMs", func() {
By("Status should not be `Frozen`")
vmObjects := virtv2.VirtualMachineList{}
err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{Namespace: conf.Namespace})
Expect(err).NotTo(HaveOccurred(), "cannot get virtual machines\nstderr: %s", err)

for _, vm := range vmObjects.Items {
Eventually(func() error {
frozen, err := CheckFileSystemFrozen(vm.Name)
if err != nil {
return nil
}
if frozen {
return errors.New("Filesystem of the Virtual Machine is frozen")
}
return nil
}).WithTimeout(
filesystemReadyTimeout,
).WithPolling(
filesystemReadyPollingInterval,
).Should(Succeed())
}
_, err := WaitForVMsUnfrozen(int64(filesystemReadyTimeout.Seconds()))
Copy link
Copy Markdown
Contributor

@Isteb4k Isteb4k May 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In fact, it's much more important for us to check the freezing event during the snapshot process rather than the unfreezing at the end of the test through Watch.

Expect(err).NotTo(HaveOccurred())
// TODO: It is a known issue that VM filesystems after snapshot not unfreezing. To prevent this error from causing noise during testing, we disabled this check. It will need to be re-enabled once this issue is fixed.
Copy link
Copy Markdown
Contributor

@Isteb4k Isteb4k May 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

t seems like the issue has been fixed, it doesn't throw anymore. Let's restore the check.

// Expect(vmsFSUnfrozen).To(BeFalse())
})
})

Expand Down
Loading