diff --git a/deploy/crds/README.adoc b/deploy/crds/README.adoc index 0425ed29f..9de32c42c 100644 --- a/deploy/crds/README.adoc +++ b/deploy/crds/README.adoc @@ -10,4 +10,7 @@ Copy from a cluster with CNV operator installed Copy from a cluster with Ansible Automation Platform operator installed == serving CRDs -Copy from a cluster with Serverless operator installed \ No newline at end of file +Copy from a cluster with Serverless operator installed + +== claws CRD +Copy from the claw-operator repo (config/crd/bases/claw.sandbox.redhat.com_claws.yaml) \ No newline at end of file diff --git a/deploy/crds/claw.sandbox.redhat.com_claws.yaml b/deploy/crds/claw.sandbox.redhat.com_claws.yaml new file mode 100644 index 000000000..a9e732e17 --- /dev/null +++ b/deploy/crds/claw.sandbox.redhat.com_claws.yaml @@ -0,0 +1,40 @@ +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1 +metadata: + name: claws.claw.sandbox.redhat.com +spec: + group: claw.sandbox.redhat.com + names: + plural: claws + singular: claw + kind: Claw + listKind: ClawList + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + description: Claw is the Schema for the claws API + type: object + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of Claw + type: object + properties: + idle: + description: Scale down replicas to put Claw into an idle mode + type: boolean + status: + description: Status defines the observed state of Claw + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} diff --git a/test/e2e/parallel/user_workloads_test.go b/test/e2e/parallel/user_workloads_test.go index 3eafc93e0..01312c0c6 100644 --- a/test/e2e/parallel/user_workloads_test.go +++ b/test/e2e/parallel/user_workloads_test.go @@ -38,13 +38,21 @@ func TestIdlerAndPriorityClass(t *testing.T) { podsToIdle := prepareWorkloads(t, await.Member1(), idler.Name, wait.WithSandboxPriorityClass()) podsNoise := prepareWorkloads(t, await.Member1(), idlerNoise.Name, wait.WithSandboxPriorityClass()) + // Create a Claw workload only in the dev namespace (the one being idled). + // Not added to prepareWorkloads to avoid exceeding the ClusterResourceQuota pod limit + // when workloads are created in multiple namespaces for the same user. + clawDeployment := createClaw(t, memberAwait, "test-idler-claw", idler.Name) + podsToIdle, err := memberAwait.WaitForPods(t, idler.Name, len(podsToIdle)+int(*clawDeployment.Spec.Replicas), + wait.PodRunning(), wait.WithPodLabel("idler", "idler"), wait.WithSandboxPriorityClass()) + require.NoError(t, err) + // Create more noise pods in non-user namespace memberAwait.CreateNamespace(t, "workloads-noise") externalNsPodsNoise := prepareWorkloads(t, await.Member1(), "workloads-noise", wait.WithOriginalPriorityClass()) // Set a short timeout for one of the idler to trigger pod idling // The idler is currently updating its status since it's already been idling the pods. So we need to keep trying to update. - idler, err := wait.For(t, memberAwait.Awaitility, &toolchainv1alpha1.Idler{}). + idler, err = wait.For(t, memberAwait.Awaitility, &toolchainv1alpha1.Idler{}). Update(idler.Name, memberAwait.Namespace, func(i *toolchainv1alpha1.Idler) { i.Spec.TimeoutSeconds = 5 }) @@ -101,6 +109,10 @@ func TestIdlerAndPriorityClass(t *testing.T) { _, err = memberAwait.WaitForAAP(t, "test-idler-aap", idler.Name, clnt.Resource(aapRes), true) require.NoError(t, err) + // Wait for the Claw resource to be idled (spec.idle: true) + _, err = memberAwait.WaitForClaw(t, "test-idler-claw", idler.Name, clnt.Resource(clawRes), true) + require.NoError(t, err) + // Wait for the InferenceService to be deleted - the expected action to idle // the workload is by deleting the InferenceService that is old enough. // The pods are idled as well, which is verified in the previous step - after some time, @@ -208,6 +220,7 @@ func createDeployment(t *testing.T, memberAwait *wait.MemberAwaitility, namespac } var aapRes = schema.GroupVersionResource{Group: "aap.ansible.com", Version: "v1alpha1", Resource: "ansibleautomationplatforms"} +var clawRes = schema.GroupVersionResource{Group: "claw.sandbox.redhat.com", Version: "v1alpha1", Resource: "claws"} var servingRuntimeRes = schema.GroupVersionResource{Group: "serving.kserve.io", Version: "v1alpha1", Resource: "servingruntimes"} var inferenceServiceRes = schema.GroupVersionResource{Group: "serving.kserve.io", Version: "v1beta1", Resource: "inferenceservices"} var dataVolumeRes = schema.GroupVersionResource{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"} @@ -244,6 +257,51 @@ func createAAP(t *testing.T, memberAwait *wait.MemberAwaitility, name, namespace return deployment } +// createClaw creates an instance of claws.claw.sandbox.redhat.com with one deployment owned by this instance +// returns the underlying deployment +func createClaw(t *testing.T, memberAwait *wait.MemberAwaitility, name, namespace string) *appsv1.Deployment { + clnt, err := dynamic.NewForConfig(memberAwait.RestConfig) + require.NoError(t, err) + + claw := clawResource(name) + createdClaw, err := clnt.Resource(clawRes).Namespace(namespace).Create(context.TODO(), claw, metav1.CreateOptions{}) + require.NoError(t, err) + + replicas := int32(2) + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(name)}, + Replicas: &replicas, + Template: podTemplateSpec(name), + }, + } + err = controllerutil.SetOwnerReference(createdClaw, deployment, scheme.Scheme) + require.NoError(t, err) + err = memberAwait.Create(t, deployment) + require.NoError(t, err) + + return deployment +} + +func clawResource(name string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "claw.sandbox.redhat.com/v1alpha1", + "kind": "Claw", + "metadata": map[string]interface{}{ + "name": name, + }, + "spec": map[string]interface{}{ + "idle": false, + }, + }, + } +} + func createReplicaSet(t *testing.T, memberAwait *wait.MemberAwaitility, namespace string) *appsv1.ReplicaSet { // Standalone ReplicaSet replicas := int32(2) diff --git a/testsupport/wait/member.go b/testsupport/wait/member.go index f23da73ad..da02eceec 100644 --- a/testsupport/wait/member.go +++ b/testsupport/wait/member.go @@ -15,6 +15,7 @@ import ( toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1" "github.com/codeready-toolchain/toolchain-common/pkg/test" appstudiov1 "github.com/codeready-toolchain/toolchain-e2e/testsupport/appstudio/api/v1alpha1" + "github.com/codeready-toolchain/toolchain-e2e/testsupport/cleanup" "github.com/davecgh/go-spew/spew" "github.com/ghodss/yaml" quotav1 "github.com/openshift/api/quota/v1" @@ -1415,6 +1416,16 @@ func (a *MemberAwaitility) Create(t *testing.T, obj client.Object) error { }) } +// CreateWithCleanup tries to create the object until success and schedules cleanup at test end. +// Workaround for https://github.com/kubernetes/kubernetes/issues/67761 +func (a *MemberAwaitility) CreateWithCleanup(t *testing.T, obj client.Object) error { + if err := a.Create(t, obj); err != nil { + return err + } + cleanup.AddCleanTasks(t, a.Client, obj) + return nil +} + // PodWaitCriterion a struct to compare with a given Pod type PodWaitCriterion struct { Match func(*corev1.Pod) bool @@ -1525,7 +1536,7 @@ func (a *MemberAwaitility) WaitForAAP(t *testing.T, name, namespace string, aapR var aap *unstructured.Unstructured err := wait.PollUntilContextTimeout(context.TODO(), a.RetryInterval, a.Timeout, true, func(ctx context.Context) (bool, error) { var err error - aap, err = aapRes.Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{}) + aap, err = aapRes.Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return false, nil @@ -1541,6 +1552,31 @@ func (a *MemberAwaitility) WaitForAAP(t *testing.T, name, namespace string, aapR return aap, err } +// WaitForClaw waits for the Claw resource to reach the expected idle state (spec.idle) +func (a *MemberAwaitility) WaitForClaw(t *testing.T, name, namespace string, clawRes dynamic.NamespaceableResourceInterface, expectedIdled bool) (*unstructured.Unstructured, error) { + t.Logf("waiting for Claw '%s' in namespace '%s'", name, namespace) + var claw *unstructured.Unstructured + err := wait.PollUntilContextTimeout(context.TODO(), a.RetryInterval, a.Timeout, true, func(ctx context.Context) (bool, error) { + var err error + claw, err = clawRes.Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + idled, found, err := unstructured.NestedBool(claw.UnstructuredContent(), "spec", "idle") + if err != nil { + return true, err + } + if !found { + return false, nil + } + return expectedIdled == idled, nil + }) + return claw, err +} + // WaitUntilInferenceServiceDeleted waits for the InferenceService resource to be deleted (idled) func (a *MemberAwaitility) WaitUntilInferenceServiceDeleted(t *testing.T, name, namespace string, inferenceServiceRes dynamic.NamespaceableResourceInterface) error { t.Logf("waiting for InferenceService '%s' to be deleted in namespace '%s'", name, namespace)