Support Claw CR idling (#1281)

* Support Claw CR idling

Author: alexeykazakov

deploy/crds/README.adoc

@@ -10,4 +10,7 @@ Copy from a cluster with CNV operator installed
 Copy from a cluster with Ansible Automation Platform operator installed
 
 == serving CRDs
-Copy from a cluster with Serverless operator installed
+Copy from a cluster with Serverless operator installed
+
+== claws CRD
+Copy from the claw-operator repo (config/crd/bases/claw.sandbox.redhat.com_claws.yaml)

deploy/crds/claw.sandbox.redhat.com_claws.yaml

@@ -0,0 +1,40 @@
+kind: CustomResourceDefinition
+apiVersion: apiextensions.k8s.io/v1
+metadata:
+  name: claws.claw.sandbox.redhat.com
+spec:
+  group: claw.sandbox.redhat.com
+  names:
+    plural: claws
+    singular: claw
+    kind: Claw
+    listKind: ClawList
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: Claw is the Schema for the claws API
+          type: object
+          properties:
+            apiVersion:
+              type: string
+            kind:
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: Spec defines the desired state of Claw
+              type: object
+              properties:
+                idle:
+                  description: Scale down replicas to put Claw into an idle mode
+                  type: boolean
+            status:
+              description: Status defines the observed state of Claw
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+      subresources:
+        status: {}

test/e2e/parallel/user_workloads_test.go

@@ -38,13 +38,21 @@ func TestIdlerAndPriorityClass(t *testing.T) {
 	podsToIdle := prepareWorkloads(t, await.Member1(), idler.Name, wait.WithSandboxPriorityClass())
 	podsNoise := prepareWorkloads(t, await.Member1(), idlerNoise.Name, wait.WithSandboxPriorityClass())
 
+	// Create a Claw workload only in the dev namespace (the one being idled).
+	// Not added to prepareWorkloads to avoid exceeding the ClusterResourceQuota pod limit
+	// when workloads are created in multiple namespaces for the same user.
+	clawDeployment := createClaw(t, memberAwait, "test-idler-claw", idler.Name)
+	podsToIdle, err := memberAwait.WaitForPods(t, idler.Name, len(podsToIdle)+int(*clawDeployment.Spec.Replicas),
+		wait.PodRunning(), wait.WithPodLabel("idler", "idler"), wait.WithSandboxPriorityClass())
+	require.NoError(t, err)
+
 	// Create more noise pods in non-user namespace
 	memberAwait.CreateNamespace(t, "workloads-noise")
 	externalNsPodsNoise := prepareWorkloads(t, await.Member1(), "workloads-noise", wait.WithOriginalPriorityClass())
 
 	// Set a short timeout for one of the idler to trigger pod idling
 	// The idler is currently updating its status since it's already been idling the pods. So we need to keep trying to update.
-	idler, err := wait.For(t, memberAwait.Awaitility, &toolchainv1alpha1.Idler{}).
+	idler, err = wait.For(t, memberAwait.Awaitility, &toolchainv1alpha1.Idler{}).
 		Update(idler.Name, memberAwait.Namespace, func(i *toolchainv1alpha1.Idler) {
 			i.Spec.TimeoutSeconds = 5
 		})
@@ -101,6 +109,10 @@ func TestIdlerAndPriorityClass(t *testing.T) {
 	_, err = memberAwait.WaitForAAP(t, "test-idler-aap", idler.Name, clnt.Resource(aapRes), true)
 	require.NoError(t, err)
 
+	// Wait for the Claw resource to be idled (spec.idle: true)
+	_, err = memberAwait.WaitForClaw(t, "test-idler-claw", idler.Name, clnt.Resource(clawRes), true)
+	require.NoError(t, err)
+
 	// Wait for the InferenceService to be deleted - the expected action to idle
 	// the workload is by deleting the InferenceService that is old enough.
 	// The pods are idled as well, which is verified in the previous step - after some time,
@@ -208,6 +220,7 @@ func createDeployment(t *testing.T, memberAwait *wait.MemberAwaitility, namespac
 }
 
 var aapRes = schema.GroupVersionResource{Group: "aap.ansible.com", Version: "v1alpha1", Resource: "ansibleautomationplatforms"}
+var clawRes = schema.GroupVersionResource{Group: "claw.sandbox.redhat.com", Version: "v1alpha1", Resource: "claws"}
 var servingRuntimeRes = schema.GroupVersionResource{Group: "serving.kserve.io", Version: "v1alpha1", Resource: "servingruntimes"}
 var inferenceServiceRes = schema.GroupVersionResource{Group: "serving.kserve.io", Version: "v1beta1", Resource: "inferenceservices"}
 var dataVolumeRes = schema.GroupVersionResource{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"}
@@ -244,6 +257,51 @@ func createAAP(t *testing.T, memberAwait *wait.MemberAwaitility, name, namespace
 	return deployment
 }
 
+// createClaw creates an instance of claws.claw.sandbox.redhat.com with one deployment owned by this instance
+// returns the underlying deployment
+func createClaw(t *testing.T, memberAwait *wait.MemberAwaitility, name, namespace string) *appsv1.Deployment {
+	clnt, err := dynamic.NewForConfig(memberAwait.RestConfig)
+	require.NoError(t, err)
+
+	claw := clawResource(name)
+	createdClaw, err := clnt.Resource(clawRes).Namespace(namespace).Create(context.TODO(), claw, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	replicas := int32(2)
+	deployment := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(name)},
+			Replicas: &replicas,
+			Template: podTemplateSpec(name),
+		},
+	}
+	err = controllerutil.SetOwnerReference(createdClaw, deployment, scheme.Scheme)
+	require.NoError(t, err)
+	err = memberAwait.Create(t, deployment)
+	require.NoError(t, err)
+
+	return deployment
+}
+
+func clawResource(name string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "claw.sandbox.redhat.com/v1alpha1",
+			"kind":       "Claw",
+			"metadata": map[string]interface{}{
+				"name": name,
+			},
+			"spec": map[string]interface{}{
+				"idle": false,
+			},
+		},
+	}
+}
+
 func createReplicaSet(t *testing.T, memberAwait *wait.MemberAwaitility, namespace string) *appsv1.ReplicaSet {
 	// Standalone ReplicaSet
 	replicas := int32(2)

testsupport/wait/member.go

@@ -15,6 +15,7 @@ import (
 	toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1"
 	"github.com/codeready-toolchain/toolchain-common/pkg/test"
 	appstudiov1 "github.com/codeready-toolchain/toolchain-e2e/testsupport/appstudio/api/v1alpha1"
+	"github.com/codeready-toolchain/toolchain-e2e/testsupport/cleanup"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/ghodss/yaml"
 	quotav1 "github.com/openshift/api/quota/v1"
@@ -1415,6 +1416,16 @@ func (a *MemberAwaitility) Create(t *testing.T, obj client.Object) error {
 	})
 }
 
+// CreateWithCleanup tries to create the object until success and schedules cleanup at test end.
+// Workaround for https://github.com/kubernetes/kubernetes/issues/67761
+func (a *MemberAwaitility) CreateWithCleanup(t *testing.T, obj client.Object) error {
+	if err := a.Create(t, obj); err != nil {
+		return err
+	}
+	cleanup.AddCleanTasks(t, a.Client, obj)
+	return nil
+}
+
 // PodWaitCriterion a struct to compare with a given Pod
 type PodWaitCriterion struct {
 	Match func(*corev1.Pod) bool
@@ -1525,7 +1536,7 @@ func (a *MemberAwaitility) WaitForAAP(t *testing.T, name, namespace string, aapR
 	var aap *unstructured.Unstructured
 	err := wait.PollUntilContextTimeout(context.TODO(), a.RetryInterval, a.Timeout, true, func(ctx context.Context) (bool, error) {
 		var err error
-		aap, err = aapRes.Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{})
+		aap, err = aapRes.Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
 		if err != nil {
 			if errors.IsNotFound(err) {
 				return false, nil
@@ -1541,6 +1552,31 @@ func (a *MemberAwaitility) WaitForAAP(t *testing.T, name, namespace string, aapR
 	return aap, err
 }
 
+// WaitForClaw waits for the Claw resource to reach the expected idle state (spec.idle)
+func (a *MemberAwaitility) WaitForClaw(t *testing.T, name, namespace string, clawRes dynamic.NamespaceableResourceInterface, expectedIdled bool) (*unstructured.Unstructured, error) {
+	t.Logf("waiting for Claw '%s' in namespace '%s'", name, namespace)
+	var claw *unstructured.Unstructured
+	err := wait.PollUntilContextTimeout(context.TODO(), a.RetryInterval, a.Timeout, true, func(ctx context.Context) (bool, error) {
+		var err error
+		claw, err = clawRes.Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return false, nil
+			}
+			return false, err
+		}
+		idled, found, err := unstructured.NestedBool(claw.UnstructuredContent(), "spec", "idle")
+		if err != nil {
+			return true, err
+		}
+		if !found {
+			return false, nil
+		}
+		return expectedIdled == idled, nil
+	})
+	return claw, err
+}
+
 // WaitUntilInferenceServiceDeleted waits for the InferenceService resource to be deleted (idled)
 func (a *MemberAwaitility) WaitUntilInferenceServiceDeleted(t *testing.T, name, namespace string, inferenceServiceRes dynamic.NamespaceableResourceInterface) error {
 	t.Logf("waiting for InferenceService '%s' to be deleted in namespace '%s'", name, namespace)