Support Claw CR idling (#745)

Author: alexeykazakov

controllers/idler/idler_controller.go

@@ -83,6 +83,8 @@ type Reconciler struct {
 // There are other AAP resource kinds which are involved in the Pod -> ... -> AnsibleAutomationPlatform ownership chain. We need to be able to get/list them.
 //+kubebuilder:rbac:groups=aap.ansible.com,resources=*,verbs=get;list
 
+//+kubebuilder:rbac:groups=claw.sandbox.redhat.com,resources=claws,verbs=get;list;patch
+
 // Reconcile reads that state of the cluster for an Idler object and makes changes based on the state read
 // and what is in the Idler.Spec
 // Note:

controllers/idler/idler_controller_test.go

@@ -232,7 +232,10 @@ func TestEnsureIdling(t *testing.T) {
 				AAPRunning(noise.aap).
 				InferenceServiceDoesNotExist(podsRunningForTooLong.inferenceService).
 				InferenceServiceExists(podsTooEarlyToKill.inferenceService).
-				InferenceServiceExists(noise.inferenceService)
+				InferenceServiceExists(noise.inferenceService).
+				ClawIdled(podsRunningForTooLong.claw).
+				ClawRunning(podsTooEarlyToKill.claw).
+				ClawRunning(noise.claw)
 
 			memberoperatortest.AssertThatIdler(t, idler.Name, fakeClients).
 				HasConditions(memberoperatortest.Running(), memberoperatortest.IdlerNotificationCreated())
@@ -333,7 +336,8 @@ func TestEnsureIdling(t *testing.T) {
 				StatefulSetScaledDown(toKill.statefulSet).
 				VMStopped(toKill.vmStopCallCounter).
 				AAPIdled(toKill.aap).
-				InferenceServiceDoesNotExist(toKill.inferenceService)
+				InferenceServiceDoesNotExist(toKill.inferenceService).
+				ClawIdled(toKill.claw)
 
 			memberoperatortest.AssertThatIdler(t, idler.Name, fakeClients).
 				ContainsCondition(memberoperatortest.FailedToIdle(strings.Split(err.Error(), ": ")[1]))
@@ -550,7 +554,8 @@ func TestEnsureIdlingFailed(t *testing.T) {
 			StatefulSetScaledDown(toKill.statefulSet).
 			VMStopped(toKill.vmStopCallCounter).
 			AAPIdled(toKill.aap).
-			InferenceServiceDoesNotExist(toKill.inferenceService)
+			InferenceServiceDoesNotExist(toKill.inferenceService).
+			ClawIdled(toKill.claw)
 	})
 }
 
@@ -884,6 +889,7 @@ type payloads struct {
 	aap                       *unstructured.Unstructured
 	servingRuntime            *unstructured.Unstructured
 	inferenceService          *unstructured.Unstructured
+	claw                      *unstructured.Unstructured
 }
 
 func (p payloads) getFirstControlledPod(ownerName string) *corev1.Pod {
@@ -1089,6 +1095,13 @@ func preparePayloads(t *testing.T, clients *memberoperatortest.FakeClientSet, na
 	inferenceService.SetCreationTimestamp(*sTime)
 	createObjectWithDynamicClient(t, clients.DynamicClient, inferenceService)
 
+	// Claw
+	clawObject := newClaw(fmt.Sprintf("%s%s-claw", namePrefix, namespace), namespace)
+	createObjectWithDynamicClient(t, clients.DynamicClient, clawObject)
+	_, clawRs := createDeployment(t, clients, namespace, namePrefix, "-claw-deployment", clawObject)
+	replicaSetsWithDeployment = append(replicaSetsWithDeployment, clawRs)
+	controlledPods = createPods(t, clients.AllNamespacesClient, clawRs, sTime, controlledPods, noRestart())
+
 	// Pods with unknown owner. They are subject of direct management by the Idler.
 	// It doesn't have to be Idler. We just need any object as the owner of the pods
 	// which is not a supported owner such as Deployment or ReplicaSet.
@@ -1144,6 +1157,7 @@ func preparePayloads(t *testing.T, clients *memberoperatortest.FakeClientSet, na
 		aap:                       aapObject,
 		servingRuntime:            servingRuntimeObject,
 		inferenceService:          inferenceService,
+		claw:                      clawObject,
 	}
 }
 
@@ -1154,6 +1168,16 @@ func newAAP(t *testing.T, idled bool, name, namespace string) *unstructured.Unst
 	return aap
 }
 
+func newClaw(name, namespace string) *unstructured.Unstructured {
+	claw := &unstructured.Unstructured{}
+	claw.SetAPIVersion("claw.sandbox.redhat.com/v1alpha1")
+	claw.SetKind("Claw")
+	claw.SetName(name)
+	claw.SetNamespace(namespace)
+	unstructured.SetNestedField(claw.Object, false, "spec", "idle") //nolint:errcheck
+	return claw
+}
+
 func newServingRuntime(name, namespace string) *unstructured.Unstructured {
 	servingRuntime := &unstructured.Unstructured{}
 	servingRuntime.SetAPIVersion("serving.kserve.io/v1alpha1")

controllers/idler/owners.go

@@ -80,6 +80,8 @@ func (i *ownerIdler) scaleOwnerToZero(ctx context.Context, pod *corev1.Pod) (str
 			err = i.stopVirtualMachine(ctx, ownerWithGVR) // Nothing to scale down. Stop instead.
 		case "AnsibleAutomationPlatform":
 			err = i.idleAAP(ctx, ownerWithGVR) // Nothing to scale down. Stop instead.
+		case "Claw":
+			err = i.idleClaw(ctx, ownerWithGVR)
 		case "ServingRuntime":
 			err = i.idleServingRuntime(ctx, ownerWithGVR) // Idle by deleting old InferenceService objects.
 		default:
@@ -178,6 +180,33 @@ func (i *ownerIdler) idleAAP(ctx context.Context, objectWithGVR *owners.ObjectWi
 	return nil
 }
 
+// idleClaw idles a Claw instance if not already idled
+func (i *ownerIdler) idleClaw(ctx context.Context, objectWithGVR *owners.ObjectWithGVR) error {
+	clawName := objectWithGVR.Object.GetName()
+	logger := log.FromContext(ctx).WithValues("name", clawName)
+	idled, _, err := unstructured.NestedBool(objectWithGVR.Object.UnstructuredContent(), "spec", "idle")
+	if err != nil {
+		logger.Error(err, "Failed to parse Claw CR to get the spec.idle field")
+	}
+	if idled {
+		logger.Info("Claw CR is already idled")
+		return nil
+	}
+	logger.Info("Idling Claw")
+
+	patch := []byte(`{"spec":{"idle":true}}`)
+	_, err = i.dynamicClient.
+		Resource(*objectWithGVR.GVR).
+		Namespace(objectWithGVR.Object.GetNamespace()).
+		Patch(ctx, clawName, types.MergePatchType, patch, metav1.PatchOptions{})
+	if err != nil {
+		return err
+	}
+
+	logger.Info("Claw idled", "name", clawName)
+	return nil
+}
+
 func (i *ownerIdler) deleteResource(ctx context.Context, objectWithGVR *owners.ObjectWithGVR) error {
 	logger := log.FromContext(ctx)
 	object := objectWithGVR.Object

controllers/idler/owners_test.go

@@ -225,6 +225,19 @@ var testConfigs = map[string]createTestConfigFunc{
 			},
 		}
 	},
+	"Claw": func(plds payloads) payloadTestConfig {
+		return payloadTestConfig{
+			// Claw -> Deployment -> ReplicaSet -> Pod
+			podOwnerName:    fmt.Sprintf("%s-deployment-replicaset", plds.claw.GetName()),
+			expectedAppName: plds.claw.GetName(),
+			ownerScaledUp: func(assertion *test.IdleablePayloadAssertion) {
+				assertion.ClawRunning(plds.claw)
+			},
+			ownerScaledDown: func(assertion *test.IdleablePayloadAssertion) {
+				assertion.ClawIdled(plds.claw)
+			},
+		}
+	},
 }
 
 var customListKinds = map[schema.GroupVersionResource]string{
@@ -596,5 +609,11 @@ func allResourcesList(t *testing.T) []*metav1.APIResourceList {
 				{Name: "inferenceservices", Namespaced: true, Kind: "InferenceService"},
 			},
 		},
+		&metav1.APIResourceList{
+			GroupVersion: "claw.sandbox.redhat.com/v1alpha1",
+			APIResources: []metav1.APIResource{
+				{Name: "claws", Namespaced: true, Kind: "Claw"},
+			},
+		},
 	)
 }

test/idler_assertion.go

@@ -378,6 +378,26 @@ func (a *IdleablePayloadAssertion) AAPRunning(aap *unstructured.Unstructured) *I
 	return a
 }
 
+var clawGVR = schema.GroupVersionResource{Group: "claw.sandbox.redhat.com", Version: "v1alpha1", Resource: "claws"}
+
+func (a *IdleablePayloadAssertion) ClawIdled(claw *unstructured.Unstructured) *IdleablePayloadAssertion {
+	actualClaw := &unstructured.Unstructured{}
+	a.getResourceFromDynamicClient(clawGVR, claw.GetNamespace(), claw.GetName(), actualClaw)
+	idled, _, err := unstructured.NestedBool(actualClaw.UnstructuredContent(), "spec", "idle")
+	require.NoError(a.t, err)
+	assert.True(a.t, idled)
+	return a
+}
+
+func (a *IdleablePayloadAssertion) ClawRunning(claw *unstructured.Unstructured) *IdleablePayloadAssertion {
+	actualClaw := &unstructured.Unstructured{}
+	a.getResourceFromDynamicClient(clawGVR, claw.GetNamespace(), claw.GetName(), actualClaw)
+	idled, _, err := unstructured.NestedBool(actualClaw.UnstructuredContent(), "spec", "idle")
+	require.NoError(a.t, err)
+	assert.False(a.t, idled)
+	return a
+}
+
 var inferenceServiceGVR = schema.GroupVersionResource{Group: "serving.kserve.io", Version: "v1beta1", Resource: "inferenceservices"}
 
 func (a *IdleablePayloadAssertion) InferenceServiceDoesNotExist(inferenceService *unstructured.Unstructured) *IdleablePayloadAssertion {