diff --git a/config/core/200-roles/clusterrole.yaml b/config/core/200-roles/clusterrole.yaml
index 5ab0e53829ff..3ad88d21028e 100644
--- a/config/core/200-roles/clusterrole.yaml
+++ b/config/core/200-roles/clusterrole.yaml
@@ -33,6 +33,9 @@ rules:
   - apiGroups: ["apps"]
     resources: ["deployments", "deployments/finalizers"] # finalizers are needed for the owner reference of the webhook
     verbs: ["get", "list", "create", "update", "delete", "patch", "watch"]
+  - apiGroups: ["*"]
+    resources: ["*/scale"]
+    verbs: ["get", "update", "patch"]
   - apiGroups: ["admissionregistration.k8s.io"]
     resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
     verbs: ["get", "list", "create", "update", "delete", "patch", "watch"]
diff --git a/pkg/reconciler/autoscaling/kpa/kpa_test.go b/pkg/reconciler/autoscaling/kpa/kpa_test.go
index 107d232ff0c8..6c7efdcbaff0 100644
--- a/pkg/reconciler/autoscaling/kpa/kpa_test.go
+++ b/pkg/reconciler/autoscaling/kpa/kpa_test.go
@@ -438,7 +438,7 @@ func TestReconcile(t *testing.T) {
 				Namespace: testNamespace,
 			},
 			Name:  deployName,
-			Patch: []byte(`[{"op":"add","path":"/spec/replicas","value":11}]`),
+			Patch: []byte(`[{"op":"replace","path":"/spec/replicas","value":11}]`),
 		}},
 	}, {
 		Name: "scale up deployment failure",
@@ -459,7 +459,7 @@ func TestReconcile(t *testing.T) {
 				Namespace: testNamespace,
 			},
 			Name:  deployName,
-			Patch: []byte(`[{"op":"add","path":"/spec/replicas","value":11}]`),
+			Patch: []byte(`[{"op":"replace","path":"/spec/replicas","value":11}]`),
 		}},
 		WantEvents: []string{
 			Eventf(corev1.EventTypeWarning, "InternalError",
@@ -735,14 +735,14 @@ func TestReconcile(t *testing.T) {
 				WithPAMetricsService(privateSvc), WithObservedGeneration(1)),
 			sks(testNamespace, testRevision, WithDeployRef(deployName), WithProxyMode, WithSKSReady),
 			metric(testNamespace, testRevision),
-			deploy(testNamespace, testRevision),
+			deploy(testNamespace, testRevision, func(d *appsv1.Deployment) { d.Spec.Replicas = ptr.Int32(1) }),
 		},
 		WantPatches: []clientgotesting.PatchActionImpl{{
 			ActionImpl: clientgotesting.ActionImpl{
 				Namespace: testNamespace,
 			},
 			Name:  deployName,
-			Patch: []byte(`[{"op":"add","path":"/spec/replicas","value":0}]`),
+			Patch: []byte(`[{"op":"replace","path":"/spec/replicas","value":0}]`),
 		}},
 	}, {
 		Name: "from serving to proxy",
@@ -777,7 +777,7 @@ func TestReconcile(t *testing.T) {
 				WithPAStatusService(testRevision), WithPAMetricsService(privateSvc), WithObservedGeneration(1)),
 			defaultSKS,
 			metric(testNamespace, testRevision),
-			deploy(testNamespace, testRevision), defaultReady,
+			deploy(testNamespace, testRevision, func(d *appsv1.Deployment) { d.Spec.Replicas = ptr.Int32(1) }), defaultReady,
 		},
 	}, {
 		Name: "activation failure",
@@ -790,7 +790,7 @@ func TestReconcile(t *testing.T) {
 				WithPAMetricsService(privateSvc)),
 			defaultSKS,
 			metric(testNamespace, testRevision),
-			deploy(testNamespace, testRevision), defaultReady,
+			deploy(testNamespace, testRevision, func(d *appsv1.Deployment) { d.Spec.Replicas = ptr.Int32(1) }), defaultReady,
 		},
 		WantStatusUpdates: []clientgotesting.UpdateActionImpl{{
 			Object: kpa(testNamespace, testRevision, markScaleTargetInitialized, WithPASKSReady, WithPAMetricsService(privateSvc),
@@ -806,7 +806,7 @@ func TestReconcile(t *testing.T) {
 				Namespace: testNamespace,
 			},
 			Name:  deployName,
-			Patch: []byte(`[{"op":"add","path":"/spec/replicas","value":0}]`),
+			Patch: []byte(`[{"op":"replace","path":"/spec/replicas","value":0}]`),
 		}},
 	}, {
 		Name: "want=-1, underscaled, PA inactive",
diff --git a/pkg/reconciler/autoscaling/kpa/scaler.go b/pkg/reconciler/autoscaling/kpa/scaler.go
index 84dba5dc81d7..247b20f813db 100644
--- a/pkg/reconciler/autoscaling/kpa/scaler.go
+++ b/pkg/reconciler/autoscaling/kpa/scaler.go
@@ -24,6 +24,9 @@ import (
 	"strconv"
 	"time"
 
+	autoscalingv1 "k8s.io/api/autoscaling/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
 	"knative.dev/pkg/apis/duck"
 	"knative.dev/pkg/injection/clients/dynamicclient"
 	"knative.dev/pkg/logging"
@@ -299,30 +302,16 @@ func (ks *scaler) handleScaleToZero(ctx context.Context, pa *autoscalingv1alpha1
 }
 
 func (ks *scaler) applyScale(ctx context.Context, pa *autoscalingv1alpha1.PodAutoscaler, desiredScale int32,
-	ps *autoscalingv1alpha1.PodScalable,
+	gvr *schema.GroupVersionResource, resourceName string,
 ) error {
 	logger := logging.FromContext(ctx)
 
-	gvr, name, err := resources.ScaleResourceArguments(pa.Spec.ScaleTargetRef)
-	if err != nil {
-		return err
-	}
-
-	psNew := ps.DeepCopy()
-	psNew.Spec.Replicas = &desiredScale
-	patch, err := duck.CreatePatch(ps, psNew)
-	if err != nil {
-		return err
-	}
-	patchBytes, err := patch.MarshalJSON()
-	if err != nil {
-		return err
-	}
+	patchBytes := []byte(fmt.Sprintf(`[{"op":"replace","path":"/spec/replicas","value":%d}]`, desiredScale))
 
-	_, err = ks.dynamicClient.Resource(*gvr).Namespace(pa.Namespace).Patch(ctx, ps.Name, types.JSONPatchType,
-		patchBytes, metav1.PatchOptions{})
+	_, err := ks.dynamicClient.Resource(*gvr).Namespace(pa.Namespace).Patch(ctx, resourceName, types.JSONPatchType,
+		patchBytes, metav1.PatchOptions{}, "scale")
 	if err != nil {
-		return fmt.Errorf("failed to apply scale %d to scale target %s: %w", desiredScale, name, err)
+		return fmt.Errorf("failed to apply scale %d to scale target %s: %w", desiredScale, resourceName, err)
 	}
 
 	logger.Debug("Successfully scaled to ", desiredScale)
@@ -364,19 +353,25 @@ func (ks *scaler) scale(ctx context.Context, pa *autoscalingv1alpha1.PodAutoscal
 		return desiredScale, nil
 	}
 
-	ps, err := resources.GetScaleResource(pa.Namespace, pa.Spec.ScaleTargetRef, ks.listerFactory)
+	gvr, name, err := resources.ScaleResourceArguments(pa.Spec.ScaleTargetRef)
+	if err != nil {
+		return desiredScale, err
+	}
+	scaleObj, err := ks.dynamicClient.Resource(*gvr).Namespace(pa.Namespace).Get(ctx, name, metav1.GetOptions{}, "scale")
 	if err != nil {
 		return desiredScale, fmt.Errorf("failed to get scale target %v: %w", pa.Spec.ScaleTargetRef, err)
 	}
+	scale := &autoscalingv1.Scale{}
 
-	currentScale := int32(1)
-	if ps.Spec.Replicas != nil {
-		currentScale = *ps.Spec.Replicas
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(scaleObj.UnstructuredContent(),
+		scale); err != nil {
+		return desiredScale, fmt.Errorf("failed to convert to autoscalingv1.Scale type: %w", err)
 	}
+	currentScale := scale.Spec.Replicas
 	if desiredScale == currentScale {
 		return desiredScale, nil
 	}
 
 	logger.Infof("Scaling from %d to %d", currentScale, desiredScale)
-	return desiredScale, ks.applyScale(ctx, pa, desiredScale, ps)
+	return desiredScale, ks.applyScale(ctx, pa, desiredScale, gvr, name)
 }