fix: Fix --adapt-from-ct for externally managed jobset and kueue (#1192)

jamOne- · web-flow · commit 367c838253be · 2026-06-25T15:28:33.000Z
* feat: skip Kueue and Jobset manifest installs if managed by Helm

* Add unit tests for helm-managed Kueue and Jobset

* Refactor is_managed_by_helm to use jsonpath for speed and target deployments explicitly

* Simplify is_managed_by_helm to only check managed-by label

* Simplify code

* Rename out to container_image in _should_install_jobset

* Rename is_managed_by_helm to is_managed_externally

* Fix pylint trailing whitespaces

* Fix pyink formatting
diff --git a/recipes/Basic_cluster_adapt.md b/recipes/Basic_cluster_adapt.md
@@ -38,6 +38,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Basic_cluster_create.md b/recipes/Basic_cluster_create.md
@@ -80,6 +80,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_RayCluster.md b/recipes/Cluster_create_RayCluster.md
@@ -83,6 +83,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_adapt_from_ct.md b/recipes/Cluster_create_adapt_from_ct.md
@@ -39,6 +39,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_for_multi-host_nodepool.md b/recipes/Cluster_create_for_multi-host_nodepool.md
@@ -82,6 +82,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_for_single-host_nodepool.md b/recipes/Cluster_create_for_single-host_nodepool.md
@@ -80,6 +80,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_private.md b/recipes/Cluster_create_private.md
@@ -88,6 +88,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster-private
 [XPK] Task: `Install Jobset on golden-cluster-private` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
@@ -369,6 +371,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster-private-ep
 [XPK] Task: `Install Jobset on golden-cluster-private-ep` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
@@ -643,6 +647,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster-private-nosubnet
 [XPK] Task: `Install Jobset on golden-cluster-private-nosubnet` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_sub-slicing.md b/recipes/Cluster_create_sub-slicing.md
@@ -83,6 +83,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_super-slicing.md b/recipes/Cluster_create_super-slicing.md
@@ -89,6 +89,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_with_CPU_and_memory_limits_above_capacity.md b/recipes/Cluster_create_with_CPU_and_memory_limits_above_capacity.md
@@ -80,6 +80,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_with_CPU_and_memory_limits_below_capacity.md b/recipes/Cluster_create_with_CPU_and_memory_limits_below_capacity.md
@@ -80,6 +80,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_with_Managed_Lustre_driver.md b/recipes/Cluster_create_with_Managed_Lustre_driver.md
@@ -91,6 +91,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_with_Managed_Lustre_driver_and_legacy_port.md b/recipes/Cluster_create_with_Managed_Lustre_driver_and_legacy_port.md
@@ -91,6 +91,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_with_gb200-4.md b/recipes/Cluster_create_with_gb200-4.md
@@ -85,6 +85,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/Cluster_create_with_shared_reservation.md b/recipes/Cluster_create_with_shared_reservation.md
@@ -83,6 +83,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/NAP_cluster-create.md b/recipes/NAP_cluster-create.md
@@ -121,6 +121,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/recipes/NAP_cluster-create_with_pathways.md b/recipes/NAP_cluster-create_with_pathways.md
@@ -124,6 +124,8 @@ data:
 [XPK] Breaking up a total of 2 commands into 1 batches
 [XPK] Pretending all the jobs succeeded
 [XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Task: `Check if Jobset is installed` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment -n jobset-system -o jsonpath='{.items[*].spec.template.spec.containers[0].image}'
 [XPK] Try 1: Install Jobset on golden-cluster
 [XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
 kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.10.1/manifests.yaml
diff --git a/src/xpk/core/cluster.py b/src/xpk/core/cluster.py
@@ -19,7 +19,7 @@
 from kubernetes import config
 from kubernetes.client.exceptions import ApiException
 
-from .kubectl_common import PatchResources, patch_controller_manager_resources
+from .kubectl_common import PatchResources, patch_controller_manager_resources, is_managed_externally
 from ..utils.console import xpk_exit, xpk_print
 from .capacity import H200_DEVICE_TYPE
 from .commands import (
@@ -36,6 +36,7 @@
 from .nodepool import recreate_nodes_in_existing_node_pools
 from .resources import get_cluster_system_characteristics
 from .system_characteristics import INSTALLER_NCCL_TCPXO, SystemCharacteristics
+from packaging.version import Version, InvalidVersion
 
 JOBSET_VERSION = 'v0.10.1'
 NRI_DEVICE_INJECTOR = 'https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nri_device_injector/nri-device-injector.yaml'
@@ -44,6 +45,44 @@
 XPK_SA = 'xpk-sa'
 
 
+def _should_install_jobset(container_image: str) -> bool:
+  if 'jobset' not in container_image:
+    return True
+
+  installed_version_str = (
+      container_image.split(':')[-1] if ':' in container_image else ''
+  )
+  try:
+    installed_version = Version(installed_version_str)
+  except InvalidVersion:
+    xpk_print(
+        'Cluster has an unknown or custom Jobset version installed. Skipping'
+        ' installation.'
+    )
+    return False
+
+  xpk_jobset_version = Version(JOBSET_VERSION.lstrip('v'))
+  if installed_version >= xpk_jobset_version:
+    xpk_print(
+        f'Cluster has Jobset version {installed_version} >='
+        f' {xpk_jobset_version}. Skipping installation.'
+    )
+    return False
+
+  _, is_external = is_managed_externally(
+      name='jobset-controller-manager', namespace='jobset-system'
+  )
+  if is_external:
+    xpk_print(
+        f'Cluster has Jobset version {installed_version} managed externally.'
+        ' Skipping upgrade to avoid conflicts.'
+    )
+    return False
+
+  xpk_print(f'Upgrading Jobset to version {JOBSET_VERSION}...')
+  return True
+
+
 # TODO(vbarr): Remove this function when jobsets gets enabled by default on
 # GKE clusters.
 def set_jobset_on_cluster(args) -> int:
@@ -55,24 +94,35 @@ def set_jobset_on_cluster(args) -> int:
   Returns:
     0 if successful and 1 otherwise.
   """
-  command = (
-      'kubectl apply --server-side --force-conflicts'
-      f' -f https://github.com/kubernetes-sigs/jobset/releases/download/{JOBSET_VERSION}/manifests.yaml'
+  check_jobset_cmd = (
+      'kubectl get deployment -n jobset-system -o'
+      " jsonpath='{.items[*].spec.template.spec.containers[0].image}'"
+  )
+  return_code, out = run_command_for_value(
+      check_jobset_cmd, 'Check if Jobset is installed'
   )
-  task = f'Install Jobset on {args.cluster}'
-  return_code = run_command_with_updates_retry(command, task)
 
-  if return_code != 0:
-    xpk_print(f'{task} returned with ERROR {return_code}.\n')
-    xpk_print(
-        "This LIKELY means you're missing Kubernetes Permissions, you can"
-        ' validate this by checking if the error references permission problems'
-        ' such as `requires one of ["container.*"] permission(s)`. Follow our'
-        ' readme:'
-        ' https://github.com/google/xpk/blob/main/README.md#troubleshooting for'
-        ' instructions on how to fix these permissions.'
+  should_install = return_code != 0 or _should_install_jobset(out)
+
+  if should_install:
+    command = (
+        'kubectl apply --server-side --force-conflicts'
+        f' -f https://github.com/kubernetes-sigs/jobset/releases/download/{JOBSET_VERSION}/manifests.yaml'
     )
-    return return_code
+    task = f'Install Jobset on {args.cluster}'
+    return_code = run_command_with_updates_retry(command, task)
+
+    if return_code != 0:
+      xpk_print(f'{task} returned with ERROR {return_code}.\n')
+      xpk_print(
+          "This LIKELY means you're missing Kubernetes Permissions, you can"
+          ' validate this by checking if the error references permission'
+          ' problems such as `requires one of ["container.*"] permission(s)`.'
+          ' Follow our readme:'
+          ' https://github.com/google/xpk/blob/main/README.md#troubleshooting'
+          ' for instructions on how to fix these permissions.'
+      )
+      return return_code
 
   if args.super_slicing:
     return patch_controller_manager_resources(
diff --git a/src/xpk/core/cluster_test.py b/src/xpk/core/cluster_test.py
@@ -226,12 +226,34 @@ def test_update_gke_cluster_with_lustre_driver_enabled_legacy_port(
 
 
 def test_set_jobset_on_cluster_not_setting_resources_by_default(
-    mock_patch_controller_manager_resources: MagicMock, command_args
+    commands_tester: CommandsTester,
+    mock_patch_controller_manager_resources: MagicMock,
+    command_args,
 ):
   result = set_jobset_on_cluster(command_args)
 
   assert result == 0
   mock_patch_controller_manager_resources.assert_not_called()
+  commands_tester.assert_command_run("kubectl apply", "manifests.yaml")
+
+
+def test_set_jobset_on_cluster_skip_when_managed_by_helm(
+    commands_tester: CommandsTester, command_args
+):
+  commands_tester.set_result_for_command(
+      (0, "jobset:v0.1.0"),
+      "kubectl get deployment -n jobset-system",
+  )
+  commands_tester.set_result_for_command(
+      (0, "Helm"),
+      "kubectl get deployment jobset-controller-manager",
+      "jsonpath=",
+  )
+
+  result = set_jobset_on_cluster(command_args)
+
+  assert result == 0
+  commands_tester.assert_command_not_run("kubectl apply", "manifests.yaml")
 
 
 def test_set_jobset_on_cluster_super_slicing_resources(
diff --git a/src/xpk/core/kubectl_common.py b/src/xpk/core/kubectl_common.py
diff --git a/src/xpk/core/kubectl_common_test.py b/src/xpk/core/kubectl_common_test.py
diff --git a/src/xpk/core/kueue_manager.py b/src/xpk/core/kueue_manager.py
diff --git a/src/xpk/core/kueue_manager_test.py b/src/xpk/core/kueue_manager_test.py
