Skip to content

Commit b767be9

Browse files
guptaakacopybara-github
authored andcommitted
Set max_slice_restarts to 1000000 in Shared Pathways Service deployment
This change configures the max_slice_restarts parameter to 1000000 when deploying the shared Pathways service. Using a very high backOffLimit for the worker pods keeps the jobset from failing on multiple client reconnections. PiperOrigin-RevId: 952999777
1 parent cfac217 commit b767be9

1 file changed

Lines changed: 3 additions & 2 deletions

File tree

pathwaysutils/experimental/shared_pathways_service/deploy_pathways_service.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import dataclasses
55
import logging
66
import math
7-
import os
87
from typing import Any
98
from absl import app
109
from absl import flags
@@ -142,7 +141,6 @@ def calculate_vms_per_slice(topology: str, chips_per_vm: int) -> int:
142141
) from e
143142

144143

145-
146144
def deploy_jobset(jobset_yaml: dict[str, Any]) -> None:
147145
"""Deploys the JobSet to the current Kubernetes cluster."""
148146
try:
@@ -185,6 +183,9 @@ def run_deployment(
185183
topology=topology,
186184
num_slices=num_slices,
187185
shared_pathways_service=True,
186+
# TODO(b/496958026): Remove this once go/sps-worker-pod-stability is
187+
# implemented
188+
max_slice_restarts=1000000,
188189
)
189190

190191
# If custom server_image is provided, mutate the templates to use it.

0 commit comments

Comments
 (0)