Skip to content

Commit fbf1023

Browse files
chzblychchzblych
authored
[None][infra] Ensure Slurm cleanup runs after interruptions (NVIDIA#15943)
Signed-off-by: Yanchao Lu <yanchaol@nvidia.com>
1 parent 5adff55 commit fbf1023

3 files changed

Lines changed: 85 additions & 18 deletions

File tree

‎jenkins/Build.groovy‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _
218

319
import groovy.transform.Field
@@ -164,6 +180,12 @@ def createKubernetesPodConfig(image, type, arch = "amd64")
164180
nodeLabelPrefix = "cpu"
165181
break
166182
}
183+
// Temporarily avoid an arm64 builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564.
184+
def blockedNodeAffinity = arch == "arm64" ? '''
185+
- key: "kubernetes.io/hostname"
186+
operator: NotIn
187+
values:
188+
- "rl300-0021.ipp2a1.colossus"''' : ""
167189
def nodeLabel = trtllm_utils.generateNodeLabel(nodeLabelPrefix)
168190
def pvcVolume = """
169191
- name: sw-tensorrt-pvc
@@ -200,6 +222,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64")
200222
values:
201223
- "core"
202224
- "qa_only"
225+
${blockedNodeAffinity}
203226
nodeSelector: ${selectors}
204227
containers:
205228
${containerConfig}

‎jenkins/BuildDockerImage.groovy‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
117
@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _
218

319
import java.lang.Exception
@@ -92,6 +108,19 @@ def createKubernetesPodConfig(type, arch = "amd64", build_wheel = false)
92108
- "rl300-0046.ipp2u1.colossus"
93109
- "rl300-0047.ipp2u1.colossus"
94110
"""
111+
} else if (arch == "arm64") {
112+
// Temporarily avoid an arm64 builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564.
113+
selectors += """
114+
affinity:
115+
nodeAffinity:
116+
requiredDuringSchedulingIgnoredDuringExecution:
117+
nodeSelectorTerms:
118+
- matchExpressions:
119+
- key: "kubernetes.io/hostname"
120+
operator: NotIn
121+
values:
122+
- "rl300-0021.ipp2a1.colossus"
123+
"""
95124
}
96125

97126
if (arch == "amd64") {

‎jenkins/L0_Test.groovy‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _
17+
@Library(['bloom-jenkins-shared-lib@dev-yanchaol-fix-step-log-retry', 'trtllm-jenkins-shared-lib@main']) _
1818

1919
import java.lang.InterruptedException
2020
import groovy.transform.Field
@@ -941,14 +941,18 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG,
941941
throw e
942942
}
943943
} finally {
944-
captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, slurmJobID, placementContext, stageName)
945-
stage("Clean Up Slurm Resource") {
946-
// Workaround to handle the interruption during clean up SLURM resources
947-
retry(3) {
948-
try {
949-
cleanUpNodeResources(pipeline, cluster, partition.clusterName, nodeName, slurmJobID)
950-
} catch (Exception e) {
951-
error "Error during clean up SLURM resources: ${e.getMessage()} and retrying."
944+
// Resource cleanup must run even if SLURM metadata capture is interrupted.
945+
try {
946+
captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, slurmJobID, placementContext, stageName)
947+
} finally {
948+
stage("Clean Up Slurm Resource") {
949+
// Workaround to handle the interruption during clean up SLURM resources
950+
retry(3) {
951+
try {
952+
cleanUpNodeResources(pipeline, cluster, partition.clusterName, nodeName, slurmJobID)
953+
} catch (Exception e) {
954+
error "Error during clean up SLURM resources: ${e.getMessage()} and retrying."
955+
}
952956
}
953957
}
954958
}
@@ -1707,15 +1711,19 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
17071711
stageIsInterrupted = true
17081712
throw e
17091713
} finally {
1710-
captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, placementContext?.slurmJobId ?: null, placementContext, stageName, jobWorkspace)
1711-
uploadResults(pipeline, cluster, partition.clusterName, jobUID, stageName, stageIsInterrupted, postTag)
1712-
stage("Clean Up Slurm Resource") {
1713-
// Workaround to handle the interruption during clean up SLURM resources
1714-
retry(3) {
1715-
try {
1716-
cleanUpSlurmResources(pipeline, cluster, partition.clusterName, jobUID)
1717-
} catch (Exception e) {
1718-
error "Error during clean up SLURM resources: ${e.getMessage()} and retrying."
1714+
// Resource cleanup must run even if metadata capture or result upload is interrupted.
1715+
try {
1716+
captureSlurmJobNodeList(pipeline, cluster, partition.clusterName, placementContext?.slurmJobId ?: null, placementContext, stageName, jobWorkspace)
1717+
uploadResults(pipeline, cluster, partition.clusterName, jobUID, stageName, stageIsInterrupted, postTag)
1718+
} finally {
1719+
stage("Clean Up Slurm Resource") {
1720+
// Workaround to handle the interruption during clean up SLURM resources
1721+
retry(3) {
1722+
try {
1723+
cleanUpSlurmResources(pipeline, cluster, partition.clusterName, jobUID)
1724+
} catch (Exception e) {
1725+
error "Error during clean up SLURM resources: ${e.getMessage()} and retrying."
1726+
}
17191727
}
17201728
}
17211729
}
@@ -2493,6 +2501,12 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod
24932501
- SYS_ADMIN"""
24942502
break
24952503
}
2504+
// Temporarily avoid an arm64 CPU builder with repeated pod DNS/JNLP failures seen in Build-SBSA #5564.
2505+
def blockedNodeAffinity = targetCloud == "kubernetes-cpu" && arch == "arm64" ? '''
2506+
- key: "kubernetes.io/hostname"
2507+
operator: NotIn
2508+
values:
2509+
- "rl300-0021.ipp2a1.colossus"''' : ""
24962510
def nodeLabel = trtllm_utils.generateNodeLabel(nodeLabelPrefix)
24972511
def pvcVolume = """
24982512
- name: sw-tensorrt-pvc
@@ -2548,6 +2562,7 @@ def createKubernetesPodConfig(image, type, arch = "amd64", gpuCount = 1, perfMod
25482562
values:
25492563
- "core"
25502564
- "qa_only"
2565+
${blockedNodeAffinity}
25512566
nodeSelector: ${selectors}
25522567
containers:
25532568
${containerConfig}

0 commit comments

Comments
 (0)