Skip to content

Commit 0044d5b

Browse files
yiqingy0yiqingy
andauthored
[TRTLLMINF-126][infra] Generate timeout result when the tests all pass in the rerun step (NVIDIA#15658)
Signed-off-by: yiqingy <yiqingy@tensorrt-llm-infra-debug-vm-01.nvidia.com> Co-authored-by: yiqingy <yiqingy@tensorrt-llm-infra-debug-vm-01.nvidia.com>
1 parent 4f1881d commit 0044d5b

1 file changed

Lines changed: 12 additions & 1 deletion

File tree

jenkins/L0_Test.groovy

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2289,7 +2289,7 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu
22892289
if (stageIsFailed && !suppressTestReporting) {
22902290
if (stageIsInterrupted) {
22912291
echo "Stage is interrupted, skip to generate terminated unexpectedly test result."
2292-
} else {
2292+
} else if (!fileExists("${stageName}/results-timeout.xml")) {
22932293
// Generate timeout test result xml if there are terminated unexpectedly tests
22942294
generateTimeoutTestResultXml(pipeline, stageName)
22952295
}
@@ -3728,6 +3728,13 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
37283728
error "Regular tests failed after rerun attempt"
37293729
}
37303730
rerunFailed = true
3731+
} else if (generateTimeoutTestResultXml(pipeline, stageName)) {
3732+
// Rerun passed but the first run had a timeout: mark this
3733+
// stage FAILURE so "[${stageName}] Run Pytest" turns red,
3734+
// not just the enclosing parent stage.
3735+
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
3736+
error "Some tests terminated unexpectedly, please check the test report."
3737+
}
37313738
}
37323739
}
37333740

@@ -3759,6 +3766,10 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
37593766
error "Some tests still failed after rerun attempts, please check the test report."
37603767
}
37613768

3769+
if (fileExists("${stageName}/results-timeout.xml") || generateTimeoutTestResultXml(pipeline, stageName)) {
3770+
error "Some tests terminated unexpectedly, please check the test report."
3771+
}
3772+
37623773
if (perfMode) {
37633774
// Only PyTorch perf stages remain; the TensorRT perf baseline was removed.
37643775
basePerfFilename = "base_perf_pytorch.csv"

0 commit comments

Comments
 (0)