From 36982dbc8be3ca1e87ae9ee454982007c68575ff Mon Sep 17 00:00:00 2001 From: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> Date: Wed, 24 Jun 2026 13:33:58 -0700 Subject: [PATCH 1/6] Allow build docker image downstream to run part of the stages Signed-off-by: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> --- jenkins/BuildDockerImage.groovy | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index 595ff8893f2c..809508dbdef4 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -484,6 +484,19 @@ def launchBuildJobs(pipeline, globalVars, imageKeyToTag) { dockerfileStage: "release", ], ] + def enabledStages = [] + if (params.buildInternalRelease) { + enabledStages += ["Build Internal release (x86_64 trtllm)", "Build Internal release (SBSA trtllm)"] + } + if (params.buildCiImage) { + enabledStages += ["Build CI Image (x86_64 tritondevel)", "Build CI Image (SBSA tritondevel)", "Build CI Image (RockyLinux8 Python310)", "Build CI Image (RockyLinux8 Python312)", "Build CI Image (SBSA Ubuntu24.04 Python312)"] + } + if (params.buildNgcRelease) { + enabledStages += ["Build NGC devel And release (x86_64)", "Build NGC devel And release (SBSA)"] + } + buildConfigs = buildConfigs.findAll { key, config -> key in enabledStages } + echo "Running stages: ${buildConfigs.keySet()}" + // Override all fields in build config with default values buildConfigs.each { key, config -> defaultBuildConfig.each { defaultKey, defaultValue -> @@ -552,6 +565,21 @@ pipeline { choices: ["build", "push"], description: "Docker image generation action. build: only perform image build step; push: build docker image and push it to artifacts" ) + booleanParam( + name: "buildInternalRelease", + defaultValue: true, + description: "Build internal release images (x86_64 and SBSA trtllm)" + ) + booleanParam( + name: "buildCiImage", + defaultValue: true, + description: "Build CI images (tritondevel and OS variant images)" + ) + booleanParam( + name: "buildNgcRelease", + defaultValue: true, + description: "Build NGC devel and release images (x86_64 and SBSA)" + ) } options { // Check the valid options at: https://www.jenkins.io/doc/book/pipeline/syntax/ From 6d71d8885696c71b5d709c773bca2a4e2a213229 Mon Sep 17 00:00:00 2001 From: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:19:35 -0700 Subject: [PATCH 2/6] Add stage to run PLC container scanning Signed-off-by: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 97f28dca435f..54365ea387bd 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1717,6 +1717,57 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Build-Docker-Images job is set explicitly. Both x86_64-Linux and SBSA-Linux sub-pipelines will be disabled." } + def plcContainerScanningJob = [ + "PLC Container Scanning": { + script { + stage("[Build-Release-Docker-Images] Remote Run") { + try { + def branch = env.gitlabBranch ? env.gitlabBranch : "main" + if (globalVars[GITHUB_PR_API_URL]) { + branch = "github-pr-" + globalVars[GITHUB_PR_API_URL].split('/').last() + } + + // Force the image tag suffix to be this L0_MergeRequest BUILD_NUMBER + // instead of the BuildDockerImages helper job's own counter. + def shortCommit = env.gitlabCommit ? env.gitlabCommit.substring(0, 7) : "undefined" + def branchTag = branch.replaceAll('/', '_') + def defaultTag = "${shortCommit}-${branchTag}-${env.BUILD_NUMBER}" + + def additionalParameters = [ + 'branch': branch, + 'action': "push", + 'triggerType': env.JOB_NAME ==~ /.*PostMerge.*/ ? "post-merge" : "pre-merge", + 'runSanityCheck': false, + 'defaultTag': defaultTag, + 'buildInternalRelease': false, + 'buildCiImage': false, + 'artifactPath': ARTIFACT_PATH, + 'uploadPath': UPLOAD_PATH + ] + launchJob(pipeline, "/LLM/helpers/BuildDockerImages", false, enableFailFast, globalVars, "x86_64", additionalParameters) + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + if (BUILD_CHECK_CHOICE == STAGE_CHOICE_IGNORE) { + catchError( + buildResult: 'SUCCESS', + stageResult: 'FAILURE') { + error "Build-Docker-Images job failed but ignored due to Jenkins configuration" + } + } else { + throw e + } + } + } + } + } + ] + if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { + stages += plcContainerScanningJob + testFilter[(TEST_STAGE_LIST)]?.remove("NGC-Container-Scanning") + echo "Will run job to build ngc containers and running in-pipeline scanning for them" + } + parallelJobs = stages.collectEntries{key, value -> [key, { script { stage(key) { From fe34b4b8b26b5df3820f668dfa8a176a47493627 Mon Sep 17 00:00:00 2001 From: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:01:41 -0700 Subject: [PATCH 3/6] Skip test Signed-off-by: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 54365ea387bd..45580965bbdc 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1458,6 +1458,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Skipping x86_64 tests (GenPostMergeBuilds mode: builds only)" return } + if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { + echo "Skipping x86_64 tests (PLC container scanning)" + return + } testStageName = "[Test-x86_64-Single-GPU] Remote Run" def singleGpuTestFailed = false @@ -1571,6 +1575,11 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) return } + if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { + echo "Skipping x86_64 tests (PLC container scanning)" + return + } + testStageName = "[Test-SBSA-Single-GPU] Remote Run" def singleGpuTestFailed = false stage(testStageName) { From 77d85881eee444072944f476b85fe12756243e0a Mon Sep 17 00:00:00 2001 From: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:04:51 -0700 Subject: [PATCH 4/6] set trigger type as post-merge to reuse wheel from build stage Signed-off-by: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 45580965bbdc..28ec3511a408 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1745,7 +1745,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) def additionalParameters = [ 'branch': branch, 'action': "push", - 'triggerType': env.JOB_NAME ==~ /.*PostMerge.*/ ? "post-merge" : "pre-merge", + 'triggerType': "post-merge", 'runSanityCheck': false, 'defaultTag': defaultTag, 'buildInternalRelease': false, From 5905296170489ddf6b200db9250f084403ec2a00 Mon Sep 17 00:00:00 2001 From: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> Date: Fri, 26 Jun 2026 00:32:14 -0700 Subject: [PATCH 5/6] Trigger PLC container scanning pipeline Signed-off-by: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 28ec3511a408..960526038435 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1751,6 +1751,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'buildInternalRelease': false, 'buildCiImage': false, 'artifactPath': ARTIFACT_PATH, + 'nspect_id': "", 'uploadPath': UPLOAD_PATH ] launchJob(pipeline, "/LLM/helpers/BuildDockerImages", false, enableFailFast, globalVars, "x86_64", additionalParameters) @@ -1768,6 +1769,34 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } } + stage("[NGC-Container-Compliance-Check] Run") { + echo "Triggering OSS Compliance (PLC) container scan for ref: " + try { + def params = [ + string(name: 'postMergePipelineName', value: env.JOB_NAME), + string(name: 'postMergeBuildNumber', value: env.BUILD_NUMBER), + string(name: 'scanMode', value: 'pre_merge'), + string(name: 'runSourceCodeScanning', value: 'false'), + string(name: 'runContainerScanning', value: 'true'), + string(name: 'runSonarQube', value: 'false'), + ] + def logger = new Logger(pipeline) + def handle = build( + job: "/LLM/helpers/PLCScanningSetup", + parameters: params, + propagate: false + ) + if (handle.result != "SUCCESS") { + error "Downstream job did not succeed" + } + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + error "OSS Compliance Check failed: ${e.getMessage()}" + } + } + } } } ] From 617144a986d756fb3522f0d1807beddf2b5c585f Mon Sep 17 00:00:00 2001 From: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:41:16 -0700 Subject: [PATCH 6/6] Clean up Signed-off-by: Yuanjing Xue <197832395+yuanjingx87@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 960526038435..b771f38a3aab 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1576,7 +1576,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } if (testFilter[(TEST_STAGE_LIST)]?.contains("NGC-Container-Scaning")) { - echo "Skipping x86_64 tests (PLC container scanning)" + echo "Skipping SBSA tests (PLC container scanning)" return } @@ -1787,7 +1787,9 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) propagate: false ) if (handle.result != "SUCCESS") { - error "Downstream job did not succeed" + catchError(buildResult: currentBuild.result ?: 'SUCCESS', stageResult: 'UNSTABLE') { + error "Risks detected on NGC Containers" + } } } catch (InterruptedException e) { throw e