[None][infra] PLC nightly pipeline update (#15274)

yuanjingx87 · tburt-nv · web-flow · commit 011e8490c145 · 2026-07-06T10:47:49.000-07:00
Signed-off-by: Yuanjing Xue &lt;197832395+yuanjingx87@users.noreply.github.com&gt;
Co-authored-by: tburt-nv &lt;195370667+tburt-nv@users.noreply.github.com&gt;
diff --git a/jenkins/TensorRT_LLM_PLC.groovy b/jenkins/TensorRT_LLM_PLC.groovy
@@ -85,6 +85,11 @@ def getLLMRepo () {
             LLM_REPO = DEFAULT_LLM_REPO
         }
     }
+    if (params.repoUrlKey == "tensorrt_llm_github_sync") {
+        withCredentials([string(credentialsId: 'github-llm-repo-sync', variable: 'GITHUB_LLM_REPO_SYNC')]) {
+            LLM_REPO = GITHUB_LLM_REPO_SYNC
+        }
+    }
     if (params.repoUrlKey == "github_fork") {
         if (!isValidGithubUser(params.forkOwner)) {
             throw new Exception("Invalid fork owner provided")
@@ -121,6 +126,14 @@ def validateRef() {
     }
 }
 
+def savePipelineScripts() {
+    // Capture the pipeline's own version of the scan scripts before the workspace
+    // is overwritten by checkoutSource() with params.ref. This ensures the script
+    // logic always matches the Groovy that is actually running.
+    checkout scm
+    sh "cp -r jenkins/scripts/pulse_in_pipeline_scanning /tmp/pulse_in_pipeline_scanning"
+}
+
 def checkoutSource ()
 {
     trtllm_utils.setupGitMirror()
@@ -131,19 +144,38 @@ def checkoutSource ()
 }
 
 def getPulseToken(serviceId, scopes) {
+    def maxRetries = 3
+    def retryDelaySec = 10
     def token
     //Configure credential 'starfleet-client-id' under Jenkins Credential Manager
     withCredentials([usernamePassword(
         credentialsId: "NSPECT_CLIENT-prod",
         usernameVariable: 'SF_CLIENT_ID',
         passwordVariable: 'SF_CLIENT_SECRET'
     )]) {
-        // Do not save AUTH_HEADER to a groovy variable since that
-        // will expose the auth_header without being masked
-        token= sh(script: """
-            AUTH_HEADER=\$(echo -n \$SF_CLIENT_ID:\$SF_CLIENT_SECRET | base64 -w0)
-            curl -s --request POST --header "Authorization: Basic \$AUTH_HEADER" --header "Content-Type: application/x-www-form-urlencoded" "https://${serviceId}.ssa.nvidia.com/token?grant_type=client_credentials&scope=${scopes}" | jq ".access_token" |  tr -d '"'
-        """, returnStdout: true).trim()
+        for (int attempt = 1; attempt <= maxRetries; attempt++) {
+            try {
+                // Do not save AUTH_HEADER to a groovy variable since that
+                // will expose the auth_header without being masked
+                token = sh(script: """
+                    AUTH_HEADER=\$(echo -n \$SF_CLIENT_ID:\$SF_CLIENT_SECRET | base64 -w0)
+                    curl -s --request POST --header "Authorization: Basic \$AUTH_HEADER" --header "Content-Type: application/x-www-form-urlencoded" "https://${serviceId}.ssa.nvidia.com/token?grant_type=client_credentials&scope=${scopes}" | jq ".access_token" |  tr -d '"'
+                """, returnStdout: true).trim()
+            } catch (Exception e) {
+                echo "getPulseToken attempt ${attempt}/${maxRetries} failed: ${e.getMessage()}"
+                token = ""
+            }
+            if (token && token != "null") {
+                break
+            }
+            if (attempt < maxRetries) {
+                echo "getPulseToken returned an empty token (attempt ${attempt}/${maxRetries}), retrying in ${retryDelaySec}s..."
+                sleep(retryDelaySec)
+            }
+        }
+    }
+    if (!token || token == "null") {
+        error("getPulseToken failed to obtain a valid token after ${maxRetries} attempts")
     }
     return token
 }
@@ -199,7 +231,7 @@ def sonarScan()
 {
     container("cpu") {
         def sonarScannerCliVer = "8.0.0.6341"
-        sh "wget https://repo1.maven.org/maven2/org/sonarsource/scanner/cli/sonar-scanner-cli/${sonarScannerCliVer}/sonar-scanner-cli-${sonarScannerCliVer}.zip"
+        sh "wget https://maven-central.storage-download.googleapis.com/maven2/org/sonarsource/scanner/cli/sonar-scanner-cli/${sonarScannerCliVer}/sonar-scanner-cli-${sonarScannerCliVer}.zip"
         sh "unzip sonar-scanner-cli-${sonarScannerCliVer}.zip"
         sh "mv sonar-scanner-${sonarScannerCliVer} ../sonar-scanner"
         sh "rm sonar-scanner-cli-${sonarScannerCliVer}.zip"
@@ -309,15 +341,22 @@ def processScanResults(ref) {
                     python3 -m venv venv
                     venv/bin/pip install requests elasticsearch==7.13.4
                 """
+                def skipArgs = ""
+                if (!params.runSourceCodeScanning) {
+                    skipArgs += " --skip-source-code"
+                }
+                if (!params.runContainerScanning) {
+                    skipArgs += " --skip-container"
+                }
                 def token = getPulseToken("4ubglassowmtsi7ogqwarmut7msn1q5ynts62fwnr1i", "public.api:read")
                 def output = withEnv(["LICENSE_CHECK_TOKEN=${token}"]) {
                     sh(script: """
-                        venv/bin/python ./jenkins/scripts/pulse_in_pipeline_scanning/main.py \
+                        venv/bin/python /tmp/pulse_in_pipeline_scanning/main.py \
                             --build-url ${pipelineUrl} \
                             --build-number ${env.BUILD_NUMBER} \
                             --ref ${ref} \
                             --report-directory ${pwd()}/scan_report \
-                            --scan-mode ${params.scanMode}
+                            --scan-mode ${params.scanMode}${skipArgs}
                     """, returnStdout: true).trim()
                 }
                 echo "Scan result: ${output}"
@@ -342,11 +381,14 @@ pipeline {
     }
     parameters {
         string(name: 'ref', defaultValue: 'main', description: 'Branch name or commit SHA (7–40 hex chars) to check out; branch push steps are skipped when a commit SHA is provided')
-        choice(name: 'repoUrlKey', choices: ['tensorrt_llm_github','tensorrt_llm_internal', 'github_fork'], description: "The repo url to process")
+        choice(name: 'repoUrlKey', choices: ['tensorrt_llm_github','tensorrt_llm_internal', 'tensorrt_llm_github_sync', 'github_fork'], description: "The repo url to process")
         string(name: 'forkOwner', defaultValue: '', description: 'Name of the fork owner, need to select \"github_fork\" for repoUrlKey, otherwise it will be ignored')
         string(name: 'postMergePipelineName', defaultValue: '', description: 'Optional: post-merge pipeline job name to associate with this scan')
         string(name: 'postMergeBuildNumber', defaultValue: '', description: 'Optional: post-merge pipeline build number to associate with this scan')
-        choice(name: 'scanMode', choices: ['monitor','release'], description: "When set to monitor, only report newly introduced dependencies. When set to release, will report all detected risks")
+        choice(name: 'scanMode', choices: ['monitor','release','pre_merge'], description: "When set to monitor, only report newly introduced dependencies. When set to release, will report all detected risks")
+        booleanParam(name: 'runSourceCodeScanning', defaultValue: true, description: 'Run Source Code OSS Scanning (lock file generation + Pulse OSS scan)')
+        booleanParam(name: 'runContainerScanning', defaultValue: true, description: 'Run Container Scanning (Pulse container scan)')
+        booleanParam(name: 'runSonarQube', defaultValue: true, description: 'Run SonarQube Code Analysis')
     }
     options {
         skipDefaultCheckout()
@@ -370,6 +412,7 @@ pipeline {
             steps {
                 script {
                     container("cpu") {
+                        savePipelineScripts()
                         installTools()
                         checkoutSource()
                         validateRef()
@@ -380,6 +423,9 @@ pipeline {
         stage('Run TRT-LLM PLC Jobs') {
             parallel {
                 stage("Source Code OSS Scanning") {
+                    when {
+                        expression { return params.runSourceCodeScanning }
+                    }
                     steps {
                         script {
                             generateLockFiles(env.LLM_REPO, env.REF)
@@ -388,13 +434,19 @@ pipeline {
                     }
                 }
                 stage("Container Scanning") {
+                    when {
+                        expression { return params.runContainerScanning }
+                    }
                     steps {
                         script {
                             pulseScanContainer(env.LLM_REPO, env.REF)
                         }
                     }
                 }
                 stage("SonarQube Code Analysis") {
+                    when {
+                        expression { return params.runSonarQube }
+                    }
                     steps {
                         script {
                             sonarScan()
diff --git a/jenkins/scripts/pulse_in_pipeline_scanning/main.py b/jenkins/scripts/pulse_in_pipeline_scanning/main.py
@@ -36,9 +36,21 @@
         "if set to release, all risks will be reported"
     ),
 )
+parser.add_argument(
+    "--skip-source-code",
+    action="store_true",
+    default=False,
+    help="Skip processing source code scanning results",
+)
+parser.add_argument(
+    "--skip-container",
+    action="store_true",
+    default=False,
+    help="Skip processing container scanning results",
+)
 args = parser.parse_args()
 
-LICENSE_CHECK_TOKEN = os.environ.get("LICENSE_CHECK_TOKEN")
+LICENSE_CHECK_TOKEN = os.environ.get("LICENSE_CHECK_TOKEN", "")
 if not LICENSE_CHECK_TOKEN:
     raise EnvironmentError("Error: Environment variable 'LICENSE_CHECK_TOKEN' is not set!")
 
@@ -59,83 +71,85 @@
 
 def process_result():
     RISKY_DEPENDENCIES = []
-
-    last_source_vulns = get_last_scan_results("source_code_vulnerability", args.ref)
-    source_vulns = submit_source_code_vulns(
-        os.path.join(args.report_directory, "source_code/vulns.json"),
-        last_source_vulns,
-        **SUBMIT_KWARG,
-    )
-    if len(source_vulns) > 0:
-        RISKY_DEPENDENCIES.append(f"{len(source_vulns)} new source code vulnerability")
-
-    last_source_licenses = get_last_scan_results("source_code_license", args.ref)
-    source_licenses = submit_source_code_licenses(
-        os.path.join(args.report_directory, "source_code/sbom.json"),
-        last_source_licenses,
-        **SUBMIT_KWARG,
-        license_check_token=LICENSE_CHECK_TOKEN,
-    )
-    if source_licenses is None:
-        RISKY_DEPENDENCIES.append("source code SBOM not found")
-        sbom_missing = True
-    else:
-        sbom_missing = False
-        if len(source_licenses) > 0:
+    source_licenses = []
+    count_container_licenses = 0
+
+    if not args.skip_source_code:
+        last_source_vulns = get_last_scan_results("source_code_vulnerability", args.ref)
+        source_vulns = submit_source_code_vulns(
+            os.path.join(args.report_directory, "source_code/vulns.json"),
+            last_source_vulns,
+            **SUBMIT_KWARG,
+        )
+        if len(source_vulns) > 0:
+            RISKY_DEPENDENCIES.append(f"{len(source_vulns)} new source code vulnerability")
+
+        last_source_licenses = get_last_scan_results("source_code_license", args.ref)
+        source_licenses = submit_source_code_licenses(
+            os.path.join(args.report_directory, "source_code/sbom.json"),
+            last_source_licenses,
+            **SUBMIT_KWARG,
+            license_check_token=LICENSE_CHECK_TOKEN,
+        )
+        if source_licenses is None:
+            RISKY_DEPENDENCIES.append("source code SBOM not found")
+        else:
+            if len(source_licenses) > 0:
+                RISKY_DEPENDENCIES.append(
+                    f"{len(source_licenses)} new source code non-permissive license"
+                )
+
+    if not args.skip_container:
+        last_container_vulns = get_last_scan_results("container_vulnerability", args.ref)
+        amd64_container_vulns = submit_container_vulns(
+            os.path.join(args.report_directory, "release_amd64/vulns.json"),
+            os.path.join(args.report_directory, "base_amd64/vulns.json"),
+            "amd64",
+            last_container_vulns,
+            **SUBMIT_KWARG,
+        )
+        arm64_container_vulns = submit_container_vulns(
+            os.path.join(args.report_directory, "release_arm64/vulns.json"),
+            os.path.join(args.report_directory, "base_arm64/vulns.json"),
+            "arm64",
+            last_container_vulns,
+            **SUBMIT_KWARG,
+        )
+        count_container_vulns = len(amd64_container_vulns + arm64_container_vulns)
+        if count_container_vulns > 0:
+            RISKY_DEPENDENCIES.append(f"{count_container_vulns} new container vulnerability")
+
+        last_container_licenses = get_last_scan_results("container_license", args.ref)
+        amd64_container_licenses = submit_container_licenses(
+            os.path.join(args.report_directory, "release_amd64/licenses.json"),
+            os.path.join(args.report_directory, "base_amd64/licenses.json"),
+            "amd64",
+            last_container_licenses,
+            **SUBMIT_KWARG,
+            license_check_token=LICENSE_CHECK_TOKEN,
+        )
+        arm64_container_licenses = submit_container_licenses(
+            os.path.join(args.report_directory, "release_arm64/licenses.json"),
+            os.path.join(args.report_directory, "base_arm64/licenses.json"),
+            "arm64",
+            last_container_licenses,
+            **SUBMIT_KWARG,
+            license_check_token=LICENSE_CHECK_TOKEN,
+        )
+        count_container_licenses = len(amd64_container_licenses + arm64_container_licenses)
+        if count_container_licenses > 0:
             RISKY_DEPENDENCIES.append(
-                f"{len(source_licenses)} new source code non-permissive license"
+                f"{count_container_licenses} new container non-permissive license"
             )
 
-    last_container_vulns = get_last_scan_results("container_vulnerability", args.ref)
-    amd64_container_vulns = submit_container_vulns(
-        os.path.join(args.report_directory, "release_amd64/vulns.json"),
-        os.path.join(args.report_directory, "base_amd64/vulns.json"),
-        "amd64",
-        last_container_vulns,
-        **SUBMIT_KWARG,
-    )
-    arm64_container_vulns = submit_container_vulns(
-        os.path.join(args.report_directory, "release_arm64/vulns.json"),
-        os.path.join(args.report_directory, "base_arm64/vulns.json"),
-        "arm64",
-        last_container_vulns,
-        **SUBMIT_KWARG,
-    )
-    count_container_vulns = len(amd64_container_vulns + arm64_container_vulns)
-    if count_container_vulns > 0:
-        RISKY_DEPENDENCIES.append(f"{count_container_vulns} new container vulnerability")
-
-    last_container_licenses = get_last_scan_results("container_license", args.ref)
-    amd64_container_licenses = submit_container_licenses(
-        os.path.join(args.report_directory, "release_amd64/licenses.json"),
-        os.path.join(args.report_directory, "base_amd64/licenses.json"),
-        "amd64",
-        last_container_licenses,
-        **SUBMIT_KWARG,
-        license_check_token=LICENSE_CHECK_TOKEN,
-    )
-    arm64_container_licenses = submit_container_licenses(
-        os.path.join(args.report_directory, "release_arm64/licenses.json"),
-        os.path.join(args.report_directory, "base_arm64/licenses.json"),
-        "arm64",
-        last_container_licenses,
-        **SUBMIT_KWARG,
-        license_check_token=LICENSE_CHECK_TOKEN,
-    )
-    count_container_licenses = len(amd64_container_licenses + arm64_container_licenses)
-    if count_container_licenses > 0:
-        RISKY_DEPENDENCIES.append(
-            f"{count_container_licenses} new container non-permissive license"
-        )
-
     if RISKY_DEPENDENCIES:
         detail = ", ".join(RISKY_DEPENDENCIES)
         status = "unstable"
         if args.scan_mode == "monitor":
             post_slack_msg(args.build_number, args.ref, detail)
         if (
             args.scan_mode == "release"
-            and not sbom_missing
+            and source_licenses is not None
             and count_container_licenses + len(source_licenses) == 0
         ):
             status = "success"
diff --git a/jenkins/scripts/pulse_in_pipeline_scanning/submit_report.py b/jenkins/scripts/pulse_in_pipeline_scanning/submit_report.py
@@ -8,7 +8,7 @@
 sys.path.append(os.path.abspath(".."))
 from utils.common import is_permissive, load_json
 from utils.es import es_post
-from utils.report import diff_licenses, get_preapproved_deps_map, get_vulns
+from utils.report import diff_licenses, get_preapproved_deps_map, get_vulns, is_preapproved
 
 ES_POST_URL = os.environ.get("TRTLLM_ES_POST_URL")
 if not ES_POST_URL:
@@ -74,9 +74,7 @@ def submit_source_code_vulns(
                 risks_to_report.append(doc)
 
             bulk_documents.append(doc)
-
-        vulnerability_count = len(bulk_documents)
-        if vulnerability_count:
+        if bulk_documents:
             _, errors = es_post(ES_POST_URL, bulk_documents)
             if errors:
                 raise RuntimeError(
@@ -156,9 +154,11 @@ def submit_source_code_licenses(
                 "s_component_type": component.get("type"),
                 "b_is_new": is_new,
             }
-            if (is_new or not only_report_new_risk) and (package_name not in map_preapproved):
+            if (is_new or not only_report_new_risk) and not is_preapproved(
+                map_preapproved, package_name, (component.get("type") or "unknown").lower()
+            ):
                 risks_to_report.append(doc)
-                sbom_documents.append(doc)
+            sbom_documents.append(doc)
         if sbom_documents:
             _, sbom_errors = es_post(ES_POST_URL, sbom_documents)
             if sbom_errors:
@@ -290,9 +290,11 @@ def submit_container_licenses(
             "s_license_ids": ",".join(license_ids),
             "b_is_new": is_new,
         }
-        if (is_new or not only_report_new_risk) and (package_name not in map_preapproved):
+        if (is_new or not only_report_new_risk) and not is_preapproved(
+            map_preapproved, package_name, (v.get("type") or "unknown").lower()
+        ):
             risks_to_report.append(doc)
-            docs.append(doc)
+        docs.append(doc)
     if docs:
         _, errors = es_post(ES_POST_URL, docs)
         if errors:
diff --git a/jenkins/scripts/pulse_in_pipeline_scanning/utils/report.py b/jenkins/scripts/pulse_in_pipeline_scanning/utils/report.py
