Reuse image sqsh file

Signed-off-by: EmmaQiaoCh <qqiao@nvidia.com>

Author: EmmaQiaoCh

jenkins/L0_Test.groovy

@@ -523,7 +523,10 @@ def cleanUpSlurmResources(def pipeline, SlurmCluster cluster, String clusterName
         Utils.exec(pipeline, script: "echo Sleeping to allow Slurm job termination; sleep 30")
 
         def cleanupCommands = [
-            "rm -rf ${cluster.scratchPath}/users/svc_tensorrt/containers/container-${slurmJobID}.sqsh || true",
+            // .sqsh is shared across jobs (named by image digest), so age-prune
+            // instead of deleting per job; reused images keep a refreshed mtime.
+            "find ${cluster.scratchPath}/users/svc_tensorrt/containers -maxdepth 1 -name 'container-*.sqsh' -mtime +3 -delete 2>/dev/null || true",
+            "find ${cluster.scratchPath}/users/svc_tensorrt/containers -maxdepth 1 \\( -name 'container-*.tmp' -o -name 'container-*.lock' \\) -mtime +1 -delete 2>/dev/null || true",
             "rm -rf ${jobWorkspace} || true",
         ].join(" ; ")
         Utils.exec(
@@ -562,7 +565,10 @@ def cleanUpNodeResources(def pipeline, SlurmCluster cluster, String clusterName,
         def entrypoint = SlurmConfig.containerRuntimeToEntrypoint[cluster.containerRuntime]
         def cleanupCommands = [
             "rm -rf /home/svc_tensorrt/bloom/scripts/agent-${nodeName}.jar /home/svc_tensorrt/bloom/scripts/${nodeName}-${entrypoint} || true",
-            "rm -rf ${cluster.scratchPath}/users/svc_tensorrt/containers/container-${slurmJobID}.sqsh || true",
+            // .sqsh is shared across jobs (named by image digest), so age-prune
+            // instead of deleting per job; reused images keep a refreshed mtime.
+            "find ${cluster.scratchPath}/users/svc_tensorrt/containers -maxdepth 1 -name 'container-*.sqsh' -mtime +3 -delete 2>/dev/null || true",
+            "find ${cluster.scratchPath}/users/svc_tensorrt/containers -maxdepth 1 \\( -name 'container-*.tmp' -o -name 'container-*.lock' \\) -mtime +1 -delete 2>/dev/null || true",
         ].join(" ; ")
         Utils.exec(
             pipeline,
@@ -1108,37 +1114,69 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                 def containerImageArg = container
                 def srunPrologue = ""
                 if (cluster.containerRuntime.toString() == "ENROOT") {
-                    def enrootImagePath = "${cluster.scratchPath}/users/svc_tensorrt/containers/container-\${SLURM_JOB_ID}.sqsh"
-                    containerImageArg = enrootImagePath
+                    def containerDir = "${cluster.scratchPath}/users/svc_tensorrt/containers"
+                    // Name the .sqsh by image digest (not job ID) so jobs sharing
+                    // an image reuse one .sqsh instead of re-running `enroot import`
+                    // per job. Path is resolved at runtime into ${enrootImagePath}.
+                    containerImageArg = "\${enrootImagePath}"
 
                     srunPrologue = """
                     export ENROOT_CACHE_PATH='/home/svc_tensorrt/.cache/enroot'
 
+                    containerDir="$containerDir"
+                    mkdir -p "\$containerDir"
+                    imageDigest=\$(printf '%s' "$container" | sha256sum | cut -d' ' -f1)
+                    export enrootImagePath="\$containerDir/container-\${imageDigest}.sqsh"
+
                     importContainerWithRetries() {
                         local docker_uri=\$1
                         local output_path=\$2
                         local max_attempts=\${3:-3}
                         local delay=\${4:-60}
                         local attempt=1
+                        local tmp_path
+
+                        # Best-effort lock so racing jobs don't all import the same
+                        # image. flock may be a no-op on some shared filesystems;
+                        # correctness still holds since the import publishes atomically.
+                        exec 9>"\${output_path}.lock" || true
+                        flock 9 || true
+
+                        if [ -f "\$output_path" ]
+                        then
+                            echo "Reusing cached container image: \$output_path"
+                            # Refresh mtime so reused images survive age-based pruning.
+                            touch "\$output_path" || true
+                            flock -u 9 || true
+                            return 0
+                        fi
 
-                        rm -f "\$output_path"
+                        # Import to a temp path, then mv to publish atomically so
+                        # other jobs never see a partial .sqsh.
+                        tmp_path="\${output_path}.\${SLURM_JOB_ID}.tmp"
+                        rm -f "\$tmp_path"
 
-                        until enroot import -o "\$output_path" -- "docker://\$docker_uri"
+                        until enroot import -o "\$tmp_path" -- "docker://\$docker_uri"
                         do
                             if ((attempt >= max_attempts))
                             then
                                 echo "enroot import failed after \$max_attempts attempts"
+                                rm -f "\$tmp_path"
+                                flock -u 9 || true
                                 return 1
                             fi
 
                             echo "enroot import failed (attempt \$attempt of \$max_attempts). Retrying in \${delay}s..."
-                            rm -f "\$output_path"
+                            rm -f "\$tmp_path"
                             sleep \$delay
-                            ((attempt++))
+                            attempt=\$((attempt + 1))
                         done
+
+                        mv -f "\$tmp_path" "\$output_path"
+                        flock -u 9 || true
                     }
 
-                    importContainerWithRetries "$container" "$enrootImagePath"
+                    importContainerWithRetries "$container" "\$enrootImagePath"
                     """.replaceAll("(?m)^\\s*", "")
                 }