[pull] main from NVIDIA:main

jenkins/L0_Test.groovy

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-@Library(['bloom-jenkins-shared-lib@dev-yanchaol-fix-step-log-retry', 'trtllm-jenkins-shared-lib@main']) _
+@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _
 
 import java.lang.InterruptedException
 import groovy.transform.Field
@@ -4559,7 +4559,7 @@ def launchTestJobs(pipeline, testFilter)
     x86SlurmTestConfigs = cbtsResizeSplits(x86SlurmTestConfigs)
     fullSet += x86SlurmTestConfigs.keySet()
 
-    parallelSlurmJobs = x86SlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
+    parallelSlurmJobs = x86SlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
         // attemptTag comes from runKubernetesPodWithInfraRetry for the outer
         // dispatcher pod (when retry is enabled — see opts below) and is
         // threaded into runLLMTestlistOnSlurm so a future re-enable of outer
@@ -4831,7 +4831,7 @@ def launchTestJobs(pipeline, testFilter)
         // singleAttempt:true disables the outer K8s pod retry; see the x86
         // SLURM closure above for the full rationale (cap nested retry budget
         // so consistently-timing-out tests don't burn ~36h on retry cascades).
-        parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
+        parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
             // attemptTag is threaded into runLLMTestlistOnSlurm as the outer
             // dispatcher pod's tag so the inner SLURM retry's postTag can't
             // collide with a previous dispatcher pod's upload. See the x86
@@ -4849,7 +4849,7 @@ def launchTestJobs(pipeline, testFilter)
 
         // Add SBSA multi node Slurm jobs
         // singleAttempt:true disables the outer K8s pod retry; see above.
-        parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
+        parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE.replace("aarch64", "x86_64"), "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
             def config = LINUX_AARCH64_CONFIG
             if (key.contains("single-device")) {
                 config = SINGLE_DEVICE_CONFIG

tensorrt_llm/media/encoding.py

@@ -24,9 +24,16 @@
 
 
 def _check_ffmpeg_available() -> bool:
-    """Return True if ffmpeg CLI is installed; cache its path."""
+    """Return True if ffmpeg CLI is installed; cache its path.
+
+    Only a successful probe is cached. A negative result is re-probed on the
+    next call: ffmpeg may be installed after import (e.g. a test fixture or
+    deployment step running ``apt-get install ffmpeg``), and permanently
+    caching the miss silently downgrades every later video encode in the
+    process to the fallback encoder.
+    """
     global _FFMPEG_PATH
-    if _FFMPEG_PATH is None:
+    if not _FFMPEG_PATH:
         ffmpeg_path = shutil.which("ffmpeg")
         if ffmpeg_path is not None:
             try:
@@ -338,9 +345,15 @@ def _write_mjpeg_avi(
 
 
 def _get_video_encoder() -> _VideoEncoder:
-    """Return the best available video encoder (cached singleton)."""
+    """Return the best available video encoder (cached singleton).
+
+    The pure-Python fallback is not cached permanently: if ffmpeg becomes
+    available later in the process lifetime, upgrade to the ffmpeg encoder.
+    """
     global _VIDEO_ENCODER
-    if _VIDEO_ENCODER is None:
+    if _VIDEO_ENCODER is None or (
+        isinstance(_VIDEO_ENCODER, _PurePythonEncoder) and _check_ffmpeg_available()
+    ):
         _VIDEO_ENCODER = _FfmpegCliEncoder() if _check_ffmpeg_available() else _PurePythonEncoder()
         logger.info(f"Using {_VIDEO_ENCODER.__class__.__name__} for video encoding")
     return _VIDEO_ENCODER

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/ltx2_lpips_golden_video.json

@@ -16,7 +16,8 @@
   "lpips_net": "alex",
   "lpips_threshold": 0.05,
   "diffusers_version": "0.38.0",
-  "tensorrt_llm_version": "1.3.0rc20",
-  "tensorrt_llm_commit": "85665f5fd331d0154a78172954846d843085e83f",
-  "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release@sha256:3308a2dc0192a8329ea02eca7b5c44f290f5e894cd8c5921099308d84c3e5691"
+  "torch_version": "2.12.0a0+0291f960b6.nv26.04.48445190",
+  "tensorrt_llm_version": "1.3.0rc21",
+  "tensorrt_llm_commit": "87aeeffe38db433d718516e8c919dd65cc92eb0a",
+  "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202606292100-15694"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan21_t2v_lpips_golden_video.json

@@ -16,7 +16,8 @@
   "lpips_net": "alex",
   "lpips_threshold": 0.05,
   "diffusers_version": "0.38.0",
-  "tensorrt_llm_version": "1.3.0rc20",
-  "tensorrt_llm_commit": "85665f5fd331d0154a78172954846d843085e83f",
-  "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release@sha256:3308a2dc0192a8329ea02eca7b5c44f290f5e894cd8c5921099308d84c3e5691"
+  "torch_version": "2.12.0a0+0291f960b6.nv26.04.48445190",
+  "tensorrt_llm_version": "1.3.0rc21",
+  "tensorrt_llm_commit": "87aeeffe38db433d718516e8c919dd65cc92eb0a",
+  "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202606292100-15694"
 }

tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/wan22_t2v_lpips_golden_video.json

@@ -16,7 +16,8 @@
   "lpips_net": "alex",
   "lpips_threshold": 0.05,
   "diffusers_version": "0.38.0",
-  "tensorrt_llm_version": "1.3.0rc20",
-  "tensorrt_llm_commit": "85665f5fd331d0154a78172954846d843085e83f",
-  "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm-staging/release@sha256:3308a2dc0192a8329ea02eca7b5c44f290f5e894cd8c5921099308d84c3e5691"
+  "torch_version": "2.12.0a0+0291f960b6.nv26.04.48445190",
+  "tensorrt_llm_version": "1.3.0rc21",
+  "tensorrt_llm_commit": "539ee226c4df7ab15802911083fe501e9d64c66e",
+  "container_image": "urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm@sha256:dad31c0b5290d836033c96d8b91f6524bdc7cc5b4d1000b4abcc57c6868ffdc0"
 }

tests/integration/defs/examples/visual_gen/test_visual_gen.py

@@ -14,12 +14,14 @@
 # limitations under the License.
 """Integration tests for VisualGen examples and visual quality checks."""
 
+import base64
 import contextlib
 import gc
 import glob
 import json
 import os
 import random
+import shutil
 import subprocess
 import sys
 import time
@@ -425,35 +427,18 @@ def _save_lpips_video_mp4(video, output_path, frame_rate):
 
     try:
         save_video(video, output_path, frame_rate=frame_rate)
-        return
     except RuntimeError as err:
         if "MP4 format requires ffmpeg" not in str(err):
             raise
-
-    import cv2
-
-    if video.dim() == 5:
-        video = video[0]
-    output_path = str(output_path)
-    output_dir = os.path.dirname(output_path)
-    if output_dir:
-        os.makedirs(output_dir, exist_ok=True)
-
-    video_np = video.detach().cpu().numpy()
-    num_frames, height, width, channels = video_np.shape
-    assert channels == 3, f"Expected RGB video with 3 channels, got {channels}"
-    writer = cv2.VideoWriter(
-        output_path,
-        cv2.VideoWriter_fourcc(*"mp4v"),
-        float(frame_rate),
-        (width, height),
-    )
-    assert writer.isOpened(), f"Failed to open MP4 writer for {output_path}"
-    try:
-        for frame in video_np:
-            writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
-    finally:
-        writer.release()
+        # Never fall back to another codec here: the goldens are H.264/x264 and
+        # LPIPS compares decoded pixels, so a silent cv2/mp4v fallback compares
+        # codec artifacts, not model output (this produced a spurious 0.059 on
+        # wan22 while the generated frames were identical to the golden).
+        pytest.fail(
+            "ffmpeg is unavailable for LPIPS video encoding; refusing to fall back "
+            "to another codec because the golden comparison would measure codec "
+            f"artifacts instead of model output: {err}"
+        )
     assert os.path.isfile(output_path), f"Visual gen did not produce {output_path}"
 
 
@@ -517,6 +502,39 @@ def _assert_lpips_below_threshold(score, threshold):
     assert score < threshold, f"LPIPS too high: {score:.6f} (expected < {threshold:.6f})"
 
 
+def _preserve_lpips_candidate_on_failure(request, score, threshold, candidate_path, artifact_name):
+    """Copy the generated candidate into pytest's --output-dir when the LPIPS gate fails.
+
+    CI archives the output dir per stage, so a threshold failure leaves behind the
+    exact CI-generated media needed to refresh the golden without guessing at
+    machine-to-machine kernel-stack drift (measured ~0.04 LPIPS across B200 hosts
+    on the same container for 1-step Wan2.1).
+    """
+    if score < threshold:
+        return
+    output_dir = request.config.getoption("--output-dir", default=None)
+    if not output_dir:
+        return
+    dest_dir = os.path.join(str(output_dir), "lpips_failure_artifacts")
+    os.makedirs(dest_dir, exist_ok=True)
+    dest = os.path.join(dest_dir, artifact_name)
+    shutil.copy2(str(candidate_path), dest)
+    print(f"[LPIPS] candidate preserved for golden refresh: {dest}")
+    # CI's per-stage results tarball only collects flat files (results.xml etc.), not
+    # this subdirectory — and junit stdout IS collected. Small candidates therefore
+    # also go into stdout as base64 so a threshold failure is diagnosable from the
+    # archived results.xml alone (this is how wan22's spurious 0.059 was traced to a
+    # silent mpeg4 codec fallback rather than a generation difference).
+    size = os.path.getsize(dest)
+    if size <= 8 * 1024 * 1024:
+        with open(dest, "rb") as fh:
+            encoded = base64.b64encode(fh.read()).decode("ascii")
+        print(f"[LPIPS-B64-BEGIN {artifact_name} {size}]")
+        for i in range(0, len(encoded), 3072):
+            print(encoded[i : i + 3072])
+        print(f"[LPIPS-B64-END {artifact_name}]")
+
+
 def _generate_flux_lpips_image(model_path, output_path):
     from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader
     from tensorrt_llm.media.encoding import save_image
@@ -561,7 +579,10 @@ def _generate_ltx2_lpips_video(output_path, *, enable_cuda_graph=False):
     _skip_if_missing(distilled_lora_path, "LTX-2 distilled LoRA")
     _disable_inductor_compile_worker_quiesce()
 
-    with _lpips_deterministic_algorithms():
+    # TorchCompileConfig(enable=False) does not suppress nested @torch.compile decorators.
+    # Wrapped here (not in the fixture) so the golden fixture and both sides of
+    # test_ltx2_cuda_graph_lpips_matches_eager run the same eager numerics.
+    with _lpips_deterministic_algorithms(), torch.compiler.set_stance("force_eager"):
         args = VisualGenArgs(
             model=checkpoint_path,
             pipeline_config={
@@ -676,19 +697,21 @@ def wan21_bf16_video_path(_visual_gen_deps, llm_venv):
     output_path = _visual_gen_output_path(llm_venv, "wan21_bf16")
     if os.path.isfile(output_path):
         return output_path
-    _generate_wan_lpips_video(
-        _lpips_model_path("Wan2.1-T2V-1.3B-Diffusers"),
-        output_path,
-        WAN21_LPIPS_PROMPT,
-        WAN21_LPIPS_NEGATIVE_PROMPT,
-        WAN21_LPIPS_HEIGHT,
-        WAN21_LPIPS_WIDTH,
-        WAN21_LPIPS_NUM_FRAMES,
-        WAN21_LPIPS_NUM_INFERENCE_STEPS,
-        WAN21_LPIPS_GUIDANCE_SCALE,
-        WAN21_LPIPS_SEED,
-        WAN_LPIPS_FRAME_RATE,
-    )
+    # TorchCompileConfig(enable=False) does not suppress nested @torch.compile decorators.
+    with torch.compiler.set_stance("force_eager"):
+        _generate_wan_lpips_video(
+            _lpips_model_path("Wan2.1-T2V-1.3B-Diffusers"),
+            output_path,
+            WAN21_LPIPS_PROMPT,
+            WAN21_LPIPS_NEGATIVE_PROMPT,
+            WAN21_LPIPS_HEIGHT,
+            WAN21_LPIPS_WIDTH,
+            WAN21_LPIPS_NUM_FRAMES,
+            WAN21_LPIPS_NUM_INFERENCE_STEPS,
+            WAN21_LPIPS_GUIDANCE_SCALE,
+            WAN21_LPIPS_SEED,
+            WAN_LPIPS_FRAME_RATE,
+        )
     return output_path
 
 
@@ -697,19 +720,21 @@ def wan22_bf16_video_path(_visual_gen_deps, llm_venv):
     output_path = _visual_gen_output_path(llm_venv, "wan22_bf16")
     if os.path.isfile(output_path):
         return output_path
-    _generate_wan_lpips_video(
-        _lpips_model_path("Wan2.2-T2V-A14B-Diffusers"),
-        output_path,
-        WAN22_LPIPS_PROMPT,
-        WAN22_LPIPS_NEGATIVE_PROMPT,
-        WAN22_LPIPS_HEIGHT,
-        WAN22_LPIPS_WIDTH,
-        WAN22_LPIPS_NUM_FRAMES,
-        WAN22_LPIPS_NUM_INFERENCE_STEPS,
-        WAN22_LPIPS_GUIDANCE_SCALE,
-        WAN22_LPIPS_SEED,
-        WAN22_LPIPS_FRAME_RATE,
-    )
+    # TorchCompileConfig(enable=False) does not suppress nested @torch.compile decorators.
+    with torch.compiler.set_stance("force_eager"):
+        _generate_wan_lpips_video(
+            _lpips_model_path("Wan2.2-T2V-A14B-Diffusers"),
+            output_path,
+            WAN22_LPIPS_PROMPT,
+            WAN22_LPIPS_NEGATIVE_PROMPT,
+            WAN22_LPIPS_HEIGHT,
+            WAN22_LPIPS_WIDTH,
+            WAN22_LPIPS_NUM_FRAMES,
+            WAN22_LPIPS_NUM_INFERENCE_STEPS,
+            WAN22_LPIPS_GUIDANCE_SCALE,
+            WAN22_LPIPS_SEED,
+            WAN22_LPIPS_FRAME_RATE,
+        )
     return output_path
 
 
@@ -855,7 +880,7 @@ def test_flux2_lpips_against_golden(tmp_path):
 
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
-def test_ltx2_lpips_against_golden(tmp_path, ltx2_two_stage_bf16_video_path):
+def test_ltx2_lpips_against_golden(request, tmp_path, ltx2_two_stage_bf16_video_path):
     golden_path = _golden_media_path(
         tmp_path, "ltx2_lpips_golden_video.mp4", "LTX-2 LPIPS golden video"
     )
@@ -867,11 +892,18 @@ def test_ltx2_lpips_against_golden(tmp_path, ltx2_two_stage_bf16_video_path):
         golden_path,
         ltx2_two_stage_bf16_video_path,
     )
+    _preserve_lpips_candidate_on_failure(
+        request,
+        score,
+        LTX2_LPIPS_THRESHOLD,
+        ltx2_two_stage_bf16_video_path,
+        "ltx2_lpips_golden_video.mp4",
+    )
     _assert_lpips_below_threshold(score, LTX2_LPIPS_THRESHOLD)
 
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
-def test_ltx2_cuda_graph_lpips_matches_eager(tmp_path):
+def test_ltx2_cuda_graph_lpips_matches_eager(_visual_gen_deps, tmp_path):
     eager_path = tmp_path / "ltx2_eager_generated.mp4"
     cuda_graph_path = tmp_path / "ltx2_cuda_graph_generated.mp4"
 
@@ -889,7 +921,7 @@ def test_ltx2_cuda_graph_lpips_matches_eager(tmp_path):
 
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
-def test_wan21_t2v_lpips_against_golden(tmp_path, wan21_bf16_video_path):
+def test_wan21_t2v_lpips_against_golden(request, tmp_path, wan21_bf16_video_path):
     golden_path = _golden_media_path(
         tmp_path, "wan21_t2v_lpips_golden_video.mp4", "Wan 2.1 LPIPS golden video"
     )
@@ -901,11 +933,18 @@ def test_wan21_t2v_lpips_against_golden(tmp_path, wan21_bf16_video_path):
         golden_path,
         wan21_bf16_video_path,
     )
+    _preserve_lpips_candidate_on_failure(
+        request,
+        score,
+        WAN_LPIPS_THRESHOLD,
+        wan21_bf16_video_path,
+        "wan21_t2v_lpips_golden_video.mp4",
+    )
     _assert_lpips_below_threshold(score, WAN_LPIPS_THRESHOLD)
 
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
-def test_wan22_t2v_lpips_against_golden(tmp_path, wan22_bf16_video_path):
+def test_wan22_t2v_lpips_against_golden(request, tmp_path, wan22_bf16_video_path):
     golden_path = _golden_media_path(
         tmp_path, "wan22_t2v_lpips_golden_video.mp4", "Wan 2.2 LPIPS golden video"
     )
@@ -917,6 +956,13 @@ def test_wan22_t2v_lpips_against_golden(tmp_path, wan22_bf16_video_path):
         golden_path,
         wan22_bf16_video_path,
     )
+    _preserve_lpips_candidate_on_failure(
+        request,
+        score,
+        WAN_LPIPS_THRESHOLD,
+        wan22_bf16_video_path,
+        "wan22_t2v_lpips_golden_video.mp4",
+    )
     _assert_lpips_below_threshold(score, WAN_LPIPS_THRESHOLD)
 
 

tests/integration/test_lists/waives.txt

@@ -288,10 +288,8 @@ examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-
 examples/test_whisper.py::test_llm_whisper_general[large-v3-disable_gemm_plugin-enable_attention_plugin-disable_weight_only-float16-nb:1-use_python_runtime] SKIP (TRTLLM-13781: legacy TensorRT examples removed; tests to be removed in follow-up PR3)
 examples/test_whisper.py::test_whisper_beam_search_generation_logits[large-v3-nb:4] SKIP (TRTLLM-13781: legacy TensorRT examples removed; tests to be removed in follow-up PR3)
 examples/test_whisper.py::test_whisper_log_probs_determinism[large-v3-bs:4-nb:4] SKIP (TRTLLM-13781: legacy TensorRT examples removed; tests to be removed in follow-up PR3)
+examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2i_lpips_against_golden SKIP (https://nvbugs/6410082)
 examples/visual_gen/test_visual_gen.py::test_cosmos3_nano_t2v_lpips_against_golden SKIP (https://nvbugs/6410082)
-examples/visual_gen/test_visual_gen.py::test_ltx2_lpips_against_golden SKIP (https://nvbugs/6410332)
-examples/visual_gen/test_visual_gen.py::test_wan21_t2v_lpips_against_golden SKIP (https://nvbugs/6410336)
-examples/visual_gen/test_visual_gen.py::test_wan22_t2v_lpips_against_golden SKIP (https://nvbugs/6401921)
 examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[attn2d_2x2] SKIP (https://nvbugs/6272644)
 examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2] SKIP (https://nvbugs/6272644)
 examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2_attn2d_2x1] SKIP (https://nvbugs/6272644)