[pull] main from NVIDIA:main

jenkins/L0_Test.groovy

@@ -4816,7 +4816,7 @@ def launchTestJobs(pipeline, testFilter)
         // singleAttempt:true disables the outer K8s pod retry; see the x86
         // SLURM closure above for the full rationale (cap nested retry budget
         // so consistently-timing-out tests don't burn ~36h on retry cascades).
-        parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "arm64"), { attemptTag, isFinalAttempt, retryContext = null ->
+        parallelSlurmJobs = SBSASlurmTestConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
             // attemptTag is threaded into runLLMTestlistOnSlurm as the outer
             // dispatcher pod's tag so the inner SLURM retry's postTag can't
             // collide with a previous dispatcher pod's upload. See the x86
@@ -4834,7 +4834,7 @@ def launchTestJobs(pipeline, testFilter)
 
         // Add SBSA multi node Slurm jobs
         // singleAttempt:true disables the outer K8s pod retry; see above.
-        parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "arm64"), { attemptTag, isFinalAttempt, retryContext = null ->
+        parallelMultiNodesSBSAJobs = multiNodesSBSAConfigs.collectEntries{key, values -> [key, [createKubernetesPodConfig(LLM_DOCKER_IMAGE, "slurm", "amd64"), { attemptTag, isFinalAttempt, retryContext = null ->
             def config = LINUX_AARCH64_CONFIG
             if (key.contains("single-device")) {
                 config = SINGLE_DEVICE_CONFIG

tensorrt_llm/_utils.py

@@ -532,6 +532,9 @@ def get_free_ports(num=1) -> List[int]:
     ports = [s.getsockname()[1] for s in sockets]
     for s in sockets:
         s.close()
+    logger.info(
+        f"[get_free_ports] pid={os.getpid()} reserved ports={ports} via "
+        f"bind-then-close (subject to TOCTOU reuse before rebinding)")
     return ports
 
 

tensorrt_llm/commands/serve.py

@@ -339,6 +339,37 @@ def _build_llm_args_from_disagg_server_cfg(other_args: Dict) -> Dict:
     return update_llm_args_with_extra_dict(llm_args, llm_args_extra_dict)
 
 
+def _diagnose_port_in_use(port: int) -> str:
+    """Describe which process currently holds the given port, best effort."""
+    try:
+        import psutil
+    except ImportError:
+        return "psutil unavailable; cannot identify the process holding the port"
+
+    details = []
+    try:
+        for conn in psutil.net_connections(kind="inet"):
+            if conn.laddr and conn.laddr.port == port:
+                holder = ""
+                if conn.pid is not None:
+                    try:
+                        proc = psutil.Process(conn.pid)
+                        holder = (f" pid={conn.pid} name={proc.name()} "
+                                  f"cmdline={' '.join(proc.cmdline())}")
+                    except (psutil.NoSuchProcess, psutil.AccessDenied):
+                        holder = f" pid={conn.pid}"
+                details.append(
+                    f"{conn.laddr.ip}:{conn.laddr.port} status={conn.status}{holder}"
+                )
+    except (psutil.Error, OSError) as e:
+        return f"failed to inspect port holders: {e}"
+
+    if not details:
+        return ("no listening socket found for this port; it may have already "
+                "been released, indicating a transient race")
+    return "; ".join(details)
+
+
 def launch_server(
         host: str,
         port: int,
@@ -367,7 +398,12 @@ def launch_server(
             if port == 0:
                 port = s.getsockname()[1]
         except OSError as e:
-            raise RuntimeError(f"Failed to bind socket to {host}:{port}: {e}")
+            holder = _diagnose_port_in_use(port)
+            logger.error(
+                f"Failed to bind server socket to {host}:{port} "
+                f"(pid={os.getpid()}): {e}. Current port holder(s): {holder}")
+            raise RuntimeError(f"Failed to bind socket to {host}:{port}: {e}. "
+                               f"Port holder(s): {holder}")
 
         if backend == 'pytorch':
             llm_args.pop("build_config", None)
@@ -582,7 +618,12 @@ def launch_visual_gen_server(
         try:
             s.bind((host, port))
         except OSError as e:
-            raise RuntimeError(f"Failed to bind socket to {host}:{port}: {e}")
+            holder = _diagnose_port_in_use(port)
+            logger.error(
+                f"Failed to bind VisualGen server socket to {host}:{port} "
+                f"(pid={os.getpid()}): {e}. Current port holder(s): {holder}")
+            raise RuntimeError(f"Failed to bind socket to {host}:{port}: {e}. "
+                               f"Port holder(s): {holder}")
 
         logger.info(f"Initializing VisualGen ({model})")
 
@@ -1391,13 +1432,20 @@ def disaggregated(
     # Inherited by child processes via env var; used for deduplication at query time.
     os.environ[DisaggLauncherEnvs.TLLM_DISAGG_DEPLOYMENT_ID] = uuid.uuid4().hex
 
+    logger.info(f"Reserving disaggregated server address "
+                f"{disagg_cfg.hostname}:{disagg_cfg.port} (pid={os.getpid()})")
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         try:
             s.bind((disagg_cfg.hostname, disagg_cfg.port))
         except OSError as e:
+            holder = _diagnose_port_in_use(disagg_cfg.port)
+            logger.error(
+                f"Failed to bind disaggregated server socket to "
+                f"{disagg_cfg.hostname}:{disagg_cfg.port} (pid={os.getpid()}): "
+                f"{e}. Current port holder(s): {holder}")
             raise RuntimeError(
-                f"Failed to bind socket to {disagg_cfg.hostname}:{disagg_cfg.port}: {e}"
-            )
+                f"Failed to bind socket to {disagg_cfg.hostname}:{disagg_cfg.port}: {e}. "
+                f"Port holder(s): {holder}")
 
         metadata_server_cfg = parse_metadata_server_config_file(
             metadata_server_config_file)

tests/integration/defs/common.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1212,16 +1212,20 @@ def get_free_port_in_ci(max_attempts=100):
     """
     global PORTS_IN_USE
 
+    pid = os.getpid()
     container_port_start = int(os.environ.get("CONTAINER_PORT_START", -1))
     container_port_num = int(os.environ.get("CONTAINER_PORT_NUM", -1))
     if container_port_start != -1 and container_port_num != -1:
+        port_range = (container_port_start,
+                      container_port_start + container_port_num - 1)
         available_ports = [
             port for port in range(container_port_start, container_port_start +
                                    container_port_num)
             if port not in PORTS_IN_USE
         ]
+        num_candidates = len(available_ports)
 
-        for _ in range(len(available_ports)):
+        for attempt in range(1, num_candidates + 1):
             # Get a random port from the available ports
             port = random.choice(available_ports)
 
@@ -1230,16 +1234,35 @@ def get_free_port_in_ci(max_attempts=100):
                 try:
                     s.bind(("localhost", port))
                     PORTS_IN_USE.add(port)
+                    print_info(
+                        f"[get_free_port_in_ci] pid={pid} allocated port={port} "
+                        f"from CI range {port_range} after {attempt} attempt(s); "
+                        f"{len(PORTS_IN_USE)} reserved in-process. The probe "
+                        f"socket is now closed, so another process may take the "
+                        f"port before the caller rebinds it (TOCTOU).")
                     return port
-                except OSError:
+                except OSError as e:
+                    print_info(
+                        f"[get_free_port_in_ci] pid={pid} candidate port={port} "
+                        f"in CI range {port_range} is busy ({e}); trying another."
+                    )
                     available_ports.remove(port)
                     continue
 
+        print_warning(
+            f"[get_free_port_in_ci] pid={pid} exhausted all {num_candidates} "
+            f"candidate ports in CI range {port_range}; falling back to a "
+            f"system-assigned ephemeral port.")
+
     # No port found in the range, try to get a random free port from the system
     for _ in range(max_attempts):
         port = get_free_port()
         if port not in PORTS_IN_USE:
             PORTS_IN_USE.add(port)
+            print_info(
+                f"[get_free_port_in_ci] pid={pid} allocated system ephemeral "
+                f"port={port}; {len(PORTS_IN_USE)} reserved in-process. Another "
+                f"process may take it before the caller rebinds it (TOCTOU).")
             return port
 
     raise Exception(

tests/integration/test_lists/waives.txt

@@ -380,7 +380,6 @@ full:H100/test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] SKI
 full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551)
 full:H20/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6345827)
 full:H20/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] SKIP (https://nvbugs/6345827)
-full:H20/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] SKIP (https://nvbugs/6313314)
 full:H20/accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=3-block_reuse=True-use_py_transceiver=False] SKIP (https://nvbugs/6344108)
 full:H20/accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_ctx_dp2_gen_tp4 SKIP (https://nvbugs/6344108)
 full:H20/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_2gpus[triton-one_model-overlap_scheduler] SKIP (https://nvbugs/6373530)

tests/unittest/_torch/sampler/test_beam_search_speculative_d2h.py

@@ -31,6 +31,7 @@
 
 import gc
 import pathlib as _pl
+from contextlib import nullcontext
 from copy import deepcopy
 from typing import Any, Iterable
 
@@ -171,9 +172,6 @@ def _run_with_env(
         monkeypatch.setattr(
             TorchSampler, "_predict_beam_search_is_likely_finishing", predictor_override
         )
-    if sampler_method_patches:
-        for name, replacement in sampler_method_patches.items():
-            monkeypatch.setattr(TorchSampler, name, replacement)
 
     gc.collect(2)
     llm = _build_llm(
@@ -184,9 +182,15 @@ def _run_with_env(
     )
     try:
         with llm:
-            return _generate(
-                llm, input_prompts, _make_sampling_params(fixed_params, stop_token_ids)
-            )
+            sampling_params = _make_sampling_params(fixed_params, stop_token_ids)
+            if sampler_method_patches:
+                # Warmup before installing the hooks.
+                _generate(llm, input_prompts, sampling_params)
+            with monkeypatch.context() if sampler_method_patches else nullcontext() as p:
+                for name, replacement in (sampler_method_patches or {}).items():
+                    p.setattr(TorchSampler, name, replacement)
+                # Run with the hooks installed.
+                return _generate(llm, input_prompts, sampling_params)
     finally:
         del llm
         gc.collect(2)