File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -1221,7 +1221,14 @@ def _profile_runners(
12211221 candidates .append (
12221222 (runner_id , runner , runner_arg_names , all_valid_tactics ))
12231223
1224- if total_pairs == 1 :
1224+ # MERGE/PARALLEL strategies rely on comparing per-rank timings to
1225+ # pick the winning tactic across ranks. If a rank's local set of
1226+ # tactics has only one entry we must still time it — the shortcut
1227+ # would record min_time=0.0, which collides with peer ranks' 0.0
1228+ # in the tie-break and silently drops slower tactics on the floor.
1229+ if total_pairs == 1 and tuning_config .distributed_tuning_strategy not in (
1230+ DistributedTuningStrategy .MERGE ,
1231+ DistributedTuningStrategy .PARALLEL ):
12251232 # Single-pair shortcut. There is nothing to compare against, so
12261233 # the timed warmup + profile-repeat loop is pure overhead. We
12271234 # fire one forward call on every rank to drive any JIT side
Original file line number Diff line number Diff line change @@ -355,7 +355,6 @@ test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W
355355test_e2e.py::test_trtllm_bench_iteration_log[TRT-streaming-meta-llama/Llama-3.1-8B-llama-3.1-model/Meta-Llama-3.1-8B] SKIP (https://nvbugs/5448523)
356356triton_server/test_triton.py::test_cpp_unit_tests[cpp-unit-tests] SKIP (https://nvbugs/5619359)
357357triton_server/test_triton.py::test_python_bls_unit_tests[python-bls-unit-tests] SKIP (https://nvbugs/5477392)
358- unittest/_torch/misc/test_autotuner.py::test_autotuner_distributed_strategy SKIP (https://nvbugs/6321874)
359358unittest/_torch/misc/test_share_tensor.py::TestShareTensor::test_share_tensor_different_dtypes SKIP (https://nvbugs/6418021)
360359unittest/_torch/modules/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/5989912)
361360unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py -m "part0" SKIP (https://nvbugs/6372711)
Original file line number Diff line number Diff line change @@ -765,9 +765,13 @@ def _distributed_worker_function(world_size, strategy):
765765 tuning_config = config_independent ,
766766 inputs = inputs )
767767
768- # Check only one file is created in the cache path
769- assert len (os .listdir (os .path .dirname (
770- cache_path ))) == 1 , "Only one rank file should be created"
768+ # Check only one cache file is created in the cache path.
769+ # The sibling ".lock" file is an implementation artifact of
770+ # _exclusive_cache_lock (see tensorrt_llm/_torch/autotuner.py) and is not
771+ # a per-rank cache file.
772+ cache_dir = os .path .dirname (cache_path )
773+ cache_files = [f for f in os .listdir (cache_dir ) if not f .endswith (".lock" )]
774+ assert len (cache_files ) == 1 , "Only one rank file should be created"
771775
772776 dist .barrier ()
773777
You can’t perform that action at this time.
0 commit comments