Skip to content

Commit ceebeea

Browse files
committed
fix(aero_realtime): disable torchvision fallback to prevent NCCL timeout
Temporarily remove torchvision from VIDEO_READER_BACKENDS during fetch_video so that decord failures raise immediately instead of falling back to torchvision which can hang 30+ min on long videos.
1 parent 4dba19a commit ceebeea

1 file changed

Lines changed: 17 additions & 5 deletions

File tree

src/lmms_engine/datasets/iterable/aero_realtime_iterable_dataset.py

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,14 @@ def _load_video_with_metadata(
145145
Uses qwen_vl_utils fetch_video with return_video_metadata=True to get
146146
frame indices and fps needed for timestamp computation.
147147
148+
Disables torchvision fallback to avoid 30-min hangs on corrupted
149+
videos that would cause NCCL timeouts.
150+
148151
Returns:
149152
Tuple of (frames, video_metadata, sample_fps).
150153
"""
151154
from qwen_vl_utils import fetch_video
155+
from qwen_vl_utils import vision_process as _vp
152156

153157
if data_folder is not None:
154158
full_path = os.path.join(data_folder, video_path)
@@ -174,11 +178,19 @@ def _load_video_with_metadata(
174178
f"Invalid video sampling strategy: {self.config.video_sampling_strategy}"
175179
)
176180

177-
video_inputs, sample_fps = fetch_video(
178-
video_dict,
179-
return_video_sample_fps=True,
180-
return_video_metadata=True,
181-
)
181+
# Temporarily remove torchvision from backends to prevent slow fallback
182+
# that can hang for 30+ minutes on problematic videos, causing NCCL timeout.
183+
_tv_backup = _vp.VIDEO_READER_BACKENDS.pop("torchvision", None)
184+
try:
185+
video_inputs, sample_fps = fetch_video(
186+
video_dict,
187+
return_video_sample_fps=True,
188+
return_video_metadata=True,
189+
)
190+
finally:
191+
if _tv_backup is not None:
192+
_vp.VIDEO_READER_BACKENDS["torchvision"] = _tv_backup
193+
182194
frames, video_metadata = video_inputs
183195
frames = frames.numpy()
184196
return frames, video_metadata, sample_fps

0 commit comments

Comments
 (0)