diff --git a/benchmarking/Dockerfile b/benchmarking/Dockerfile index fa960c28a9..14d9a23ca2 100644 --- a/benchmarking/Dockerfile +++ b/benchmarking/Dockerfile @@ -27,6 +27,9 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Add video benchmark codec support to the benchmark image. +RUN bash /opt/Curator/docker/common/install_h264_support.sh --with-libopenh264 + # Ensure all extras and dependency groups (including test) are synced RUN cd /opt/Curator \ && uv sync --extra all --all-groups \ diff --git a/benchmarking/nightly-benchmark.yaml b/benchmarking/nightly-benchmark.yaml index cf331b949f..2b69a96bbb 100644 --- a/benchmarking/nightly-benchmark.yaml +++ b/benchmarking/nightly-benchmark.yaml @@ -877,6 +877,7 @@ entries: --video-dir={dataset:videos,mp4} --model-dir={dataset:videos_model_weights,files} --video-limit=1000 + --transcode-encoder=libopenh264 timeout_s: 800 sink_data: - name: slack @@ -900,6 +901,7 @@ entries: --video-dir={dataset:videos,mp4} --model-dir={dataset:videos_model_weights,files} --video-limit=1000 + --transcode-encoder=libopenh264 timeout_s: 800 sink_data: - name: slack @@ -923,6 +925,7 @@ entries: --no-generate-embeddings --video-dir={dataset:videos,mp4} --video-limit=1000 + --transcode-encoder=libopenh264 timeout_s: 400 sink_data: - name: slack @@ -947,6 +950,7 @@ entries: --no-generate-embeddings --video-dir={dataset:videos,mp4} --video-limit=1000 + --transcode-encoder=libopenh264 timeout_s: 800 sink_data: - name: slack @@ -974,6 +978,7 @@ entries: --no-generate-embeddings --enhance-captions --video-limit=256 + --transcode-encoder=libopenh264 timeout_s: 1800 sink_data: - name: slack @@ -1000,6 +1005,7 @@ entries: --no-generate-embeddings --enhance-captions --video-limit=256 + --transcode-encoder=libopenh264 timeout_s: 1800 sink_data: - name: slack @@ -1089,6 +1095,7 @@ entries: --aesthetic-threshold=3.5 --transnetv2-frame-decoder-mode ffmpeg_cpu --video-limit=1000 + --transcode-encoder=libopenh264 timeout_s: 800 sink_data: - name: slack @@ -1116,6 +1123,7 @@ entries: --aesthetic-threshold=3.5 --transnetv2-frame-decoder-mode ffmpeg_cpu --video-limit=1000 + --transcode-encoder=libopenh264 timeout_s: 1800 sink_data: - name: slack diff --git a/benchmarking/tools/ci_benchmark_launcher.sh b/benchmarking/tools/ci_benchmark_launcher.sh index 77fd661105..b9f4802cba 100644 --- a/benchmarking/tools/ci_benchmark_launcher.sh +++ b/benchmarking/tools/ci_benchmark_launcher.sh @@ -28,7 +28,7 @@ apt-get update -qq && apt-get install -y --no-install-recommends lynx if [[ "${ENTRY_NAME}" == audio_* ]]; then apt-get install -y --no-install-recommends ffmpeg elif [[ "${ENTRY_NAME}" == video_* ]]; then - bash /opt/Curator/docker/common/install_ffmpeg.sh + bash /opt/Curator/docker/common/install_h264_support.sh --with-libopenh264 fi cd /opt/Curator diff --git a/docker/common/install_ffmpeg.sh b/docker/common/install_ffmpeg.sh index 81a4652c20..75a6794eec 100644 --- a/docker/common/install_ffmpeg.sh +++ b/docker/common/install_ffmpeg.sh @@ -31,62 +31,73 @@ done export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y \ - libcrypt-dev \ autoconf \ automake \ build-essential \ + ca-certificates \ cmake \ - libaom-dev \ - libass-dev \ - libdav1d-dev \ - libdrm-dev \ - libfreetype6-dev \ - libgnutls28-dev \ + libcrypt-dev \ libnuma-dev \ - libopenh264-dev \ libtool \ - libva-dev \ - libvorbis-dev \ libvpx-dev \ - libwebp-dev \ nasm \ pkg-config \ - vainfo \ wget \ yasm \ zlib1g-dev -# Install NVCODEC +# Install nv-codec-headers (NVENC + NVDEC bridge to the NVIDIA driver) wget -O /tmp/nv-codec-headers.tar.gz https://github.com/FFmpeg/nv-codec-headers/releases/download/n${NVCODEC_VERSION}/nv-codec-headers-${NVCODEC_VERSION}.tar.gz tar xzvf /tmp/nv-codec-headers.tar.gz -C /tmp/ cd /tmp/nv-codec-headers-${NVCODEC_VERSION} make make install -# Install FFMPEG -wget -O /tmp/ffmpeg-snapshot.tar.bz2 https://www.ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 +# Build FFmpeg ${FFMPEG_VERSION} from the upstream release tarball: +# - --disable-everything strips ALL components by default (encoders, +# decoders, muxers, demuxers, parsers, bsfs, hwaccels, filters, protocols) +# and we re-enable only what's needed. +# - --enable-version3 selects LGPLv3+. +# - Encoders: only NVENC (h264/hevc/av1) and libvpx-vp9 + rawvideo. +# - Decoders: NVDEC variants for h264/hevc/av1/vp9 + software vp8/vp9 + +# mpeg1/2/4 + libvpx_vp9 + rawvideo. NO software h264/hevc/av1. +# - Shared-linked: installs libav*.so to /usr/local/lib for command-line +# tools and any optional source-built consumers. +cd /tmp +wget -O /tmp/ffmpeg-snapshot.tar.bz2 "https://www.ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2" tar xjvf /tmp/ffmpeg-snapshot.tar.bz2 -C /tmp/ -cd /tmp/ffmpeg-${FFMPEG_VERSION} -PATH="/usr/local/cuda/bin:$PATH" ./configure \ - --prefix=/usr/local \ - --enable-nonfree \ - --enable-cuda-nvcc \ - --enable-libnpp \ - --enable-libopenh264 \ - --enable-libaom \ - --enable-libdav1d \ - --enable-libvorbis \ - --enable-libvpx \ - --enable-libwebp \ - --enable-vaapi \ - --extra-cflags=-I/usr/local/cuda/include \ - --extra-ldflags=-L/usr/local/cuda/lib64 \ - --extra-libs=-lpthread \ - --extra-libs=-lm \ - --disable-static \ +cd "/tmp/ffmpeg-${FFMPEG_VERSION}" +PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" ./configure \ + --prefix="/usr/local" \ --enable-shared \ + --disable-static \ + --extra-cflags="-I/usr/local/cuda/include" \ + --extra-ldflags="-L/usr/local/cuda/lib64" \ + --extra-libs="-lpthread -lm" \ + --ld="g++" \ + --enable-version3 \ + --disable-everything \ + --disable-network \ --disable-doc \ - --disable-debug + --disable-ffplay \ + --disable-vaapi \ + --disable-vdpau \ + --disable-dxva2 \ + --disable-libdrm \ + --enable-encoder=rawvideo,libvpx_vp9,h264_nvenc,hevc_nvenc,av1_nvenc \ + --enable-decoder=rawvideo,libvpx_vp9,vp9,vp8,h264_cuvid,hevc_cuvid,av1_cuvid,mpeg1video,mpeg2video,mpeg4 \ + --enable-muxer=mp4,rawvideo,image2pipe \ + --enable-demuxer=mov,mp4,m4a,3gp,3g2,mj2,avi,matroska,webm,image2,image2pipe \ + --enable-parser=h264,hevc,av1,vp8,vp9 \ + --enable-bsf=h264_mp4toannexb,hevc_mp4toannexb \ + --enable-protocol=file,pipe \ + --enable-filter=scale,format,null,copy \ + --enable-libvpx \ + --enable-cuda \ + --enable-cuvid \ + --enable-nvdec \ + --enable-nvenc \ + --enable-ffnvcodec make -j$(nproc) make install ldconfig diff --git a/docker/common/install_h264_support.sh b/docker/common/install_h264_support.sh new file mode 100755 index 0000000000..27b21aaf86 --- /dev/null +++ b/docker/common/install_h264_support.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Opt-in installer that adds software h264/hevc/av1 decoder support to a +# Curator container by building FFmpeg from source with those decoders added +# to the strict allowlist. +# +# The release Curator image may omit system ffmpeg entirely. When installed +# with install_ffmpeg.sh, Curator's strict FFmpeg build routes h264/hevc/av1 +# decode through NVDEC only. That breaks ffprobe-based metadata extraction in +# CPU-only Ray actors (VideoReader, ClipWriter), which is why this opt-in +# exists. +# +# Run inside a running container, e.g.: +# docker exec bash /opt/Curator/docker/common/install_h264_support.sh +# +# Behaviour: +# - Builds FFmpeg from the upstream release tarball, same version as install_ffmpeg.sh, +# with the existing allowlist plus --enable-decoder=h264,hevc,av1. +# - Optionally also enables the libopenh264 software h264 ENCODER (Cisco's +# free-license OpenH264 binary; opt-in via --with-libopenh264). +# - Replaces /usr/local/bin/{ffmpeg,ffprobe} in place. +# - Default stays LGPLv3 (only FFmpeg-internal native decoders); with +# --with-libopenh264 the resulting binary additionally links Cisco's +# OpenH264 (BSD-2-Clause; see https://www.openh264.org/BINARY_LICENSE.txt). +# - Takes ~5-10 min. +# +# Keep FFMPEG_VERSION and NVCODEC_VERSION in sync with docker/common/install_ffmpeg.sh. + +set -euo pipefail + +FFMPEG_VERSION=8.0.1 +NVCODEC_VERSION=12.1.14.0 +WITH_LIBOPENH264=0 + +usage() { + cat <<'EOF' +Usage: install_h264_support.sh [--with-libopenh264] [--FFMPEG_VERSION=] + [--NVCODEC_VERSION=] + +Builds FFmpeg from source with software h264/hevc/av1 decoders enabled. + +Options: + --with-libopenh264 Also enable the libopenh264 software h264 ENCODER + (Cisco's OpenH264 binary; required by Curator's + --transcode-encoder=libopenh264 path). + --FFMPEG_VERSION= FFmpeg upstream release version (default: 8.0.1). + --NVCODEC_VERSION= nv-codec-headers release version (default: 12.1.14.0). + -h, --help Show this help. + +License notice: + Default mode enables only FFmpeg's internal h264/hevc/av1 decoders (LGPL). + With --with-libopenh264 the build additionally links Cisco's OpenH264 + binary (BSD-2-Clause + Cisco-distributed binary license; see + https://www.openh264.org/BINARY_LICENSE.txt). By running this script you + are responsible for any license obligations the resulting binaries impose + on your distribution. +EOF +} + +for arg in "$@"; do + case $arg in + --with-libopenh264) WITH_LIBOPENH264=1 ;; + --FFMPEG_VERSION=?*) FFMPEG_VERSION="${arg#*=}" ;; + --NVCODEC_VERSION=?*) NVCODEC_VERSION="${arg#*=}" ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown argument: $arg" >&2; usage >&2; exit 2 ;; + esac +done + +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: must be run as root inside the container." >&2 + exit 1 +fi + +echo "==> install_h264_support.sh: rebuilding ffmpeg ${FFMPEG_VERSION}" +echo " Decoders added: h264, hevc, av1 (software)" +if [ "$WITH_LIBOPENH264" -eq 1 ]; then + echo " Encoder added: libopenh264 (Cisco OpenH264 binary)" +fi + echo " NOTE: This expands the container's codec footprint beyond Curator's" + echo " default strict FFmpeg allowlist. License obligations of the resulting binaries" +echo " are the user's responsibility." +echo + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt_packages=( + autoconf automake build-essential ca-certificates cmake + libcrypt-dev libnuma-dev libtool libvpx-dev nasm pkg-config + wget yasm zlib1g-dev +) +if [ "$WITH_LIBOPENH264" -eq 1 ]; then + apt_packages+=(libopenh264-dev) +fi +apt-get install -y "${apt_packages[@]}" + +if [ ! -f /usr/local/include/ffnvcodec/dynlink_loader.h ]; then + wget -O /tmp/nv-codec-headers.tar.gz \ + "https://github.com/FFmpeg/nv-codec-headers/releases/download/n${NVCODEC_VERSION}/nv-codec-headers-${NVCODEC_VERSION}.tar.gz" + tar xzf /tmp/nv-codec-headers.tar.gz -C /tmp/ + (cd "/tmp/nv-codec-headers-${NVCODEC_VERSION}" && make && make install) +fi + +cd /tmp +rm -rf "ffmpeg-${FFMPEG_VERSION}" ffmpeg-snapshot.tar.bz2 +wget -O /tmp/ffmpeg-snapshot.tar.bz2 \ + "https://www.ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2" +tar xjvf /tmp/ffmpeg-snapshot.tar.bz2 -C /tmp/ +cd "/tmp/ffmpeg-${FFMPEG_VERSION}" + +# Configure mirrors install_ffmpeg.sh exactly, with the decoder allowlist +# extended to include software h264/hevc/av1, and optionally the libopenh264 +# software h264 encoder. +encoder_list="rawvideo,libvpx_vp9,h264_nvenc,hevc_nvenc,av1_nvenc" +extra_configure_flags=() +if [ "$WITH_LIBOPENH264" -eq 1 ]; then + encoder_list="${encoder_list},libopenh264" + extra_configure_flags+=(--enable-libopenh264) +fi + +PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" ./configure \ + --prefix="/usr/local" \ + --enable-shared \ + --disable-static \ + --extra-cflags="-I/usr/local/cuda/include" \ + --extra-ldflags="-L/usr/local/cuda/lib64" \ + --extra-libs="-lpthread -lm" \ + --ld="g++" \ + --enable-version3 \ + --disable-everything \ + --disable-network \ + --disable-doc \ + --disable-ffplay \ + --disable-vaapi \ + --disable-vdpau \ + --disable-dxva2 \ + --disable-libdrm \ + --enable-encoder="${encoder_list}" \ + --enable-decoder=rawvideo,libvpx_vp9,vp9,vp8,h264_cuvid,hevc_cuvid,av1_cuvid,mpeg1video,mpeg2video,mpeg4,h264,hevc,av1 \ + --enable-muxer=mp4,rawvideo,image2pipe \ + --enable-demuxer=mov,mp4,m4a,3gp,3g2,mj2,avi,matroska,webm,image2,image2pipe \ + --enable-parser=h264,hevc,av1,vp8,vp9 \ + --enable-bsf=h264_mp4toannexb,hevc_mp4toannexb \ + --enable-protocol=file,pipe \ + --enable-filter=scale,format,null,copy \ + --enable-libvpx \ + --enable-cuda \ + --enable-cuvid \ + --enable-nvdec \ + --enable-nvenc \ + --enable-ffnvcodec \ + "${extra_configure_flags[@]}" +make -j"$(nproc)" +make install +ldconfig + +cd / +rm -rf /tmp/ffmpeg* /tmp/nv-codec-headers* +echo +if [ "$WITH_LIBOPENH264" -eq 1 ]; then + echo "==> Done. /usr/local/bin/{ffmpeg,ffprobe} now include software h264/hevc/av1" + echo " decoders and the libopenh264 software h264 encoder." +else + echo "==> Done. /usr/local/bin/{ffmpeg,ffprobe} now include software h264/hevc/av1 decoders." +fi diff --git a/fern/versions/main/pages/curate-video/process-data/transcoding.mdx b/fern/versions/main/pages/curate-video/process-data/transcoding.mdx index 5762a9a9da..e832fbe9b6 100644 --- a/fern/versions/main/pages/curate-video/process-data/transcoding.mdx +++ b/fern/versions/main/pages/curate-video/process-data/transcoding.mdx @@ -59,12 +59,12 @@ python -m ray_curator.examples.video.video_split_clip_example \ | Encoder | Hardware | Description | | --- | --- | --- | -| `libx264` | CPU | Widely available, high quality, CPU-based. | -| `libopenh264` | CPU | Good quality and throughput balance. Often faster than `libx264` at similar presets. | | `h264_nvenc` | NVIDIA GPU (NVENC) | Uses NVENC for high-throughput H.264 encoding on NVIDIA GPU hardware. | +| `libvpx-vp9` | CPU | VP9 software encoder. Use as a fallback on GPUs without NVENC hardware, such as A100 and H100. | +| `libopenh264` | CPU | Optional H.264 software encoder. Requires `install_h264_support.sh --with-libopenh264` or another FFmpeg build that includes it. | -On systems with supported NVIDIA GPU hardware and an `ffmpeg` build with NVENC, `h264_nvenc` can significantly increase throughput. Refer to the verification steps below to confirm NVENC availability. +On systems with supported NVIDIA GPU hardware and an `ffmpeg` build with NVENC, `h264_nvenc` can significantly increase throughput. On GPUs without NVENC hardware, use `libvpx-vp9`, or opt in to `libopenh264` if H.264 output is required. ### Verify `ffmpeg`/NVENC Support @@ -87,7 +87,7 @@ Use `ClipTranscodingStage` to control encoder choice, batching, and acceleration from nemo_curator.stages.video.clipping.clip_extraction_stages import ClipTranscodingStage transcode = ClipTranscodingStage( - encoder="h264_nvenc", # or "libopenh264", "libx264" + encoder="h264_nvenc", # or "libvpx-vp9", "libopenh264" use_hwaccel=True, # enable NVENC when using h264_nvenc encoder_threads=1, # CPU thread count for CPU encoders encode_batch_size=16, # number of clips per encode batch @@ -102,8 +102,8 @@ transcode = ClipTranscodingStage( | Parameter | Description | | --- | --- | -| `encoder` | Selects the encoding backend. Recommended defaults: `libopenh264` (CPU) or `h264_nvenc` (GPU). | -| `use_hwaccel` | Enable when using GPU encoders like `h264_nvenc`. | +| `encoder` | Selects the encoding backend. Supported values are `h264_nvenc`, `libvpx-vp9`, and `libopenh264`. | +| `use_hwaccel` | Enable only when using `h264_nvenc`. | | `encoder_threads` | CPU threads per worker for CPU encoders. Increase to use more CPU. | | `encode_batch_size` | Batching size for clips; larger batches can improve throughput. | | `use_input_bit_rate` | If True, attempts to reuse the input bit rate; otherwise, the encoder uses its default rate control. | @@ -117,4 +117,5 @@ Refer to the quickstart options in [Get Started with Video Curation](/get-starte - "Encoder not found": Your `ffmpeg` build may lack the encoder; verify with `ffmpeg -encoders`. - "No NVENC capable devices found": Install NVIDIA drivers/CUDA and ensure the GPU is visible in `nvidia-smi`. +- `SoftwareCodecMissingError`: install software H.264/HEVC/AV1 support with `bash /opt/Curator/docker/common/install_h264_support.sh`, or use `--transcode-encoder libvpx-vp9`. - Output mismatch or low quality: Revisit encoder defaults; set explicit bit rate/quality settings as needed, or enable `use_input_bit_rate`. diff --git a/fern/versions/main/pages/get-started/installation.mdx b/fern/versions/main/pages/get-started/installation.mdx index 6903fe9a20..babf449155 100644 --- a/fern/versions/main/pages/get-started/installation.mdx +++ b/fern/versions/main/pages/get-started/installation.mdx @@ -139,12 +139,12 @@ docker run --gpus all -it --rm nemo-curator:latest ### Install FFmpeg and Encoders (Required for Video) -Curator’s video pipelines rely on `FFmpeg` for decoding and encoding. If you plan to encode clips (for example, using `--transcode-encoder libopenh264` or `h264_nvenc`), install `FFmpeg` with the corresponding encoders. +Curator's video pipelines rely on `FFmpeg` for command-line decoding, encoding, and metadata extraction. The default container does not include system `ffmpeg`; install it only for video or audio workflows that need it. -Use the maintained script in the repository to build and install `FFmpeg` with `libopenh264` and NVIDIA NVENC support. The script enables `--enable-libopenh264`, `--enable-cuda-nvcc`, and `--enable-libnpp`. +Use the maintained script in the repository to build and install a strict-allowlist `FFmpeg` with NVIDIA NVENC/NVDEC and VP9 support. The default build excludes software H.264/HEVC/AV1 decoders and software H.264 encoders. - Script source: [docker/common/install_ffmpeg.sh](https://github.com/NVIDIA-NeMo/Curator/blob/main/docker/common/install_ffmpeg.sh) @@ -158,22 +158,40 @@ sudo bash install_ffmpeg.sh -Confirm that `FFmpeg` is on your `PATH` and that at least one H.264 encoder is available: +Confirm that `FFmpeg` is on your `PATH` and that the expected encoders are available: ```bash ffmpeg -hide_banner -version | head -n 5 -ffmpeg -encoders | grep -E "h264_nvenc|libopenh264|libx264" | cat +ffmpeg -encoders | grep -E "h264_nvenc|libvpx-vp9|libopenh264" | cat ``` -If encoders are missing, reinstall `FFmpeg` with the required options or use the Debian/Ubuntu script above. +If encoders are missing, reinstall `FFmpeg` with the required options or use the Debian/Ubuntu script above. `libopenh264` appears only after opting in with `install_h264_support.sh --with-libopenh264`. -**FFmpeg build requires CUDA toolkit (nvcc):** If you encounter `ERROR: failed checking for nvcc` during FFmpeg installation, ensure that the CUDA toolkit is installed and `nvcc` is available on your `PATH`. You can verify with `nvcc --version`. If using the NeMo Curator container, FFmpeg is pre-installed with NVENC support. +**FFmpeg build requires CUDA toolkit (nvcc):** If you encounter `ERROR: failed checking for nvcc` during FFmpeg installation, ensure that the CUDA toolkit is installed and `nvcc` is available on your `PATH`. You can verify with `nvcc --version`. +#### Software H.264/HEVC/AV1 Codec Support (Advanced) + +Curator's strict FFmpeg build routes H.264/HEVC/AV1 decode through NVDEC and excludes software H.264 encoders by default. You may need extra software codec support when CPU-only stages run `ffprobe` on H.264/HEVC/AV1 inputs, or when using `--transcode-encoder=libopenh264`. + +Inside a container, run: + +```bash +# Add software h264/hevc/av1 decoders only +bash /opt/Curator/docker/common/install_h264_support.sh + +# Add those decoders plus the libopenh264 software h264 encoder +bash /opt/Curator/docker/common/install_h264_support.sh --with-libopenh264 +``` + + +With `--with-libopenh264`, the resulting FFmpeg binary links Cisco OpenH264. You are responsible for any license obligations imposed by the resulting binaries. + + --- ## Package Extras diff --git a/fern/versions/main/pages/get-started/video.mdx b/fern/versions/main/pages/get-started/video.mdx index 6517a5cc1d..177d302d41 100644 --- a/fern/versions/main/pages/get-started/video.mdx +++ b/fern/versions/main/pages/get-started/video.mdx @@ -56,9 +56,9 @@ To use NeMo Curator's video curation capabilities, ensure your system meets thes - Reduced configuration (lower batch sizes, FP8): ~21GB VRAM #### Software Dependencies -* **FFmpeg 8.0+** with H.264 encoding support - - GPU encoder: `h264_nvenc` (recommended for performance) - - CPU encoders: `libopenh264` or `libx264` (fallback options) +* **FFmpeg 8.0+** with one of the following encoders: + - GPU encoder: `h264_nvenc` (recommended for performance; requires an NVENC-equipped GPU — note that A100 and H100 do **not** include NVENC) + - CPU encoder: `libvpx-vp9` (for non-NVENC GPUs; produces VP9 in `.mp4`) If `uv` is not installed, refer to the [Installation Guide](/get-started/installation) for setup instructions, or install it quickly with: @@ -120,12 +120,12 @@ For details on container environments and configurations, see [Container Environ ## Install FFmpeg and Encoders -Curator’s video pipelines rely on `FFmpeg` for decoding and encoding. If you plan to encode clips (for example, using `--transcode-encoder libopenh264` or `h264_nvenc`), install `FFmpeg` with the corresponding encoders. +Curator’s video pipelines rely on `FFmpeg` for decoding and encoding. If you plan to encode clips (using `--transcode-encoder h264_nvenc` or `--transcode-encoder libvpx-vp9`), install `FFmpeg` with NVENC and `libvpx-vp9` support. The maintained install script bundles both. -Use the maintained script in the repository to build and install `FFmpeg` with `libopenh264` and NVIDIA NVENC support. The script enables `--enable-libopenh264`, `--enable-cuda-nvcc`, and `--enable-libnpp`. +Use the maintained script in the repository to build and install `FFmpeg` with NVIDIA NVENC and `libvpx-vp9` support. The script enables dynamic CUDA/NVIDIA codec support through `--enable-cuda`, `--enable-cuvid`, `--enable-nvdec`, `--enable-nvenc`, and `--enable-ffnvcodec`, plus `--enable-libvpx`. - Script source: [docker/common/install_ffmpeg.sh](https://github.com/NVIDIA-NeMo/Curator/blob/main/docker/common/install_ffmpeg.sh) @@ -139,11 +139,11 @@ sudo bash install_ffmpeg.sh -Confirm that `FFmpeg` is on your `PATH` and that at least one H.264 encoder is available: +Confirm that `FFmpeg` is on your `PATH` and that the expected video encoders are available: ```bash ffmpeg -hide_banner -version | head -n 5 -ffmpeg -encoders | grep -E "h264_nvenc|libopenh264|libx264" | cat +ffmpeg -encoders | grep -E "h264_nvenc|libvpx-vp9|libopenh264" | cat ``` If encoders are missing, reinstall `FFmpeg` with the required options or use the Debian/Ubuntu script above. @@ -154,6 +154,10 @@ If encoders are missing, reinstall `FFmpeg` with the required options or use the Refer to [Clip Encoding](/curate-video/process-data/transcoding) to choose encoders and verify NVENC support on your system. + +On GPUs without NVENC hardware, such as A100 and H100, use `--transcode-encoder libvpx-vp9` or install software H.264 support and use `--transcode-encoder libopenh264`. + + ### Available Models Embeddings convert each video clip into a numeric vector that captures visual and semantic content. Curator uses these vectors to: @@ -217,7 +221,7 @@ Organize input videos and output locations before running the pipeline. ## Run the Splitting Pipeline Example -Use the example script from https://github.com/NVIDIA-NeMo/Curator/tree/main/tutorials/video/getting-started to read videos, split into clips, and write outputs. This runs a Ray pipeline with `XennaExecutor` under the hood. +Use the [example script](https://github.com/NVIDIA-NeMo/Curator/tree/main/tutorials/video/getting-started) to read videos, split into clips, and write outputs. This runs a Ray pipeline with `XennaExecutor` under the hood. ```bash python tutorials/video/getting-started/video_split_clip_example.py \ @@ -227,7 +231,7 @@ python tutorials/video/getting-started/video_split_clip_example.py \ --splitting-algorithm fixed_stride \ --fixed-stride-split-duration 10.0 \ --embedding-algorithm cosmos-embed1-224p \ - --transcode-encoder libopenh264 \ + --transcode-encoder h264_nvenc \ --verbose ``` @@ -235,7 +239,7 @@ python tutorials/video/getting-started/video_split_clip_example.py \ 1. Reads all video files from `$DATA_DIR` 2. Splits each video into 10-second clips using fixed stride 3. Generates embeddings using Cosmos-Embed1-224p model -4. Encodes clips using libopenh264 codec +4. Encodes clips using h264_nvenc codec 5. Writes output clips and metadata to `$OUT_DIR` @@ -246,8 +250,8 @@ python tutorials/video/getting-started/video_split_clip_example.py \ --splitting-algorithm fixed_stride --fixed-stride-split-duration 10.0 --embedding-algorithm cosmos-embed1-224p - --transcode-encoder libopenh264' > my_config.txt - + --transcode-encoder h264_nvenc' > my_config.txt + python tutorials/video/getting-started/video_split_clip_example.py @my_config.txt @@ -262,14 +266,14 @@ python tutorials/video/getting-started/video_split_clip_example.py \ | **Embedding** | | `--embedding-algorithm` | `cosmos-embed1-224p`, `cosmos-embed1-336p`, `cosmos-embed1-448p` | Embedding model to use | | **Encoding** | -| `--transcode-encoder` | `h264_nvenc`, `libopenh264`, `libx264` | Video encoder for output clips | -| `--transcode-use-hwaccel` | Flag | Enable hardware acceleration for encoding | +| `--transcode-encoder` | `h264_nvenc`, `libvpx-vp9`, `libopenh264` | Video encoder for output clips. Use `libvpx-vp9` (CPU) on GPUs without NVENC such as A100/H100. `libopenh264` is accepted but requires a user-installed FFmpeg build — see [BYO H.264](../admin/installation.md#bring-your-own-h264-software-encoder-advanced). | +| `--transcode-use-hwaccel` | Flag | Enable hardware acceleration for encoding (only valid with `h264_nvenc`). | | **Optional Features** | | `--generate-captions` | Flag | Generate text captions for each clip | | `--generate-previews` | Flag | Create preview images for each clip | | `--verbose` | Flag | Enable detailed logging output | -### Understanding Pipeline Output +### Understand Pipeline Output After successful execution, the output directory will contain: diff --git a/nemo_curator/stages/video/clipping/clip_extraction_stages.py b/nemo_curator/stages/video/clipping/clip_extraction_stages.py index 67f9a751d0..8fdee32d53 100644 --- a/nemo_curator/stages/video/clipping/clip_extraction_stages.py +++ b/nemo_curator/stages/video/clipping/clip_extraction_stages.py @@ -30,13 +30,29 @@ from nemo_curator.utils import grouping from nemo_curator.utils.operation_utils import make_pipeline_temporary_dir +SUPPORTED_ENCODERS = ("h264_nvenc", "libvpx-vp9", "libopenh264") + +_BYO_H264_DOCS_URL = ( + "https://github.com/NVIDIA-NeMo/Curator/blob/main/fern/versions/main/pages/get-started/installation.mdx" + "#software-h264hevcav1-codec-support-advanced" +) + @dataclass class ClipTranscodingStage(ProcessingStage[VideoTask, VideoTask]): """Stage that transcodes video clips into a standardized format. - This stage handles the conversion of video clips using FFmpeg, supporting both - software (libx264, libopenh264) and hardware (NVENC) encoding with configurable parameters. + This stage handles the conversion of video clips using FFmpeg. Supported + encoders: + + - ``h264_nvenc`` — hardware H.264 via NVENC (recommended; requires an + NVENC-equipped NVIDIA GPU — note that A100/H100 do not include NVENC). + - ``libvpx-vp9`` — royalty-free VP9 software encoder (CPU fallback for + non-NVENC GPUs). Significantly slower; emits a perf advisory. + - ``libopenh264`` — H.264 software encoder. Not bundled with Curator's + FFmpeg build for licensing reasons; users must install it themselves. + The stage probes for it at setup time and raises a clear error pointing + to the docs if it is not available. Args: num_cpus_per_worker: Number of CPUs per worker for Xenna scheduling. Does not affect Ray Data CPU scheduling; use ray_data_num_cpus for that. @@ -44,7 +60,7 @@ class ClipTranscodingStage(ProcessingStage[VideoTask, VideoTask]): encoder_threads: Number of threads per encoder. encode_batch_size: Number of clips to encode in parallel. nb_streams_per_gpu: Number of streams per GPU. - use_hwaccel: Whether to use hardware acceleration. + use_hwaccel: Whether to use hardware acceleration. Only valid with `h264_nvenc`. use_input_bit_rate: Whether to use input video bit rate. num_clips_per_chunk: Number of clips per chunk. If the number of clips is larger than this, the clips will be split into chunks, and created VideoTasks for each chunk. verbose: Whether to print verbose logs. @@ -53,7 +69,7 @@ class ClipTranscodingStage(ProcessingStage[VideoTask, VideoTask]): """ num_cpus_per_worker: float = 6.0 - encoder: str = "libx264" + encoder: str = "h264_nvenc" encoder_threads: int = 1 encode_batch_size: int = 16 nb_streams_per_gpu: int = 3 @@ -75,11 +91,49 @@ def setup(self, worker_metadata: WorkerMetadata | None = None) -> None: # noqa: worker_metadata (WorkerMetadata, optional): Information about the worker (provided by some backends) """ if not shutil.which("ffmpeg"): - msg = "ClipTranscodingStage requires 'ffmpeg' built with libopenh264/NVENC support. See docker/common/install_ffmpeg.sh." + msg = ( + "Could not find `ffmpeg` on PATH. ClipTranscodingStage requires " + "FFmpeg built with supported video encoders. See docker/common/install_ffmpeg.sh." + ) raise RuntimeError(msg) - if self.encoder not in {"libopenh264", "libx264", "h264_nvenc"}: - error_msg = f"Expected encoder of `libopenh264`, `libx264`, or `h264_nvenc`. Got {self.encoder}" + if self.encoder not in SUPPORTED_ENCODERS: + error_msg = f"Expected encoder in {SUPPORTED_ENCODERS}. Got {self.encoder}" + raise ValueError(error_msg) + if self.encoder == "libvpx-vp9" and self.use_hwaccel: + error_msg = "use_hwaccel is not supported with libvpx-vp9 (CPU encoder)" raise ValueError(error_msg) + if self.encoder == "libopenh264": + self._verify_libopenh264_available() + + @staticmethod + def _verify_libopenh264_available() -> None: + """Probe the local FFmpeg build for libopenh264 support.""" + ffmpeg_bin = shutil.which("ffmpeg") + if ffmpeg_bin is None: + error_msg = ( + "Could not find `ffmpeg` on PATH while verifying libopenh264 support. " + f"Install FFmpeg and ensure it is on PATH. See {_BYO_H264_DOCS_URL}" + ) + raise RuntimeError(error_msg) + try: + result = subprocess.run( # noqa: S603 + [ffmpeg_bin, "-hide_banner", "-encoders"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except subprocess.TimeoutExpired as e: + error_msg = f"`ffmpeg -encoders` timed out while verifying libopenh264 support. See {_BYO_H264_DOCS_URL}" + raise RuntimeError(error_msg) from e + if "libopenh264" not in result.stdout: + error_msg = ( + "encoder='libopenh264' was requested but the local FFmpeg build " + "does not include it. Curator does not ship libopenh264 due to " + "its patent-license redistribution model. To enable it, install " + f"a libopenh264-enabled FFmpeg yourself — see {_BYO_H264_DOCS_URL}" + ) + raise RuntimeError(error_msg) def __post_init__(self) -> None: """Post-initialization method called after all fields are set.""" @@ -97,6 +151,14 @@ def __post_init__(self) -> None: # so Xenna scheduling is unaffected. self.ray_data_num_cpus = 1.0 + if self.encoder == "libvpx-vp9": + logger.warning( + "ClipTranscodingStage: libvpx-vp9 is significantly slower than " + "h264_nvenc and libopenh264. If your GPU has NVENC, prefer " + "encoder='h264_nvenc'. To use libopenh264 instead, see " + f"{_BYO_H264_DOCS_URL}" + ) + def inputs(self) -> tuple[list[str], list[str]]: return ["data"], ["source_bytes"] @@ -245,11 +307,8 @@ def _add_decoder_threads(self, command: list[str]) -> None: def _add_hwaccel_options(self, command: list[str]) -> None: """Add hardware acceleration options to command.""" - if self.use_hwaccel: - if self.encoder == "h264_nvenc": - command.extend(["-hwaccel", "cuda", "-hwaccel_output_format", "cuda"]) - else: - command.extend(["-hwaccel", "auto"]) + if self.use_hwaccel and self.encoder == "h264_nvenc": + command.extend(["-hwaccel", "cuda", "-hwaccel_output_format", "cuda"]) def _add_input_options(self, command: list[str], clip: Clip, video_filename: str, index: int) -> None: """Add input options to command.""" @@ -276,6 +335,8 @@ def _add_video_encoding_options(self, command: list[str], use_bit_rate: str | No if self.encoder == "h264_nvenc": self._add_nvenc_options(command, force_pix_fmt) + elif self.encoder == "libvpx-vp9": + self._add_libvpx_vp9_options(command, use_bit_rate, force_pix_fmt) def _add_nvenc_options(self, command: list[str], force_pix_fmt: bool) -> None: """Add NVENC-specific encoding options.""" @@ -301,6 +362,28 @@ def _add_nvenc_options(self, command: list[str], force_pix_fmt: bool) -> None: if force_pix_fmt: command.extend(["-pix_fmt", "yuv420p"]) + def _add_libvpx_vp9_options(self, command: list[str], use_bit_rate: str | None, force_pix_fmt: bool) -> None: + """Add libvpx-vp9 (CPU) encoding options.""" + # Constant-quality mode when no explicit bitrate is requested. + # libvpx-vp9 requires `-b:v 0` to honor `-crf` exactly. + if use_bit_rate is None: + command.extend(["-b:v", "0", "-crf", "31"]) + command.extend( + [ + "-deadline", + "good", + "-cpu-used", + "4", + "-row-mt", + "1", + "-tile-columns", + "2", + ] + ) + + if force_pix_fmt: + command.extend(["-pix_fmt", "yuv420p"]) + def _add_output_options(self, command: list[str], clip: Clip, index: int) -> None: """Add output options to command.""" # Add encoder threads diff --git a/nemo_curator/stages/video/filtering/motion_vector_backend.py b/nemo_curator/stages/video/filtering/motion_vector_backend.py index 8db276f271..e99829e28f 100644 --- a/nemo_curator/stages/video/filtering/motion_vector_backend.py +++ b/nemo_curator/stages/video/filtering/motion_vector_backend.py @@ -181,6 +181,10 @@ def decode_for_motion( # noqa: C901 DecodedData object containing motion vectors and frame dimensions. """ + # PyAV's Python log callback can deadlock in Ray workers when FFmpeg decoder + # threads log during codec-context teardown. + av.logging.restore_default_callback() + with cast("av.container.InputContainer", av.open(video, metadata_errors="ignore")) as input_container: stream = input_container.streams.video[0] ctx = stream.codec_context diff --git a/nemo_curator/stages/video/io/video_reader.py b/nemo_curator/stages/video/io/video_reader.py index 9d5300fb5e..acff2456b1 100644 --- a/nemo_curator/stages/video/io/video_reader.py +++ b/nemo_curator/stages/video/io/video_reader.py @@ -25,6 +25,7 @@ from nemo_curator.tasks.file_group import FileGroupTask from nemo_curator.tasks.video import Video, VideoTask from nemo_curator.utils.client_utils import FSPath, is_remote_url +from nemo_curator.utils.decoder_utils import SoftwareCodecMissingError @dataclass @@ -171,6 +172,9 @@ def _extract_and_validate_metadata(self, video: Video) -> bool: """ try: video.populate_metadata() + except SoftwareCodecMissingError as e: + logger.error(f"Skipping {video.input_video}: software codec missing for this stage. {e}") + return False except Exception as e: # noqa: BLE001 logger.warning(f"Failed to extract metadata for {video.input_video}: {e}") return False diff --git a/nemo_curator/utils/decoder_utils.py b/nemo_curator/utils/decoder_utils.py index 3d3441f251..88286c888d 100644 --- a/nemo_curator/utils/decoder_utils.py +++ b/nemo_curator/utils/decoder_utils.py @@ -38,6 +38,61 @@ from nemo_curator.utils.operation_utils import make_pipeline_named_temporary_file +class SoftwareCodecMissingError(RuntimeError): + """Raised when ffprobe fails to open a stream because the FFmpeg build lacks + a software decoder for the input codec (for example, a strict Curator + FFmpeg install with NVDEC-only h264/hevc/av1 decoders). + + Carries the detected codec name (e.g. ``"h264"``) so callers can produce a + targeted, actionable message. + """ + + def __init__(self, message: str, codec: str | None = None) -> None: + super().__init__(message) + self.codec = codec + + +# MP4 sample-description FOURCCs for codecs that Curator's strict FFmpeg +# install can decode only via NVDEC. Used by the heuristic header sniff below. +_MP4_GPU_ONLY_CODEC_TAGS: dict[bytes, str] = { + b"avc1": "h264", + b"avc3": "h264", + b"hev1": "hevc", + b"hvc1": "hevc", + b"av01": "av1", +} + + +def _detect_codec_from_mp4_header(path: Path, *, scan_bytes: int = 1_048_576) -> str | None: + """Heuristically detect the video codec of an MP4/MOV file by scanning its + first ``scan_bytes`` for known sample-description FOURCC tags. + + This avoids invoking ffprobe (which is what's failing) and is intentionally + permissive: a substring match in the header is enough for diagnostic + purposes. Returns the codec name or ``None`` if no known tag was found or + the file could not be read. + """ + try: + with Path(path).open("rb") as fh: + head = fh.read(scan_bytes) + except OSError: + return None + for tag, codec in _MP4_GPU_ONLY_CODEC_TAGS.items(): + if tag in head: + return codec + return None + + +# Substrings in ffprobe's stderr that indicate the failure was a codec/CUDA +# initialization problem rather than e.g. a missing/corrupt file. +_CODEC_OPEN_FAILURE_SIGNALS: tuple[str, ...] = ( + "CUDA_ERROR_NO_DEVICE", + "no CUDA-capable device", + "Failed loading nvcuvid", + "Cannot load libnvcuvid", +) + + class Resolution(NamedTuple): """Container for video frame dimensions. @@ -112,7 +167,7 @@ def to_str(self) -> str: return f"{self.extraction_policy!s}-{int(self.target_fps * 1000)}" -def extract_video_metadata(video: str | bytes) -> VideoMetadata: +def extract_video_metadata(video: str | bytes) -> VideoMetadata: # noqa: C901, PLR0912 """Extract metadata from a video file using ffprobe. Args: @@ -142,7 +197,22 @@ def extract_video_metadata(video: str | bytes) -> VideoMetadata: error_msg = f"{real_video_path} not found!" raise FileNotFoundError(error_msg) cmd.append(real_video_path.as_posix()) - result = subprocess.run(cmd, input=inp, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) # noqa: UP022, S603 + try: + result = subprocess.run(cmd, input=inp, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) # noqa: UP022, S603 + except subprocess.CalledProcessError as e: + stderr = (e.stderr or b"").decode(errors="replace") + if any(signal in stderr for signal in _CODEC_OPEN_FAILURE_SIGNALS): + codec = _detect_codec_from_mp4_header(real_video_path) + msg = ( + f"ffprobe could not open the video codec for {real_video_path}" + f"{f' (detected {codec})' if codec else ''}. " + "The active ffprobe appears to use NVDEC-only h264/hevc/av1 decoders, " + "and no GPU is visible to this stage. To process h264/hevc/av1 inputs, " + "install full ffmpeg inside the container with:\n" + " bash /opt/Curator/docker/common/install_h264_support.sh" + ) + raise SoftwareCodecMissingError(msg, codec=codec) from e + raise video_info = json.loads(result.stdout) video_stream, audio_codec = None, None diff --git a/tests/stages/video/clipping/test_clip_transcoding_stage.py b/tests/stages/video/clipping/test_clip_transcoding_stage.py index dd494cd841..b1ff11b7fe 100644 --- a/tests/stages/video/clipping/test_clip_transcoding_stage.py +++ b/tests/stages/video/clipping/test_clip_transcoding_stage.py @@ -22,7 +22,6 @@ import pytest from nemo_curator.backends.base import WorkerMetadata -from nemo_curator.stages.resources import Resources from nemo_curator.stages.video.clipping.clip_extraction_stages import ClipTranscodingStage from nemo_curator.tasks.video import Clip, Video, VideoMetadata, VideoTask @@ -51,7 +50,7 @@ def setup_method(self) -> None: """Set up test fixtures.""" self.stage = ClipTranscodingStage( num_cpus_per_worker=4.0, - encoder="libx264", + encoder="h264_nvenc", encoder_threads=2, encode_batch_size=8, use_hwaccel=False, @@ -104,9 +103,71 @@ def test_setup_invalid_encoder(self) -> None: """Test setup with invalid encoder raises ValueError.""" stage = ClipTranscodingStage(encoder="invalid_encoder") - with pytest.raises(ValueError, match="Expected encoder of"): + with pytest.raises(ValueError, match="Expected encoder in"): stage.setup() + def test_setup_libvpx_vp9_valid(self) -> None: + """Test setup accepts libvpx-vp9 as a valid encoder.""" + stage = ClipTranscodingStage(encoder="libvpx-vp9", use_hwaccel=False) + # Should not raise + stage.setup() + + def test_setup_libvpx_vp9_with_hwaccel_raises(self) -> None: + """Test setup rejects libvpx-vp9 combined with use_hwaccel=True.""" + stage = ClipTranscodingStage(encoder="libvpx-vp9", use_hwaccel=True) + + with pytest.raises(ValueError, match="use_hwaccel is not supported with libvpx-vp9"): + stage.setup() + + @patch("nemo_curator.stages.video.clipping.clip_extraction_stages.subprocess.run") + @patch("nemo_curator.stages.video.clipping.clip_extraction_stages.shutil.which") + def test_setup_libopenh264_available_passes(self, mock_which: MagicMock, mock_run: MagicMock) -> None: + """libopenh264 is accepted when the local FFmpeg build advertises it.""" + mock_which.return_value = "/usr/local/bin/ffmpeg" + mock_run.return_value = MagicMock(stdout="V..... libopenh264 OpenH264 H.264", returncode=0) + stage = ClipTranscodingStage(encoder="libopenh264") + stage.setup() # should not raise + mock_run.assert_called_once() + # Verify the probe used `ffmpeg -hide_banner -encoders` with the resolved path + cmd = mock_run.call_args.args[0] + assert cmd[0] == "/usr/local/bin/ffmpeg" + assert "-encoders" in cmd + + @patch("nemo_curator.stages.video.clipping.clip_extraction_stages.subprocess.run") + @patch("nemo_curator.stages.video.clipping.clip_extraction_stages.shutil.which") + def test_setup_libopenh264_unavailable_raises(self, mock_which: MagicMock, mock_run: MagicMock) -> None: + """libopenh264 raises a clear error when the local FFmpeg build lacks it.""" + mock_which.return_value = "/usr/local/bin/ffmpeg" + mock_run.return_value = MagicMock(stdout="V..... h264_nvenc NVIDIA NVENC", returncode=0) + stage = ClipTranscodingStage(encoder="libopenh264") + with pytest.raises(RuntimeError, match=r"libopenh264.*does not include it"): + stage.setup() + + @patch("nemo_curator.stages.video.clipping.clip_extraction_stages.shutil.which") + def test_setup_libopenh264_ffmpeg_missing_raises(self, mock_which: MagicMock) -> None: + """A missing FFmpeg binary surfaces as a RuntimeError pointing to the docs.""" + mock_which.return_value = None + stage = ClipTranscodingStage(encoder="libopenh264") + with pytest.raises(RuntimeError, match=r"Could not find `ffmpeg` on PATH"): + stage.setup() + + def test_post_init_libvpx_vp9_emits_perf_warning(self) -> None: + """Constructing a libvpx-vp9 stage logs the perf advisory.""" + import nemo_curator.stages.video.clipping.clip_extraction_stages as ces + + with patch.object(ces, "logger") as mock_logger: + ClipTranscodingStage(encoder="libvpx-vp9") + mock_logger.warning.assert_called_once() + assert "libvpx-vp9 is significantly slower" in mock_logger.warning.call_args[0][0] + + def test_post_init_h264_nvenc_no_perf_warning(self) -> None: + """h264_nvenc construction does not trigger the VP9 perf advisory.""" + import nemo_curator.stages.video.clipping.clip_extraction_stages as ces + + with patch.object(ces, "logger") as mock_logger: + ClipTranscodingStage(encoder="h264_nvenc") + assert not any("libvpx-vp9 is significantly slower" in str(c) for c in mock_logger.warning.call_args_list) + def test_ray_stage_spec(self) -> None: """Test that ray_stage_spec returns the correct values.""" spec = self.stage.ray_stage_spec() @@ -117,14 +178,6 @@ def test_ray_stage_spec(self) -> None: assert RayStageSpecKeys.IS_FANOUT_STAGE in spec assert spec[RayStageSpecKeys.IS_FANOUT_STAGE] is True - def test_resources_cpu_encoder(self) -> None: - """Test resource requirements for CPU encoders.""" - stage = ClipTranscodingStage(encoder="libx264", use_hwaccel=False, num_cpus_per_worker=6.0) - - resources = stage.resources - assert isinstance(resources, Resources) - assert resources.cpus == 6.0 - def test_process_no_clips(self) -> None: """Test processing when video has no clips.""" self.mock_video.clips = [] @@ -304,6 +357,22 @@ def test_resources_fractional_gpu_allocation(self) -> None: stage_multi = ClipTranscodingStage(use_hwaccel=True, encoder="h264_nvenc", nb_streams_per_gpu=4) assert stage_multi.resources.gpus == 0.25 + def test_resources_libvpx_vp9_uses_cpu(self) -> None: + """Test that libvpx-vp9 allocates CPU resources, not GPU.""" + stage = ClipTranscodingStage(encoder="libvpx-vp9", use_hwaccel=False, num_cpus_per_worker=8.0) + assert stage.resources.cpus == 8.0 + assert stage.resources.gpus == 0 + + def test_add_hwaccel_options_libvpx_vp9_ignored(self) -> None: + """Test that hwaccel options are not added for libvpx-vp9 even if requested.""" + command: list[str] = [] + # Bypass setup-time validation by constructing without use_hwaccel, + # then assert the command builder is also defensive. + stage = ClipTranscodingStage(encoder="libvpx-vp9", use_hwaccel=False) + stage.use_hwaccel = True # simulate misconfiguration + stage._add_hwaccel_options(command) + assert "-hwaccel" not in command + def test_add_input_options(self) -> None: """Test adding input options to FFmpeg command.""" command = [] @@ -321,7 +390,7 @@ def test_add_input_options(self) -> None: assert "-map" in command assert "0:v:0" in command assert "-c:v" in command - assert "libx264" in command + assert "h264_nvenc" in command def test_add_video_encoding_options_no_bitrate(self) -> None: """Test adding video encoding options without bit rate.""" @@ -342,6 +411,50 @@ def test_add_video_encoding_options_with_bitrate(self) -> None: assert "-b:v" in command assert "5000K" in command + def test_add_video_encoding_options_libvpx_vp9_crf_mode(self) -> None: + """Test that libvpx-vp9 emits CRF-mode options when no bitrate is given.""" + stage = ClipTranscodingStage(encoder="libvpx-vp9") + command: list[str] = [] + + stage._add_video_encoding_options(command, None, False) + + # CRF mode: -b:v 0 plus -crf + assert "-b:v" in command + assert "0" in command + assert "-crf" in command + # VP9 threading/speed knobs + assert "-row-mt" in command + assert "-tile-columns" in command + assert "-deadline" in command + assert "-cpu-used" in command + # Must not contain NVENC-only flags + assert "-rc:v" not in command + assert "-cq:v" not in command + assert "-tune" not in command + + def test_add_video_encoding_options_libvpx_vp9_with_bitrate(self) -> None: + """Test that libvpx-vp9 honors an explicit bitrate (skips CRF mode).""" + stage = ClipTranscodingStage(encoder="libvpx-vp9") + command: list[str] = [] + + stage._add_video_encoding_options(command, "5000K", False) + + # Bitrate path: -b:v 5000K, no -crf + assert "5000K" in command + assert "-crf" not in command + # Threading/speed knobs still present + assert "-row-mt" in command + + def test_add_video_encoding_options_libvpx_vp9_force_pix_fmt(self) -> None: + """Test that libvpx-vp9 forces yuv420p when force_pix_fmt is True.""" + stage = ClipTranscodingStage(encoder="libvpx-vp9") + command: list[str] = [] + + stage._add_video_encoding_options(command, None, True) + + assert "-pix_fmt" in command + assert "yuv420p" in command + def test_add_output_options(self) -> None: """Test adding output options to FFmpeg command.""" command = [] @@ -580,14 +693,14 @@ def test_edge_case_empty_clips(self) -> None: class TestClipTranscodingStageRayDataResources: def test_cpu_path_sets_ray_data_num_cpus_to_1(self): - stage = ClipTranscodingStage() + stage = ClipTranscodingStage(encoder="libvpx-vp9") assert stage.ray_data_num_cpus == 1.0 assert stage.resources.cpus == stage.num_cpus_per_worker def test_cpu_path_ray_stage_spec_includes_ray_num_cpus(self): from nemo_curator.backends.utils import RayStageSpecKeys - stage = ClipTranscodingStage() + stage = ClipTranscodingStage(encoder="libvpx-vp9") spec = stage.ray_stage_spec() assert spec[RayStageSpecKeys.RAY_NUM_CPUS] == 1.0 assert spec[RayStageSpecKeys.IS_FANOUT_STAGE] is True diff --git a/tests/stages/video/filtering/test_motion_vector_backend.py b/tests/stages/video/filtering/test_motion_vector_backend.py index 46d8188d8b..66cfa7384b 100644 --- a/tests/stages/video/filtering/test_motion_vector_backend.py +++ b/tests/stages/video/filtering/test_motion_vector_backend.py @@ -211,7 +211,7 @@ def test_successful_decode(self): """Test successful decode operation.""" mock_video = io.BytesIO(b"mock_video_data") - with patch("av.open") as mock_open: + with patch("av.logging.restore_default_callback") as mock_restore_default_callback, patch("av.open") as mock_open: # Mock motion vector side data mock_side_data = Mock() mock_side_data.type = Mock() @@ -260,11 +260,14 @@ def test_successful_decode(self): mock_open.return_value = mock_container + # PyAV >= 15 ships av.sidedata.sidedata.Type.MOTION_VECTORS as a real enum + # member, so we no longer need (and cannot) patch it in. result = decode_for_motion(mock_video) assert isinstance(result, DecodedData) assert len(result.frames) == 1 assert result.frame_size == torch.Size([480, 640, 3]) + mock_restore_default_callback.assert_called_once_with() def test_no_motion_vectors(self): """Test decode with no motion vectors.""" @@ -355,6 +358,8 @@ def test_custom_parameters(self): mock_open.return_value = mock_container + # PyAV >= 15 ships av.sidedata.sidedata.Type.MOTION_VECTORS as a real enum + # member, so we no longer need (and cannot) patch it in. result = decode_for_motion(mock_video, thread_count=8, target_fps=5.0, target_duration_ratio=0.3) assert isinstance(result, DecodedData) diff --git a/tests/utils/test_decoder_utils.py b/tests/utils/test_decoder_utils.py index f8ccf0a8a0..1d2345a20c 100644 --- a/tests/utils/test_decoder_utils.py +++ b/tests/utils/test_decoder_utils.py @@ -17,6 +17,7 @@ import io import json import pathlib +import subprocess import tempfile from unittest.mock import Mock, patch @@ -27,7 +28,9 @@ FrameExtractionPolicy, FrameExtractionSignature, Resolution, + SoftwareCodecMissingError, VideoMetadata, + _detect_codec_from_mp4_header, _make_video_stream, decode_video_cpu, extract_frames, @@ -252,6 +255,114 @@ def test_extract_video_metadata_file_not_found(self) -> None: with pytest.raises(FileNotFoundError, match="not found"): extract_video_metadata(non_existent_path) + @pytest.mark.parametrize( + "stderr_signal", + [ + b"[CUDA @ 0x0] cu->cuInit(0) failed -> CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected", + b"[h264_cuvid] no CUDA-capable device is detected", + b"[h264_cuvid] Cannot load libnvcuvid.so.1", + b"[h264_cuvid] Failed loading nvcuvid.", + ], + ) + @patch("subprocess.run") + def test_extract_video_metadata_raises_software_codec_missing( + self, mock_subprocess: Mock, stderr_signal: bytes + ) -> None: + """When ffprobe fails because the codec/CUDA cannot be opened, raise + SoftwareCodecMissingError with a hint pointing at install_h264_support.sh.""" + mock_subprocess.side_effect = subprocess.CalledProcessError( + returncode=1, cmd=["ffprobe"], stderr=stderr_signal + ) + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp: + # Minimal mp4 header containing an avc1 (h264) sample-description tag + # so that _detect_codec_from_mp4_header reports "h264". + tmp.write(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2avc1mp41") + tmp.write(b"\x00" * 64) + tmp_path = tmp.name + try: + with pytest.raises(SoftwareCodecMissingError) as excinfo: + extract_video_metadata(tmp_path) + assert excinfo.value.codec == "h264" + assert "install_h264_support.sh" in str(excinfo.value) + finally: + pathlib.Path(tmp_path).unlink() + + @pytest.mark.parametrize( + "stderr_signal", + [ + # A generic "Could not open codec" without any CUDA signal must NOT + # be remapped — common cause is a corrupt file or unsupported codec + # profile, neither of which install_h264_support.sh fixes. + b"[vp9 @ 0x0] Could not open codec for input stream 0", + b"some other generic ffprobe failure", + b"Invalid data found when processing input", + ], + ) + @patch("subprocess.run") + def test_extract_video_metadata_reraises_unrelated_failure( + self, mock_subprocess: Mock, stderr_signal: bytes + ) -> None: + """ffprobe failures unrelated to NVDEC/CUDA must not be remapped.""" + mock_subprocess.side_effect = subprocess.CalledProcessError( + returncode=1, cmd=["ffprobe"], stderr=stderr_signal + ) + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp: + tmp_path = tmp.name + try: + with pytest.raises(subprocess.CalledProcessError): + extract_video_metadata(tmp_path) + finally: + pathlib.Path(tmp_path).unlink() + + +class TestSoftwareCodecMissingError: + """SoftwareCodecMissingError carries an actionable message and the detected codec.""" + + def test_inherits_runtime_error_and_carries_codec(self) -> None: + err = SoftwareCodecMissingError("oops", codec="hevc") + assert isinstance(err, RuntimeError) + assert err.codec == "hevc" + assert str(err) == "oops" + + def test_codec_defaults_to_none(self) -> None: + err = SoftwareCodecMissingError("oops") + assert err.codec is None + + +class TestDetectCodecFromMp4Header: + """_detect_codec_from_mp4_header is a heuristic FOURCC scan over the file head.""" + + @pytest.mark.parametrize( + ("tag", "expected"), + [ + (b"avc1", "h264"), + (b"avc3", "h264"), + (b"hev1", "hevc"), + (b"hvc1", "hevc"), + (b"av01", "av1"), + ], + ) + def test_detects_known_tags(self, tag: bytes, expected: str) -> None: + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp: + tmp.write(b"\x00" * 32 + tag + b"\x00" * 32) + tmp_path = pathlib.Path(tmp.name) + try: + assert _detect_codec_from_mp4_header(tmp_path) == expected + finally: + tmp_path.unlink() + + def test_returns_none_for_unknown_content(self) -> None: + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp: + tmp.write(b"plain bytes with no codec FOURCC") + tmp_path = pathlib.Path(tmp.name) + try: + assert _detect_codec_from_mp4_header(tmp_path) is None + finally: + tmp_path.unlink() + + def test_returns_none_for_unreadable_path(self) -> None: + assert _detect_codec_from_mp4_header(pathlib.Path("/path/that/does/not/exist.mp4")) is None + @patch("subprocess.run") @patch("nemo_curator.utils.decoder_utils.make_pipeline_named_temporary_file") def test_extract_video_metadata_no_video_stream(self, mock_temp_file: Mock, mock_subprocess: Mock) -> None: diff --git a/tutorials/video/getting-started/README.md b/tutorials/video/getting-started/README.md index c4a4f95a4f..bafdc6d89b 100644 --- a/tutorials/video/getting-started/README.md +++ b/tutorials/video/getting-started/README.md @@ -50,10 +50,10 @@ python video_split_clip_example.py \ --transnetv2-min-length-s 2.0 \ --transnetv2-max-length-s 10.0 \ --embedding-algorithm cosmos-embed1-224p \ - --transcode-encoder libopenh264 \ + --transcode-encoder h264_nvenc \ --verbose ``` -This example demonstrates a more advanced workflow than the minimal example by using scene-aware splitting with the TransNetV2 algorithm (which detects scene boundaries instead of fixed intervals), applies the Cosmos-Embed1 embedding model to each clip, transcodes the output using the `libopenh264` encoder, and enables verbose logging for more detailed output. +This example demonstrates a more advanced workflow than the minimal example by using scene-aware splitting with the TransNetV2 algorithm (which detects scene boundaries instead of fixed intervals), applies the Cosmos-Embed1 embedding model to each clip, transcodes the output using the `h264_nvenc` encoder, and enables verbose logging for more detailed output. On GPUs without NVENC (such as A100/H100), pass `--transcode-encoder libvpx-vp9` instead — VP9 is a royalty-free CPU encoder that produces clips in the same `.mp4` container. **Full pipeline with captions and filtering**: ```bash diff --git a/tutorials/video/getting-started/video_split_clip_example.py b/tutorials/video/getting-started/video_split_clip_example.py index 6d4e554da3..446d273b05 100644 --- a/tutorials/video/getting-started/video_split_clip_example.py +++ b/tutorials/video/getting-started/video_split_clip_example.py @@ -13,6 +13,10 @@ # limitations under the License. import argparse +import re +import shutil +import subprocess +import sys from nemo_curator.backends.xenna import XennaExecutor from nemo_curator.pipeline import Pipeline @@ -232,7 +236,57 @@ def create_video_splitting_pipeline(args: argparse.Namespace) -> Pipeline: # no return pipeline +# Encoders that produce h264 clip output. ClipWriter's metadata extraction +# runs ffprobe in a CPU-only Ray actor, so it needs a software h264 decoder +# (NVDEC-only h264 won't work without GPU visibility in that actor). +_H264_PRODUCING_ENCODERS = frozenset({"h264_nvenc", "libopenh264"}) + +# Matches the ` V..... h264 ` row in `ffmpeg -decoders`, excluding `h264_cuvid` etc. +_H264_SW_DECODER_LINE = re.compile(r"^\s+V\S*\s+h264\s") + + +def _h264_software_decoder_available() -> bool: + if shutil.which("ffmpeg") is None: + return False + try: + out = subprocess.run( + ["ffmpeg", "-hide_banner", "-decoders"], # noqa: S607 + check=True, + capture_output=True, + text=True, + timeout=10, + ).stdout + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError): + return False + return any(_H264_SW_DECODER_LINE.match(line) for line in out.splitlines()) + + +def _preflight_check_h264_decoder(encoder: str) -> None: + """Fail-fast if the chosen transcode encoder produces h264 but the system + ffmpeg lacks a software h264 decoder — ClipWriter would otherwise crash on + every transcoded clip in a CPU-only Ray actor. + """ + if encoder not in _H264_PRODUCING_ENCODERS: + return + if _h264_software_decoder_available(): + return + msg = ( + f"\nERROR: --transcode-encoder={encoder} produces h264 clips, but the " + "container's ffmpeg does not include a software h264 decoder.\n" + "ClipWriter's metadata extraction (ffprobe in a CPU-only Ray actor) " + "will fail on every transcoded clip.\n\n" + "Fix one of:\n" + " 1. Install software h264/hevc/av1 decoders inside the container:\n" + " bash /opt/Curator/docker/common/install_h264_support.sh\n" + " 2. Pick a transcode encoder whose output codec the system ffmpeg " + "can software-decode (e.g. --transcode-encoder libvpx-vp9).\n" + ) + print(msg, file=sys.stderr) + sys.exit(2) + + def main(args: argparse.Namespace) -> None: + _preflight_check_h264_decoder(args.transcode_encoder) pipeline = create_video_splitting_pipeline(args) # Print pipeline description @@ -380,9 +434,15 @@ def create_video_splitting_argparser() -> argparse.ArgumentParser: # noqa: PLR0 parser.add_argument( "--transcode-encoder", type=str, - default="libopenh264", - choices=["libopenh264", "h264_nvenc", "libx264"], - help="Codec for transcoding clips; None to skip transcoding.", + default="h264_nvenc", + choices=["h264_nvenc", "libvpx-vp9", "libopenh264"], + help=( + "Codec for transcoding clips. Use `h264_nvenc` on NVENC-equipped GPUs; " + "use `libvpx-vp9` (CPU) as a royalty-free fallback on GPUs without NVENC " + "such as A100/H100; `libopenh264` is accepted but requires a user-" + "installed FFmpeg build (Curator does not ship it — see the " + "Bring-Your-Own H.264 docs)." + ), ) parser.add_argument( "--transcode-encoder-threads",