ffmpeg v8.1 and OpenVINO 2026.1 upgrade

README.md

@@ -21,6 +21,8 @@
     - [Why is iVSR needed](#12-why-is-ivsr-needed)
     - [iVSR Components](#13-ivsr-components)
     - [Capabilities of iVSR](#14-capabilities-of-ivsr)
+        - [Video Super Resolution (VSR)](#141-video-super-resolution-vsr)
+        - [Smart Video Processing (SVP)](#142-smart-video-processing-svp)
 2. [Setup iVSR env on linux](#2-setup-ivsr-env-on-linux)
     - [Install GPU kernel packages(Optional)](#21-optional-install-gpu-kernel-packages)
     - [Install dependencies and build iVSR manually](#22-install-dependencies-and-build-ivsr-manually)
@@ -60,7 +62,7 @@ For a detailed introduction to the iVSR SDK API, please refer to [this introduct
 We've also included a `vsr_sample` as a demonstration of its usage.
 
 In order to support the widely-used media processing solution FFmpeg, we've provided an iVSR SDK plugin to simplify integration.<br>
-This plugin is integrated into FFmpeg's `dnn_processing` filter in the [FFmpeg documentation](https://ffmpeg.org/ffmpeg-filters.html#dnn_005fprocessing-1) in the libavfilter library, serving as a new `ivsr` backend to this filter. Please note that the patches provided in this project are specifically for FFmpeg n7.1.<br>
+This plugin is integrated into FFmpeg's `dnn_processing` filter in the [FFmpeg documentation](https://ffmpeg.org/ffmpeg-filters.html#dnn_005fprocessing-1) in the libavfilter library, serving as a new `ivsr` backend to this filter. The patches provided in this project target **FFmpeg n8.1**.<br>
 
 ### 1.3.3 OpenVINO patches and extension
 In [this folder](./ivsr_ov/based_on_openvino_2022.3/patches), you'll find patches for OpenVINO that enable the Enhanced BasicVSR model. These patches utilize OpenVINO's [Custom OpenVINO™ Operations](https://docs.openvino.ai/2024/documentation/openvino-extensibility/custom-openvino-operations.html) feature, which allows users to support models with custom operations not inherently supported by OpenVINO.<br>
@@ -129,7 +131,7 @@ The software was validated on:
 - Host OS: Linux-based OS (Ubuntu 22.04 or Rocky Linux 9.3)
 - Docker-based OS: Ubuntu 22.04 or Rocky Linux 9.3
 - OpenVINO: [2022.3](https://github.com/openvinotoolkit/openvino/tree/2022.3.0), [2023.2](https://github.com/openvinotoolkit/openvino/tree/2023.2.0), or [2024.5](https://github.com/openvinotoolkit/openvino/tree/2024.5.0)
-- FFmpeg: [n7.1](https://github.com/FFmpeg/FFmpeg/tree/n7.1)
+- FFmpeg: [n8.1](https://github.com/FFmpeg/FFmpeg/tree/n8.1)
 
 Building iVSR requires the installation of the GPU driver (optional), OpenCV, OpenVINO, and FFmpeg.  
 We provide **three** ways to install requirements and build iVSR SDK & iVSR FFmpeg plugin:<br>

build.sh

@@ -6,20 +6,20 @@
 set -e
 
 base_dir=$(pwd)
-ov_version=2022.3
+ov_version=2026.1
 
 # Function to display usage information
 usage() {
-    echo "Usage: $0  --ov_version [2022.3|2023.2|2024.5]"
+    echo "Usage: $0  --ov_version [2022.3|2024.5|2026.1]"
     exit 1
 }
 
 prepare_dependencies() {
   echo "Preparing dependencies..."
   sudo apt-get install -y --no-install-recommends \
     curl ca-certificates gpg-agent software-properties-common
-  
-  sudo apt-get install -y --no-install-recommends --fix-missing \
+
+  sudo apt-get install -y --no-install-recommends \
     autoconf \
     automake \
     build-essential \
@@ -52,10 +52,14 @@ install_openvino_from_source() {
   git submodule update --init --recursive
 
   ## applying ov22.3 patches to enable Enhanced BasicVSR model
-  for patch_file in $(find ../patches -iname "*.patch" | sort -n);do
-      echo "Applying: ${patch_file}"
-      git am --whitespace=fix ${patch_file}
-  done
+  if git tag --points-at HEAD 2>/dev/null | grep -q "^${ov_branch}$"; then
+    for patch_file in $(find ../patches -iname "*.patch" | sort -n);do
+        echo "Applying: ${patch_file}"
+        git am --whitespace=fix ${patch_file}
+    done
+  else
+    echo "OpenVINO patches already applied, skipping..."
+  fi
 
   mkdir -p build && cd build && \
   cmake \
@@ -95,8 +99,10 @@ build_install_ivsr_sdk() {
 
   ivsr_sdk_dir=${base_dir}/ivsr_sdk/
   cd ${ivsr_sdk_dir}
+  rm -rf build
   mkdir -p build && cd build && cmake \
     -DENABLE_LOG=OFF -DENABLE_PERF=OFF -DENABLE_THREADPROCESS=ON \
+    -DENABLE_IRGUARD=${enable_irguard} \
     -DCMAKE_BUILD_TYPE=Release .. && \
   make -j $(nproc --all)
   sudo make install
@@ -109,20 +115,51 @@ build_ffmpeg() {
     DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \
     ca-certificates tar g++ wget pkg-config nasm yasm libglib2.0-dev flex bison gobject-introspection libgirepository1.0-dev \
     python3-dev libx11-dev libxv-dev libxt-dev libasound2-dev libpango1.0-dev libtheora-dev libvisual-0.4-dev libgl1-mesa-dev \
-    libcurl4-gnutls-dev librtmp-dev libx264-dev libx265-dev libde265-dev libva-dev libtbb-dev
-
-  # Add commands to build FFMPEG from source
+    libcurl4-gnutls-dev librtmp-dev libx264-dev libx265-dev libde265-dev libva-dev libtbb-dev \
+    patchutils
+
   ffmpeg_dir=$base_dir/ivsr_ffmpeg_plugin/ffmpeg
+  ffmpeg_tag=n8.1
   ffmpeg_repo=https://github.com/FFmpeg/FFmpeg.git
 
-  if [ ! -d "${ffmpeg_dir}" ]; then
-    git clone --depth 1 --branch n7.1 ${ffmpeg_repo} ${ffmpeg_dir}
+  if [ ! -d "${ffmpeg_dir}/.git" ]; then
+    git clone --depth 1 --branch ${ffmpeg_tag} ${ffmpeg_repo} ${ffmpeg_dir}
     git config --global --add safe.directory ${ffmpeg_dir}
   fi
 
-  # Apply patches
-  cd ${ffmpeg_dir} && cp -rf $base_dir/ivsr_ffmpeg_plugin/patches/*.patch .
-  git am --whitespace=fix *.patch
+  cd ${ffmpeg_dir}
+  git am --abort 2>/dev/null || true
+  # Ensure the target tag is locally reachable (handles a prior clone at a different tag)
+  if ! git rev-parse "${ffmpeg_tag}" >/dev/null 2>&1; then
+    git fetch --depth 1 origin "refs/tags/${ffmpeg_tag}:refs/tags/${ffmpeg_tag}"
+  fi
+  git checkout -f "${ffmpeg_tag}"
+
+  # ---------------------------------------------------------------
+  # Apply all iVSR patches for n8.1.
+  # Patch 0001: strip configure, dnn_interface.c, and swscale_unscaled.c
+  #   hunks — those three files are fully covered by patch 0004.
+  # Patch 0004: n8.1-native patch for configure, dnn_interface.c and
+  #   swscale_unscaled.c (exact n8.1 context; no sed required).
+  # Patches 0002/0003: fix-ups for dnn_backend_ivsr.c (added by 0001).
+  # ---------------------------------------------------------------
+  rm -f *.patch
+  cp "$base_dir/ivsr_ffmpeg_plugin/patches/"*.patch .
+
+  filterdiff \
+    -x '*/configure' \
+    -x '*/dnn_interface.c' \
+    -x '*/swscale_unscaled.c' \
+    0001-*.patch | \
+    git apply --3way --ignore-whitespace -
+
+  git apply --ignore-whitespace 0004-*.patch
+
+  # Stage new files added by 0001 so 3-way merge can resolve them
+  git add -A
+
+  git apply --3way --whitespace=fix 0002-*.patch
+  git apply --3way --whitespace=fix 0003-*.patch
 
   ./configure \
       --enable-gpl \
@@ -143,14 +180,33 @@ build_ffmpeg() {
 install_openvino_from_apt() {
   echo "Installing OpenVINO from apt..."
   local version=$1
+  local key_url=https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+  local keyring=/etc/apt/trusted.gpg.d/intel.gpg
 
-  wget -qO - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | sudo apt-key add -
+  # Download GPG key with timeout; fail loudly if it doesn't work
+  if ! wget --timeout=30 --tries=3 -O /tmp/intel-sw-products.pub "${key_url}"; then
+    echo "ERROR: Failed to download Intel GPG key from ${key_url}" >&2
+    exit 1
+  fi
+  sudo gpg --output "${keyring}" --dearmor /tmp/intel-sw-products.pub
+  rm -f /tmp/intel-sw-products.pub
 
-  echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu22 main" | sudo tee /etc/apt/sources.list.d/intel-openvino-2023.list
-  echo "deb https://apt.repos.intel.com/openvino/2024 ubuntu22 main" | sudo tee /etc/apt/sources.list.d/intel-openvino-2024.list
+  if [ ! -s "${keyring}" ]; then
+    echo "ERROR: Intel GPG keyring is empty after dearmor step." >&2
+    exit 1
+  fi
+
+  # Unified repo (all versions from 2024 onwards live here; no year in path)
+  # Detect Ubuntu codename suffix expected by the repo (ubuntu22 / ubuntu24)
+  local ubuntu_major
+  ubuntu_major=$(lsb_release -rs | cut -d. -f1)
+  local dist="ubuntu${ubuntu_major}"
+
+  echo "deb https://apt.repos.intel.com/openvino ${dist} main" \
+    | sudo tee /etc/apt/sources.list.d/intel-openvino.list
 
   sudo -E apt-get update && \
-    DEBIAN_FRONTEND=noninteractive sudo -E apt-get install -y openvino-$version.0
+    DEBIAN_FRONTEND=noninteractive sudo -E apt-get install -y openvino-${version}.0
 }
 
 main() {
@@ -165,6 +221,12 @@ main() {
     shift
   done
 
+  # irguard model protection is required for OV 2022.3 and 2024.5
+  case "${ov_version}" in
+    2022.3|2024.5) enable_irguard=ON ;;
+    *)             enable_irguard=OFF ;;
+  esac
+
   prepare_dependencies
   config_git_users
   if [ "$ov_version" = "2022.3" ]; then

docs/docker_image_build.md

@@ -20,15 +20,44 @@ sudo systemctl restart docker
 
 ```bash
 cd ./ivsr_ffmpeg_plugin
-./build_docker.sh --enable_ov_patch [true|false] --ov_version [2022.3|2023.2|2024.5|2024.5s] --os_version [rockylinux9|ubuntu22]
+./build_docker.sh --enable_ov_patch [true|false] --ov_version [2022.3|2023.2|2024.5|2024.5s|2026.1] --os_version [rockylinux9|ubuntu22]
 ```
-- `enable_ov_patch`: Set as `true` or `false` to enable or disable the application of OpenVINO 2022.3 patches, which are needed to support the Enhanced BasicVSR model.
-- `ov_version`: Set the OpenVINO version to `2022.3`, `2023.2`, `2024.5`, or `2024.5s`. These versions will be built and installed. The `2024.5s` version means installing OpenVINO 2024.5 via apt or yum, not building and installing from source code. iVSR currently supports OpenVINO 2022.3, 2023.2, and 2024.5. However, the patches to enable the Enhanced BasicVSR model are only for OpenVINO 2022.3.
-- `os_version`: Set OS version of Docker image to `ubuntu22` (Ubuntu 22.04) or `rockylinux9` (Rocky Linux 9.3) to build docker image based on specific OS.
 
-If the docker image builds successfully, you can see a docker image named `ffmpeg_ivsr_sdk_${os_version}_ov${ov_version}` such as `ffmpeg_ivsr_sdk_ubuntu22_ov2022.3` or `ffmpeg_ivsr_sdk_rockylinux9_ov2022.3` in the output of `docker image ls`.
+- `enable_ov_patch`: Set to `true` or `false` to enable or disable OpenVINO 2022.3 patches, which are needed to support the Enhanced BasicVSR model. Ignored for all versions other than `2022.3`.
+- `ov_version`: OpenVINO version to install. `2022.3`, `2023.2`, and `2024.5` are built from source; `2024.5s` and `2026.1` install via apt. The patches to enable the Enhanced BasicVSR model are only available for `2022.3`.
+- `os_version`: Base OS — `ubuntu22` (Ubuntu 22.04) or `rockylinux9` (Rocky Linux 9.3).
 
-### 4. Start Docker Container
+If the docker image builds successfully, you can see a docker image named `ffmpeg_ivsr_sdk_${os_version}_ov${ov_version}` in the output of `docker image ls`:
+
+```bash
+docker image ls | grep ffmpeg_ivsr_sdk
+```
+
+For example:
+
+```
+ffmpeg_ivsr_sdk_ubuntu22_ov2026.1    latest   ...
+ffmpeg_ivsr_sdk_ubuntu22_ov2024.5s   latest   ...
+ffmpeg_ivsr_sdk_rockylinux9_ov2022.3 latest   ...
+```
+
+### 4. Sanity-check the image (no model required)
+
+Verify the `ivsr` backend is registered in the built FFmpeg:
+
+```bash
+docker run --rm ffmpeg_ivsr_sdk_<os_version>_ov<ov_version>:latest \
+    ffmpeg -filters 2>&1 | grep ivsr
+```
+
+Verify the installed OpenVINO version:
+
+```bash
+docker run --rm ffmpeg_ivsr_sdk_<os_version>_ov<ov_version>:latest \
+    python3 -c "import openvino; print(openvino.__version__)"
+```
+
+### 5. Start Docker Container
 
 ```bash
 # The backslash at the end of the line indicates that the command continues on the next line
@@ -39,10 +68,46 @@ sudo docker run -itd --name ffmpeg_ivsr_sdk_container --privileged \
   -e no_proxy=$no_proxy \
   --shm-size=128g \
   --device=/dev/dri:/dev/dri \
-  ffmpeg_ivsr_sdk_<os_version>_<ov_version>:latest bash
+  ffmpeg_ivsr_sdk_<os_version>_ov<ov_version>:latest bash
 
 # Open another shell terminal to interact with the running container
 sudo docker exec -it ffmpeg_ivsr_sdk_container bash
 ```
 
-Note `--device=/dev/dri:/dev/dri` is specified in the command to add the host gpu device to container.<br>
+`--device=/dev/dri:/dev/dri` passes the host GPU through to the container for GPU inference (`device=GPU`).
+
+### 6. Run inference
+
+Mount your model directory and a test video, then run the desired model. The examples below use `docker run --rm` for a one-shot test; replace the image tag, model path, and video path as needed.
+
+**Enhanced EDSR (2× upscale, `rgb24` input, `model_type=2`):**
+
+```bash
+docker run --rm \
+  -v /path/to/models:/models:ro \
+  -v /path/to/input.mp4:/input.mp4:ro \
+  -v /path/to/output:/output \
+  ffmpeg_ivsr_sdk_<os_version>_ov<ov_version>:latest \
+  ffmpeg -hide_banner -y -i /input.mp4 \
+    -vf 'format=rgb24,dnn_processing=dnn_backend=ivsr:model=/models/<edsr_model.xml>:input=input:output=output:nif=1:device=CPU:model_type=2:normalize_factor=255.0' \
+    -pix_fmt yuv420p /output/edsr_out.mp4
+```
+
+Expected: output resolution is 2× the input (e.g. 1280×720 → 2560×1440).
+
+**SVP-Basic Y-channel (pre-filter / bitrate reduction, `yuv420p` input, `model_type=1`):**
+
+```bash
+docker run --rm \
+  -v /path/to/models:/models:ro \
+  -v /path/to/input.mp4:/input.mp4:ro \
+  -v /path/to/output:/output \
+  ffmpeg_ivsr_sdk_<os_version>_ov<ov_version>:latest \
+  ffmpeg -hide_banner -y -i /input.mp4 \
+    -vf 'format=yuv420p,dnn_processing=dnn_backend=ivsr:model=/models/<svp_model.xml>:input=input:output=output:nif=1:device=CPU:model_type=1' \
+    -pix_fmt yuv420p /output/svp_out.mp4
+```
+
+Expected: output resolution matches input; bitrate is reduced compared to re-encoding without SVP.
+
+For additional FFmpeg filter options and command-line examples for all supported models, see the [FFmpeg plugin README](../ivsr_ffmpeg_plugin/README.md).

ivsr_ffmpeg_plugin/README.md

@@ -1,11 +1,15 @@
 # iVSR FFmpeg plugin - iVSR SDK based
 The folder `ivsr_ffmpeg_plugin` enables model inference using FFmpeg with iVSR SDK as backend. It provides additional `ivsr` backend for the DNN interface called by the `dnn_processing` filter.<br>
-The patches included in `patches` folder are specifically for FFmpeg n7.1.<br>
+The patches included in `patches` folder are specifically for FFmpeg n8.1.<br>
 
 <div align=center>
 <img src="./figs/ffmpeg_ivsr_sdk_backend.png" width = 80% height = 80% />
 </div>
 
+## Docker image build and test
+
+For full instructions on building the Docker image, see [docs/docker_image_build.md](../docs/docker_image_build.md).
+
 ## How to run inference with FFmpeg-plugin
 To run inference with iVSR SDK, you need to specify `ivsr` as the backend for the `dnn_processing` filter. Here is an example of how to do it: `dnn_processing=dnn_backend=ivsr`. <br>
 Additionally, there are other parameters that you can use. These parameters are listed in the table below:<br>
@@ -18,12 +22,12 @@ Additionally, there are other parameters that you can use. These parameters are
 |output|output name of the model|NULL|output|
 |device|device for inference task|CPU|CPU or GPU|
 |model_type|type for models|0|0 for Enhanced BasicVSR, 1 for SVP models, 2 for Enhanced EDSR, 3 for one CUSTOM VSR, 4 for TSENet|
-|normalize_factor|factor for normalization|1.0|255.0 for Enhanced EDSR, 1.0 for other models supported in current version|
-|num_streams|number of execution streams for the throughput mode (now valid only for GPU devices).|1|use `benchmark_app` (a tool provided by OpenVINO Toolkit), to get the appropriate value for the best throughput|
-|extension|extension lib file full path, required for loading Enhanced BasicVSR model|
-|op_xml|custom op xml file full path, required for loading Enhanced BasicVSR model|
-|nif|number of input frames in batch sent to the DNN backend|1|3 for Enhanced BasicVSR, 1 for other models supported in current version|
-|nireq|number of request|0|use the default setting or set it to match the number of cpu cores|
+|normalize_factor|normalizing factor for models that do not require input normalization to [0, 1]|1.0|255.0 for Enhanced EDSR, 1.0 for all other models|
+|num_streams|number of execution streams for throughput mode (valid only for GPU devices)|1|use `benchmark_app` to determine the best value|
+|extension|extension lib file full path, required for Enhanced BasicVSR|—|—|
+|op_xml|custom op xml file full path, required for Enhanced BasicVSR|—|—|
+|nif|number of input frames in batch sent to the DNN backend|1|3 for Enhanced BasicVSR, 1 for other models|
+|nireq|number of infer requests|0|leave as default or set to match CPU core count|
 
 Here are some examples of FFmpeg command lines to run inference with the supported models using the `ivsr` backend.<br>
 

ivsr_ffmpeg_plugin/dockerfiles/rockylinux9/Dockerfile

@@ -167,7 +167,7 @@ RUN cmake .. \
 #build ffmpeg with iVSR SDK backend
 RUN dnf -y --enablerepo=crb install nasm
 RUN dnf -y --enablerepo=devel install yasm
-RUN dnf -y install diffutils
+RUN dnf -y install diffutils patchutils
 
 # build libx264
 WORKDIR ${WORKSPACE}
@@ -195,16 +195,23 @@ ARG FFMPEG_IVSR_SDK_PLUGIN_DIR=${IVSR_DIR}/ivsr_ffmpeg_plugin
 ARG FFMPEG_DIR=${FFMPEG_IVSR_SDK_PLUGIN_DIR}/ffmpeg
 
 ARG FFMPEG_REPO=https://github.com/FFmpeg/FFmpeg.git
-ARG FFMPEG_VERSION=n7.1
+ARG FFMPEG_VERSION=n8.1
 WORKDIR ${FFMPEG_DIR}
 RUN git clone ${FFMPEG_REPO} ${FFMPEG_DIR} && \
     git checkout ${FFMPEG_VERSION}
 COPY ./ivsr_ffmpeg_plugin/patches/*.patch ${FFMPEG_DIR}/
-RUN { set -e; \
-  for patch_file in $(find -iname "*.patch" | sort -n); do \
-    echo "Applying: ${patch_file}"; \
-    git am --whitespace=fix ${patch_file}; \
-  done; }
+# Patch 0001 was written for n7.1 context; exclude the three files covered by
+# the n8.1-native patch 0004, then apply 0004 for those files, and apply 0002/0003.
+RUN filterdiff \
+        -x '*/configure' \
+        -x '*/dnn_interface.c' \
+        -x '*/swscale_unscaled.c' \
+        0001-*.patch | \
+        git apply --3way --ignore-whitespace - && \
+    git apply --ignore-whitespace 0004-*.patch && \
+    git add -A && \
+    git apply --3way --whitespace=fix 0002-*.patch && \
+    git apply --3way --whitespace=fix 0003-*.patch
 
 RUN ./configure \
 --extra-cflags=-fopenmp \