Probe VAAPI device initialization

Author: tces1

internal/transcode/ffmpeg_args_test.go

@@ -2,8 +2,11 @@ package transcode
 
 import (
 	"errors"
+	"fmt"
+	"os"
 	"path/filepath"
 	"slices"
+	"strings"
 	"testing"
 )
 
@@ -172,6 +175,52 @@ func TestResolveHardwareDecodeFallsBackToSoftwareWhenProbeFails(t *testing.T) {
 	}
 }
 
+func TestDefaultHardwareProbeRejectsVAAPIWhenDeviceInitializationFails(t *testing.T) {
+	tempDir := t.TempDir()
+	ffmpegPath := filepath.Join(tempDir, "ffmpeg")
+	callsPath := filepath.Join(tempDir, "calls")
+	script := fmt.Sprintf(`#!/bin/sh
+echo "$@" >> %q
+case " $* " in
+  *" -hwaccels "*)
+    printf 'Hardware acceleration methods:\nvaapi\n'
+    exit 0
+    ;;
+  *" -init_hw_device "*)
+    echo 'Failed to initialise VAAPI connection' >&2
+    exit 1
+    ;;
+esac
+exit 0
+`, callsPath)
+	if err := os.WriteFile(ffmpegPath, []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	devicePath := filepath.Join(tempDir, "renderD128")
+	if err := os.WriteFile(devicePath, nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	err := defaultHardwareProbe(ffmpegPath, FFmpegOptions{
+		HardwareDecode: "vaapi",
+		HardwareDevice: devicePath,
+	})
+
+	if err == nil {
+		t.Fatal("expected VAAPI device initialization failure")
+	}
+	if !strings.Contains(err.Error(), "vaapi device init probe failed") {
+		t.Fatalf("error = %v", err)
+	}
+	calls, err := os.ReadFile(callsPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(calls), "-init_hw_device vaapi=probe:"+devicePath) {
+		t.Fatalf("ffmpeg calls should initialize the configured VAAPI device, calls=%s", calls)
+	}
+}
+
 func TestNewManagerFallsBackToSoftwareDecodeWhenHardwareProbeFails(t *testing.T) {
 	manager := NewManager(Options{
 		FFmpegPath:     "/usr/bin/ffmpeg",

internal/transcode/manager.go

@@ -751,7 +751,10 @@ func defaultHardwareProbe(ffmpegPath string, options FFmpegOptions) error {
 		if err := probeFFmpegHWAccel(ffmpegPath, "vaapi"); err != nil {
 			return err
 		}
-		return probeDevice(options.HardwareDevice)
+		if err := probeDevice(options.HardwareDevice); err != nil {
+			return err
+		}
+		return probeFFmpegVAAPIDeviceInit(ffmpegPath, options.HardwareDevice)
 	default:
 		return fmt.Errorf("unsupported hardware decode mode %q", options.HardwareDecode)
 	}
@@ -776,6 +779,32 @@ func probeFFmpegHWAccel(ffmpegPath, name string) error {
 	return fmt.Errorf("ffmpeg does not list %s hwaccel", name)
 }
 
+func probeFFmpegVAAPIDeviceInit(ffmpegPath, device string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	output, err := exec.CommandContext(ctx, ffmpegPath,
+		"-hide_banner",
+		"-loglevel", "error",
+		"-init_hw_device", "vaapi=probe:"+device,
+		"-f", "lavfi",
+		"-i", "nullsrc=s=16x16:d=0.1",
+		"-frames:v", "1",
+		"-f", "null",
+		"-",
+	).CombinedOutput()
+	if ctx.Err() != nil {
+		return fmt.Errorf("vaapi device init probe timed out")
+	}
+	if err != nil {
+		if message := strings.TrimSpace(string(output)); message != "" {
+			return fmt.Errorf("vaapi device init probe failed: %w: %s", err, message)
+		}
+		return fmt.Errorf("vaapi device init probe failed: %w", err)
+	}
+	return nil
+}
+
 func probeDevice(device string) error {
 	if strings.TrimSpace(device) == "" {
 		return errors.New("hardware device is required")