Fix VAAPI H264 input surface format

Author: tces1

internal/transcode/ffmpeg_args_test.go

@@ -163,8 +163,9 @@ func TestBuildFFmpegArgsUsesFullVAAPITranscodePipeline(t *testing.T) {
 	if codecIndex < 0 || args[codecIndex+1] != "h264_vaapi" {
 		t.Fatalf("expected VAAPI H.264 encoder, args=%v", args)
 	}
-	if slices.Contains(args, "-vf") {
-		t.Fatalf("full VAAPI pipeline should not scale, args=%v", args)
+	vfIndex := slices.Index(args, "-vf")
+	if vfIndex < 0 || args[vfIndex+1] != "scale_vaapi=format=nv12" {
+		t.Fatalf("full VAAPI pipeline should convert surfaces to NV12 with VAAPI, args=%v", args)
 	}
 	for _, softwareOnly := range []string{"libx264", "-preset", "-tune", "-level"} {
 		if slices.Contains(args, softwareOnly) {

internal/transcode/manager.go

@@ -1248,6 +1248,7 @@ func appendVideoTranscodeArgs(args []string, options FFmpegOptions) []string {
 	switch hardwarePipeline(options) {
 	case "vaapi-full":
 		return append(args,
+			"-vf", vaapiFormatFilter(),
 			"-c:v", "h264_vaapi",
 			"-low_power", "1",
 			"-g", strconv.Itoa(lowLatencyGOP),
@@ -1286,6 +1287,10 @@ func vaapiScaleFilter(width, height int) string {
 	return fmt.Sprintf("scale_vaapi=w=%d:h=%d:force_original_aspect_ratio=decrease:force_divisible_by=2:format=nv12", width, height)
 }
 
+func vaapiFormatFilter() string {
+	return "scale_vaapi=format=nv12"
+}
+
 func softwareScaleFilter(width, height int) string {
 	return fmt.Sprintf("scale=w=%d:h=%d:force_original_aspect_ratio=decrease:force_divisible_by=2", width, height)
 }