Skip to content

Commit 7a6052f

Browse files
ddennedyddennedy
committed
Add hwaccel scale filter with preview scaling
1 parent 5646c2f commit 7a6052f

2 files changed

Lines changed: 276 additions & 13 deletions

File tree

‎src/framework/mlt_consumer.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -694,6 +694,9 @@ mlt_frame mlt_consumer_get_frame(mlt_consumer self)
694694
mlt_properties_set(frame_properties,
695695
"consumer.color_range",
696696
mlt_properties_get(properties, "color_range"));
697+
mlt_properties_set(frame_properties,
698+
"consumer.scale",
699+
mlt_properties_get(properties, "scale"));
697700

698701
if (mlt_properties_get(properties, "mlt_color_trc")) {
699702
// Add a normalize filter to convert the mlt_color_trc to color_trc

‎src/modules/avformat/producer_avformat.c‎

Lines changed: 273 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ struct producer_avformat_s
143143
int device_type;
144144
char device[128];
145145
AVBufferRef *device_ctx;
146+
int filters_initialized;
146147
} hwaccel;
147148
};
148149
typedef struct producer_avformat_s *producer_avformat;
@@ -819,6 +820,7 @@ static mlt_image_format pick_image_format(enum AVPixelFormat pix_fmt,
819820
case AV_PIX_FMT_YUV420P:
820821
case AV_PIX_FMT_YUVJ420P:
821822
case AV_PIX_FMT_YUVA420P:
823+
case AV_PIX_FMT_NV12:
822824
return mlt_image_yuv420p;
823825
case AV_PIX_FMT_RGB24:
824826
case AV_PIX_FMT_BGR24:
@@ -830,6 +832,7 @@ static mlt_image_format pick_image_format(enum AVPixelFormat pix_fmt,
830832
case AV_PIX_FMT_BAYER_RGGB16LE:
831833
return mlt_image_rgb;
832834
case AV_PIX_FMT_YUV420P10LE:
835+
case AV_PIX_FMT_P010LE:
833836
return mlt_image_yuv420p10;
834837
case AV_PIX_FMT_YUV422P10LE:
835838
case AV_PIX_FMT_YUV444P10LE:
@@ -946,6 +949,211 @@ static int get_basic_info(producer_avformat self, mlt_profile profile, const cha
946949
return error;
947950
}
948951

952+
static int setup_hwaccel_filters(producer_avformat self,
953+
mlt_producer producer,
954+
const char *filter_name,
955+
double scale_factor)
956+
{
957+
mlt_properties properties = MLT_PRODUCER_PROPERTIES(producer);
958+
AVStream *stream = self->video_format->streams[self->video_index];
959+
AVRational frame_rate = guess_frame_rate(self, stream);
960+
961+
int scaled_width = (int) (self->video_frame->width * scale_factor);
962+
int scaled_height = (int) (self->video_frame->height * scale_factor);
963+
char scale_args[32];
964+
scaled_width += scaled_width % 2;
965+
scaled_height += scaled_height % 2;
966+
snprintf(scale_args, sizeof(scale_args), "w=%d:h=%d", scaled_width, scaled_height);
967+
968+
mlt_log_verbose(MLT_PRODUCER_SERVICE(producer),
969+
"Attempting to set up %s filter: %dx%d -> %dx%d\n",
970+
filter_name,
971+
self->video_frame->width,
972+
self->video_frame->height,
973+
scaled_width,
974+
scaled_height);
975+
976+
self->vfilter_graph = avfilter_graph_alloc();
977+
if (!self->vfilter_graph) {
978+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to allocate filter graph\n");
979+
return -1;
980+
}
981+
982+
// Create buffer source with hardware frames context
983+
const AVFilter *buffersrc = avfilter_get_by_name("buffer");
984+
self->vfilter_in = avfilter_graph_alloc_filter(self->vfilter_graph, buffersrc, "hw_buffer");
985+
if (!self->vfilter_in) {
986+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to allocate buffer source\n");
987+
avfilter_graph_free(&self->vfilter_graph);
988+
return -1;
989+
}
990+
991+
// Set options on the filter context
992+
int ret = av_opt_set_int(self->vfilter_in,
993+
"width",
994+
self->video_frame->width,
995+
AV_OPT_SEARCH_CHILDREN);
996+
if (ret >= 0)
997+
ret = av_opt_set_int(self->vfilter_in,
998+
"height",
999+
self->video_frame->height,
1000+
AV_OPT_SEARCH_CHILDREN);
1001+
if (ret >= 0)
1002+
ret = av_opt_set_pixel_fmt(self->vfilter_in,
1003+
"pix_fmt",
1004+
self->video_frame->format,
1005+
AV_OPT_SEARCH_CHILDREN);
1006+
if (ret >= 0)
1007+
ret = av_opt_set_q(self->vfilter_in, "time_base", stream->time_base, AV_OPT_SEARCH_CHILDREN);
1008+
if (ret >= 0)
1009+
ret = av_opt_set_q(self->vfilter_in, "frame_rate", frame_rate, AV_OPT_SEARCH_CHILDREN);
1010+
1011+
AVRational sar;
1012+
sar.num = mlt_properties_get_int(properties, "meta.media.sample_aspect_num");
1013+
sar.den = FFMAX(mlt_properties_get_int(properties, "meta.media.sample_aspect_den"), 1);
1014+
if (ret >= 0)
1015+
ret = av_opt_set_q(self->vfilter_in, "sar", sar, AV_OPT_SEARCH_CHILDREN);
1016+
1017+
if (ret < 0) {
1018+
mlt_log_error(MLT_PRODUCER_SERVICE(producer),
1019+
"Failed to set buffer source options: %d\n",
1020+
ret);
1021+
avfilter_graph_free(&self->vfilter_graph);
1022+
return ret;
1023+
}
1024+
1025+
// Set hw_frames_ctx before initializing
1026+
AVBufferSrcParameters *params = av_buffersrc_parameters_alloc();
1027+
if (!params) {
1028+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to allocate buffer src params\n");
1029+
avfilter_graph_free(&self->vfilter_graph);
1030+
return -1;
1031+
}
1032+
1033+
params->hw_frames_ctx = av_buffer_ref(self->video_frame->hw_frames_ctx);
1034+
ret = av_buffersrc_parameters_set(self->vfilter_in, params);
1035+
av_free(params);
1036+
if (ret < 0) {
1037+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to set hw_frames_ctx: %d\n", ret);
1038+
avfilter_graph_free(&self->vfilter_graph);
1039+
return ret;
1040+
}
1041+
1042+
// Initialize the filter
1043+
ret = avfilter_init_str(self->vfilter_in, NULL);
1044+
if (ret < 0) {
1045+
mlt_log_error(MLT_PRODUCER_SERVICE(producer),
1046+
"Failed to initialize buffer source: %d\n",
1047+
ret);
1048+
avfilter_graph_free(&self->vfilter_graph);
1049+
return ret;
1050+
}
1051+
1052+
// Create scale filter
1053+
const AVFilter *scale_filter = avfilter_get_by_name(filter_name);
1054+
if (!scale_filter) {
1055+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "%s filter not available\n", filter_name);
1056+
avfilter_graph_free(&self->vfilter_graph);
1057+
return -1;
1058+
}
1059+
1060+
AVFilterContext *scale_ctx = NULL;
1061+
ret = avfilter_graph_create_filter(&scale_ctx,
1062+
scale_filter,
1063+
filter_name,
1064+
scale_args,
1065+
NULL,
1066+
self->vfilter_graph);
1067+
if (ret < 0) {
1068+
mlt_log_error(MLT_PRODUCER_SERVICE(producer),
1069+
"Failed to create %s filter: %d\n",
1070+
filter_name,
1071+
ret);
1072+
avfilter_graph_free(&self->vfilter_graph);
1073+
return ret;
1074+
}
1075+
1076+
// Create buffer sink
1077+
const AVFilter *buffersink = avfilter_get_by_name("buffersink");
1078+
ret = avfilter_graph_create_filter(&self->vfilter_out,
1079+
buffersink,
1080+
"hw_buffersink",
1081+
NULL,
1082+
NULL,
1083+
self->vfilter_graph);
1084+
if (ret < 0) {
1085+
mlt_log_error(MLT_PRODUCER_SERVICE(producer),
1086+
"Failed to create buffer sink filter: %d\n",
1087+
ret);
1088+
avfilter_graph_free(&self->vfilter_graph);
1089+
return ret;
1090+
}
1091+
1092+
// Link filters: buffer -> scale -> buffersink
1093+
ret = avfilter_link(self->vfilter_in, 0, scale_ctx, 0);
1094+
if (ret < 0) {
1095+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to link buffer to scale: %d\n", ret);
1096+
avfilter_graph_free(&self->vfilter_graph);
1097+
return ret;
1098+
}
1099+
1100+
ret = avfilter_link(scale_ctx, 0, self->vfilter_out, 0);
1101+
if (ret < 0) {
1102+
mlt_log_error(MLT_PRODUCER_SERVICE(producer),
1103+
"Failed to link scale to buffersink: %d\n",
1104+
ret);
1105+
avfilter_graph_free(&self->vfilter_graph);
1106+
return ret;
1107+
}
1108+
1109+
// Configure the filter graph
1110+
ret = avfilter_graph_config(self->vfilter_graph, NULL);
1111+
if (ret < 0) {
1112+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to configure filter graph: %d\n", ret);
1113+
avfilter_graph_free(&self->vfilter_graph);
1114+
return ret;
1115+
}
1116+
1117+
mlt_log_info(MLT_PRODUCER_SERVICE(producer),
1118+
"%s filter initialized: %dx%d -> %dx%d\n",
1119+
filter_name,
1120+
self->video_frame->width,
1121+
self->video_frame->height,
1122+
scaled_width,
1123+
scaled_height);
1124+
1125+
return 0;
1126+
}
1127+
1128+
static int apply_hwaccel_filters(producer_avformat self, mlt_producer producer)
1129+
{
1130+
int ret = av_buffersrc_add_frame(self->vfilter_in, self->video_frame);
1131+
if (ret < 0) {
1132+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to add frame to filter: %d\n", ret);
1133+
// Disable filter on error to prevent repeated failures
1134+
avfilter_graph_free(&self->vfilter_graph);
1135+
self->vfilter_in = NULL;
1136+
self->vfilter_out = NULL;
1137+
return ret;
1138+
}
1139+
1140+
AVFrame *filtered_frame = av_frame_alloc();
1141+
ret = av_buffersink_get_frame(self->vfilter_out, filtered_frame);
1142+
if (ret >= 0) {
1143+
av_frame_unref(self->video_frame);
1144+
av_frame_move_ref(self->video_frame, filtered_frame);
1145+
av_frame_free(&filtered_frame);
1146+
} else {
1147+
mlt_log_error(MLT_PRODUCER_SERVICE(producer), "Failed to get filtered frame: %d\n", ret);
1148+
av_frame_free(&filtered_frame);
1149+
// Disable filter on error to prevent repeated failures
1150+
avfilter_graph_free(&self->vfilter_graph);
1151+
self->vfilter_in = NULL;
1152+
self->vfilter_out = NULL;
1153+
}
1154+
return ret;
1155+
}
1156+
9491157
static int setup_video_filters(producer_avformat self)
9501158
{
9511159
mlt_properties properties = MLT_PRODUCER_PROPERTIES(self->parent);
@@ -1963,7 +2171,8 @@ static void convert_image(producer_avformat self,
19632171
mlt_log_timings_begin();
19642172

19652173
mlt_log_debug(MLT_PRODUCER_SERVICE(self->parent),
1966-
"%s @ %dx%d colorspace %d->%d range %d->%d\n",
2174+
"%s->%s @ %dx%d colorspace %d->%d range %d->%d\n",
2175+
av_get_pix_fmt_name(pix_fmt),
19672176
mlt_image_format_name(*format),
19682177
width,
19692178
height,
@@ -2380,8 +2589,9 @@ static int producer_get_image(mlt_frame frame,
23802589
codec_params = stream->codecpar;
23812590

23822591
// Only change the requested image format for special cases
2383-
*format = pick_image_format(self->vfilter_out ? av_buffersink_get_format(self->vfilter_out)
2384-
: codec_params->format,
2592+
*format = pick_image_format((self->vfilter_out && !self->hwaccel.filters_initialized)
2593+
? av_buffersink_get_format(self->vfilter_out)
2594+
: codec_params->format,
23852595
*format);
23862596

23872597
// Duplicate the last image if necessary
@@ -2539,6 +2749,52 @@ static int producer_get_image(mlt_frame frame,
25392749
} else {
25402750
if (self->hwaccel.device_ctx
25412751
&& self->video_frame->format == self->hwaccel.pix_fmt) {
2752+
// Check if we need to set up hardware scaling filter (only once)
2753+
double consumer_scale = mlt_properties_get_double(frame_properties,
2754+
"consumer.scale");
2755+
consumer_scale = consumer_scale * profile->height
2756+
/ self->video_frame->height;
2757+
if (consumer_scale > 0.0 && consumer_scale < 1.0) {
2758+
#if HAVE_FFMPEG_VULKAN
2759+
if (self->hwaccel.pix_fmt == AV_PIX_FMT_VULKAN
2760+
&& !self->hwaccel.filters_initialized
2761+
&& !self->vfilter_graph) {
2762+
setup_hwaccel_filters(self,
2763+
producer,
2764+
"scale_vulkan",
2765+
consumer_scale);
2766+
self->hwaccel.filters_initialized = 1;
2767+
}
2768+
#endif
2769+
if (self->hwaccel.pix_fmt == AV_PIX_FMT_VAAPI
2770+
&& !self->hwaccel.filters_initialized
2771+
&& !self->vfilter_graph) {
2772+
setup_hwaccel_filters(self,
2773+
producer,
2774+
"scale_vaapi",
2775+
consumer_scale);
2776+
self->hwaccel.filters_initialized = 1;
2777+
}
2778+
}
2779+
2780+
// Apply hardware scale filter if initialized successfully
2781+
// Only apply if frame is still in hardware format
2782+
#if HAVE_FFMPEG_VULKAN
2783+
if (self->hwaccel.pix_fmt == AV_PIX_FMT_VULKAN
2784+
&& self->video_frame->format == AV_PIX_FMT_VULKAN
2785+
&& self->hwaccel.filters_initialized && self->vfilter_graph
2786+
&& self->vfilter_in && self->vfilter_out) {
2787+
apply_hwaccel_filters(self, producer);
2788+
}
2789+
#endif
2790+
if (self->hwaccel.pix_fmt == AV_PIX_FMT_VAAPI
2791+
&& self->video_frame->format == AV_PIX_FMT_VAAPI
2792+
&& self->hwaccel.filters_initialized && self->vfilter_graph
2793+
&& self->vfilter_in && self->vfilter_out) {
2794+
apply_hwaccel_filters(self, producer);
2795+
}
2796+
2797+
// Transfer from hardware to software frame
25422798
AVFrame *sw_video_frame = av_frame_alloc();
25432799
int transfer_data_result
25442800
= av_hwframe_transfer_data(sw_video_frame, self->video_frame, 0);
@@ -2651,19 +2907,23 @@ static int producer_get_image(mlt_frame frame,
26512907
#else
26522908
self->video_frame->top_field_first = self->top_field_first;
26532909
#endif
2654-
if ((self->autorotate || mlt_properties_get(properties, "filtergraph"))
2655-
&& !setup_filters(self) && self->vfilter_graph) {
2656-
int ret = av_buffersrc_add_frame(self->vfilter_in, self->video_frame);
2657-
if (ret < 0) {
2658-
got_picture = 0;
2659-
break;
2660-
}
2661-
while (ret >= 0) {
2662-
ret = av_buffersink_get_frame_flags(self->vfilter_out, self->video_frame, 0);
2910+
if ((self->autorotate || mlt_properties_exists(properties, "filtergraph"))
2911+
&& !self->hwaccel.filters_initialized) {
2912+
if (!setup_filters(self) && self->vfilter_graph) {
2913+
int ret = av_buffersrc_add_frame(self->vfilter_in, self->video_frame);
26632914
if (ret < 0) {
2664-
ret = 0;
2915+
got_picture = 0;
26652916
break;
26662917
}
2918+
while (ret >= 0) {
2919+
ret = av_buffersink_get_frame_flags(self->vfilter_out,
2920+
self->video_frame,
2921+
0);
2922+
if (ret < 0) {
2923+
ret = 0;
2924+
break;
2925+
}
2926+
}
26672927
}
26682928
}
26692929

0 commit comments

Comments
 (0)