Merge pull request #14 from membraneframework/support-pixel-format

Support pixel format in RawVideo

Author: FelonEkonom

README.md

@@ -17,7 +17,7 @@ The package can be installed by adding `membrane_transcoder_plugin` to your list
 ```elixir
 def deps do
   [
-    {:membrane_transcoder_plugin, "~> 0.3.2"}
+    {:membrane_transcoder_plugin, "~> 0.3.3"}
   ]
 end
 ```

lib/transcoder.ex

@@ -20,6 +20,12 @@ defmodule Membrane.Transcoder do
   * `Membrane.RawAudio`
   * `Membrane.RemoteStream{content_format: Membrane.Opus}` (only as an input stream)
   * `Membrane.RemoteStream{content_format: Membrane.MPEGAudio}` (only as an input stream)
+
+  While `#{inspect(__MODULE__)}` can transcode between different stream formats, it can also be used
+  to change some parameters of the stream format.
+  Now, the only supported stream parameters are:
+  * `:pixel_format` in `Membrane.RawVideo`
+  * `:alignment` and `:stream_structure` in `Membrane.H264` and `Membrane.H265`
   """
   use Membrane.Bin
 
@@ -51,6 +57,16 @@ defmodule Membrane.Transcoder do
   @type stream_format_module ::
           H264 | H265 | VP8 | VP9 | RawVideo | AAC | Opus | Membrane.MPEGAudio | RawAudio
 
+  @typedoc """
+  Describes a tuple consisting of a stream format module and its options.
+
+  An alternative to `t:#{inspect(__MODULE__)}.stream_format/0`.
+
+  Allows you to specify some fields of the output stream format, without the need to
+  set all keys required by the struct.
+  """
+  @type stream_format_tuple :: {stream_format_module(), keyword()}
+
   @typedoc """
   Describes a function which can be used to provide output format based on the input format.
   """
@@ -69,6 +85,7 @@ defmodule Membrane.Transcoder do
                 spec:
                   stream_format()
                   | stream_format_module()
+                  | stream_format_tuple()
                   | stream_format_resolver(),
                 description: """
                 An option specifying desired output format.
@@ -206,6 +223,9 @@ defmodule Membrane.Transcoder do
       module when is_atom(module) ->
         %{state | output_stream_format: struct(module)}
 
+      {module, opts} when is_atom(module) and is_list(opts) ->
+        %{state | output_stream_format: struct(module, opts)}
+
       resolver when is_function(resolver) ->
         %{state | output_stream_format: resolver.(state.input_stream_format)}
         |> resolve_output_stream_format()

lib/transcoder/audio.ex

@@ -58,10 +58,7 @@ defmodule Membrane.Transcoder.Audio do
   defguard is_mp3_compliant(format)
            when is_map_key(format, :sample_rate) and format.sample_rate == 44_100 and
                   is_map_key(format, :sample_format) and format.sample_format == :s32le and
-                  is_map_key(
-                    format,
-                    :channels
-                  ) and format.channels == 2
+                  is_map_key(format, :channels) and format.channels == 2
 
   @spec plug_audio_transcoding(
           ChildrenSpec.builder(),

lib/transcoder/video.ex

@@ -4,6 +4,7 @@ defmodule Membrane.Transcoder.Video do
   import Membrane.ChildrenSpec
   require Membrane.Logger
   alias Membrane.{ChildrenSpec, H264, H265, RawVideo, RemoteStream, VP8, VP9}
+  alias Membrane.FFmpeg.SWScale
 
   @type video_stream_format :: VP8.t() | VP9.t() | H264.t() | H265.t() | RawVideo.t()
 
@@ -72,6 +73,16 @@ defmodule Membrane.Transcoder.Video do
     })
   end
 
+  defp do_plug_video_transcoding(
+         builder,
+         %RawVideo{} = input_format,
+         %RawVideo{} = output_format,
+         _transcoding_policy
+       ) do
+    builder
+    |> maybe_plug_swscale_converter(input_format, output_format)
+  end
+
   defp do_plug_video_transcoding(
          builder,
          %format_module{},
@@ -96,6 +107,7 @@ defmodule Membrane.Transcoder.Video do
   defp do_plug_video_transcoding(builder, input_format, output_format, _transcoding_policy) do
     builder
     |> maybe_plug_parser_and_decoder(input_format)
+    |> maybe_plug_swscale_converter(input_format, output_format)
     |> maybe_plug_encoder_and_parser(output_format)
   end
 
@@ -135,6 +147,38 @@ defmodule Membrane.Transcoder.Video do
     builder
   end
 
+  defp maybe_plug_swscale_converter(builder, input_format, %RawVideo{} = output_format) do
+    case input_format do
+      _any when output_format.pixel_format == nil ->
+        builder
+
+      %RawVideo{pixel_format: pixel_format} when pixel_format == output_format.pixel_format ->
+        builder
+
+      _input_format ->
+        builder
+        |> child(:raw_video_converter, %SWScale.Converter{format: output_format.pixel_format})
+    end
+  end
+
+  defp maybe_plug_swscale_converter(builder, input_format, %h26x{}) when h26x in [H264, H265] do
+    case input_format do
+      %RawVideo{pixel_format: pixel_format} when pixel_format in [:I420, :I422] ->
+        builder
+
+      %h26x{} when h26x in [H264, H265] ->
+        builder
+
+      _input_format ->
+        builder
+        |> child(:raw_video_converter, %SWScale.Converter{format: :I420})
+    end
+  end
+
+  defp maybe_plug_swscale_converter(builder, _input_format, _output_format) do
+    builder
+  end
+
   defp maybe_plug_encoder_and_parser(builder, %H264{} = h264) do
     builder
     |> child(:h264_encoder, %H264.FFmpeg.Encoder{preset: :ultrafast})

mix.exs

@@ -1,7 +1,7 @@
 defmodule Membrane.Transcoder.Plugin.Mixfile do
   use Mix.Project
 
-  @version "0.3.2"
+  @version "0.3.3"
   @github_url "https://github.com/membraneframework/membrane_transcoder_plugin"
 
   def project do
@@ -46,6 +46,7 @@ defmodule Membrane.Transcoder.Plugin.Mixfile do
       {:membrane_h264_ffmpeg_plugin, "~> 0.32.0"},
       {:membrane_h265_ffmpeg_plugin, "~> 0.4.2"},
       {:membrane_ffmpeg_swresample_plugin, "~> 0.20.0"},
+      {:membrane_ffmpeg_swscale_plugin, "~> 0.16.2"},
       {:membrane_timestamp_queue, "~> 0.2.2"},
       {:membrane_h264_format, "~> 0.6.1"},
       {:membrane_h265_format, "~> 0.2.0"},

mix.lock

@@ -26,6 +26,7 @@
   "membrane_common_c": {:hex, :membrane_common_c, "0.16.0", "caf3f29d2f5a1d32d8c2c122866110775866db2726e4272be58e66dfdf4bce40", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:shmex, "~> 0.5.0", [hex: :shmex, repo: "hexpm", optional: false]}, {:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "a3c7e91de1ce1f8b23b9823188a5d13654d317235ea0ca781c05353ed3be9b1c"},
   "membrane_core": {:hex, :membrane_core, "1.2.2", "14c1c5f6b5cae0defb849309bc0b18b5d284e2e4b9c35b9486077a3d0763b8cd", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:qex, "~> 0.3", [hex: :qex, repo: "hexpm", optional: false]}, {:ratio, "~> 3.0 or ~> 4.0", [hex: :ratio, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f53bf0101d0583e959534895117d47c615c7fe5fcf37f8f1e4a3f7a5ca468a93"},
   "membrane_ffmpeg_swresample_plugin": {:hex, :membrane_ffmpeg_swresample_plugin, "0.20.2", "2e669f0b25418d10b51a73bc52d2e12e4a3a26b416c5c1199d852c3f781a18b3", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.2", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:mockery, "~> 2.1", [hex: :mockery, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "6c8d3bcd61d568dd94cabb9b45f29e8926e0076e4432d8f419378e004e02147c"},
+  "membrane_ffmpeg_swscale_plugin": {:hex, :membrane_ffmpeg_swscale_plugin, "0.16.2", "581909312d6d12ed560ee99caa1b1674a339760ab2ad6835d243326806c23da1", [:mix], [{:bundlex, "~> 1.2", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.1", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_video_format, "~> 0.4.1", [hex: :membrane_raw_video_format, repo: "hexpm", optional: false]}], "hexpm", "46c185dacff1e1b404d0ceb74d0d5224f0931fe1e8b951cc3776ebd099e39afc"},
   "membrane_file_plugin": {:hex, :membrane_file_plugin, "0.17.2", "650e134c2345d946f930082fac8bac9f5aba785a7817d38a9a9da41ffc56fa92", [:mix], [{:logger_backends, "~> 1.0", [hex: :logger_backends, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "df50c6040004cd7b901cf057bd7e99c875bbbd6ae574efc93b2c753c96f43b9d"},
   "membrane_funnel_plugin": {:hex, :membrane_funnel_plugin, "0.9.1", "9e108f4ef9d905ebff2da3ba5e58a5b756b58812f4fa68bd576add68fda310a0", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "39fdef1bf29eac949f65a37ea941f997c22ed042c55af044d27a781b63e82f6b"},
   "membrane_h264_ffmpeg_plugin": {:hex, :membrane_h264_ffmpeg_plugin, "0.32.4", "5548a37642125d37b9ae4df26d59ef5397781b84101b7b60fe8b4fdb70dc536d", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.1", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_video_format, "~> 0.4.1", [hex: :membrane_raw_video_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "6878c763b2f0e0fd1651e8cbac3707b3c58a51858acceca7ad0d1c1786f5e3ec"},

test/integration_test.exs

@@ -14,7 +14,7 @@ defmodule Membrane.Transcoder.IntegrationTest do
     %{input_format: VP8, input_file: "video_vp8.ivf", preprocess: &Preprocessors.parse_vpx/1},
     %{input_format: VP9, input_file: "video_vp9.ivf", preprocess: &Preprocessors.parse_vpx/1}
   ]
-  @video_outputs [RawVideo, H264, H265, VP8, VP9]
+  @video_outputs [RawVideo, {RawVideo, pixel_format: :RGB}, H264, H265, VP8, VP9]
   @video_cases for input <- @video_inputs,
                    output <- @video_outputs,
                    do: Map.put(input, :output_format, output)
@@ -57,7 +57,19 @@ defmodule Membrane.Transcoder.IntegrationTest do
 
       Testing.Pipeline.execute_actions(pid, spec: spec)
 
-      assert_sink_stream_format(pid, :sink, %unquote(test_case.output_format){})
+      case unquote(test_case.output_format) do
+        {module, opts} when is_atom(module) ->
+          assert_sink_stream_format(pid, :sink, %^module{} = received_format)
+
+          for {key, value} <- opts do
+            assert Map.get(received_format, key) == value
+          end
+
+        module when is_atom(module) ->
+          assert_sink_stream_format(pid, :sink, received_format)
+          assert received_format.__struct__ == module
+      end
+
       Testing.Pipeline.terminate(pid)
     end
   end)

test/test_helper.exs

@@ -1 +1,8 @@
-ExUnit.start(capture_log: true)
+# System.schedulers_online() * 2 is the default value of max_cases
+
+max_cases =
+  if System.get_env("CIRCLECI") == "true",
+    do: 1,
+    else: System.schedulers_online() * 2
+
+ExUnit.start(capture_log: true, max_cases: max_cases)