Support pixel format conversions in raw video

Author: FelonEkonom

README.md

@@ -17,7 +17,7 @@ The package can be installed by adding `membrane_transcoder_plugin` to your list
 ```elixir
 def deps do
   [
-    {:membrane_transcoder_plugin, "~> 0.3.2"}
+    {:membrane_transcoder_plugin, "~> 0.3.3"}
   ]
 end
 ```

lib/transcoder.ex

@@ -20,6 +20,12 @@ defmodule Membrane.Transcoder do
   * `Membrane.RawAudio`
   * `Membrane.RemoteStream{content_format: Membrane.Opus}` (only as an input stream)
   * `Membrane.RemoteStream{content_format: Membrane.MPEGAudio}` (only as an input stream)
+
+  While `#{inspect(__MODULE__)}` can transcode between different stream formats, it can also be used
+  to change some parameters of the stream format.
+  Now, the only supported stream parameters are:
+  * `:pixel_format` in `Membrane.RawVideo`
+  * `:alignment` and `:stream_structure` in `Membrane.H264` and `Membrane.H265`
   """
   use Membrane.Bin
 
@@ -51,6 +57,16 @@ defmodule Membrane.Transcoder do
   @type stream_format_module ::
           H264 | H265 | VP8 | VP9 | RawVideo | AAC | Opus | Membrane.MPEGAudio | RawAudio
 
+  @typedoc """
+  Describes a tuple consisting of a stream format module and its options.
+
+  An alternative to `t:#{inspect(__MODULE__)}.stream_format/0`.
+
+  Allows you to specify some fields of the output stream format, without the need to
+  set all keys required by the struct.
+  """
+  @type stream_format_tuple :: {stream_format_module(), keyword()}
+
   @typedoc """
   Describes a function which can be used to provide output format based on the input format.
   """
@@ -206,6 +222,9 @@ defmodule Membrane.Transcoder do
       module when is_atom(module) ->
         %{state | output_stream_format: struct(module)}
 
+      {module, opts} when is_atom(module) and is_list(opts) ->
+        %{state | output_stream_format: struct(module, opts)}
+
       resolver when is_function(resolver) ->
         %{state | output_stream_format: resolver.(state.input_stream_format)}
         |> resolve_output_stream_format()

lib/transcoder/audio.ex

@@ -58,10 +58,7 @@ defmodule Membrane.Transcoder.Audio do
   defguard is_mp3_compliant(format)
            when is_map_key(format, :sample_rate) and format.sample_rate == 44_100 and
                   is_map_key(format, :sample_format) and format.sample_format == :s32le and
-                  is_map_key(
-                    format,
-                    :channels
-                  ) and format.channels == 2
+                  is_map_key(format, :channels) and format.channels == 2
 
   @spec plug_audio_transcoding(
           ChildrenSpec.builder(),

lib/transcoder/video.ex

@@ -4,6 +4,7 @@ defmodule Membrane.Transcoder.Video do
   import Membrane.ChildrenSpec
   require Membrane.Logger
   alias Membrane.{ChildrenSpec, H264, H265, RawVideo, RemoteStream, VP8, VP9}
+  alias Membrane.FFmpeg.SWScale
 
   @type video_stream_format :: VP8.t() | VP9.t() | H264.t() | H265.t() | RawVideo.t()
 
@@ -72,6 +73,16 @@ defmodule Membrane.Transcoder.Video do
     })
   end
 
+  defp do_plug_video_transcoding(
+         builder,
+         %RawVideo{} = input_format,
+         %RawVideo{} = output_format,
+         _transcoding_policy
+       ) do
+    builder
+    |> maybe_plug_swscale_converter(input_format, output_format)
+  end
+
   defp do_plug_video_transcoding(
          builder,
          %format_module{},
@@ -96,6 +107,7 @@ defmodule Membrane.Transcoder.Video do
   defp do_plug_video_transcoding(builder, input_format, output_format, _transcoding_policy) do
     builder
     |> maybe_plug_parser_and_decoder(input_format)
+    |> maybe_plug_swscale_converter(input_format, output_format)
     |> maybe_plug_encoder_and_parser(output_format)
   end
 
@@ -135,6 +147,21 @@ defmodule Membrane.Transcoder.Video do
     builder
   end
 
+  defp maybe_plug_swscale_converter(builder, input_format, output_format) do
+    # output pixel format is nil when the transcoder outptu format was set to
+    # Membrane.RawVideo module without specifying pixel_format field
+
+    if pixel_format(output_format) not in [nil, pixel_format(input_format)] do
+      builder
+      |> child(:raw_video_converter, %SWScale.Converter{format: pixel_format(output_format)})
+    else
+      builder
+    end
+  end
+
+  defp pixel_format(%RawVideo{pixel_format: pixel_format}), do: pixel_format
+  defp pixel_format(_encoded_video), do: :I420
+
   defp maybe_plug_encoder_and_parser(builder, %H264{} = h264) do
     builder
     |> child(:h264_encoder, %H264.FFmpeg.Encoder{preset: :ultrafast})

mix.exs

@@ -1,7 +1,7 @@
 defmodule Membrane.Transcoder.Plugin.Mixfile do
   use Mix.Project
 
-  @version "0.3.2"
+  @version "0.3.3"
   @github_url "https://github.com/membraneframework/membrane_transcoder_plugin"
 
   def project do
@@ -46,6 +46,7 @@ defmodule Membrane.Transcoder.Plugin.Mixfile do
       {:membrane_h264_ffmpeg_plugin, "~> 0.32.0"},
       {:membrane_h265_ffmpeg_plugin, "~> 0.4.2"},
       {:membrane_ffmpeg_swresample_plugin, "~> 0.20.0"},
+      {:membrane_ffmpeg_swscale_plugin, "~> 0.16.2"},
       {:membrane_timestamp_queue, "~> 0.2.2"},
       {:membrane_h264_format, "~> 0.6.1"},
       {:membrane_h265_format, "~> 0.2.0"},

mix.lock

@@ -26,6 +26,7 @@
   "membrane_common_c": {:hex, :membrane_common_c, "0.16.0", "caf3f29d2f5a1d32d8c2c122866110775866db2726e4272be58e66dfdf4bce40", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:shmex, "~> 0.5.0", [hex: :shmex, repo: "hexpm", optional: false]}, {:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "a3c7e91de1ce1f8b23b9823188a5d13654d317235ea0ca781c05353ed3be9b1c"},
   "membrane_core": {:hex, :membrane_core, "1.2.2", "14c1c5f6b5cae0defb849309bc0b18b5d284e2e4b9c35b9486077a3d0763b8cd", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:qex, "~> 0.3", [hex: :qex, repo: "hexpm", optional: false]}, {:ratio, "~> 3.0 or ~> 4.0", [hex: :ratio, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f53bf0101d0583e959534895117d47c615c7fe5fcf37f8f1e4a3f7a5ca468a93"},
   "membrane_ffmpeg_swresample_plugin": {:hex, :membrane_ffmpeg_swresample_plugin, "0.20.2", "2e669f0b25418d10b51a73bc52d2e12e4a3a26b416c5c1199d852c3f781a18b3", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.2", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_audio_format, "~> 0.12.0", [hex: :membrane_raw_audio_format, repo: "hexpm", optional: false]}, {:mockery, "~> 2.1", [hex: :mockery, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "6c8d3bcd61d568dd94cabb9b45f29e8926e0076e4432d8f419378e004e02147c"},
+  "membrane_ffmpeg_swscale_plugin": {:hex, :membrane_ffmpeg_swscale_plugin, "0.16.2", "581909312d6d12ed560ee99caa1b1674a339760ab2ad6835d243326806c23da1", [:mix], [{:bundlex, "~> 1.2", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.1", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_video_format, "~> 0.4.1", [hex: :membrane_raw_video_format, repo: "hexpm", optional: false]}], "hexpm", "46c185dacff1e1b404d0ceb74d0d5224f0931fe1e8b951cc3776ebd099e39afc"},
   "membrane_file_plugin": {:hex, :membrane_file_plugin, "0.17.2", "650e134c2345d946f930082fac8bac9f5aba785a7817d38a9a9da41ffc56fa92", [:mix], [{:logger_backends, "~> 1.0", [hex: :logger_backends, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "df50c6040004cd7b901cf057bd7e99c875bbbd6ae574efc93b2c753c96f43b9d"},
   "membrane_funnel_plugin": {:hex, :membrane_funnel_plugin, "0.9.1", "9e108f4ef9d905ebff2da3ba5e58a5b756b58812f4fa68bd576add68fda310a0", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "39fdef1bf29eac949f65a37ea941f997c22ed042c55af044d27a781b63e82f6b"},
   "membrane_h264_ffmpeg_plugin": {:hex, :membrane_h264_ffmpeg_plugin, "0.32.4", "5548a37642125d37b9ae4df26d59ef5397781b84101b7b60fe8b4fdb70dc536d", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_common_c, "~> 0.16.0", [hex: :membrane_common_c, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.1", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:membrane_raw_video_format, "~> 0.4.1", [hex: :membrane_raw_video_format, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "6878c763b2f0e0fd1651e8cbac3707b3c58a51858acceca7ad0d1c1786f5e3ec"},

test/integration_test.exs

@@ -14,7 +14,7 @@ defmodule Membrane.Transcoder.IntegrationTest do
     %{input_format: VP8, input_file: "video_vp8.ivf", preprocess: &Preprocessors.parse_vpx/1},
     %{input_format: VP9, input_file: "video_vp9.ivf", preprocess: &Preprocessors.parse_vpx/1}
   ]
-  @video_outputs [RawVideo, H264, H265, VP8, VP9]
+  @video_outputs [RawVideo, {RawVideo, pixel_format: :RGB}, H264, H265, VP8, VP9]
   @video_cases for input <- @video_inputs,
                    output <- @video_outputs,
                    do: Map.put(input, :output_format, output)
@@ -37,6 +37,11 @@ defmodule Membrane.Transcoder.IntegrationTest do
   @test_cases @video_cases ++ @audio_cases
 
   Enum.map(@test_cases, fn test_case ->
+    if test_case.input_format == RawVideo and
+         test_case.output_format == {RawVideo, pixel_format: :RGB} do
+      @tag :xd
+    end
+
     test "if transcoder supports #{inspect(test_case.input_format)} input and #{inspect(test_case.output_format)} output" do
       pid = Testing.Pipeline.start_link_supervised!()
 
@@ -57,7 +62,19 @@ defmodule Membrane.Transcoder.IntegrationTest do
 
       Testing.Pipeline.execute_actions(pid, spec: spec)
 
-      assert_sink_stream_format(pid, :sink, %unquote(test_case.output_format){})
+      case unquote(test_case.output_format) do
+        {module, opts} when is_atom(module) ->
+          assert_sink_stream_format(pid, :sink, %^module{} = received_format)
+
+          for {key, value} <- opts do
+            assert Map.get(received_format, key) == value
+          end
+
+        module when is_atom(module) ->
+          assert_sink_stream_format(pid, :sink, received_format)
+          assert received_format.__struct__ == module
+      end
+
       Testing.Pipeline.terminate(pid)
     end
   end)