Merge remote-tracking branch 'origin/master' into transcoding_policy

Author: FelonEkonom

examples/vp8_to_h264.exs

@@ -28,4 +28,4 @@ defmodule Example do
 end
 
 File.mkdir("tmp")
-Example.convert(Path.join("./test/fixtures", "video.ivf"), Path.join("./tmp", "video.h264"))
+Example.convert(Path.join("./test/fixtures", "video_vp8.ivf"), Path.join("./tmp", "video.h264"))

lib/transcoder.ex

@@ -8,13 +8,18 @@ defmodule Membrane.Transcoder do
   * `Membrane.H264`
   * `Membrane.H265`
   * `Membrane.VP8`
+  * `Membrane.VP9`
   * `Membrane.RawVideo`
+  * `Membrane.RemoteStream{content_type: Membrane.VP8}` (only as an input stream)
+  * `Membrane.RemoteStream{content_type: Membrane.VP9}` (only as an input stream)
 
   The following audio stream formats are supported:
   * `Membrane.AAC`
   * `Membrane.Opus`
+  * `Membrane.MPEGAudio`
   * `Membrane.RawAudio`
   * `Membrane.RemoteStream{content_type: Membrane.Opus}` (only as an input stream)
+  * `Membrane.RemoteStream{content_type: Membrane.MPEGAudio}` (only as an input stream)
   """
   use Membrane.Bin
 
@@ -23,7 +28,7 @@ defmodule Membrane.Transcoder do
   require Membrane.Logger
 
   alias __MODULE__.{Audio, Video}
-  alias Membrane.{AAC, Funnel, H264, H265, Opus, RawAudio, RawVideo, RemoteStream, VP8}
+  alias Membrane.{AAC, Funnel, H264, H265, Opus, RawAudio, RawVideo, RemoteStream, VP8, VP9}
 
   @typedoc """
   Describes stream formats acceptable on the bin's input and output.
@@ -32,24 +37,30 @@ defmodule Membrane.Transcoder do
           H264.t()
           | H265.t()
           | VP8.t()
+          | VP9.t()
           | RawVideo.t()
           | AAC.t()
           | Opus.t()
+          | Membrane.MPEGAudio.t()
           | RemoteStream.t()
           | RawAudio.t()
 
   @typedoc """
   Describes stream format modules that can be used to define inputs and outputs of the bin.
   """
-  @type stream_format_module :: H264 | H265 | VP8 | RawVideo | AAC | Opus | RawAudio
+  @type stream_format_module ::
+          H264 | H265 | VP8 | VP9 | RawVideo | AAC | Opus | Membrane.MPEGAudio | RawAudio
 
   @typedoc """
   Describes a function which can be used to provide output format based on the input format.
   """
   @type stream_format_resolver :: (stream_format() -> stream_format() | stream_format_module())
 
   def_input_pad :input,
-    accepted_format: format when Audio.is_audio_format(format) or Video.is_video_format(format)
+    accepted_format:
+      format
+      when Audio.is_audio_format(format) or Video.is_video_format(format) or
+             format.__struct__ == RemoteStream
 
   def_output_pad :output,
     accepted_format: format when Audio.is_audio_format(format) or Video.is_video_format(format)
@@ -99,12 +110,22 @@ defmodule Membrane.Transcoder do
                 If the transition from the input stream format to the output stream format is not
                 possible without decoding or encoding the stream, an error will be raised.
                 """
+              ],
+              assumed_input_stream_format: [
+                spec: %Membrane.RemoteStream{content_format: Membrane.MPEGAudio} | nil,
+                default: nil,
+                description: """
+                Allows to override stream format of the input stream with
+                `%Membrane.RemoteStream{content_format: Membrane.MPEGAudio}`
+                If nil, the input stream format won't be overriden.
+                """
               ]
 
   @impl true
   def handle_init(_ctx, opts) do
     spec = [
       bin_input()
+      |> maybe_override_input_stream_format(opts.assumed_input_stream_format)
       |> child(:connector, %Membrane.Connector{notify_on_stream_format?: true}),
       child(:output_funnel, Funnel)
       |> bin_output()
@@ -120,6 +141,28 @@ defmodule Membrane.Transcoder do
     {[spec: spec], state}
   end
 
+  defp maybe_override_input_stream_format(
+         builder,
+         %Membrane.RemoteStream{content_format: Membrane.MPEGAudio} = stream_format
+       ) do
+    builder
+    |> child(:stream_format_changer, %Membrane.Transcoder.StreamFormatChanger{
+      stream_format: stream_format
+    })
+  end
+
+  defp maybe_override_input_stream_format(builder, nil) do
+    builder
+  end
+
+  defp maybe_override_input_stream_format(_builder, stream_format) do
+    raise """
+    The only input stream format that can be assumed is \
+    `%Membrane.RemoteStream{content_format: Membrane.MPEGAudio}`, while you wanted to assume: \
+    #{inspect(stream_format)}
+    """
+  end
+
   @impl true
   def handle_child_notification({:stream_format, _pad, format}, :connector, _ctx, state)
       when state.input_stream_format == nil do

lib/transcoder/audio.ex

@@ -3,9 +3,8 @@ defmodule Membrane.Transcoder.Audio do
 
   import Membrane.ChildrenSpec
   require Membrane.Logger
-  alias Membrane.{AAC, ChildrenSpec, Opus, RawAudio, RemoteStream}
+  alias Membrane.{AAC, ChildrenSpec, MPEGAudio, Opus, RawAudio, RemoteStream}
 
-  @opus_sample_rate 48_000
   @aac_sample_rates [
     96_000,
     88_200,
@@ -21,13 +20,31 @@ defmodule Membrane.Transcoder.Audio do
     8000
   ]
 
-  @type audio_stream_format :: AAC.t() | Opus.t() | RawAudio.t()
+  @type audio_stream_format :: AAC.t() | Opus.t() | Membrane.MPEGAudio.t() | RawAudio.t()
 
   defguard is_audio_format(format)
            when is_struct(format) and
-                  (format.__struct__ in [AAC, Opus, RawAudio] or
-                     (format.__struct__ == RemoteStream and format.content_format == Opus and
-                        format.type == :packetized))
+                  (format.__struct__ in [AAC, Opus, MPEGAudio, RawAudio] or
+                     (format.__struct__ == RemoteStream and
+                        format.content_format == Opus and
+                        format.type == :packetized) or
+                     (format.__struct__ == RemoteStream and format.content_format == MPEGAudio))
+
+  defguard is_opus_compliant(format)
+           when is_map_key(format, :content_type) and format.content_type == :s16le and
+                  is_map_key(format, :sample_rate) and format.sample_rate == 48_000
+
+  defguard is_aac_compliant(format)
+           when is_map_key(format, :content_type) and format.content_type == :s16le and
+                  is_map_key(format, :sample_rate) and format.sample_rate in @aac_sample_rates
+
+  defguard is_mp3_compliant(format)
+           when is_map_key(format, :sample_rate) and format.sample_rate == 44_100 and
+                  is_map_key(format, :sample_format) and format.sample_format == :s32le and
+                  is_map_key(
+                    format,
+                    :channels
+                  ) and format.channels == 2
 
   @spec plug_audio_transcoding(
           ChildrenSpec.builder(),
@@ -99,34 +116,50 @@ defmodule Membrane.Transcoder.Audio do
     builder |> child(:aac_decoder, AAC.FDK.Decoder)
   end
 
+  defp maybe_plug_decoder(builder, %MPEGAudio{}) do
+    builder |> child(:mp3_decoder, Membrane.MP3.MAD.Decoder)
+  end
+
+  defp maybe_plug_decoder(builder, %RemoteStream{content_format: MPEGAudio}) do
+    builder |> child(:mp3_decoder, Membrane.MP3.MAD.Decoder)
+  end
+
   defp maybe_plug_decoder(builder, %RawAudio{}) do
     builder
   end
 
-  defp maybe_plug_resampler(builder, %{sample_rate: sample_rate} = input_format, %Opus{})
-       when sample_rate != @opus_sample_rate do
+  defp maybe_plug_resampler(builder, input_format, %Opus{})
+       when not is_opus_compliant(input_format) do
     builder
     |> child(:resampler, %Membrane.FFmpeg.SWResample.Converter{
       output_stream_format: %RawAudio{
         sample_format: :s16le,
-        sample_rate: @opus_sample_rate,
-        channels: input_format.channels
+        sample_rate: 48_000,
+        channels: 1
       }
     })
   end
 
-  defp maybe_plug_resampler(builder, %{sample_rate: sample_rate} = input_format, %AAC{})
-       when sample_rate not in @aac_sample_rates do
+  defp maybe_plug_resampler(builder, input_format, %AAC{})
+       when not is_aac_compliant(input_format) do
     builder
     |> child(:resampler, %Membrane.FFmpeg.SWResample.Converter{
       output_stream_format: %RawAudio{
         sample_format: :s16le,
         sample_rate: 44_100,
-        channels: input_format.channels
+        channels: 1
       }
     })
   end
 
+  defp maybe_plug_resampler(builder, input_format, %MPEGAudio{})
+       when not is_mp3_compliant(input_format) do
+    builder
+    |> child(:resampler, %Membrane.FFmpeg.SWResample.Converter{
+      output_stream_format: %RawAudio{sample_rate: 44_100, sample_format: :s32le, channels: 2}
+    })
+  end
+
   defp maybe_plug_resampler(builder, _input_format, _output_format) do
     builder
   end
@@ -139,6 +172,10 @@ defmodule Membrane.Transcoder.Audio do
     builder |> child(:aac_encoder, AAC.FDK.Encoder)
   end
 
+  defp maybe_plug_encoder(builder, %MPEGAudio{}) do
+    builder |> child(:mp3_encoder, Membrane.MP3.Lame.Encoder)
+  end
+
   defp maybe_plug_encoder(builder, %RawAudio{}) do
     builder
   end

lib/transcoder/stream_format_changer.ex

@@ -0,0 +1,34 @@
+defmodule Membrane.Transcoder.StreamFormatChanger do
+  @moduledoc false
+  use Membrane.Filter
+
+  def_input_pad :input, accepted_format: %Membrane.RemoteStream{}
+  def_output_pad :output, accepted_format: _any
+
+  def_options stream_format: [
+                spec: Membrane.StreamFormat.t(),
+                description: """
+                Stream format that will be sent on `handle_playing`.
+                """
+              ]
+
+  @impl true
+  def handle_init(_ctx, opts) do
+    {[], %{stream_format: opts.stream_format}}
+  end
+
+  @impl true
+  def handle_playing(_ctx, state) do
+    {[stream_format: {:output, state.stream_format}], state}
+  end
+
+  @impl true
+  def handle_buffer(:input, buffer, _ctx, state) do
+    {[buffer: {:output, buffer}], state}
+  end
+
+  @impl true
+  def handle_stream_format(:input, _stream_format, _ctx, state) do
+    {[], state}
+  end
+end

lib/transcoder/video.ex

@@ -3,14 +3,14 @@ defmodule Membrane.Transcoder.Video do
 
   import Membrane.ChildrenSpec
   require Membrane.Logger
-  alias Membrane.{ChildrenSpec, H264, H265, RawVideo, RemoteStream, VP8}
+  alias Membrane.{ChildrenSpec, H264, H265, RawVideo, RemoteStream, VP8, VP9}
 
-  @type video_stream_format :: VP8.t() | H264.t() | H265.t() | RawVideo.t()
+  @type video_stream_format :: VP8.t() | VP9.t() | H264.t() | H265.t() | RawVideo.t()
 
   defguard is_video_format(format)
            when is_struct(format) and
-                  (format.__struct__ in [VP8, H264, H265, RawVideo] or
-                     (format.__struct__ == RemoteStream and format.content_format == VP8 and
+                  (format.__struct__ in [VP8, VP9, H264, H265, RawVideo] or
+                     (format.__struct__ == RemoteStream and format.content_format in [VP8, VP9] and
                         format.type == :packetized))
 
   @spec plug_video_transcoding(
@@ -24,22 +24,22 @@ defmodule Membrane.Transcoder.Video do
     do_plug_video_transcoding(builder, input_format, output_format, transcoding_policy)
   end
 
-  defp do_plug_video_transcoding(
-         builder,
-         %h26x{},
-         %h26x{} = output_format,
-         transcoding_policy
-       )
-       when h26x in [H264, H265] and transcoding_policy in [:if_needed, :never] do
-    parser =
-      h26x
-      |> Module.concat(Parser)
-      |> struct!(
-        output_stream_structure: stream_structure_type(output_format),
-        output_alignment: output_format.alignment
-      )
+  defp do_plug_video_transcoding(builder, %H264{}, %H264{} = output_format, transcoding_policy)
+       when transcoding_policy in [:if_needed, :never] do
+    builder
+    |> child(:h264_parser, %H264.Parser{
+      output_stream_structure: stream_structure_type(output_format),
+      output_alignment: output_format.alignment
+    })
+  end
 
-    builder |> child(:h264_parser, parser)
+  defp do_plug_video_transcoding(builder, %H265{}, %H265{} = output_format, transcoding_policy)
+       when transcoding_policy in [:if_needed, :never] do
+    builder
+    |> child(:h265_parser, %H265.Parser{
+      output_stream_structure: stream_structure_type(output_format),
+      output_alignment: output_format.alignment
+    })
   end
 
   defp do_plug_video_transcoding(
@@ -87,15 +87,18 @@ defmodule Membrane.Transcoder.Video do
     |> child(:h265_decoder, %H265.FFmpeg.Decoder{})
   end
 
-  defp maybe_plug_parser_and_decoder(builder, %VP8{}) do
-    builder |> child(:vp8_decoder, %VP8.Decoder{})
+  defp maybe_plug_parser_and_decoder(builder, %vpx{}) when vpx in [VP8, VP9] do
+    decoder_module = Module.concat(vpx, Decoder)
+    builder |> child(:vp8_decoder, decoder_module)
   end
 
   defp maybe_plug_parser_and_decoder(builder, %RemoteStream{
-         content_format: VP8,
+         content_format: vpx,
          type: :packetized
-       }) do
-    builder |> child(:vp8_decoder, %VP8.Decoder{})
+       })
+       when vpx in [VP8, VP9] do
+    decoder_module = Module.concat(vpx, Decoder)
+    builder |> child(:vp8_decoder, decoder_module)
   end
 
   defp maybe_plug_parser_and_decoder(builder, %RawVideo{}) do
@@ -131,6 +134,17 @@ defmodule Membrane.Transcoder.Video do
     builder |> child(:vp8_encoder, %VP8.Encoder{g_threads: number_of_threads, cpu_used: 15})
   end
 
+  defp maybe_plug_encoder_and_parser(builder, %VP9{}) do
+    cpu_quota = :erlang.system_info(:cpu_quota)
+
+    number_of_threads =
+      if cpu_quota != :unknown,
+        do: cpu_quota,
+        else: :erlang.system_info(:logical_processors_available)
+
+    builder |> child(:vp8_encoder, %VP9.Encoder{g_threads: number_of_threads, cpu_used: 15})
+  end
+
   defp maybe_plug_encoder_and_parser(builder, %RawVideo{}) do
     builder
   end

mix.exs

@@ -53,6 +53,9 @@ defmodule Membrane.Transcoder.Plugin.Mixfile do
       {:membrane_opus_format, "~> 0.3.0"},
       {:membrane_aac_format, "~> 0.8.0"},
       {:membrane_funnel_plugin, "~> 0.9.1"},
+      {:membrane_mpegaudio_format, "~> 0.3.0"},
+      {:membrane_mp3_mad_plugin, "~> 0.18.4"},
+      {:membrane_mp3_lame_plugin, "~> 0.18.3"},
       {:ex_doc, ">= 0.0.0", only: :dev, runtime: false},
       {:dialyxir, ">= 0.0.0", only: :dev, runtime: false},
       {:credo, ">= 0.0.0", only: :dev, runtime: false},