From 9011e3f7a9868b5aaf446e9f8c84bb2af971600c Mon Sep 17 00:00:00 2001 From: noarkhh Date: Fri, 31 Oct 2025 10:33:10 +0100 Subject: [PATCH 01/17] Implement a message endpoint --- lib/boombox.ex | 3 ++- lib/boombox/server.ex | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/boombox.ex b/lib/boombox.ex index 292b070..b8b9d71 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -440,7 +440,8 @@ defmodule Boombox do Boombox.Server.start( packet_serialization: false, stop_application: false, - communication_medium: server_communication_medium + communication_medium: server_communication_medium, + parent_pid: self() ) Boombox.Server.run(pid, Map.to_list(opts)) diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index d6b4c27..a3452ae 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -27,6 +27,8 @@ defmodule Boombox.Server do @type communication_medium :: :calls | :messages + @type communication_medium :: :calls | :messages + @type opts :: [ name: GenServer.name(), packet_serialization: boolean(), @@ -427,21 +429,26 @@ defmodule Boombox.Server do @spec get_boombox_mode(boombox_opts()) :: boombox_mode() defp get_boombox_mode(boombox_opts) do - case Map.new(boombox_opts) do - %{input: {:stream, _input_opts}, output: {:stream, _output_opts}} -> - raise ArgumentError, "Elixir endpoint on both input and output is not supported" + cond do + elixir_endpoint?(boombox_opts[:input]) and elixir_endpoint?(boombox_opts[:output]) -> + raise ArgumentError, "Using an elixir endpoint on both input and output is not supported" - %{input: {:stream, _input_opts}} -> + elixir_endpoint?(boombox_opts[:input]) -> :consuming - %{output: {:stream, _output_opts}} -> + elixir_endpoint?(boombox_opts[:output]) -> :producing - _other -> + true -> :standalone end end + defp elixir_endpoint?({type, _opts}) when type in [:reader, :writer, :message], + do: true + + defp elixir_endpoint?(_io), do: false + @spec consuming_boombox_run(boombox_opts(), pid()) :: :ok defp consuming_boombox_run(boombox_opts, server_pid) do Stream.resource( From 233c155c861d06c5ae3f86fa4d93a30993f4ef60 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Fri, 31 Oct 2025 16:14:59 +0100 Subject: [PATCH 02/17] refactor wip --- lib/boombox.ex | 90 +++++++++---------------------------- lib/boombox/internal_bin.ex | 5 ++- lib/boombox/pipeline.ex | 54 ++++++++++++++++++++++ lib/boombox/server.ex | 4 +- test/boombox_test.exs | 2 + 5 files changed, 81 insertions(+), 74 deletions(-) diff --git a/lib/boombox.ex b/lib/boombox.ex index b8b9d71..9c24b1c 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -11,6 +11,9 @@ defmodule Boombox do alias Membrane.HTTPAdaptiveStream alias Membrane.RTP + alias Boombox.Pipeline + + @elixir_endpoints [:stream, :message, :writer, :reader] defmodule Writer do @moduledoc """ @@ -159,12 +162,6 @@ defmodule Boombox do @type elixir_output :: {:stream | :reader | :message, out_raw_data_opts()} @typep procs :: %{pipeline: pid(), supervisor: pid()} - @typep opts_map :: %{ - input: input() | elixir_input(), - output: output() | elixir_output(), - parent: pid() - } - @doc """ Runs boombox with given input and output. @@ -217,13 +214,13 @@ defmodule Boombox do case opts do %{input: {:stream, _stream_opts}} -> - procs = start_pipeline(opts) - source = await_source_ready() + procs = Pipeline.start_pipeline(opts) + source = Pipeline.await_source_ready() consume_stream(stream, source, procs) %{output: {:stream, _stream_opts}} -> - procs = start_pipeline(opts) - sink = await_sink_ready() + procs = Pipeline.start_pipeline(opts) + sink = Pipeline.await_sink_ready() produce_stream(sink, procs) %{input: {:writer, _writer_opts}} -> @@ -242,8 +239,8 @@ defmodule Boombox do opts -> opts - |> start_pipeline() - |> await_pipeline() + |> Pipeline.start_pipeline() + |> Pipeline.await_pipeline() end end @@ -282,8 +279,8 @@ defmodule Boombox do case opts do %{input: {:stream, _stream_opts}} -> - procs = start_pipeline(opts) - source = await_source_ready() + procs = Pipeline.start_pipeline(opts) + source = Pipeline.await_source_ready() Task.async(fn -> Process.monitor(procs.supervisor) @@ -291,8 +288,8 @@ defmodule Boombox do end) %{output: {:stream, _stream_opts}} -> - procs = start_pipeline(opts) - sink = await_sink_ready() + procs = Pipeline.start_pipeline(opts) + sink = Pipeline.await_sink_ready() produce_stream(sink, procs) %{input: {:writer, _writer_opts}} -> @@ -312,23 +309,23 @@ defmodule Boombox do # In case of rtmp, rtmps, rtp, rtsp, we need to wait for the tcp/udp server to be ready # before returning from async/2. %{input: {protocol, _opts}} when protocol in [:rtmp, :rtmps, :rtp, :rtsp, :srt] -> - procs = start_pipeline(opts) + procs = Pipeline.start_pipeline(opts) task = Task.async(fn -> Process.monitor(procs.supervisor) - await_pipeline(procs) + Pipeline.await_pipeline(procs) end) await_external_resource_ready() task opts -> - procs = start_pipeline(opts) + procs = Pipeline.start_pipeline(opts) Task.async(fn -> Process.monitor(procs.supervisor) - await_pipeline(procs) + Pipeline.await_pipeline(procs) end) end end @@ -429,7 +426,7 @@ defmodule Boombox do end end - defp elixir_endpoint?({type, _opts}) when type in [:reader, :writer, :stream, :message], + defp elixir_endpoint?({type, _opts}) when type in @elixir_endpoints, do: true defp elixir_endpoint?(_io), do: false @@ -479,7 +476,7 @@ defmodule Boombox do _state -> send(source, :boombox_eos) - await_pipeline(procs) + Pipeline.await_pipeline(procs) end end @@ -503,54 +500,12 @@ defmodule Boombox do end end, fn - %{procs: procs} -> terminate_pipeline(procs) + %{procs: procs} -> Pipeline.terminate_pipeline(procs) :eos -> :ok end ) end - @spec start_pipeline(opts_map()) :: procs() - defp start_pipeline(opts) do - opts = - opts - |> Map.update!(:input, &resolve_stream_endpoint(&1, self())) - |> Map.update!(:output, &resolve_stream_endpoint(&1, self())) - |> Map.put(:parent, self()) - - {:ok, supervisor, pipeline} = - Membrane.Pipeline.start_link(Boombox.Pipeline, opts) - - Process.monitor(supervisor) - %{supervisor: supervisor, pipeline: pipeline} - end - - @spec terminate_pipeline(procs) :: :ok - defp terminate_pipeline(procs) do - Membrane.Pipeline.terminate(procs.pipeline) - await_pipeline(procs) - end - - @spec await_pipeline(procs) :: :ok - defp await_pipeline(%{supervisor: supervisor}) do - receive do - {:DOWN, _monitor, :process, ^supervisor, _reason} -> :ok - end - end - - @spec await_source_ready() :: pid() - defp await_source_ready() do - receive do - {:boombox_ex_stream_source, source} -> source - end - end - - @spec await_sink_ready() :: pid() - defp await_sink_ready() do - receive do - {:boombox_ex_stream_sink, sink} -> sink - end - end - # Waits for the external resource to be ready. # This is used to wait for the tcp/udp server to be ready before returning from async/2. # It is used for rtmp, rtmps, rtp, rtsp. @@ -580,9 +535,4 @@ defmodule Boombox do :ok end - - defp resolve_stream_endpoint({:stream, stream_options}, parent), - do: {:stream, parent, stream_options} - - defp resolve_stream_endpoint(endpoint, _parent), do: endpoint end diff --git a/lib/boombox/internal_bin.ex b/lib/boombox/internal_bin.ex index 2165288..25b327b 100644 --- a/lib/boombox/internal_bin.ex +++ b/lib/boombox/internal_bin.ex @@ -453,7 +453,7 @@ defmodule Boombox.InternalBin do end defp create_input({:stream, stream_process, params}, _ctx, _state) do - Boombox.InternalBin.ElixirStream.create_input(stream_process, params) + Boombox.InternalBin.ElixirEndpoints.create_input(stream_process, params, :pull) end defp create_input({:h264, location, opts}, _ctx, _state) do @@ -687,9 +687,10 @@ defmodule Boombox.InternalBin do is_input_realtime = input_realtime?(state.input) result = - Boombox.InternalBin.ElixirStream.link_output( + Boombox.InternalBin.ElixirEndpoints.link_output( stream_process, params, + :pull, track_builders, spec_builder, is_input_realtime diff --git a/lib/boombox/pipeline.ex b/lib/boombox/pipeline.ex index e37103b..377abcc 100644 --- a/lib/boombox/pipeline.ex +++ b/lib/boombox/pipeline.ex @@ -1,6 +1,54 @@ defmodule Boombox.Pipeline do @moduledoc false use Membrane.Pipeline + @elixir_endpoints [:stream, :message, :writer, :reader] + + @type opts_map :: %{ + input: Boombox.input() | Boombox.elixir_input(), + output: Boombox.output() | Boombox.elixir_output() + } + + @spec start_pipeline(opts_map()) :: Boombox.procs() + def start_pipeline(opts) do + opts = + opts + |> Map.update!(:input, &resolve_stream_endpoint(&1, self())) + |> Map.update!(:output, &resolve_stream_endpoint(&1, self())) + |> Map.put(:parent, self()) + + {:ok, supervisor, pipeline} = + Membrane.Pipeline.start_link(Boombox.Pipeline, opts) + + Process.monitor(supervisor) + %{supervisor: supervisor, pipeline: pipeline} + end + + @spec terminate_pipeline(Boombox.procs()) :: :ok + def terminate_pipeline(procs) do + Membrane.Pipeline.terminate(procs.pipeline) + await_pipeline(procs) + end + + @spec await_pipeline(Boombox.procs()) :: :ok + def await_pipeline(%{supervisor: supervisor}) do + receive do + {:DOWN, _monitor, :process, ^supervisor, _reason} -> :ok + end + end + + @spec await_source_ready() :: pid() + def await_source_ready() do + receive do + {:boombox_ex_stream_source, source} -> source + end + end + + @spec await_sink_ready() :: pid() + def await_sink_ready() do + receive do + {:boombox_ex_stream_sink, sink} -> sink + end + end @impl true def handle_init(_ctx, opts) do @@ -23,4 +71,10 @@ defmodule Boombox.Pipeline do def handle_child_notification(:processing_finished, :boombox, _ctx, state) do {[terminate: :normal], state} end + + defp resolve_stream_endpoint({endpoint_type, opts}, parent) + when endpoint_type in @elixir_endpoints, + do: {endpoint_type, parent, opts} + + defp resolve_stream_endpoint(endpoint, _parent), do: endpoint end diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index a3452ae..fefa363 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -37,8 +37,8 @@ defmodule Boombox.Server do ] @type boombox_opts :: [ - input: Boombox.input() | {:writer, Boombox.in_raw_data_opts()}, - output: Boombox.output() | {:reader, Boombox.out_raw_data_opts()} + input: Boombox.input() | {:writer | :message, Boombox.in_raw_data_opts()}, + output: Boombox.output() | {:reader | :message, Boombox.out_raw_data_opts()} ] @typedoc """ diff --git a/test/boombox_test.exs b/test/boombox_test.exs index 15f04c8..50067cb 100644 --- a/test/boombox_test.exs +++ b/test/boombox_test.exs @@ -474,6 +474,7 @@ defmodule BoomboxTest do [:stream, :writer, :message] |> Enum.each(fn elixir_endpoint -> + @tag :bouncing_bubble_elixir_webrtc_mp4 @tag String.to_atom("bouncing_bubble_#{elixir_endpoint}_webrtc_mp4") async_test "bouncing bubble -> #{elixir_endpoint} -> webrtc -> mp4", %{tmp_dir: tmp} do signaling = Membrane.WebRTC.Signaling.new() @@ -548,6 +549,7 @@ defmodule BoomboxTest do [:stream, :reader, :message] |> Enum.each(fn elixir_endpoint -> + @tag :mp4_elixir_resampled_pcm @tag String.to_atom("mp4_#{elixir_endpoint}_resampled_pcm") async_test "mp4 -> #{elixir_endpoint} -> resampled PCM" do boombox = From e4bf357419b44eaab67f1de915dbc613f49b2d19 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Thu, 6 Nov 2025 11:00:10 +0100 Subject: [PATCH 03/17] WIP --- lib/boombox.ex | 2 +- lib/boombox/server.ex | 122 +++++++++++++++++++++++++------- python/src/boombox/boombox.py | 16 +++-- python/src/boombox/endpoints.py | 14 ++-- 4 files changed, 116 insertions(+), 38 deletions(-) diff --git a/lib/boombox.ex b/lib/boombox.ex index 9c24b1c..002e4fb 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -431,7 +431,7 @@ defmodule Boombox do defp elixir_endpoint?(_io), do: false - @spec start_server(opts_map(), :messages | :calls) :: boombox_server() + @spec start_server(Pipeline.opts_map(), :messages | :calls) :: boombox_server() defp start_server(opts, server_communication_medium) do {:ok, pid} = Boombox.Server.start( diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index fefa363..60aa30a 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -27,8 +27,6 @@ defmodule Boombox.Server do @type communication_medium :: :calls | :messages - @type communication_medium :: :calls | :messages - @type opts :: [ name: GenServer.name(), packet_serialization: boolean(), @@ -99,12 +97,31 @@ defmodule Boombox.Server do boombox_pid: pid() | nil, boombox_mode: Boombox.Server.boombox_mode() | nil, communication_medium: Boombox.Server.communication_medium(), - parent_pid: pid() + parent_pid: pid(), + membrane_source_pid: pid() | nil, + membrane_source_demand: non_neg_integer(), + membrane_sink_pid: pid() | nil, + pipeline_supervisor_pid: pid() | nil, + pid_to_reply_to: pid() | nil } - @enforce_keys [:packet_serialization, :stop_application, :communication_medium, :parent_pid] - - defstruct @enforce_keys ++ [boombox_pid: nil, boombox_mode: nil] + @enforce_keys [ + :packet_serialization, + :stop_application, + :communication_medium, + :parent_pid + ] + + defstruct @enforce_keys ++ + [ + boombox_pid: nil, + boombox_mode: nil, + membrane_source_pid: nil, + membrane_source_demand: 0, + membrane_sink_pid: nil, + pipeline_supervisor_pid: nil, + pid_to_reply_to: nil + ] end @doc """ @@ -200,8 +217,8 @@ defmodule Boombox.Server do end @impl true - def handle_call(request, _from, state) do - {response, state} = handle_request(request, state) + def handle_call(request, from, state) do + {response, state} = handle_request(request, from, state) {:reply, response, state} end @@ -248,6 +265,21 @@ defmodule Boombox.Server do {:noreply, state} end + @impl true + def handle_info({:boombox_ex_stream_source, source}, state) do + {:noreply, %State{state | membrane_source_pid: source}} + end + + @impl true + def handle_info({:boombox_ex_stream_sink, sink}, state) do + {:noreply, %State{state | membrane_sink_pid: sink}} + end + + @impl true + def handle_info({:boombox_demand, demand}, state) do + {:noreply, %State{state | membrane_source_demand: state.membrane_source_demand + demand}} + end + @impl true def handle_info({:DOWN, _ref, :process, pid, reason}, %State{boombox_pid: pid} = state) do reason = @@ -259,6 +291,20 @@ defmodule Boombox.Server do {:stop, reason, state} end + @impl true + def handle_info( + {:DOWN, _ref, :process, pid, reason}, + %State{pipeline_supervisor_pid: pid} = state + ) do + reason = + case reason do + :normal -> :normal + reason -> {:boombox_crash, reason} + end + + {:stop, reason, state} + end + @impl true def handle_info(info, state) do Logger.warning("Ignoring message #{inspect(info)}") @@ -283,8 +329,13 @@ defmodule Boombox.Server do end end - @spec handle_request({:run, boombox_opts()}, State.t()) :: {boombox_mode(), State.t()} - defp handle_request({:run, boombox_opts}, %State{} = state) do + defp handle_request(request, from \\ nil, state) + + @spec handle_request({:run, boombox_opts()}, GenServer.from() | nil, State.t()) :: + {boombox_mode(), State.t()} + defp handle_request({:run, boombox_opts}, _from, state) do + boombox_mode = get_boombox_mode(boombox_opts) + boombox_opts = boombox_opts |> Enum.map(fn @@ -294,9 +345,8 @@ defmodule Boombox.Server do other -> other end) - boombox_mode = get_boombox_mode(boombox_opts) - server_pid = self() + procs = Boombox.Pipeline.start_pipeline(Map.new(boombox_opts)) boombox_process_fun = case boombox_mode do @@ -313,25 +363,33 @@ defmodule Boombox.Server do boombox_pid = spawn(boombox_process_fun) Process.monitor(boombox_pid) - {boombox_mode, %State{state | boombox_pid: boombox_pid, boombox_mode: boombox_mode}} + {boombox_mode, + %State{ + state + | boombox_pid: boombox_pid, + boombox_mode: boombox_mode, + pipeline_supervisor_pid: procs.supervisor + }} end - @spec handle_request(:get_pid, State.t()) :: {pid(), State.t()} - defp handle_request(:get_pid, state) do + @spec handle_request(:get_pid, GenServer.from() | nil, State.t()) :: {pid(), State.t()} + defp handle_request(:get_pid, _from, state) do {self(), state} end - defp handle_request(_request, %State{boombox_pid: nil} = state) do + defp handle_request(_request, _from, %State{boombox_pid: nil} = state) do {{:error, :boombox_not_running}, state} end @spec handle_request( {:consume_packet, serialized_boombox_packet() | Boombox.Packet.t()}, + GenServer.from() | nil, State.t() ) :: {:ok | :finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} defp handle_request( {:consume_packet, packet}, + from, %State{boombox_mode: :consuming, boombox_pid: boombox_pid} = state ) do packet = @@ -341,6 +399,12 @@ defmodule Boombox.Server do packet end + demand = + if state.membrane_source_demand == 1 do + send(state.membrane_source_pid, packet) + {:noreply, %State{state | membrane_source_demand: 0}} + end + send(boombox_pid, {:consume_packet, packet}) receive do @@ -352,14 +416,19 @@ defmodule Boombox.Server do end end - defp handle_request({:consume_packet, _packet}, %State{boombox_mode: _other_mode} = state) do + defp handle_request( + {:consume_packet, _packet}, + _from, + %State{boombox_mode: _other_mode} = state + ) do {{:error, :incompatible_mode}, state} end - @spec handle_request(:finish_consuming, State.t()) :: + @spec handle_request(:finish_consuming, GenServer.from() | nil, State.t()) :: {:finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} defp handle_request( :finish_consuming, + _from, %State{boombox_mode: :consuming, boombox_pid: boombox_pid} = state ) do send(boombox_pid, :finish_consuming) @@ -370,15 +439,16 @@ defmodule Boombox.Server do end end - defp handle_request(:finish_consuming, %State{boombox_mode: _other_mode} = state) do + defp handle_request(:finish_consuming, _from, %State{boombox_mode: _other_mode} = state) do {{:error, :incompatible_mode}, state} end - @spec handle_request(:produce_packet, State.t()) :: + @spec handle_request(:produce_packet, GenServer.from() | nil, State.t()) :: {{:ok | :finished, serialized_boombox_packet() | Boombox.Packet.t()} | {:error, :incompatible_mode | :boombox_not_running}, State.t()} defp handle_request( :produce_packet, + _from, %State{boombox_mode: :producing, boombox_pid: boombox_pid} = state ) do send(boombox_pid, :produce_packet) @@ -399,15 +469,16 @@ defmodule Boombox.Server do {{response_type, packet}, state} end - defp handle_request(:produce_packet, %State{boombox_mode: _other_mode} = state) do + defp handle_request(:produce_packet, _from, %State{boombox_mode: _other_mode} = state) do {{:error, :incompatible_mode}, state} end - @spec handle_request(:finish_producing, State.t()) :: + @spec handle_request(:finish_producing, GenServer.from() | nil, State.t()) :: {{:finished, serialized_boombox_packet() | Boombox.Packet.t()} | {:error, :incompatible_mode}, State.t()} defp handle_request( :finish_producing, + _from, %State{boombox_mode: :producing, boombox_pid: boombox_pid} = state ) do send(boombox_pid, :finish_producing) @@ -418,12 +489,13 @@ defmodule Boombox.Server do end end - defp handle_request(:finish_producing, %State{boombox_mode: _other_mode} = state) do + defp handle_request(:finish_producing, _from, %State{boombox_mode: _other_mode} = state) do {{:error, :incompatible_mode}, state} end - @spec handle_request(term(), State.t()) :: {{:error, :invalid_request}, State.t()} - defp handle_request(_invalid_request, state) do + @spec handle_request(term(), GenServer.from() | nil, State.t()) :: + {{:error, :invalid_request}, State.t()} + defp handle_request(_invalid_request, _from, state) do {{:error, :invalid_request}, state} end diff --git a/python/src/boombox/boombox.py b/python/src/boombox/boombox.py index a512e77..38a0c09 100644 --- a/python/src/boombox/boombox.py +++ b/python/src/boombox/boombox.py @@ -24,7 +24,7 @@ from ._vendor.term import Atom, Pid from .endpoints import BoomboxEndpoint, AudioSampleFormat -from typing import Generator, ClassVar, Optional, Any, get_args +from typing import Generator, ClassVar, Literal, Optional, Any, get_args from typing_extensions import override @@ -120,7 +120,9 @@ def __init__( self._download_elixir_boombox_release() - self._erlang_process = subprocess.Popen([self._server_release_path, "start"], env=env) + self._erlang_process = subprocess.Popen( + [self._server_release_path, "start"], env=env + ) atexit.register(lambda: self._erlang_process.kill()) super().__init__(True) @@ -132,8 +134,8 @@ def __init__( self.get_node().monitor_process(self.pid_, self._receiver) boombox_arg = [ - (Atom("input"), self._serialize_endpoint(input)), - (Atom("output"), self._serialize_endpoint(output)), + (Atom("input"), self._serialize_endpoint(input, "input")), + (Atom("output"), self._serialize_endpoint(output, "output")), ] self._call((Atom("run"), boombox_arg)) @@ -507,11 +509,13 @@ def _serialize_packet(packet: AudioPacket | VideoPacket) -> dict[Atom, Any]: } @staticmethod - def _serialize_endpoint(endpoint: BoomboxEndpoint | str) -> Any: + def _serialize_endpoint( + endpoint: BoomboxEndpoint | str, direction: Literal["input", "output"] + ) -> Any: if isinstance(endpoint, str): return endpoint.encode() else: - return endpoint.serialize() + return endpoint.serialize(direction) @dataclasses.dataclass diff --git a/python/src/boombox/endpoints.py b/python/src/boombox/endpoints.py index a37016a..c04e179 100644 --- a/python/src/boombox/endpoints.py +++ b/python/src/boombox/endpoints.py @@ -81,7 +81,7 @@ def get_atom_fields(self) -> set[str]: # def validate_direction(self, direction: Literal['input', 'output']) -> # bool: ... - def serialize(self) -> tuple: + def serialize(self, direction: Literal['input', 'output']) -> tuple: """Serializes itself to an Elixir-compatible term. To allow Pyrlang to send the endpoint definition to Elixir it first @@ -111,11 +111,11 @@ def serialize(self) -> tuple: if f.kw_only and self.__dict__[f.name] is not None ] if keyword_fields: - return (self.get_endpoint_name(), *required_field_values, keyword_fields) + return (self.get_endpoint_name(direction), *required_field_values, keyword_fields) else: - return (self.get_endpoint_name(), *required_field_values) + return (self.get_endpoint_name(direction), *required_field_values) - def get_endpoint_name(self) -> Atom: + def get_endpoint_name(self, direction: Literal["input", "output"]) -> Atom: """:meta private:""" return Atom(self.__class__.__name__.lower()) @@ -179,8 +179,10 @@ class RawData(BoomboxEndpoint): is_live: bool | None = None @override - def get_endpoint_name(self) -> Atom: - return Atom("stream") + def get_endpoint_name(self, direction) -> Atom: + match direction: + case "input": return Atom("writer") + case "output": return Atom("reader") @override def get_atom_fields(self) -> set[str]: From a946dd90f15ec2faef9629efa15aac4d8a1b7acc Mon Sep 17 00:00:00 2001 From: noarkhh Date: Fri, 14 Nov 2025 15:44:12 +0100 Subject: [PATCH 04/17] wip --- lib/boombox.ex | 12 +- lib/boombox/internal_bin/elixir_endpoints.ex | 162 ++++++++++++++++++ .../elixir_endpoints/pull_sink.ex | 35 ++++ .../elixir_endpoints/pull_source.ex | 27 +++ .../elixir_endpoints/push_sink.ex | 28 +++ .../elixir_endpoints/push_source.ex | 24 +++ .../internal_bin/elixir_endpoints/sink.ex | 84 +++++++++ .../internal_bin/elixir_endpoints/source.ex | 95 ++++++++++ .../internal_bin/elixir_stream/sink.ex | 15 +- .../internal_bin/elixir_stream/source.ex | 18 +- lib/boombox/pipeline.ex | 4 +- lib/boombox/server.ex | 34 ++-- 12 files changed, 511 insertions(+), 27 deletions(-) create mode 100644 lib/boombox/internal_bin/elixir_endpoints.ex create mode 100644 lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex create mode 100644 lib/boombox/internal_bin/elixir_endpoints/pull_source.ex create mode 100644 lib/boombox/internal_bin/elixir_endpoints/push_sink.ex create mode 100644 lib/boombox/internal_bin/elixir_endpoints/push_source.ex create mode 100644 lib/boombox/internal_bin/elixir_endpoints/sink.ex create mode 100644 lib/boombox/internal_bin/elixir_endpoints/source.ex diff --git a/lib/boombox.ex b/lib/boombox.ex index 002e4fb..1c916ed 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -453,8 +453,8 @@ defmodule Boombox do fn %Boombox.Packet{} = packet, %{demand: 0} = state -> receive do - {:boombox_demand, demand} -> - send(source, packet) + {:boombox_demand, ^source, demand} -> + send(source, {:boombox_packet, self(), packet}) {:cont, %{state | demand: demand - 1}} {:DOWN, _monitor, :process, supervisor, _reason} @@ -463,7 +463,7 @@ defmodule Boombox do end %Boombox.Packet{} = packet, %{demand: demand} = state -> - send(source, packet) + send(source, {:boombox_packet, self(), packet}) {:cont, %{state | demand: demand - 1}} value, _state -> @@ -475,7 +475,7 @@ defmodule Boombox do :ok _state -> - send(source, :boombox_eos) + send(source, {:boombox_close, self()}) Pipeline.await_pipeline(procs) end end @@ -487,10 +487,10 @@ defmodule Boombox do %{sink: sink, procs: procs} end, fn %{sink: sink, procs: procs} = state -> - send(sink, :boombox_demand) + send(sink, {:boombox_demand, self()}) receive do - %Boombox.Packet{} = packet -> + {:boombox_packet, ^sink, %Boombox.Packet{} = packet} -> verify_packet!(packet) {[packet], state} diff --git a/lib/boombox/internal_bin/elixir_endpoints.ex b/lib/boombox/internal_bin/elixir_endpoints.ex new file mode 100644 index 0000000..0bb5d2b --- /dev/null +++ b/lib/boombox/internal_bin/elixir_endpoints.ex @@ -0,0 +1,162 @@ +defmodule Boombox.InternalBin.ElixirEndpoints do + @moduledoc false + + import Membrane.ChildrenSpec + require Membrane.Pad, as: Pad + + alias __MODULE__.{PullSink, PushSink, PullSource, PushSource} + alias Boombox.InternalBin.Ready + alias Membrane.FFmpeg.SWScale + + @options_audio_keys [:audio_format, :audio_rate, :audio_channels] + + # the size of the toilet capacity is supposed to handle more or less + # the burst of packets from one segment of Live HLS stream + @realtimer_toilet_capacity 10_000 + + @type flow_control_mode :: :push | :pull + + @spec create_input(pid(), Boombox.in_raw_data_opts(), flow_control_mode()) :: Ready.t() + def create_input(producer, options, flow_control_mode) do + options = parse_options(options, :input) + + builders = + [:audio, :video] + |> Enum.filter(&(options[&1] != false)) + |> Map.new(fn + :video -> + {:video, + get_child(:elixir_stream_source) + |> via_out(Pad.ref(:output, :video)) + |> child(%SWScale.Converter{format: :I420}) + |> child(%Membrane.H264.FFmpeg.Encoder{profile: :baseline, preset: :ultrafast})} + + :audio -> + {:audio, + get_child(:elixir_stream_source) + |> via_out(Pad.ref(:output, :audio))} + end) + + source_definition = + case flow_control_mode do + :push -> %PushSource{producer: producer} + :pull -> %PullSource{producer: producer} + end + + spec_builder = child(:elixir_stream_source, source_definition) + + %Ready{track_builders: builders, spec_builder: spec_builder} + end + + @spec link_output( + pid(), + Boombox.out_raw_data_opts(), + flow_control_mode(), + Boombox.InternalBin.track_builders(), + Membrane.ChildrenSpec.t(), + boolean() + ) :: Ready.t() + def link_output( + consumer, + options, + flow_control_mode, + track_builders, + spec_builder, + is_input_realtime + ) do + options = parse_options(options, :output) + pace_control = Map.get(options, :pace_control, true) + + {track_builders, to_ignore} = + Map.split_with(track_builders, fn {kind, _builder} -> options[kind] != false end) + + sink_definition = + case flow_control_mode do + :push -> %PushSink{consumer: consumer} + :pull -> %PullSink{consumer: consumer} + end + + spec = + [ + spec_builder, + child(:elixir_stream_sink, sink_definition), + Enum.map(track_builders, fn + {:audio, builder} -> + builder + |> child(:elixir_stream_audio_transcoder, %Membrane.Transcoder{ + output_stream_format: Membrane.RawAudio + }) + |> maybe_plug_resampler(options) + |> maybe_plug_realtimer(:audio, pace_control, is_input_realtime) + |> via_in(Pad.ref(:input, :audio)) + |> get_child(:elixir_stream_sink) + + {:video, builder} -> + builder + |> child(:elixir_stream_video_transcoder, %Membrane.Transcoder{ + output_stream_format: Membrane.RawVideo + }) + |> child(:elixir_stream_rgb_converter, %SWScale.Converter{ + format: :RGB, + output_width: options[:video_width], + output_height: options[:video_height] + }) + |> maybe_plug_realtimer(:video, pace_control, is_input_realtime) + |> via_in(Pad.ref(:input, :video)) + |> get_child(:elixir_stream_sink) + end), + Enum.map(to_ignore, fn {_track, builder} -> builder |> child(Membrane.Debug.Sink) end) + ] + + %Ready{actions: [spec: spec], eos_info: Map.keys(track_builders)} + end + + defp maybe_plug_realtimer(builder, kind, pace_control, is_input_realtime) + + defp maybe_plug_realtimer(builder, kind, true, false) do + builder + |> via_in(:input, toilet_capacity: @realtimer_toilet_capacity) + |> child({:elixir_stream, kind, :realtimer}, Membrane.Realtimer) + end + + defp maybe_plug_realtimer(builder, _kind, _pace_control, _is_input_realtime), do: builder + @spec parse_options(Boombox.in_raw_data_opts(), :input) :: map() + @spec parse_options(Boombox.out_raw_data_opts(), :output) :: map() + defp parse_options(options, direction) do + audio = Keyword.get(options, :audio) + + audio_keys = + if direction == :output and audio != false and + Enum.any?(@options_audio_keys, &Keyword.has_key?(options, &1)), + do: @options_audio_keys, + else: [] + + options = + options + |> Keyword.validate!( + [:video, :audio, :video_width, :video_height, :pace_control, :is_live] ++ audio_keys + ) + |> Map.new() + + if options.audio == false and options.video == false do + raise "Got audio and video options set to false. At least one track must be enabled." + end + + options + end + + defp maybe_plug_resampler(builder, %{ + audio_format: format, + audio_rate: rate, + audio_channels: channels + }) do + format = %Membrane.RawAudio{sample_format: format, sample_rate: rate, channels: channels} + + builder + |> child(%Membrane.FFmpeg.SWResample.Converter{output_stream_format: format}) + end + + defp maybe_plug_resampler(builder, _options) do + builder + end +end diff --git a/lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex b/lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex new file mode 100644 index 0000000..55528af --- /dev/null +++ b/lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex @@ -0,0 +1,35 @@ +defmodule Boombox.InternalBin.ElixirEndpoints.PullSink do + @moduledoc false + use Membrane.Sink + + alias Boombox.InternalBin.ElixirEndpoints.Sink + + def_input_pad :input, + accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), + availability: :on_request, + flow_control: :manual, + demand_unit: :buffers + + def_options consumer: [spec: pid()] + + @impl true + defdelegate handle_init(ctx, opts), to: Sink + + @impl true + defdelegate handle_pad_added(pad, ctx, state), to: Sink + + @impl true + defdelegate handle_playing(ctx, state), to: Sink + + @impl true + defdelegate handle_info(info, ctx, state), to: Sink + + @impl true + defdelegate handle_stream_format(pad, stream_format, ctx, state), to: Sink + + @impl true + defdelegate handle_buffer(pad, buffer, ctx, state), to: Sink + + @impl true + defdelegate handle_end_of_stream(pad, ctx, state), to: Sink +end diff --git a/lib/boombox/internal_bin/elixir_endpoints/pull_source.ex b/lib/boombox/internal_bin/elixir_endpoints/pull_source.ex new file mode 100644 index 0000000..4b3afb7 --- /dev/null +++ b/lib/boombox/internal_bin/elixir_endpoints/pull_source.ex @@ -0,0 +1,27 @@ +defmodule Boombox.InternalBin.ElixirEndpoints.PullSource do + @moduledoc false + use Membrane.Source + + alias Boombox.InternalBin.ElixirEndpoints.Source + + def_output_pad :output, + accepted_format: any_of(Membrane.RawVideo, Membrane.RawAudio), + availability: :on_request, + flow_control: :manual + + def_options producer: [ + spec: pid() + ] + + @impl true + defdelegate handle_init(ctx, opts), to: Source + + @impl true + defdelegate handle_playing(ctx, state), to: Source + + @impl true + defdelegate handle_demand(pad, size, unit, ctx, state), to: Source + + @impl true + defdelegate handle_info(info, ctx, state), to: Source +end diff --git a/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex b/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex new file mode 100644 index 0000000..e9f30b6 --- /dev/null +++ b/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex @@ -0,0 +1,28 @@ +defmodule Boombox.InternalBin.ElixirEndpoints.PushSink do + @moduledoc false + use Membrane.Sink + + alias Boombox.InternalBin.ElixirEndpoints.Sink + + def_input_pad :input, + accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), + availability: :on_request, + flow_control: :auto + + def_options consumer: [spec: pid()] + + @impl true + defdelegate handle_init(ctx, opts), to: Sink + + @impl true + defdelegate handle_pad_added(pad, ctx, state), to: Sink + + @impl true + defdelegate handle_playing(ctx, state), to: Sink + + @impl true + defdelegate handle_stream_format(pad, stream_format, ctx, state), to: Sink + + @impl true + defdelegate handle_end_of_stream(pad, ctx, state), to: Sink +end diff --git a/lib/boombox/internal_bin/elixir_endpoints/push_source.ex b/lib/boombox/internal_bin/elixir_endpoints/push_source.ex new file mode 100644 index 0000000..df71d0a --- /dev/null +++ b/lib/boombox/internal_bin/elixir_endpoints/push_source.ex @@ -0,0 +1,24 @@ +defmodule Boombox.InternalBin.ElixirEndpoints.PushSource do + @moduledoc false + use Membrane.Source + + alias Boombox.InternalBin.ElixirEndpoints.Source + + def_output_pad :output, + accepted_format: any_of(Membrane.RawVideo, Membrane.RawAudio), + availability: :on_request, + flow_control: :push + + def_options producer: [ + spec: pid() + ] + + @impl true + defdelegate handle_init(ctx, opts), to: Source + + @impl true + defdelegate handle_playing(ctx, state), to: Source + + @impl true + defdelegate handle_info(info, ctx, state), to: Source +end diff --git a/lib/boombox/internal_bin/elixir_endpoints/sink.ex b/lib/boombox/internal_bin/elixir_endpoints/sink.ex new file mode 100644 index 0000000..6f7741b --- /dev/null +++ b/lib/boombox/internal_bin/elixir_endpoints/sink.ex @@ -0,0 +1,84 @@ +defmodule Boombox.InternalBin.ElixirEndpoints.Sink do + @moduledoc false + alias Membrane.Pad + require Membrane.Pad + + def handle_init(_ctx, opts) do + {[], Map.merge(Map.from_struct(opts), %{last_pts: %{}, audio_format: nil})} + end + + def handle_pad_added(Pad.ref(:input, kind), _ctx, state) do + {[], %{state | last_pts: Map.put(state.last_pts, kind, 0)}} + end + + def handle_playing(_ctx, state) do + send(state.consumer, {:boombox_elixir_sink, self()}) + {[], state} + end + + def handle_info({:boombox_demand, consumer}, _ctx, %{consumer: consumer} = state) do + if state.last_pts == %{} do + {[], state} + else + {kind, _pts} = + Enum.min_by(state.last_pts, fn {_kind, pts} -> pts end) + + {[demand: Pad.ref(:input, kind)], state} + end + end + + def handle_info(info, _ctx, state) do + dbg(info) + dbg(state) + raise "aaaaa" + end + + def handle_stream_format(Pad.ref(:input, :audio), stream_format, _ctx, state) do + audio_format = %{ + audio_format: stream_format.sample_format, + audio_rate: stream_format.sample_rate, + audio_channels: stream_format.channels + } + + {[], %{state | audio_format: audio_format}} + end + + def handle_stream_format(_pad, _stream_format, _ctx, state) do + {[], state} + end + + def handle_buffer(Pad.ref(:input, :video), buffer, ctx, state) do + state = %{state | last_pts: %{state.last_pts | video: buffer.pts}} + %{width: width, height: height} = ctx.pads[Pad.ref(:input, :video)].stream_format + + {:ok, image} = + Vix.Vips.Image.new_from_binary(buffer.payload, width, height, 3, :VIPS_FORMAT_UCHAR) + + send(state.consumer, %Boombox.Packet{ + payload: image, + pts: buffer.pts, + kind: :video + }) + + {[], state} + end + + def handle_buffer(Pad.ref(:input, :audio), buffer, _ctx, state) do + state = %{state | last_pts: %{state.last_pts | audio: buffer.pts}} + + packet = %Boombox.Packet{ + payload: buffer.payload, + pts: buffer.pts, + kind: :audio, + format: state.audio_format + } + + send(state.consumer, {:boombox_packet, self(), packet}) + + {[], state} + end + + def handle_end_of_stream(Pad.ref(:input, kind), _ctx, state) do + {[], %{state | last_pts: Map.delete(state.last_pts, kind)}} + end +end diff --git a/lib/boombox/internal_bin/elixir_endpoints/source.ex b/lib/boombox/internal_bin/elixir_endpoints/source.ex new file mode 100644 index 0000000..df1b4ae --- /dev/null +++ b/lib/boombox/internal_bin/elixir_endpoints/source.ex @@ -0,0 +1,95 @@ +defmodule Boombox.InternalBin.ElixirEndpoints.Source do + @moduledoc false + alias Membrane.Pad + require Membrane.Pad + + def handle_init(_ctx, opts) do + state = %{ + producer: opts.producer, + audio_format: nil, + video_dims: nil + } + + {[], state} + end + + def handle_playing(_ctx, state) do + send(state.producer, {:boombox_elixir_source, self()}) + {[], state} + end + + def handle_demand(Pad.ref(:output, _id), _size, _unit, ctx, state) do + demands = Enum.map(ctx.pads, fn {_pad, %{demand: demand}} -> demand end) + + if Enum.all?(demands, &(&1 > 0)) do + send(state.producer, {:boombox_demand, self(), Enum.sum(demands)}) + end + + {[], state} + end + + def handle_info( + {:boombox_packet, producer, %Boombox.Packet{kind: :video} = packet}, + _ctx, + %{producer: producer} = state + ) do + image = packet.payload |> Image.flatten!() |> Image.to_colorspace!(:srgb) + video_dims = %{width: Image.width(image), height: Image.height(image)} + {:ok, payload} = Vix.Vips.Image.write_to_binary(image) + buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} + + if video_dims == state.video_dims do + {[buffer: {Pad.ref(:output, :video), buffer}], state} + else + stream_format = %Membrane.RawVideo{ + width: video_dims.width, + height: video_dims.height, + pixel_format: :RGB, + aligned: true, + framerate: nil + } + + {[ + stream_format: {Pad.ref(:output, :video), stream_format}, + buffer: {Pad.ref(:output, :video), buffer} + ], %{state | video_dims: video_dims}} + end + end + + def handle_info( + {:boombox_packet, producer, %Boombox.Packet{kind: :audio} = packet}, + _ctx, + %{producer: producer} = state + ) do + %Boombox.Packet{payload: payload, format: format} = packet + buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} + + case format do + empty_format when empty_format == %{} and state.audio_format == nil -> + raise "No audio stream format provided" + + empty_format when empty_format == %{} -> + {[buffer: {Pad.ref(:output, :audio), buffer}], state} + + unchanged_format when unchanged_format == state.audio_format -> + {[buffer: {Pad.ref(:output, :audio), buffer}], state} + + new_format -> + stream_format = %Membrane.RawAudio{ + sample_format: new_format.audio_format, + sample_rate: new_format.audio_rate, + channels: new_format.audio_channels + } + + {[ + stream_format: {Pad.ref(:output, :audio), stream_format}, + buffer: {Pad.ref(:output, :audio), buffer} + ], %{state | audio_format: format}} + end + end + + def handle_info({:boombox_close, producer}, ctx, %{producer: producer} = state) do + actions = Enum.map(ctx.pads, fn {ref, _data} -> {:end_of_stream, ref} end) + {actions, state} + end +end diff --git a/lib/boombox/internal_bin/elixir_stream/sink.ex b/lib/boombox/internal_bin/elixir_stream/sink.ex index 7595e5c..bd93933 100644 --- a/lib/boombox/internal_bin/elixir_stream/sink.ex +++ b/lib/boombox/internal_bin/elixir_stream/sink.ex @@ -27,7 +27,7 @@ defmodule Boombox.InternalBin.ElixirStream.Sink do end @impl true - def handle_info(:boombox_demand, _ctx, state) do + def handle_info({:boombox_demand, consumer}, _ctx, %{consumer: consumer} = state) do if state.last_pts == %{} do {[], state} else @@ -38,6 +38,13 @@ defmodule Boombox.InternalBin.ElixirStream.Sink do end end + @impl true + def handle_info(info, _ctx, state) do + dbg(info) + dbg(state) + raise "aaaaa" + end + @impl true def handle_stream_format(Pad.ref(:input, :audio), stream_format, _ctx, state) do audio_format = %{ @@ -75,12 +82,14 @@ defmodule Boombox.InternalBin.ElixirStream.Sink do def handle_buffer(Pad.ref(:input, :audio), buffer, _ctx, state) do state = %{state | last_pts: %{state.last_pts | audio: buffer.pts}} - send(state.consumer, %Boombox.Packet{ + packet = %Boombox.Packet{ payload: buffer.payload, pts: buffer.pts, kind: :audio, format: state.audio_format - }) + } + + send(state.consumer, {:boombox_packet, self(), packet}) {[], state} end diff --git a/lib/boombox/internal_bin/elixir_stream/source.ex b/lib/boombox/internal_bin/elixir_stream/source.ex index 877c32c..fbe584b 100644 --- a/lib/boombox/internal_bin/elixir_stream/source.ex +++ b/lib/boombox/internal_bin/elixir_stream/source.ex @@ -25,7 +25,7 @@ defmodule Boombox.InternalBin.ElixirStream.Source do @impl true def handle_playing(_ctx, state) do - send(state.producer, {:boombox_ex_stream_source, self()}) + send(state.producer, {:boombox_elixir_source, self()}) {[], state} end @@ -34,14 +34,18 @@ defmodule Boombox.InternalBin.ElixirStream.Source do demands = Enum.map(ctx.pads, fn {_pad, %{demand: demand}} -> demand end) if Enum.all?(demands, &(&1 > 0)) do - send(state.producer, {:boombox_demand, Enum.sum(demands)}) + send(state.producer, {:boombox_demand, self(), Enum.sum(demands)}) end {[], state} end @impl true - def handle_info(%Boombox.Packet{kind: :video} = packet, _ctx, state) do + def handle_info( + {:boombox_packet, producer, %Boombox.Packet{kind: :video} = packet}, + _ctx, + %{producer: producer} = state + ) do image = packet.payload |> Image.flatten!() |> Image.to_colorspace!(:srgb) video_dims = %{width: Image.width(image), height: Image.height(image)} {:ok, payload} = Vix.Vips.Image.write_to_binary(image) @@ -66,7 +70,11 @@ defmodule Boombox.InternalBin.ElixirStream.Source do end @impl true - def handle_info(%Boombox.Packet{kind: :audio} = packet, _ctx, state) do + def handle_info( + {:boombox_packet, producer, %Boombox.Packet{kind: :audio} = packet}, + _ctx, + %{producer: producer} = state + ) do %Boombox.Packet{payload: payload, format: format} = packet buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} @@ -95,7 +103,7 @@ defmodule Boombox.InternalBin.ElixirStream.Source do end @impl true - def handle_info(:boombox_eos, ctx, state) do + def handle_info({:boombox_close, producer}, ctx, %{producer: producer} = state) do actions = Enum.map(ctx.pads, fn {ref, _data} -> {:end_of_stream, ref} end) {actions, state} end diff --git a/lib/boombox/pipeline.ex b/lib/boombox/pipeline.ex index 377abcc..6b3f057 100644 --- a/lib/boombox/pipeline.ex +++ b/lib/boombox/pipeline.ex @@ -39,14 +39,14 @@ defmodule Boombox.Pipeline do @spec await_source_ready() :: pid() def await_source_ready() do receive do - {:boombox_ex_stream_source, source} -> source + {:boombox_elixir_source, source} -> source end end @spec await_sink_ready() :: pid() def await_sink_ready() do receive do - {:boombox_ex_stream_sink, sink} -> sink + {:boombox_elixir_sink, sink} -> sink end end diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index 60aa30a..e782ac6 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -16,6 +16,14 @@ defmodule Boombox.Server do # tuple to the `sender` when finished. # The packets that Boombox is consuming and producing are in the form of # `t:serialized_boombox_packet/0` or `t:Boombox.Packet.t/0`, depending on set options. + # + # Avaliable actions: + # write buffer to boombox synchronously + # write buffer to boombox asynchronously + # read buffer from boombox synchronously + # receive buffer from boombox asynchronously (message) + # close boombox for wrtiting + # close boombox for reading use GenServer @@ -26,6 +34,7 @@ defmodule Boombox.Server do @type t :: GenServer.server() @type communication_medium :: :calls | :messages + @type flow_control :: :push | :pull @type opts :: [ name: GenServer.name(), @@ -102,6 +111,7 @@ defmodule Boombox.Server do membrane_source_demand: non_neg_integer(), membrane_sink_pid: pid() | nil, pipeline_supervisor_pid: pid() | nil, + pipeline_pid: pid() | nil, pid_to_reply_to: pid() | nil } @@ -120,6 +130,7 @@ defmodule Boombox.Server do membrane_source_demand: 0, membrane_sink_pid: nil, pipeline_supervisor_pid: nil, + pipeline_pid: nil, pid_to_reply_to: nil ] end @@ -266,12 +277,12 @@ defmodule Boombox.Server do end @impl true - def handle_info({:boombox_ex_stream_source, source}, state) do + def handle_info({:boombox_elixir_source, source}, state) do {:noreply, %State{state | membrane_source_pid: source}} end @impl true - def handle_info({:boombox_ex_stream_sink, sink}, state) do + def handle_info({:boombox_elixir_sink, sink}, state) do {:noreply, %State{state | membrane_sink_pid: sink}} end @@ -346,7 +357,8 @@ defmodule Boombox.Server do end) server_pid = self() - procs = Boombox.Pipeline.start_pipeline(Map.new(boombox_opts)) + # procs = Boombox.Pipeline.start_pipeline(Map.new(boombox_opts)) + procs = %{supervisor: nil, pipeline: nil} boombox_process_fun = case boombox_mode do @@ -368,7 +380,8 @@ defmodule Boombox.Server do state | boombox_pid: boombox_pid, boombox_mode: boombox_mode, - pipeline_supervisor_pid: procs.supervisor + pipeline_supervisor_pid: procs.supervisor, + pipeline_pid: procs.pipeline }} end @@ -385,8 +398,7 @@ defmodule Boombox.Server do {:consume_packet, serialized_boombox_packet() | Boombox.Packet.t()}, GenServer.from() | nil, State.t() - ) :: - {:ok | :finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} + ) :: {:ok | :finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} defp handle_request( {:consume_packet, packet}, from, @@ -399,11 +411,11 @@ defmodule Boombox.Server do packet end - demand = - if state.membrane_source_demand == 1 do - send(state.membrane_source_pid, packet) - {:noreply, %State{state | membrane_source_demand: 0}} - end + # demand = + # if state.membrane_source_demand == 1 do + # send(state.membrane_source_pid, packet) + # {:noreply, %State{state | membrane_source_demand: 0}} + # end send(boombox_pid, {:consume_packet, packet}) From 2e11129cc5784520b68ac7801b0fb1587ae95b74 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Fri, 14 Nov 2025 18:22:02 +0100 Subject: [PATCH 05/17] Make the refactor work for pull flow control --- lib/boombox.ex | 15 +- lib/boombox/internal_bin.ex | 43 +- .../internal_bin/elixir_endpoints/sink.ex | 6 - .../internal_bin/elixir_endpoints/source.ex | 2 +- lib/boombox/internal_bin/elixir_stream.ex | 140 ------ .../internal_bin/elixir_stream/sink.ex | 101 ---- .../internal_bin/elixir_stream/source.ex | 110 ---- lib/boombox/pipeline.ex | 7 +- lib/boombox/server.ex | 473 +++++++++++------- test/boombox_test.exs | 9 +- 10 files changed, 330 insertions(+), 576 deletions(-) delete mode 100644 lib/boombox/internal_bin/elixir_stream.ex delete mode 100644 lib/boombox/internal_bin/elixir_stream/sink.ex delete mode 100644 lib/boombox/internal_bin/elixir_stream/source.ex diff --git a/lib/boombox.ex b/lib/boombox.ex index 1c916ed..cee44c5 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -161,7 +161,6 @@ defmodule Boombox do @type elixir_output :: {:stream | :reader | :message, out_raw_data_opts()} - @typep procs :: %{pipeline: pid(), supervisor: pid()} @doc """ Runs boombox with given input and output. @@ -362,7 +361,7 @@ defmodule Boombox do Can be called only when using `:reader` endpoint on output. """ @spec read(Reader.t()) :: - {:ok | :finished, Boombox.Packet.t()} | {:error, :incompatible_mode} + {:ok, Boombox.Packet.t()} | :finished | {:error, :incompatible_mode} def read(reader) do Boombox.Server.produce_packet(reader.server_reference) end @@ -387,15 +386,13 @@ defmodule Boombox do of type `:finished` has been received. When using `:reader` endpoint on output informs Boombox that no more packets will be read - from it with `read/1` and that it should terminate accordingly. This function will then - return one last packet. + from it with `read/1` and that it should terminate accordingly. When using `:writer` endpoint on input informs Boombox that it will not be provided any more packets with `write/2` and should terminate accordingly. """ - @spec close(Writer.t()) :: :finished | {:error, :incompatible_mode} - @spec close(Reader.t()) :: {:finished, Boombox.Packet.t()} | {:error, :incompatible_mode} + @spec close(Writer.t() | Reader.t()) :: :finished | {:error, :incompatible_mode} def close(%Writer{} = writer) do Boombox.Server.finish_consuming(writer.server_reference) end @@ -445,7 +442,7 @@ defmodule Boombox do pid end - @spec consume_stream(Enumerable.t(), pid(), procs()) :: term() + @spec consume_stream(Enumerable.t(), pid(), Pipeline.procs()) :: term() defp consume_stream(stream, source, procs) do Enum.reduce_while( stream, @@ -475,12 +472,12 @@ defmodule Boombox do :ok _state -> - send(source, {:boombox_close, self()}) + send(source, {:boombox_eos, self()}) Pipeline.await_pipeline(procs) end end - @spec produce_stream(pid(), procs()) :: Enumerable.t() + @spec produce_stream(pid(), Pipeline.procs()) :: Enumerable.t() defp produce_stream(sink, procs) do Stream.resource( fn -> diff --git a/lib/boombox/internal_bin.ex b/lib/boombox/internal_bin.ex index 25b327b..6065c48 100644 --- a/lib/boombox/internal_bin.ex +++ b/lib/boombox/internal_bin.ex @@ -9,6 +9,8 @@ defmodule Boombox.InternalBin do alias Membrane.Transcoder + @elixir_endpoint_types [:stream, :message, :reader, :writer] + @type input :: Boombox.input() | {:stream, pid(), Boombox.in_raw_data_opts()} @@ -427,12 +429,15 @@ defmodule Boombox.InternalBin do case result do %Ready{actions: actions} = result when ready_status != nil -> - proceed(ctx, %{ - state - | status: ready_status, - last_result: result, - actions_acc: actions_acc ++ actions - }) + proceed( + ctx, + %{ + state + | status: ready_status, + last_result: result, + actions_acc: actions_acc ++ actions + } + ) %Wait{actions: actions} when wait_status != nil -> {actions_acc ++ actions, %{state | actions_acc: [], status: wait_status}} @@ -452,8 +457,12 @@ defmodule Boombox.InternalBin do Boombox.InternalBin.RTSP.create_input(uri) end - defp create_input({:stream, stream_process, params}, _ctx, _state) do - Boombox.InternalBin.ElixirEndpoints.create_input(stream_process, params, :pull) + defp create_input({type, process, params}, _ctx, _state) when type in @elixir_endpoint_types do + Boombox.InternalBin.ElixirEndpoints.create_input( + process, + params, + elixir_endpoint_flow_control(type) + ) end defp create_input({:h264, location, opts}, _ctx, _state) do @@ -683,14 +692,15 @@ defmodule Boombox.InternalBin do {result, state} end - defp link_output({:stream, stream_process, params}, track_builders, spec_builder, _ctx, state) do + defp link_output({type, process, params}, track_builders, spec_builder, _ctx, state) + when type in @elixir_endpoint_types do is_input_realtime = input_realtime?(state.input) result = Boombox.InternalBin.ElixirEndpoints.link_output( - stream_process, + process, params, - :pull, + elixir_endpoint_flow_control(type), track_builders, spec_builder, is_input_realtime @@ -840,7 +850,8 @@ defmodule Boombox.InternalBin do {:rtp, opts} -> if Keyword.keyword?(opts), do: value - {:stream, stream_process, opts} when is_pid(stream_process) -> + {elixir_endpoint, process, opts} + when is_pid(process) and elixir_endpoint in @elixir_endpoint_types -> if Keyword.keyword?(opts), do: value {:srt, server_awaiting_accept} @@ -948,6 +959,14 @@ defmodule Boombox.InternalBin do defp stream?({:stream, _pid, _opts}), do: true defp stream?(_endpoint), do: false + @spec elixir_endpoint_flow_control(:stream | :message | :reader | :writer) :: :pull | :push + defp elixir_endpoint_flow_control(type) do + cond do + type in [:stream, :writer, :reader] -> :pull + type == :message -> :push + end + end + @spec handles_keyframe_requests?(input()) :: boolean() defp handles_keyframe_requests?(input) do stream?(input) or webrtc?(input) diff --git a/lib/boombox/internal_bin/elixir_endpoints/sink.ex b/lib/boombox/internal_bin/elixir_endpoints/sink.ex index 6f7741b..77db246 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/sink.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/sink.ex @@ -27,12 +27,6 @@ defmodule Boombox.InternalBin.ElixirEndpoints.Sink do end end - def handle_info(info, _ctx, state) do - dbg(info) - dbg(state) - raise "aaaaa" - end - def handle_stream_format(Pad.ref(:input, :audio), stream_format, _ctx, state) do audio_format = %{ audio_format: stream_format.sample_format, diff --git a/lib/boombox/internal_bin/elixir_endpoints/source.ex b/lib/boombox/internal_bin/elixir_endpoints/source.ex index df1b4ae..6e3f468 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/source.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/source.ex @@ -88,7 +88,7 @@ defmodule Boombox.InternalBin.ElixirEndpoints.Source do end end - def handle_info({:boombox_close, producer}, ctx, %{producer: producer} = state) do + def handle_info({:boombox_eos, producer}, ctx, %{producer: producer} = state) do actions = Enum.map(ctx.pads, fn {ref, _data} -> {:end_of_stream, ref} end) {actions, state} end diff --git a/lib/boombox/internal_bin/elixir_stream.ex b/lib/boombox/internal_bin/elixir_stream.ex deleted file mode 100644 index 9770b7a..0000000 --- a/lib/boombox/internal_bin/elixir_stream.ex +++ /dev/null @@ -1,140 +0,0 @@ -defmodule Boombox.InternalBin.ElixirStream do - @moduledoc false - - import Membrane.ChildrenSpec - require Membrane.Pad, as: Pad - - alias __MODULE__.{Sink, Source} - alias Boombox.InternalBin.Ready - alias Membrane.FFmpeg.SWScale - - @options_audio_keys [:audio_format, :audio_rate, :audio_channels] - - # the size of the toilet capacity is supposed to handle more or less - # the burst of packets from one segment of Live HLS stream - @realtimer_toilet_capacity 10_000 - - @spec create_input(producer :: pid, options :: Boombox.in_raw_data_opts()) :: Ready.t() - def create_input(producer, options) do - options = parse_options(options, :input) - - builders = - [:audio, :video] - |> Enum.filter(&(options[&1] != false)) - |> Map.new(fn - :video -> - {:video, - get_child(:elixir_stream_source) - |> via_out(Pad.ref(:output, :video)) - |> child(%SWScale.Converter{format: :I420}) - |> child(%Membrane.H264.FFmpeg.Encoder{profile: :baseline, preset: :ultrafast})} - - :audio -> - {:audio, - get_child(:elixir_stream_source) - |> via_out(Pad.ref(:output, :audio))} - end) - - spec_builder = child(:elixir_stream_source, %Source{producer: producer}) - - %Ready{track_builders: builders, spec_builder: spec_builder} - end - - @spec link_output( - consumer :: pid, - options :: Boombox.out_raw_data_opts(), - Boombox.InternalBin.track_builders(), - Membrane.ChildrenSpec.t(), - boolean() - ) :: Ready.t() - def link_output(consumer, options, track_builders, spec_builder, is_input_realtime) do - options = parse_options(options, :output) - pace_control = Map.get(options, :pace_control, true) - - {track_builders, to_ignore} = - Map.split_with(track_builders, fn {kind, _builder} -> options[kind] != false end) - - spec = - [ - spec_builder, - child(:elixir_stream_sink, %Sink{consumer: consumer}), - Enum.map(track_builders, fn - {:audio, builder} -> - builder - |> child(:elixir_stream_audio_transcoder, %Membrane.Transcoder{ - output_stream_format: Membrane.RawAudio - }) - |> maybe_plug_resampler(options) - |> maybe_plug_realtimer(:audio, pace_control, is_input_realtime) - |> via_in(Pad.ref(:input, :audio)) - |> get_child(:elixir_stream_sink) - - {:video, builder} -> - builder - |> child(:elixir_stream_video_transcoder, %Membrane.Transcoder{ - output_stream_format: Membrane.RawVideo - }) - |> child(:elixir_stream_rgb_converter, %SWScale.Converter{ - format: :RGB, - output_width: options[:video_width], - output_height: options[:video_height] - }) - |> maybe_plug_realtimer(:video, pace_control, is_input_realtime) - |> via_in(Pad.ref(:input, :video)) - |> get_child(:elixir_stream_sink) - end), - Enum.map(to_ignore, fn {_track, builder} -> builder |> child(Membrane.Debug.Sink) end) - ] - - %Ready{actions: [spec: spec], eos_info: Map.keys(track_builders)} - end - - defp maybe_plug_realtimer(builder, kind, pace_control, is_input_realtime) - - defp maybe_plug_realtimer(builder, kind, true, false) do - builder - |> via_in(:input, toilet_capacity: @realtimer_toilet_capacity) - |> child({:elixir_stream, kind, :realtimer}, Membrane.Realtimer) - end - - defp maybe_plug_realtimer(builder, _kind, _pace_control, _is_input_realtime), do: builder - @spec parse_options(Boombox.in_raw_data_opts(), :input) :: map() - @spec parse_options(Boombox.out_raw_data_opts(), :output) :: map() - defp parse_options(options, direction) do - audio = Keyword.get(options, :audio) - - audio_keys = - if direction == :output and audio != false and - Enum.any?(@options_audio_keys, &Keyword.has_key?(options, &1)), - do: @options_audio_keys, - else: [] - - options = - options - |> Keyword.validate!( - [:video, :audio, :video_width, :video_height, :pace_control, :is_live] ++ audio_keys - ) - |> Map.new() - - if options.audio == false and options.video == false do - raise "Got audio and video options set to false. At least one track must be enabled." - end - - options - end - - defp maybe_plug_resampler(builder, %{ - audio_format: format, - audio_rate: rate, - audio_channels: channels - }) do - format = %Membrane.RawAudio{sample_format: format, sample_rate: rate, channels: channels} - - builder - |> child(%Membrane.FFmpeg.SWResample.Converter{output_stream_format: format}) - end - - defp maybe_plug_resampler(builder, _options) do - builder - end -end diff --git a/lib/boombox/internal_bin/elixir_stream/sink.ex b/lib/boombox/internal_bin/elixir_stream/sink.ex deleted file mode 100644 index bd93933..0000000 --- a/lib/boombox/internal_bin/elixir_stream/sink.ex +++ /dev/null @@ -1,101 +0,0 @@ -defmodule Boombox.InternalBin.ElixirStream.Sink do - @moduledoc false - use Membrane.Sink - - def_input_pad :input, - accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), - availability: :on_request, - flow_control: :manual, - demand_unit: :buffers - - def_options consumer: [spec: pid()] - - @impl true - def handle_init(_ctx, opts) do - {[], Map.merge(Map.from_struct(opts), %{last_pts: %{}, audio_format: nil})} - end - - @impl true - def handle_pad_added(Pad.ref(:input, kind), _ctx, state) do - {[], %{state | last_pts: Map.put(state.last_pts, kind, 0)}} - end - - @impl true - def handle_playing(_ctx, state) do - send(state.consumer, {:boombox_ex_stream_sink, self()}) - {[], state} - end - - @impl true - def handle_info({:boombox_demand, consumer}, _ctx, %{consumer: consumer} = state) do - if state.last_pts == %{} do - {[], state} - else - {kind, _pts} = - Enum.min_by(state.last_pts, fn {_kind, pts} -> pts end) - - {[demand: Pad.ref(:input, kind)], state} - end - end - - @impl true - def handle_info(info, _ctx, state) do - dbg(info) - dbg(state) - raise "aaaaa" - end - - @impl true - def handle_stream_format(Pad.ref(:input, :audio), stream_format, _ctx, state) do - audio_format = %{ - audio_format: stream_format.sample_format, - audio_rate: stream_format.sample_rate, - audio_channels: stream_format.channels - } - - {[], %{state | audio_format: audio_format}} - end - - @impl true - def handle_stream_format(_pad, _stream_format, _ctx, state) do - {[], state} - end - - @impl true - def handle_buffer(Pad.ref(:input, :video), buffer, ctx, state) do - state = %{state | last_pts: %{state.last_pts | video: buffer.pts}} - %{width: width, height: height} = ctx.pads[Pad.ref(:input, :video)].stream_format - - {:ok, image} = - Vix.Vips.Image.new_from_binary(buffer.payload, width, height, 3, :VIPS_FORMAT_UCHAR) - - send(state.consumer, %Boombox.Packet{ - payload: image, - pts: buffer.pts, - kind: :video - }) - - {[], state} - end - - @impl true - def handle_buffer(Pad.ref(:input, :audio), buffer, _ctx, state) do - state = %{state | last_pts: %{state.last_pts | audio: buffer.pts}} - - packet = %Boombox.Packet{ - payload: buffer.payload, - pts: buffer.pts, - kind: :audio, - format: state.audio_format - } - - send(state.consumer, {:boombox_packet, self(), packet}) - - {[], state} - end - - @impl true - def handle_end_of_stream(Pad.ref(:input, kind), _ctx, state) do - {[], %{state | last_pts: Map.delete(state.last_pts, kind)}} - end -end diff --git a/lib/boombox/internal_bin/elixir_stream/source.ex b/lib/boombox/internal_bin/elixir_stream/source.ex deleted file mode 100644 index fbe584b..0000000 --- a/lib/boombox/internal_bin/elixir_stream/source.ex +++ /dev/null @@ -1,110 +0,0 @@ -defmodule Boombox.InternalBin.ElixirStream.Source do - @moduledoc false - use Membrane.Source - - def_output_pad :output, - accepted_format: any_of(Membrane.RawVideo, Membrane.RawAudio), - availability: :on_request, - flow_control: :manual, - demand_unit: :buffers - - def_options producer: [ - spec: pid() - ] - - @impl true - def handle_init(_ctx, opts) do - state = %{ - producer: opts.producer, - audio_format: nil, - video_dims: nil - } - - {[], state} - end - - @impl true - def handle_playing(_ctx, state) do - send(state.producer, {:boombox_elixir_source, self()}) - {[], state} - end - - @impl true - def handle_demand(Pad.ref(:output, _id), _size, _unit, ctx, state) do - demands = Enum.map(ctx.pads, fn {_pad, %{demand: demand}} -> demand end) - - if Enum.all?(demands, &(&1 > 0)) do - send(state.producer, {:boombox_demand, self(), Enum.sum(demands)}) - end - - {[], state} - end - - @impl true - def handle_info( - {:boombox_packet, producer, %Boombox.Packet{kind: :video} = packet}, - _ctx, - %{producer: producer} = state - ) do - image = packet.payload |> Image.flatten!() |> Image.to_colorspace!(:srgb) - video_dims = %{width: Image.width(image), height: Image.height(image)} - {:ok, payload} = Vix.Vips.Image.write_to_binary(image) - buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} - - if video_dims == state.video_dims do - {[buffer: {Pad.ref(:output, :video), buffer}], state} - else - stream_format = %Membrane.RawVideo{ - width: video_dims.width, - height: video_dims.height, - pixel_format: :RGB, - aligned: true, - framerate: nil - } - - {[ - stream_format: {Pad.ref(:output, :video), stream_format}, - buffer: {Pad.ref(:output, :video), buffer} - ], %{state | video_dims: video_dims}} - end - end - - @impl true - def handle_info( - {:boombox_packet, producer, %Boombox.Packet{kind: :audio} = packet}, - _ctx, - %{producer: producer} = state - ) do - %Boombox.Packet{payload: payload, format: format} = packet - buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} - - case format do - empty_format when empty_format == %{} and state.audio_format == nil -> - raise "No audio stream format provided" - - empty_format when empty_format == %{} -> - {[buffer: {Pad.ref(:output, :audio), buffer}], state} - - unchanged_format when unchanged_format == state.audio_format -> - {[buffer: {Pad.ref(:output, :audio), buffer}], state} - - new_format -> - stream_format = %Membrane.RawAudio{ - sample_format: new_format.audio_format, - sample_rate: new_format.audio_rate, - channels: new_format.audio_channels - } - - {[ - stream_format: {Pad.ref(:output, :audio), stream_format}, - buffer: {Pad.ref(:output, :audio), buffer} - ], %{state | audio_format: format}} - end - end - - @impl true - def handle_info({:boombox_close, producer}, ctx, %{producer: producer} = state) do - actions = Enum.map(ctx.pads, fn {ref, _data} -> {:end_of_stream, ref} end) - {actions, state} - end -end diff --git a/lib/boombox/pipeline.ex b/lib/boombox/pipeline.ex index 6b3f057..9e05e5d 100644 --- a/lib/boombox/pipeline.ex +++ b/lib/boombox/pipeline.ex @@ -7,8 +7,9 @@ defmodule Boombox.Pipeline do input: Boombox.input() | Boombox.elixir_input(), output: Boombox.output() | Boombox.elixir_output() } + @type procs :: %{pipeline: pid(), supervisor: pid()} - @spec start_pipeline(opts_map()) :: Boombox.procs() + @spec start_pipeline(opts_map()) :: procs() def start_pipeline(opts) do opts = opts @@ -23,13 +24,13 @@ defmodule Boombox.Pipeline do %{supervisor: supervisor, pipeline: pipeline} end - @spec terminate_pipeline(Boombox.procs()) :: :ok + @spec terminate_pipeline(procs()) :: :ok def terminate_pipeline(procs) do Membrane.Pipeline.terminate(procs.pipeline) await_pipeline(procs) end - @spec await_pipeline(Boombox.procs()) :: :ok + @spec await_pipeline(procs()) :: :ok def await_pipeline(%{supervisor: supervisor}) do receive do {:DOWN, _monitor, :process, ^supervisor, _reason} -> :ok diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index e782ac6..b0915ca 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -103,16 +103,17 @@ defmodule Boombox.Server do @type t :: %__MODULE__{ packet_serialization: boolean(), stop_application: boolean(), - boombox_pid: pid() | nil, boombox_mode: Boombox.Server.boombox_mode() | nil, communication_medium: Boombox.Server.communication_medium(), parent_pid: pid(), membrane_source_pid: pid() | nil, membrane_source_demand: non_neg_integer(), membrane_sink_pid: pid() | nil, + procs: Boombox.Pipeline.procs() | nil, pipeline_supervisor_pid: pid() | nil, pipeline_pid: pid() | nil, - pid_to_reply_to: pid() | nil + ghosted_client: GenServer.from() | pid() | nil, + limbo_packet: Boombox.Packet.t() | Boombox.Server.serialized_boombox_packet() | nil } @enforce_keys [ @@ -124,14 +125,15 @@ defmodule Boombox.Server do defstruct @enforce_keys ++ [ - boombox_pid: nil, boombox_mode: nil, membrane_source_pid: nil, membrane_source_demand: 0, membrane_sink_pid: nil, + procs: nil, pipeline_supervisor_pid: nil, pipeline_pid: nil, - pid_to_reply_to: nil + ghosted_client: nil, + limbo_packet: nil ] end @@ -199,7 +201,8 @@ defmodule Boombox.Server do Can be called only when Boombox is in `:producing` mode. """ @spec produce_packet(t()) :: - {:ok | :finished, serialized_boombox_packet() | Boombox.Packet.t()} + {:ok, serialized_boombox_packet() | Boombox.Packet.t()} + | :finished | {:error, :incompatible_mode} def produce_packet(server) do GenServer.call(server, :produce_packet) @@ -209,9 +212,7 @@ defmodule Boombox.Server do Informs Boombox that no more packets will be read and shouldn't be produced. Can be called only when Boombox is in `:producing` mode. """ - @spec finish_producing(t()) :: - {:finished, serialized_boombox_packet() | Boombox.Packet.t()} - | {:error, :incompatible_mode} + @spec finish_producing(t()) :: :finished | {:error, :incompatible_mode} def finish_producing(server) do GenServer.call(server, :finish_producing) end @@ -229,27 +230,30 @@ defmodule Boombox.Server do @impl true def handle_call(request, from, state) do - {response, state} = handle_request(request, from, state) - {:reply, response, state} + handle_request(request, from, state) end @impl true def handle_info({:call, sender, request}, state) do - {response, state} = handle_request(request, state) - send(sender, {:response, response}) - {:noreply, state} - end + {reply_action, response, state} = handle_request(request, sender, state) + + if reply_action == :reply do + reply(sender, response) + end - @impl true - def handle_info( - {:boombox_packet, sender_pid, %Boombox.Packet{} = packet}, - %State{communication_medium: :messages} = state - ) do - {response, state} = handle_request({:consume_packet, packet}, state) - if response == :finished, do: send(sender_pid, :boombox_finished) {:noreply, state} end + # @impl true + # def handle_info( + # {:boombox_packet, sender_pid, %Boombox.Packet{} = packet}, + # %State{communication_medium: :messages} = state + # ) do + # {response, state} = handle_request({:consume_packet, packet}, state) + # if response == :finished, do: send(sender_pid, :boombox_finished) + # {:noreply, state} + # end + @impl true def handle_info({:boombox_close, sender_pid}, %State{communication_medium: :messages} = state) do handle_request(:finish_consuming, state) @@ -257,56 +261,100 @@ defmodule Boombox.Server do {:noreply, state} end - @impl true - def handle_info( - {:packet_produced, packet, boombox_pid}, - %State{communication_medium: :messages, boombox_pid: boombox_pid} = state - ) do - send(state.parent_pid, {:boombox_packet, self(), packet}) - {:noreply, state} - end - - @impl true - def handle_info( - {:finished, packet, boombox_pid}, - %State{communication_medium: :messages, boombox_pid: boombox_pid} = state - ) do - send(state.parent_pid, {:boombox_packet, self(), packet}) - send(state.parent_pid, {:boombox_finished, self()}) - {:noreply, state} - end + # @impl true + # def handle_info( + # {:packet_produced, packet, boombox_pid}, + # %State{communication_medium: :messages, boombox_pid: boombox_pid} = state + # ) do + # send(state.parent_pid, {:boombox_packet, self(), packet}) + # {:noreply, state} + # end + + # @impl true + # def handle_info( + # {:finished, packet, boombox_pid}, + # %State{communication_medium: :messages, boombox_pid: boombox_pid} = state + # ) do + # send(state.parent_pid, {:boombox_packet, self(), packet}) + # send(state.parent_pid, {:boombox_finished, self()}) + # {:noreply, state} + # end @impl true def handle_info({:boombox_elixir_source, source}, state) do + state = + if state.ghosted_client != nil do + send(source, {:boombox_packet, self(), state.limbo_packet}) + reply(state.ghosted_client, :ok) + + %State{state | ghosted_client: nil, limbo_packet: nil} + else + state + end + {:noreply, %State{state | membrane_source_pid: source}} end @impl true def handle_info({:boombox_elixir_sink, sink}, state) do + if state.ghosted_client != nil do + send(sink, {:boombox_demand, self()}) + end + {:noreply, %State{state | membrane_sink_pid: sink}} end @impl true - def handle_info({:boombox_demand, demand}, state) do + def handle_info({:boombox_demand, source, demand}, %State{membrane_source_pid: source} = state) do + state = + if state.ghosted_client != nil do + send(state.membrane_source_pid, {:boombox_packet, self(), state.limbo_packet}) + reply(state.ghosted_client, :ok) + %State{state | ghosted_client: nil, limbo_packet: nil} + else + state + end + {:noreply, %State{state | membrane_source_demand: state.membrane_source_demand + demand}} end @impl true - def handle_info({:DOWN, _ref, :process, pid, reason}, %State{boombox_pid: pid} = state) do - reason = - case reason do - :normal -> :normal - reason -> {:boombox_crash, reason} - end + def handle_info({:boombox_packet, sink, packet}, %State{membrane_sink_pid: sink} = state) do + if state.ghosted_client != nil do + packet = + if state.packet_serialization do + serialize_packet(packet) + else + packet + end - {:stop, reason, state} + reply(state.ghosted_client, {:ok, packet}) + {:noreply, %State{state | ghosted_client: nil}} + else + {:noreply, state} + end end + # @impl true + # def handle_info({:DOWN, _ref, :process, pid, reason}, %State{boombox_pid: pid} = state) do + # reason = + # case reason do + # :normal -> :normal + # reason -> {:boombox_crash, reason} + # end + + # {:stop, reason, state} + # end + @impl true def handle_info( {:DOWN, _ref, :process, pid, reason}, %State{pipeline_supervisor_pid: pid} = state ) do + if state.ghosted_client != nil do + GenServer.reply(state.ghosted_client, :finished) + end + reason = case reason do :normal -> :normal @@ -342,67 +390,80 @@ defmodule Boombox.Server do defp handle_request(request, from \\ nil, state) - @spec handle_request({:run, boombox_opts()}, GenServer.from() | nil, State.t()) :: - {boombox_mode(), State.t()} + @spec handle_request({:run, boombox_opts()}, GenServer.from() | pid() | nil, State.t()) :: + {:reply, boombox_mode(), State.t()} defp handle_request({:run, boombox_opts}, _from, state) do boombox_mode = get_boombox_mode(boombox_opts) - boombox_opts = - boombox_opts - |> Enum.map(fn - {direction, {:message, opts}} -> {direction, {:stream, opts}} - {direction, {:writer, opts}} -> {direction, {:stream, opts}} - {direction, {:reader, opts}} -> {direction, {:stream, opts}} - other -> other - end) - - server_pid = self() - # procs = Boombox.Pipeline.start_pipeline(Map.new(boombox_opts)) - procs = %{supervisor: nil, pipeline: nil} + # boombox_opts = + # boombox_opts + # |> Enum.map(fn + # {direction, {:message, opts}} -> {direction, {:stream, opts}} + # {direction, {:writer, opts}} -> {direction, {:stream, opts}} + # {direction, {:reader, opts}} -> {direction, {:stream, opts}} + # other -> other + # end) + + # server_pid = self() + procs = Boombox.Pipeline.start_pipeline(Map.new(boombox_opts)) + # procs = %{supervisor: nil, pipeline: nil} + + # boombox_process_fun = + # state = + # case boombox_mode do + # :consuming -> + # IO.inspect("aaa") + + # receive do + # {:boombox_elixir_source, source} -> %State{state | membrane_source_pid: source} + # end - boombox_process_fun = - case boombox_mode do - :consuming -> - fn -> consuming_boombox_run(boombox_opts, server_pid) end + # IO.inspect(state) - :producing -> - fn -> producing_boombox_run(boombox_opts, server_pid, state.communication_medium) end + # :producing -> + # receive do + # {:boombox_elixir_sink, sink} -> %State{state | membrane_sink_pid: sink} + # end - :standalone -> - fn -> standalone_boombox_run(boombox_opts) end - end + # :standalone -> + # state + # end - boombox_pid = spawn(boombox_process_fun) - Process.monitor(boombox_pid) + # boombox_pid = spawn(boombox_process_fun) + # Process.monitor(boombox_pid) - {boombox_mode, + {:reply, boombox_mode, %State{ state - | boombox_pid: boombox_pid, - boombox_mode: boombox_mode, + | boombox_mode: boombox_mode, pipeline_supervisor_pid: procs.supervisor, - pipeline_pid: procs.pipeline + pipeline_pid: procs.pipeline, + procs: procs }} end - @spec handle_request(:get_pid, GenServer.from() | nil, State.t()) :: {pid(), State.t()} + @spec handle_request(:get_pid, GenServer.from() | pid() | nil, State.t()) :: + {:reply, pid(), State.t()} defp handle_request(:get_pid, _from, state) do - {self(), state} + {:reply, self(), state} end - defp handle_request(_request, _from, %State{boombox_pid: nil} = state) do - {{:error, :boombox_not_running}, state} + defp handle_request(_request, _from, %State{procs: nil} = state) do + {:reply, {:error, :boombox_not_running}, state} end @spec handle_request( {:consume_packet, serialized_boombox_packet() | Boombox.Packet.t()}, GenServer.from() | nil, State.t() - ) :: {:ok | :finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} + ) :: + {:reply, :ok | :finished | {:error, :incompatible_mode | :boombox_not_running}, + State.t()} + | {:noreply, State.t()} defp handle_request( {:consume_packet, packet}, from, - %State{boombox_mode: :consuming, boombox_pid: boombox_pid} = state + %State{boombox_mode: :consuming} = state ) do packet = if state.packet_serialization do @@ -411,21 +472,33 @@ defmodule Boombox.Server do packet end - # demand = + cond do + state.membrane_source_pid == nil -> + {:noreply, %State{state | ghosted_client: from, limbo_packet: packet}} + + state.membrane_source_demand == 0 -> + {:noreply, %State{state | ghosted_client: from, limbo_packet: packet}} + + true -> + send(state.membrane_source_pid, {:boombox_packet, self(), packet}) + {:reply, :ok, %State{state | membrane_source_demand: state.membrane_source_demand - 1}} + end + # if state.membrane_source_demand == 1 do - # send(state.membrane_source_pid, packet) - # {:noreply, %State{state | membrane_source_demand: 0}} + # {:noreply, %State{state | membrane_source_demand: 0, ghosted_client: from}} + # else + # {:reply, :ok, %State{state | membrane_source_demand: state.membrane_source_demand - 1}} # end - send(boombox_pid, {:consume_packet, packet}) + # send(boombox_pid, {:consume_packet, packet}) - receive do - {:packet_consumed, ^boombox_pid} -> - {:ok, state} + # receive do + # {:packet_consumed, ^boombox_pid} -> + # {:ok, state} - {:finished, ^boombox_pid} -> - {:finished, state} - end + # {:finished, ^boombox_pid} -> + # {:finished, state} + # end end defp handle_request( @@ -433,82 +506,104 @@ defmodule Boombox.Server do _from, %State{boombox_mode: _other_mode} = state ) do - {{:error, :incompatible_mode}, state} + {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:finish_consuming, GenServer.from() | nil, State.t()) :: - {:finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} + @spec handle_request(:finish_consuming, GenServer.from() | pid() | nil, State.t()) :: + {:reply, :finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} defp handle_request( :finish_consuming, - _from, - %State{boombox_mode: :consuming, boombox_pid: boombox_pid} = state + from, + %State{boombox_mode: :consuming} = state ) do - send(boombox_pid, :finish_consuming) - - receive do - {:finished, ^boombox_pid} -> - {:finished, state} - end + send(state.membrane_source_pid, {:boombox_eos, self()}) + reply(from, :finished) + Boombox.Pipeline.await_pipeline(state.procs) + # Boombox.Pipeline.terminate_pipeline(state.procs) + # send(boombox_pid, :finish_consuming) + + # receive do + # {:finished, ^boombox_pid} -> + # {:finished, state} + # end + {:noreply, state} end defp handle_request(:finish_consuming, _from, %State{boombox_mode: _other_mode} = state) do - {{:error, :incompatible_mode}, state} + {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:produce_packet, GenServer.from() | nil, State.t()) :: - {{:ok | :finished, serialized_boombox_packet() | Boombox.Packet.t()} - | {:error, :incompatible_mode | :boombox_not_running}, State.t()} + @spec handle_request(:produce_packet, GenServer.from() | pid() | nil, State.t()) :: + {:reply, {:error, :incompatible_mode | :boombox_not_running}, State.t()} + | {:noreply, State.t()} defp handle_request( :produce_packet, - _from, - %State{boombox_mode: :producing, boombox_pid: boombox_pid} = state + from, + %State{boombox_mode: :producing} = state ) do - send(boombox_pid, :produce_packet) + if state.membrane_sink_pid != nil do + send(state.membrane_sink_pid, {:boombox_demand, self()}) + end - {response_type, packet} = - receive do - {:packet_produced, packet, ^boombox_pid} -> {:ok, packet} - {:finished, packet, ^boombox_pid} -> {:finished, packet} - end + # send(boombox_pid, :produce_packet) - packet = - if state.packet_serialization do - serialize_packet(packet) - else - packet - end + # {response_type, packet} = + # receive do + # {:packet_produced, packet, ^boombox_pid} -> {:ok, packet} + # {:finished, packet, ^boombox_pid} -> {:finished, packet} + # end - {{response_type, packet}, state} + # packet = + # if state.packet_serialization do + # serialize_packet(packet) + # else + # packet + # end + + # {{response_type, packet}, state} + + {:noreply, %State{state | ghosted_client: from}} end defp handle_request(:produce_packet, _from, %State{boombox_mode: _other_mode} = state) do - {{:error, :incompatible_mode}, state} + {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:finish_producing, GenServer.from() | nil, State.t()) :: - {{:finished, serialized_boombox_packet() | Boombox.Packet.t()} - | {:error, :incompatible_mode}, State.t()} + @spec handle_request(:finish_producing, GenServer.from() | pid() | nil, State.t()) :: + {:reply, :finished | {:error, :incompatible_mode}, State.t()} defp handle_request( :finish_producing, - _from, - %State{boombox_mode: :producing, boombox_pid: boombox_pid} = state + from, + %State{boombox_mode: :producing} = state ) do - send(boombox_pid, :finish_producing) + reply(from, :finished) + Boombox.Pipeline.terminate_pipeline(state.procs) + # send(boombox_pid, :finish_producing) - receive do - {:finished, packet, ^boombox_pid} -> - {{:finished, packet}, state} - end + # receive do + # {:finished, packet, ^boombox_pid} -> + # {{:finished, packet}, state} + # end + {:noreply, :finished, state} end defp handle_request(:finish_producing, _from, %State{boombox_mode: _other_mode} = state) do - {{:error, :incompatible_mode}, state} + {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(term(), GenServer.from() | nil, State.t()) :: - {{:error, :invalid_request}, State.t()} + @spec handle_request(term(), GenServer.from() | pid() | nil, State.t()) :: + {:reply, {:error, :invalid_request}, State.t()} defp handle_request(_invalid_request, _from, state) do - {{:error, :invalid_request}, state} + {:reply, {:error, :invalid_request}, state} + end + + @spec reply(GenServer.from() | pid(), term()) :: :ok + defp reply(pid, reply_content) when is_pid(pid) do + send(pid, {:response, reply_content}) + end + + defp reply({pid, _tag} = client, reply_content) when is_pid(pid) do + GenServer.reply(client, reply_content) end @spec get_boombox_mode(boombox_opts()) :: boombox_mode() @@ -528,62 +623,62 @@ defmodule Boombox.Server do end end - defp elixir_endpoint?({type, _opts}) when type in [:reader, :writer, :message], + defp elixir_endpoint?({type, _opts}) when type in [:stream, :reader, :writer, :message], do: true defp elixir_endpoint?(_io), do: false - @spec consuming_boombox_run(boombox_opts(), pid()) :: :ok - defp consuming_boombox_run(boombox_opts, server_pid) do - Stream.resource( - fn -> true end, - fn is_first_iteration -> - if not is_first_iteration do - send(server_pid, {:packet_consumed, self()}) - end - - receive do - {:consume_packet, packet} -> - {[packet], false} - - :finish_consuming -> - {:halt, false} - end - end, - fn _is_first_iteration -> send(server_pid, {:finished, self()}) end - ) - |> Boombox.run(boombox_opts) - end - - @spec producing_boombox_run(boombox_opts(), pid(), communication_medium()) :: :ok - defp producing_boombox_run(boombox_opts, server_pid, communication_medium) do - last_packet = - Boombox.run(boombox_opts) - |> Enum.reduce_while(nil, fn new_packet, last_produced_packet -> - if last_produced_packet != nil do - send(server_pid, {:packet_produced, last_produced_packet, self()}) - end - - action = - if communication_medium == :calls do - receive do - :produce_packet -> :cont - :finish_producing -> :halt - end - else - :cont - end - - {action, new_packet} - end) - - send(server_pid, {:finished, last_packet, self()}) - end - - @spec standalone_boombox_run(boombox_opts()) :: :ok - defp standalone_boombox_run(boombox_opts) do - Boombox.run(boombox_opts) - end + # @spec consuming_boombox_run(boombox_opts(), pid()) :: :ok + # defp consuming_boombox_run(boombox_opts, server_pid) do + # Stream.resource( + # fn -> true end, + # fn is_first_iteration -> + # if not is_first_iteration do + # send(server_pid, {:packet_consumed, self()}) + # end + + # receive do + # {:consume_packet, packet} -> + # {[packet], false} + + # :finish_consuming -> + # {:halt, false} + # end + # end, + # fn _is_first_iteration -> send(server_pid, {:finished, self()}) end + # ) + # |> Boombox.run(boombox_opts) + # end + + # @spec producing_boombox_run(boombox_opts(), pid(), communication_medium()) :: :ok + # defp producing_boombox_run(boombox_opts, server_pid, communication_medium) do + # last_packet = + # Boombox.run(boombox_opts) + # |> Enum.reduce_while(nil, fn new_packet, last_produced_packet -> + # if last_produced_packet != nil do + # send(server_pid, {:packet_produced, last_produced_packet, self()}) + # end + + # action = + # if communication_medium == :calls do + # receive do + # :produce_packet -> :cont + # :finish_producing -> :halt + # end + # else + # :cont + # end + + # {action, new_packet} + # end) + + # send(server_pid, {:finished, last_packet, self()}) + # end + + # @spec standalone_boombox_run(boombox_opts()) :: :ok + # defp standalone_boombox_run(boombox_opts) do + # Boombox.run(boombox_opts) + # end @spec deserialize_packet(serialized_boombox_packet()) :: Packet.t() defp deserialize_packet(%{payload: {:audio, payload}} = serialized_packet) do diff --git a/test/boombox_test.exs b/test/boombox_test.exs index 50067cb..438621c 100644 --- a/test/boombox_test.exs +++ b/test/boombox_test.exs @@ -572,11 +572,10 @@ defmodule BoomboxTest do :reader -> Stream.unfold(:ok, fn :ok -> - {result, packet} = Boombox.read(boombox) - {packet, result} - - :finished -> - nil + case Boombox.read(boombox) do + {:ok, packet} -> {packet, :ok} + :finished -> nil + end end) :message -> From 9319e2673b77ca156e4386a87a462d58e3d2276b Mon Sep 17 00:00:00 2001 From: noarkhh Date: Thu, 20 Nov 2025 11:28:33 +0100 Subject: [PATCH 06/17] Refactor working --- lib/boombox.ex | 8 +- .../elixir_endpoints/push_sink.ex | 3 + lib/boombox/pipeline.ex | 6 + lib/boombox/server.ex | 134 +++++++++++++----- test/boombox_test.exs | 23 ++- 5 files changed, 116 insertions(+), 58 deletions(-) diff --git a/lib/boombox.ex b/lib/boombox.ex index cee44c5..a6d54c0 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -179,12 +179,10 @@ defmodule Boombox do write media packets to boombox with `write/2` and to finish writing with `close/1`. * `:message` - this function returns a PID of a process to communicate with. The process accepts the following types of messages: - - `{:boombox_packet, sender_pid :: pid(), packet :: Boombox.Packet.t()}` - provides boombox + - `{:boombox_packet, packet :: Boombox.Packet.t()}` - provides boombox with a media packet. The process will a `{:boombox_finished, boombox_pid :: pid()}` message to `sender_pid` if it has finished processing packets and should not be provided any more. - - `{:boombox_close, sender_pid :: pid()}` - tells boombox that no more packets will be - provided and that it should terminate. The process will reply by sending - `{:boombox_finished, boombox_pid :: pid()}` to `sender_pid` + - `:boombox_close` - tells boombox that no more packets will be provided and that it should terminate. Output endpoints with special behaviours: * `:stream` - this function will return a `Stream` that contains `Boombox.Packet`s @@ -207,7 +205,7 @@ defmodule Boombox do @spec run(Enumerable.t() | nil, input: input() | elixir_input(), output: output() | elixir_output() - ) :: :ok | Enumerable.t() | Writer.t() | Reader.t() + ) :: :ok | Enumerable.t() | Writer.t() | Reader.t() | pid() def run(stream \\ nil, opts) do opts = validate_opts!(stream, opts) diff --git a/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex b/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex index e9f30b6..35faba3 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex @@ -23,6 +23,9 @@ defmodule Boombox.InternalBin.ElixirEndpoints.PushSink do @impl true defdelegate handle_stream_format(pad, stream_format, ctx, state), to: Sink + @impl true + defdelegate handle_buffer(pad, buffer, ctx, state), to: Sink + @impl true defdelegate handle_end_of_stream(pad, ctx, state), to: Sink end diff --git a/lib/boombox/pipeline.ex b/lib/boombox/pipeline.ex index 9e05e5d..2c07efc 100644 --- a/lib/boombox/pipeline.ex +++ b/lib/boombox/pipeline.ex @@ -62,6 +62,12 @@ defmodule Boombox.Pipeline do {[spec: spec], %{parent: opts.parent}} end + @impl true + def handle_playing(_ctx, state) do + send(state.parent, {:pipeline_playing, self()}) + {[], state} + end + @impl true def handle_child_notification(:external_resource_ready, _element, _context, state) do send(state.parent, :external_resource_ready) diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index b0915ca..76a9ce2 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -279,38 +279,40 @@ defmodule Boombox.Server do # send(state.parent_pid, {:boombox_finished, self()}) # {:noreply, state} # end + # + # @impl true + # def handle_info({:pipeline_playing, pipeline_pid}, %State{pipeline_pid: pipeline_pid} = state) do + # reply(state.ghosted_client, state.boombox_mode) + # {:noreply, %State{state | ghosted_client: nil}} + # end @impl true - def handle_info({:boombox_elixir_source, source}, state) do + def handle_info({boombox_elixir_element, pid}, state) + when boombox_elixir_element in [:boombox_elixir_source, :boombox_elixir_sink] do state = if state.ghosted_client != nil do - send(source, {:boombox_packet, self(), state.limbo_packet}) - reply(state.ghosted_client, :ok) + reply(state.ghosted_client, state.boombox_mode) - %State{state | ghosted_client: nil, limbo_packet: nil} + %State{state | ghosted_client: nil} else state end - {:noreply, %State{state | membrane_source_pid: source}} - end - - @impl true - def handle_info({:boombox_elixir_sink, sink}, state) do - if state.ghosted_client != nil do - send(sink, {:boombox_demand, self()}) - end + state = + case boombox_elixir_element do + :boombox_elixir_source -> %State{state | membrane_source_pid: pid} + :boombox_elixir_sink -> %State{state | membrane_sink_pid: pid} + end - {:noreply, %State{state | membrane_sink_pid: sink}} + {:noreply, state} end @impl true def handle_info({:boombox_demand, source, demand}, %State{membrane_source_pid: source} = state) do state = if state.ghosted_client != nil do - send(state.membrane_source_pid, {:boombox_packet, self(), state.limbo_packet}) reply(state.ghosted_client, :ok) - %State{state | ghosted_client: nil, limbo_packet: nil} + %State{state | ghosted_client: nil} else state end @@ -319,14 +321,15 @@ defmodule Boombox.Server do end @impl true - def handle_info({:boombox_packet, sink, packet}, %State{membrane_sink_pid: sink} = state) do + def handle_info( + {:boombox_packet, sink, packet}, + %State{membrane_sink_pid: sink, communication_medium: :calls} = state + ) do if state.ghosted_client != nil do packet = - if state.packet_serialization do - serialize_packet(packet) - else - packet - end + if state.packet_serialization, + do: serialize_packet(packet), + else: packet reply(state.ghosted_client, {:ok, packet}) {:noreply, %State{state | ghosted_client: nil}} @@ -335,6 +338,58 @@ defmodule Boombox.Server do end end + @impl true + def handle_info( + {:boombox_packet, sink, packet}, + %State{membrane_sink_pid: sink, communication_medium: :messages} = state + ) do + packet = + if state.packet_serialization, + do: serialize_packet(packet), + else: packet + + send(state.parent_pid, {:boombox_packet, self(), packet}) + + {:noreply, state} + end + + @impl true + def handle_info( + {:boombox_packet, packet}, + %State{ + communication_medium: :messages, + boombox_mode: :consuming, + membrane_source_pid: source + } = state + ) + when is_pid(source) do + packet = + if state.packet_serialization do + deserialize_packet(packet) + else + packet + end + + send(state.membrane_source_pid, {:boombox_packet, self(), packet}) + + {:noreply, state} + end + + @impl true + def handle_info( + :boombox_close, + %State{ + communication_medium: :messages, + boombox_mode: :consuming, + membrane_source_pid: source + } = state + ) + when is_pid(source) do + send(state.membrane_source_pid, {:boombox_eos, self()}) + + {:noreply, state} + end + # @impl true # def handle_info({:DOWN, _ref, :process, pid, reason}, %State{boombox_pid: pid} = state) do # reason = @@ -351,8 +406,14 @@ defmodule Boombox.Server do {:DOWN, _ref, :process, pid, reason}, %State{pipeline_supervisor_pid: pid} = state ) do - if state.ghosted_client != nil do - GenServer.reply(state.ghosted_client, :finished) + case state.communication_medium do + :calls -> + if state.ghosted_client != nil do + GenServer.reply(state.ghosted_client, :finished) + end + + :messages -> + send(state.parent_pid, {:boombox_finished, self()}) end reason = @@ -392,7 +453,7 @@ defmodule Boombox.Server do @spec handle_request({:run, boombox_opts()}, GenServer.from() | pid() | nil, State.t()) :: {:reply, boombox_mode(), State.t()} - defp handle_request({:run, boombox_opts}, _from, state) do + defp handle_request({:run, boombox_opts}, from, state) do boombox_mode = get_boombox_mode(boombox_opts) # boombox_opts = @@ -406,6 +467,7 @@ defmodule Boombox.Server do # server_pid = self() procs = Boombox.Pipeline.start_pipeline(Map.new(boombox_opts)) + # procs = %{supervisor: nil, pipeline: nil} # boombox_process_fun = @@ -432,13 +494,14 @@ defmodule Boombox.Server do # boombox_pid = spawn(boombox_process_fun) # Process.monitor(boombox_pid) - {:reply, boombox_mode, + {:noreply, %State{ state | boombox_mode: boombox_mode, pipeline_supervisor_pid: procs.supervisor, pipeline_pid: procs.pipeline, - procs: procs + procs: procs, + ghosted_client: from }} end @@ -472,16 +535,13 @@ defmodule Boombox.Server do packet end - cond do - state.membrane_source_pid == nil -> - {:noreply, %State{state | ghosted_client: from, limbo_packet: packet}} - - state.membrane_source_demand == 0 -> - {:noreply, %State{state | ghosted_client: from, limbo_packet: packet}} + send(state.membrane_source_pid, {:boombox_packet, self(), packet}) + state = %State{state | membrane_source_demand: state.membrane_source_demand - 1} - true -> - send(state.membrane_source_pid, {:boombox_packet, self(), packet}) - {:reply, :ok, %State{state | membrane_source_demand: state.membrane_source_demand - 1}} + if state.membrane_source_demand == 0 do + {:noreply, %State{state | ghosted_client: from}} + else + {:reply, :ok, state} end # if state.membrane_source_demand == 1 do @@ -541,9 +601,7 @@ defmodule Boombox.Server do from, %State{boombox_mode: :producing} = state ) do - if state.membrane_sink_pid != nil do - send(state.membrane_sink_pid, {:boombox_demand, self()}) - end + send(state.membrane_sink_pid, {:boombox_demand, self()}) # send(boombox_pid, :produce_packet) diff --git a/test/boombox_test.exs b/test/boombox_test.exs index 438621c..2fd071c 100644 --- a/test/boombox_test.exs +++ b/test/boombox_test.exs @@ -488,10 +488,10 @@ defmodule BoomboxTest do max_y = Image.height(bg) - Image.height(overlay) fps = 60 - image_sink = + image_sink = fn stream -> case unquote(elixir_endpoint) do :stream -> - &Boombox.run(&1, + Boombox.run(stream, input: {:stream, video: :image, audio: false}, output: {:webrtc, signaling} ) @@ -503,10 +503,8 @@ defmodule BoomboxTest do output: {:webrtc, signaling} ) - fn stream -> - Enum.each(stream, &Boombox.write(writer, &1)) - Boombox.close(writer) - end + Enum.each(stream, &Boombox.write(writer, &1)) + Boombox.close(writer) :message -> server = @@ -515,16 +513,10 @@ defmodule BoomboxTest do output: {:webrtc, signaling} ) - fn stream -> - Enum.each(stream, &send(server, {:boombox_packet, self(), &1})) - send(server, {:boombox_close, self()}) - - receive do - {:boombox_finished, ^server} -> - :ok - end - end + Enum.each(stream, &send(server, {:boombox_packet, &1})) + send(server, :boombox_close) end + end Task.async(fn -> Stream.iterate({_x = 300, _y = 0, _dx = 1, _dy = 2, _pts = 0}, fn {x, y, dx, dy, pts} -> @@ -543,6 +535,7 @@ defmodule BoomboxTest do output = Path.join(tmp, "output.mp4") Boombox.run(input: {:webrtc, signaling}, output: output) + Compare.compare(output, "test/fixtures/ref_bouncing_bubble.mp4", kinds: [:video]) end end) From 8f9fb5dcf096cf92e52c9d5ecdfb96499054abc5 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Thu, 20 Nov 2025 15:38:36 +0100 Subject: [PATCH 07/17] Fix receiving finished --- lib/boombox.ex | 8 +- lib/boombox/server.ex | 416 +++++++++++------------------------------- 2 files changed, 111 insertions(+), 313 deletions(-) diff --git a/lib/boombox.ex b/lib/boombox.ex index a6d54c0..344aa48 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -352,8 +352,8 @@ defmodule Boombox do @doc """ Reads a packet from Boombox. - If returned with `:ok`, then this function can be called - again to request the next packet, and if returned with `:finished`, then Boombox finished it's + If returned with `:ok`, then this function can be called again to request the + next packet, and if returned with `:finished`, then Boombox finished it's operation and will not produce any more packets. Can be called only when using `:reader` endpoint on output. @@ -369,7 +369,7 @@ defmodule Boombox do Returns `:ok` if more packets can be provided, and `:finished` when Boombox finished consuming and will not accept any more packets. Returns - synchronously once the packet has been processed by Boombox. + synchronously once the packet has been ingested Boombox is ready for more packets. Can be called only when using `:writer` endpoint on input. """ @@ -390,7 +390,7 @@ defmodule Boombox do any more packets with `write/2` and should terminate accordingly. """ - @spec close(Writer.t() | Reader.t()) :: :finished | {:error, :incompatible_mode} + @spec close(Writer.t() | Reader.t()) :: :ok | {:error, :incompatible_mode} def close(%Writer{} = writer) do Boombox.Server.finish_consuming(writer.server_reference) end diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index 76a9ce2..83d23a5 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -106,14 +106,13 @@ defmodule Boombox.Server do boombox_mode: Boombox.Server.boombox_mode() | nil, communication_medium: Boombox.Server.communication_medium(), parent_pid: pid(), - membrane_source_pid: pid() | nil, + membrane_sink: pid() | nil, + membrane_source: pid() | nil, membrane_source_demand: non_neg_integer(), - membrane_sink_pid: pid() | nil, - procs: Boombox.Pipeline.procs() | nil, - pipeline_supervisor_pid: pid() | nil, - pipeline_pid: pid() | nil, + pipeline_supervisor: pid() | nil, + pipeline: pid() | nil, ghosted_client: GenServer.from() | pid() | nil, - limbo_packet: Boombox.Packet.t() | Boombox.Server.serialized_boombox_packet() | nil + pipeline_exit_reason: term() } @enforce_keys [ @@ -126,14 +125,13 @@ defmodule Boombox.Server do defstruct @enforce_keys ++ [ boombox_mode: nil, - membrane_source_pid: nil, + membrane_sink: nil, + membrane_source: nil, membrane_source_demand: 0, - membrane_sink_pid: nil, - procs: nil, - pipeline_supervisor_pid: nil, - pipeline_pid: nil, + pipeline_supervisor: nil, + pipeline: nil, ghosted_client: nil, - limbo_packet: nil + pipeline_exit_reason: nil ] end @@ -233,82 +231,67 @@ defmodule Boombox.Server do handle_request(request, from, state) end + # Imitating calls with messages @impl true def handle_info({:call, sender, request}, state) do - {reply_action, response, state} = handle_request(request, sender, state) + case handle_request(request, sender, state) do + {:reply, reply, state} -> + reply(sender, reply) + {:noreply, state} - if reply_action == :reply do - reply(sender, response) + {:stop, reason, reply, state} -> + reply(sender, reply) + {:stop, reason, state} + + other -> + other end + end + + # Message API - writing packets + @impl true + def handle_info( + {:boombox_packet, packet}, + %State{communication_medium: :messages, boombox_mode: :consuming} = state + ) do + packet = + if state.packet_serialization, + do: deserialize_packet(packet), + else: packet + + send(state.membrane_source, {:boombox_packet, self(), packet}) {:noreply, state} end - # @impl true - # def handle_info( - # {:boombox_packet, sender_pid, %Boombox.Packet{} = packet}, - # %State{communication_medium: :messages} = state - # ) do - # {response, state} = handle_request({:consume_packet, packet}, state) - # if response == :finished, do: send(sender_pid, :boombox_finished) - # {:noreply, state} - # end - + # Message API - closing for writing @impl true - def handle_info({:boombox_close, sender_pid}, %State{communication_medium: :messages} = state) do - handle_request(:finish_consuming, state) - send(sender_pid, {:boombox_finished, self()}) + def handle_info( + :boombox_close, + %State{communication_medium: :messages, boombox_mode: :consuming} = state + ) do + send(state.membrane_source, {:boombox_eos, self()}) {:noreply, state} end - # @impl true - # def handle_info( - # {:packet_produced, packet, boombox_pid}, - # %State{communication_medium: :messages, boombox_pid: boombox_pid} = state - # ) do - # send(state.parent_pid, {:boombox_packet, self(), packet}) - # {:noreply, state} - # end - - # @impl true - # def handle_info( - # {:finished, packet, boombox_pid}, - # %State{communication_medium: :messages, boombox_pid: boombox_pid} = state - # ) do - # send(state.parent_pid, {:boombox_packet, self(), packet}) - # send(state.parent_pid, {:boombox_finished, self()}) - # {:noreply, state} - # end - # - # @impl true - # def handle_info({:pipeline_playing, pipeline_pid}, %State{pipeline_pid: pipeline_pid} = state) do - # reply(state.ghosted_client, state.boombox_mode) - # {:noreply, %State{state | ghosted_client: nil}} - # end - @impl true def handle_info({boombox_elixir_element, pid}, state) when boombox_elixir_element in [:boombox_elixir_source, :boombox_elixir_sink] do - state = - if state.ghosted_client != nil do - reply(state.ghosted_client, state.boombox_mode) + reply(state.ghosted_client, state.boombox_mode) - %State{state | ghosted_client: nil} - else - state - end + state = %State{state | ghosted_client: nil} state = case boombox_elixir_element do - :boombox_elixir_source -> %State{state | membrane_source_pid: pid} - :boombox_elixir_sink -> %State{state | membrane_sink_pid: pid} + :boombox_elixir_source -> %State{state | membrane_source: pid} + :boombox_elixir_sink -> %State{state | membrane_sink: pid} end {:noreply, state} end @impl true - def handle_info({:boombox_demand, source, demand}, %State{membrane_source_pid: source} = state) do + def handle_info({:boombox_demand, source, demand}, %State{membrane_source: source} = state) do state = if state.ghosted_client != nil do reply(state.ghosted_client, :ok) @@ -323,7 +306,7 @@ defmodule Boombox.Server do @impl true def handle_info( {:boombox_packet, sink, packet}, - %State{membrane_sink_pid: sink, communication_medium: :calls} = state + %State{membrane_sink: sink, communication_medium: :calls} = state ) do if state.ghosted_client != nil do packet = @@ -341,7 +324,7 @@ defmodule Boombox.Server do @impl true def handle_info( {:boombox_packet, sink, packet}, - %State{membrane_sink_pid: sink, communication_medium: :messages} = state + %State{membrane_sink: sink, communication_medium: :messages} = state ) do packet = if state.packet_serialization, @@ -353,76 +336,30 @@ defmodule Boombox.Server do {:noreply, state} end - @impl true - def handle_info( - {:boombox_packet, packet}, - %State{ - communication_medium: :messages, - boombox_mode: :consuming, - membrane_source_pid: source - } = state - ) - when is_pid(source) do - packet = - if state.packet_serialization do - deserialize_packet(packet) - else - packet - end - - send(state.membrane_source_pid, {:boombox_packet, self(), packet}) - - {:noreply, state} - end - - @impl true - def handle_info( - :boombox_close, - %State{ - communication_medium: :messages, - boombox_mode: :consuming, - membrane_source_pid: source - } = state - ) - when is_pid(source) do - send(state.membrane_source_pid, {:boombox_eos, self()}) - - {:noreply, state} - end - - # @impl true - # def handle_info({:DOWN, _ref, :process, pid, reason}, %State{boombox_pid: pid} = state) do - # reason = - # case reason do - # :normal -> :normal - # reason -> {:boombox_crash, reason} - # end - - # {:stop, reason, state} - # end - @impl true def handle_info( {:DOWN, _ref, :process, pid, reason}, - %State{pipeline_supervisor_pid: pid} = state + %State{pipeline_supervisor: pid} = state ) do + reason = + case reason do + :normal -> :normal + reason -> {:boombox_crash, reason} + end + case state.communication_medium do :calls -> if state.ghosted_client != nil do - GenServer.reply(state.ghosted_client, :finished) + reply(state.ghosted_client, :finished) + {:stop, reason, state} + else + {:noreply, %State{state | pipeline_exit_reason: reason}} end :messages -> send(state.parent_pid, {:boombox_finished, self()}) + {:stop, reason, state} end - - reason = - case reason do - :normal -> :normal - reason -> {:boombox_crash, reason} - end - - {:stop, reason, state} end @impl true @@ -434,8 +371,8 @@ defmodule Boombox.Server do @impl true def terminate(reason, state) do if state.stop_application do - # Stop the application after the process terminates, allowing it to exit with the original - # reason, not :shutdown coming from the top. + # Stop the application after the process terminates, allowing for the process to exit with the original + # reason, not :shutdown coming from Application. pid = self() spawn(fn -> @@ -449,75 +386,39 @@ defmodule Boombox.Server do end end - defp handle_request(request, from \\ nil, state) - - @spec handle_request({:run, boombox_opts()}, GenServer.from() | pid() | nil, State.t()) :: + @spec handle_request({:run, boombox_opts()}, GenServer.from() | pid(), State.t()) :: {:reply, boombox_mode(), State.t()} defp handle_request({:run, boombox_opts}, from, state) do boombox_mode = get_boombox_mode(boombox_opts) - # boombox_opts = - # boombox_opts - # |> Enum.map(fn - # {direction, {:message, opts}} -> {direction, {:stream, opts}} - # {direction, {:writer, opts}} -> {direction, {:stream, opts}} - # {direction, {:reader, opts}} -> {direction, {:stream, opts}} - # other -> other - # end) - - # server_pid = self() - procs = Boombox.Pipeline.start_pipeline(Map.new(boombox_opts)) - - # procs = %{supervisor: nil, pipeline: nil} - - # boombox_process_fun = - # state = - # case boombox_mode do - # :consuming -> - # IO.inspect("aaa") - - # receive do - # {:boombox_elixir_source, source} -> %State{state | membrane_source_pid: source} - # end - - # IO.inspect(state) - - # :producing -> - # receive do - # {:boombox_elixir_sink, sink} -> %State{state | membrane_sink_pid: sink} - # end - - # :standalone -> - # state - # end - - # boombox_pid = spawn(boombox_process_fun) - # Process.monitor(boombox_pid) + %{supervisor: pipeline_supervisor, pipeline: pipeline} = + boombox_opts + |> Map.new() + |> Boombox.Pipeline.start_pipeline() {:noreply, %State{ state | boombox_mode: boombox_mode, - pipeline_supervisor_pid: procs.supervisor, - pipeline_pid: procs.pipeline, - procs: procs, + pipeline_supervisor: pipeline_supervisor, + pipeline: pipeline, ghosted_client: from }} end - @spec handle_request(:get_pid, GenServer.from() | pid() | nil, State.t()) :: + @spec handle_request(:get_pid, GenServer.from() | pid(), State.t()) :: {:reply, pid(), State.t()} defp handle_request(:get_pid, _from, state) do {:reply, self(), state} end - defp handle_request(_request, _from, %State{procs: nil} = state) do + defp handle_request(_request, _from, %State{pipeline: nil} = state) do {:reply, {:error, :boombox_not_running}, state} end @spec handle_request( {:consume_packet, serialized_boombox_packet() | Boombox.Packet.t()}, - GenServer.from() | nil, + GenServer.from() | pid(), State.t() ) :: {:reply, :ok | :finished | {:error, :incompatible_mode | :boombox_not_running}, @@ -529,36 +430,23 @@ defmodule Boombox.Server do %State{boombox_mode: :consuming} = state ) do packet = - if state.packet_serialization do - deserialize_packet(packet) - else - packet - end + if state.packet_serialization, + do: deserialize_packet(packet), + else: packet - send(state.membrane_source_pid, {:boombox_packet, self(), packet}) + send(state.membrane_source, {:boombox_packet, self(), packet}) state = %State{state | membrane_source_demand: state.membrane_source_demand - 1} - if state.membrane_source_demand == 0 do - {:noreply, %State{state | ghosted_client: from}} - else - {:reply, :ok, state} - end - - # if state.membrane_source_demand == 1 do - # {:noreply, %State{state | membrane_source_demand: 0, ghosted_client: from}} - # else - # {:reply, :ok, %State{state | membrane_source_demand: state.membrane_source_demand - 1}} - # end - - # send(boombox_pid, {:consume_packet, packet}) + cond do + state.pipeline_exit_reason != nil -> + {:stop, state.pipeline_exit_reason, :finished, state} - # receive do - # {:packet_consumed, ^boombox_pid} -> - # {:ok, state} + state.membrane_source_demand == 0 -> + {:noreply, %State{state | ghosted_client: from}} - # {:finished, ^boombox_pid} -> - # {:finished, state} - # end + true -> + {:reply, :ok, state} + end end defp handle_request( @@ -569,87 +457,49 @@ defmodule Boombox.Server do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:finish_consuming, GenServer.from() | pid() | nil, State.t()) :: - {:reply, :finished | {:error, :incompatible_mode | :boombox_not_running}, State.t()} - defp handle_request( - :finish_consuming, - from, - %State{boombox_mode: :consuming} = state - ) do - send(state.membrane_source_pid, {:boombox_eos, self()}) - reply(from, :finished) - Boombox.Pipeline.await_pipeline(state.procs) - # Boombox.Pipeline.terminate_pipeline(state.procs) - # send(boombox_pid, :finish_consuming) - - # receive do - # {:finished, ^boombox_pid} -> - # {:finished, state} - # end - {:noreply, state} + @spec handle_request(:finish_consuming, GenServer.from() | pid(), State.t()) :: + {:reply, :ok | :finished | {:error, :incompatible_mode}, State.t()} + defp handle_request(:finish_consuming, _from, %State{boombox_mode: :consuming} = state) do + if state.pipeline_exit_reason != nil do + {:stop, state.pipeline_exit_reason, :finished, state} + else + send(state.membrane_source, {:boombox_eos, self()}) + {:reply, :ok, state} + end end defp handle_request(:finish_consuming, _from, %State{boombox_mode: _other_mode} = state) do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:produce_packet, GenServer.from() | pid() | nil, State.t()) :: + @spec handle_request(:produce_packet, GenServer.from() | pid(), State.t()) :: {:reply, {:error, :incompatible_mode | :boombox_not_running}, State.t()} | {:noreply, State.t()} - defp handle_request( - :produce_packet, - from, - %State{boombox_mode: :producing} = state - ) do - send(state.membrane_sink_pid, {:boombox_demand, self()}) - - # send(boombox_pid, :produce_packet) - - # {response_type, packet} = - # receive do - # {:packet_produced, packet, ^boombox_pid} -> {:ok, packet} - # {:finished, packet, ^boombox_pid} -> {:finished, packet} - # end - - # packet = - # if state.packet_serialization do - # serialize_packet(packet) - # else - # packet - # end - - # {{response_type, packet}, state} - - {:noreply, %State{state | ghosted_client: from}} + defp handle_request(:produce_packet, from, %State{boombox_mode: :producing} = state) do + if state.pipeline_exit_reason != nil do + {:stop, state.pipeline_exit_reason, :finished, state} + else + send(state.membrane_sink, {:boombox_demand, self()}) + {:noreply, %State{state | ghosted_client: from}} + end end defp handle_request(:produce_packet, _from, %State{boombox_mode: _other_mode} = state) do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:finish_producing, GenServer.from() | pid() | nil, State.t()) :: - {:reply, :finished | {:error, :incompatible_mode}, State.t()} - defp handle_request( - :finish_producing, - from, - %State{boombox_mode: :producing} = state - ) do - reply(from, :finished) - Boombox.Pipeline.terminate_pipeline(state.procs) - # send(boombox_pid, :finish_producing) - - # receive do - # {:finished, packet, ^boombox_pid} -> - # {{:finished, packet}, state} - # end - {:noreply, :finished, state} + @spec handle_request(:finish_producing, GenServer.from() | pid(), State.t()) :: + {:reply, :ok | {:error, :incompatible_mode}, State.t()} + defp handle_request(:finish_producing, _from, %State{boombox_mode: :producing} = state) do + Membrane.Pipeline.terminate(state.pipeline, asynchronous?: true) + {:reply, :ok, state} end defp handle_request(:finish_producing, _from, %State{boombox_mode: _other_mode} = state) do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(term(), GenServer.from() | pid() | nil, State.t()) :: + @spec handle_request(term(), GenServer.from() | pid(), State.t()) :: {:reply, {:error, :invalid_request}, State.t()} defp handle_request(_invalid_request, _from, state) do {:reply, {:error, :invalid_request}, state} @@ -686,58 +536,6 @@ defmodule Boombox.Server do defp elixir_endpoint?(_io), do: false - # @spec consuming_boombox_run(boombox_opts(), pid()) :: :ok - # defp consuming_boombox_run(boombox_opts, server_pid) do - # Stream.resource( - # fn -> true end, - # fn is_first_iteration -> - # if not is_first_iteration do - # send(server_pid, {:packet_consumed, self()}) - # end - - # receive do - # {:consume_packet, packet} -> - # {[packet], false} - - # :finish_consuming -> - # {:halt, false} - # end - # end, - # fn _is_first_iteration -> send(server_pid, {:finished, self()}) end - # ) - # |> Boombox.run(boombox_opts) - # end - - # @spec producing_boombox_run(boombox_opts(), pid(), communication_medium()) :: :ok - # defp producing_boombox_run(boombox_opts, server_pid, communication_medium) do - # last_packet = - # Boombox.run(boombox_opts) - # |> Enum.reduce_while(nil, fn new_packet, last_produced_packet -> - # if last_produced_packet != nil do - # send(server_pid, {:packet_produced, last_produced_packet, self()}) - # end - - # action = - # if communication_medium == :calls do - # receive do - # :produce_packet -> :cont - # :finish_producing -> :halt - # end - # else - # :cont - # end - - # {action, new_packet} - # end) - - # send(server_pid, {:finished, last_packet, self()}) - # end - - # @spec standalone_boombox_run(boombox_opts()) :: :ok - # defp standalone_boombox_run(boombox_opts) do - # Boombox.run(boombox_opts) - # end - @spec deserialize_packet(serialized_boombox_packet()) :: Packet.t() defp deserialize_packet(%{payload: {:audio, payload}} = serialized_packet) do %Boombox.Packet{ From 93524de83e25ba5739aa448ee3a4ac6ef9e9669c Mon Sep 17 00:00:00 2001 From: noarkhh Date: Thu, 20 Nov 2025 15:58:04 +0100 Subject: [PATCH 08/17] Fix typespecs --- lib/boombox/server.ex | 6 ++++-- python/src/boombox/boombox.py | 34 +++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index 83d23a5..a9d8937 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -421,9 +421,9 @@ defmodule Boombox.Server do GenServer.from() | pid(), State.t() ) :: - {:reply, :ok | :finished | {:error, :incompatible_mode | :boombox_not_running}, - State.t()} + {:reply, :ok | {:error, :incompatible_mode | :boombox_not_running}, State.t()} | {:noreply, State.t()} + | {:stop, term(), :finished, State.t()} defp handle_request( {:consume_packet, packet}, from, @@ -459,6 +459,7 @@ defmodule Boombox.Server do @spec handle_request(:finish_consuming, GenServer.from() | pid(), State.t()) :: {:reply, :ok | :finished | {:error, :incompatible_mode}, State.t()} + | {:stop, term(), :finished, State.t()} defp handle_request(:finish_consuming, _from, %State{boombox_mode: :consuming} = state) do if state.pipeline_exit_reason != nil do {:stop, state.pipeline_exit_reason, :finished, state} @@ -475,6 +476,7 @@ defmodule Boombox.Server do @spec handle_request(:produce_packet, GenServer.from() | pid(), State.t()) :: {:reply, {:error, :incompatible_mode | :boombox_not_running}, State.t()} | {:noreply, State.t()} + | {:stop, term(), :finished, State.t()} defp handle_request(:produce_packet, from, %State{boombox_mode: :producing} = state) do if state.pipeline_exit_reason != nil do {:stop, state.pipeline_exit_reason, :finished, state} diff --git a/python/src/boombox/boombox.py b/python/src/boombox/boombox.py index 38a0c09..0f9a19e 100644 --- a/python/src/boombox/boombox.py +++ b/python/src/boombox/boombox.py @@ -100,6 +100,7 @@ class Boombox(process.Process): _terminated: asyncio.Future _finished: bool _erlang_process: subprocess.Popen + _boombox_mode: Atom _python_node_name = f"{uuid.uuid4()}@127.0.0.1" _cookie = str(uuid.uuid4()) @@ -137,7 +138,7 @@ def __init__( (Atom("input"), self._serialize_endpoint(input, "input")), (Atom("output"), self._serialize_endpoint(output, "output")), ] - self._call((Atom("run"), boombox_arg)) + self._boombox_mode = self._call((Atom("run"), boombox_arg)) def read(self) -> Generator[AudioPacket | VideoPacket, None, None]: """Read media packets produced by Boombox. @@ -161,8 +162,7 @@ def read(self) -> Generator[AudioPacket | VideoPacket, None, None]: match self._call(Atom("produce_packet")): case (Atom("ok"), packet): yield self._deserialize_packet(packet) - case (Atom("finished"), packet): - yield self._deserialize_packet(packet) + case Atom("finished"): return case (Atom("error"), Atom("incompatible_mode")): raise RuntimeError("Output not defined with an RawData endpoint.") @@ -214,12 +214,12 @@ def write(self, packet: AudioPacket | VideoPacket) -> bool: raise RuntimeError(f"Unknown response: {other}") def close(self, wait: bool = True, kill: bool = False) -> None: - """Closes Boombox for writing. + """Closes Boombox for writing or reading. - Enabled only if Boombox has been initialized with input defined with an - :py:class:`.RawData` endpoint. + Enabled only if Boombox has been initialized with input or output defined + with a :py:class:`.RawData` endpoint. - This method informs Boombox that it shouldn't expect any more packets. + This method informs Boombox that it shouldn't expect or produce any more packets. Parameters ---------- @@ -236,17 +236,25 @@ def close(self, wait: bool = True, kill: bool = False) -> None: Raises ------ RuntimeError - If Boombox's input was not defined with an :py:class:`.RawData` - endpoint. + If neither of Boombox's input or output was defined with a + :py:class:`.RawData` endpoint. """ - match self._call(Atom("finish_consuming")): - case Atom("finished"): + match self._boombox_mode: + case Atom("consuming"): + request = Atom("finish_consuming") + case Atom("producing"): + request = Atom("finish_producing") + case other: + raise RuntimeError( + "Can't close boombox if not using a RawData endpoint" + ) + + match self._call(request): + case Atom("ok"): if kill: self.kill() elif wait: self.wait() - case (Atom("error"), Atom("incompatible_mode")): - raise RuntimeError("Input should be defined with an RawData endpoint.") case other: raise RuntimeError(f"Unknown response: {other}") From 7f15c5ba537f7ca8de49c2bd6dd9d5b76a508770 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Wed, 26 Nov 2025 10:43:13 +0100 Subject: [PATCH 09/17] Fix sink --- lib/boombox/internal_bin/elixir_endpoints/sink.ex | 6 ++++-- lib/boombox/pipeline.ex | 6 ------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/boombox/internal_bin/elixir_endpoints/sink.ex b/lib/boombox/internal_bin/elixir_endpoints/sink.ex index 77db246..4cd1772 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/sink.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/sink.ex @@ -48,11 +48,13 @@ defmodule Boombox.InternalBin.ElixirEndpoints.Sink do {:ok, image} = Vix.Vips.Image.new_from_binary(buffer.payload, width, height, 3, :VIPS_FORMAT_UCHAR) - send(state.consumer, %Boombox.Packet{ + packet = %Boombox.Packet{ payload: image, pts: buffer.pts, kind: :video - }) + } + + send(state.consumer, {:boombox_packet, self(), packet}) {[], state} end diff --git a/lib/boombox/pipeline.ex b/lib/boombox/pipeline.ex index 2c07efc..9e05e5d 100644 --- a/lib/boombox/pipeline.ex +++ b/lib/boombox/pipeline.ex @@ -62,12 +62,6 @@ defmodule Boombox.Pipeline do {[spec: spec], %{parent: opts.parent}} end - @impl true - def handle_playing(_ctx, state) do - send(state.parent, {:pipeline_playing, self()}) - {[], state} - end - @impl true def handle_child_notification(:external_resource_ready, _element, _context, state) do send(state.parent, :external_resource_ready) From a4c8e9b82133a93b50b4675e80f2d75ecb1131e2 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Wed, 26 Nov 2025 14:12:26 +0100 Subject: [PATCH 10/17] Handle termination requests correctly --- lib/boombox/server.ex | 71 +++++++++++++--------- python/examples/anonymization_demo.py | 10 ++- python/src/boombox/_vendor/pyrlang/node.py | 2 +- python/src/boombox/_vendor/pyrlang/rex.py | 2 +- python/src/boombox/boombox.py | 53 +++++++++++----- 5 files changed, 88 insertions(+), 50 deletions(-) diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index a9d8937..2e20566 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -111,8 +111,9 @@ defmodule Boombox.Server do membrane_source_demand: non_neg_integer(), pipeline_supervisor: pid() | nil, pipeline: pid() | nil, - ghosted_client: GenServer.from() | pid() | nil, - pipeline_exit_reason: term() + ghosted_client: GenServer.from() | Process.dest() | nil, + pipeline_termination_reason: term(), + termination_requested: boolean() } @enforce_keys [ @@ -131,7 +132,8 @@ defmodule Boombox.Server do pipeline_supervisor: nil, pipeline: nil, ghosted_client: nil, - pipeline_exit_reason: nil + pipeline_termination_reason: nil, + termination_requested: false ] end @@ -349,11 +351,16 @@ defmodule Boombox.Server do case state.communication_medium do :calls -> - if state.ghosted_client != nil do - reply(state.ghosted_client, :finished) - {:stop, reason, state} - else - {:noreply, %State{state | pipeline_exit_reason: reason}} + cond do + state.ghosted_client != nil -> + reply(state.ghosted_client, :finished) + {:stop, reason, state} + + state.termination_requested -> + {:stop, reason, state} + + true -> + {:noreply, %State{state | pipeline_termination_reason: reason}} end :messages -> @@ -386,7 +393,7 @@ defmodule Boombox.Server do end end - @spec handle_request({:run, boombox_opts()}, GenServer.from() | pid(), State.t()) :: + @spec handle_request({:run, boombox_opts()}, GenServer.from() | Process.dest(), State.t()) :: {:reply, boombox_mode(), State.t()} defp handle_request({:run, boombox_opts}, from, state) do boombox_mode = get_boombox_mode(boombox_opts) @@ -406,7 +413,7 @@ defmodule Boombox.Server do }} end - @spec handle_request(:get_pid, GenServer.from() | pid(), State.t()) :: + @spec handle_request(:get_pid, GenServer.from() | Process.dest(), State.t()) :: {:reply, pid(), State.t()} defp handle_request(:get_pid, _from, state) do {:reply, self(), state} @@ -418,7 +425,7 @@ defmodule Boombox.Server do @spec handle_request( {:consume_packet, serialized_boombox_packet() | Boombox.Packet.t()}, - GenServer.from() | pid(), + GenServer.from() | Process.dest(), State.t() ) :: {:reply, :ok | {:error, :incompatible_mode | :boombox_not_running}, State.t()} @@ -438,8 +445,8 @@ defmodule Boombox.Server do state = %State{state | membrane_source_demand: state.membrane_source_demand - 1} cond do - state.pipeline_exit_reason != nil -> - {:stop, state.pipeline_exit_reason, :finished, state} + state.pipeline_termination_reason != nil -> + {:stop, state.pipeline_termination_reason, :finished, state} state.membrane_source_demand == 0 -> {:noreply, %State{state | ghosted_client: from}} @@ -457,15 +464,15 @@ defmodule Boombox.Server do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:finish_consuming, GenServer.from() | pid(), State.t()) :: - {:reply, :ok | :finished | {:error, :incompatible_mode}, State.t()} - | {:stop, term(), :finished, State.t()} + @spec handle_request(:finish_consuming, GenServer.from() | Process.dest(), State.t()) :: + {:reply, :ok | {:error, :incompatible_mode}, State.t()} + | {:stop, term(), :ok, State.t()} defp handle_request(:finish_consuming, _from, %State{boombox_mode: :consuming} = state) do - if state.pipeline_exit_reason != nil do - {:stop, state.pipeline_exit_reason, :finished, state} + if state.pipeline_termination_reason != nil do + {:stop, state.pipeline_termination_reason, :ok, state} else send(state.membrane_source, {:boombox_eos, self()}) - {:reply, :ok, state} + {:reply, :ok, %State{state | termination_requested: true}} end end @@ -473,13 +480,13 @@ defmodule Boombox.Server do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:produce_packet, GenServer.from() | pid(), State.t()) :: + @spec handle_request(:produce_packet, GenServer.from() | Process.dest(), State.t()) :: {:reply, {:error, :incompatible_mode | :boombox_not_running}, State.t()} | {:noreply, State.t()} | {:stop, term(), :finished, State.t()} defp handle_request(:produce_packet, from, %State{boombox_mode: :producing} = state) do - if state.pipeline_exit_reason != nil do - {:stop, state.pipeline_exit_reason, :finished, state} + if state.pipeline_termination_reason != nil do + {:stop, state.pipeline_termination_reason, :finished, state} else send(state.membrane_sink, {:boombox_demand, self()}) {:noreply, %State{state | ghosted_client: from}} @@ -490,30 +497,34 @@ defmodule Boombox.Server do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(:finish_producing, GenServer.from() | pid(), State.t()) :: + @spec handle_request(:finish_producing, GenServer.from() | Process.dest(), State.t()) :: {:reply, :ok | {:error, :incompatible_mode}, State.t()} defp handle_request(:finish_producing, _from, %State{boombox_mode: :producing} = state) do Membrane.Pipeline.terminate(state.pipeline, asynchronous?: true) - {:reply, :ok, state} + {:reply, :ok, %State{state | termination_requested: true}} end defp handle_request(:finish_producing, _from, %State{boombox_mode: _other_mode} = state) do {:reply, {:error, :incompatible_mode}, state} end - @spec handle_request(term(), GenServer.from() | pid(), State.t()) :: + @spec handle_request(term(), GenServer.from() | Process.dest(), State.t()) :: {:reply, {:error, :invalid_request}, State.t()} defp handle_request(_invalid_request, _from, state) do {:reply, {:error, :invalid_request}, state} end - @spec reply(GenServer.from() | pid(), term()) :: :ok - defp reply(pid, reply_content) when is_pid(pid) do - send(pid, {:response, reply_content}) + @spec reply(GenServer.from() | Process.dest(), term()) :: :ok + defp reply(dest, reply_content) when is_pid(dest) or is_port(dest) or is_atom(dest) do + send(dest, {:response, reply_content}) + end + + defp reply({name, node} = dest, reply_content) when is_atom(name) and is_atom(node) do + send(dest, {:response, reply_content}) end - defp reply({pid, _tag} = client, reply_content) when is_pid(pid) do - GenServer.reply(client, reply_content) + defp reply({pid, _tag} = genserver_from, reply_content) when is_pid(pid) do + GenServer.reply(genserver_from, reply_content) end @spec get_boombox_mode(boombox_opts()) :: boombox_mode() diff --git a/python/examples/anonymization_demo.py b/python/examples/anonymization_demo.py index 28303bc..621d0fb 100644 --- a/python/examples/anonymization_demo.py +++ b/python/examples/anonymization_demo.py @@ -26,6 +26,7 @@ import queue import threading import time +import logging from typing import NoReturn @@ -66,6 +67,7 @@ def read_packets(boombox: Boombox, packet_queue: queue.Queue) -> None: packet_queue.put(packet) if packet_queue.qsize() > MAX_QUEUE_SIZE: packet_queue.get() + print("dupsko") def resize_frame(frame: np.ndarray, scale_factor: float) -> np.ndarray: @@ -267,6 +269,9 @@ def main(): SERVER_ADDRESS = "localhost" SERVER_PORT = 8000 + logging.basicConfig() + Boombox.logger.setLevel(logging.INFO) + threading.Thread( target=run_server, args=(SERVER_ADDRESS, SERVER_PORT), daemon=True ).start() @@ -322,6 +327,7 @@ def main(): args=(input_boombox, packet_queue), daemon=True, ) + print("c") reading_thread.start() print("Input boombox initialized.") @@ -363,7 +369,7 @@ def main(): if should_anonymize: packet.payload = distort_audio(packet.payload, packet.sample_rate) - output_boombox.write(packet) + print(output_boombox.write(packet)) if isinstance(packet, VideoPacket): video_read_end_time = time.time() * 1000 @@ -398,7 +404,7 @@ def main(): render_transcription(transcription_lines, frame) packet.payload = frame - output_boombox.write(packet) + print(output_boombox.write(packet)) video_read_start_time = time.time() * 1000 output_boombox.close() diff --git a/python/src/boombox/_vendor/pyrlang/node.py b/python/src/boombox/_vendor/pyrlang/node.py index c3375f3..038d495 100644 --- a/python/src/boombox/_vendor/pyrlang/node.py +++ b/python/src/boombox/_vendor/pyrlang/node.py @@ -288,7 +288,7 @@ def _send_local_registered(self, receiver, message) -> None: receiver, receiver_obj, message) receiver_obj.deliver_message(msg=message) else: - LOG.warning("Send to unknown %s ignored", receiver) + LOG.info("Send to unknown %s ignored", receiver) def _send_local(self, receiver, message) -> None: """ Try find a process by pid and drop a message into its ``inbox_``. diff --git a/python/src/boombox/_vendor/pyrlang/rex.py b/python/src/boombox/_vendor/pyrlang/rex.py index d8582d5..1fb81c0 100644 --- a/python/src/boombox/_vendor/pyrlang/rex.py +++ b/python/src/boombox/_vendor/pyrlang/rex.py @@ -42,7 +42,7 @@ def handle_cast(self, msg): @info(1, lambda msg: True) def handle_info(self, msg): - LOG.error("rex unhandled info msg: %s", msg) + LOG.info("rex unhandled info msg: %s", msg) def act_on_msg(msg): diff --git a/python/src/boombox/boombox.py b/python/src/boombox/boombox.py index 0f9a19e..a0648d2 100644 --- a/python/src/boombox/boombox.py +++ b/python/src/boombox/boombox.py @@ -29,7 +29,7 @@ RELEASES_URL = "https://github.com/membraneframework/boombox/releases" -PACKAGE_NAME = "boomboxlib" +PACKAGE_NAME = "boomboxlibb" class Boombox(process.Process): @@ -85,8 +85,15 @@ class Boombox(process.Process): Definition of an input or output of Boombox. Can be provided explicitly by an appropriate :py:class:`.BoomboxEndpoint` or a string of a path to a file or an URL, that Boombox will attempt to interpret as an endpoint. + + Attributes + ---------- + logger : ClassVar[logging.Logger] + Logger used in this class """ + logger: ClassVar[logging.Logger] + _python_node_name: ClassVar[str] _cookie: ClassVar[str] @@ -105,6 +112,7 @@ class Boombox(process.Process): _python_node_name = f"{uuid.uuid4()}@127.0.0.1" _cookie = str(uuid.uuid4()) _node = node.Node(node_name=_python_node_name, cookie=_cookie) + logger = logging.getLogger(__name__) threading.Thread(target=_node.run, daemon=True).start() def __init__( @@ -131,7 +139,9 @@ def __init__( self._terminated = self.get_node().get_loop().create_future() self._finished = False self._receiver = (self._erlang_node_name, Atom("boombox_server")) + print("a") self._receiver = self._call(Atom("get_pid")) + print("b") self.get_node().monitor_process(self.pid_, self._receiver) boombox_arg = [ @@ -158,16 +168,22 @@ def read(self) -> Generator[AudioPacket | VideoPacket, None, None]: RuntimeError If Boombox's output was not defined by an :py:class:`.RawData` endpoint. """ - while True: - match self._call(Atom("produce_packet")): - case (Atom("ok"), packet): - yield self._deserialize_packet(packet) - case Atom("finished"): - return - case (Atom("error"), Atom("incompatible_mode")): - raise RuntimeError("Output not defined with an RawData endpoint.") - case other: - raise RuntimeError(f"Unknown response: {other}") + try: + while True: + match self._call(Atom("produce_packet")): + case (Atom("ok"), packet): + yield self._deserialize_packet(packet) + case Atom("finished"): + return + case (Atom("error"), Atom("incompatible_mode")): + raise RuntimeError( + "Output not defined with an RawData endpoint." + ) + case other: + raise RuntimeError(f"Unknown response: {other}") + finally: + # pass + self.close() def write(self, packet: AudioPacket | VideoPacket) -> bool: """Write packets to Boombox. @@ -187,7 +203,7 @@ def write(self, packet: AudioPacket | VideoPacket) -> bool: Returns ------- finished : bool - Informs if Boombox has finished accepting packets and closed its + If true then Boombox has finished accepting packets and closed its input for any further ones. Once it finishes processing the previously provided packet, it will terminate. @@ -251,6 +267,7 @@ def close(self, wait: bool = True, kill: bool = False) -> None: match self._call(request): case Atom("ok"): + print("uuuu") if kill: self.kill() elif wait: @@ -284,8 +301,12 @@ def handle_one_inbox_message(self, msg: Any) -> None: if not self._response.done(): self._response.set_result(response) case (Atom("DOWN"), _, Atom("process"), _, Atom("normal")): + print(self._boombox_mode) + print("1") self._terminated.set_result(Atom("normal")) case (Atom("DOWN"), _, Atom("process"), _, reason): + print(self._boombox_mode) + print("2") self._terminated.set_result(reason) if not self._response.done(): self._response.set_exception( @@ -342,9 +363,9 @@ def update_to(self, b=1, bsize=1, tsize=None): self._server_release_path = os.path.join(self._data_dir, "bin", "server") if os.path.exists(self._server_release_path): - logging.info("Elixir boombox release already present.") + self.logger.info("Elixir boombox release already present.") return - logging.info("Elixir boombox release not found, downloading...") + self.logger.info("Elixir boombox release not found, downloading...") if self._version == "dev": release_url = os.path.join(RELEASES_URL, "latest/download") @@ -369,13 +390,13 @@ def update_to(self, b=1, bsize=1, tsize=None): unit_scale=True, unit_divisor=1024, miniters=1, - desc=f"Downloading {release_tarball}", + desc=f"Downloading {release_tarball} from {release_url}", ) as t: urllib.request.urlretrieve( download_url, filename=tarball_path, reporthook=t.update_to ) - logging.info("Download complete. Extracting...") + self.logger.info("Download complete. Extracting...") with tarfile.open(tarball_path) as tar: tar.extractall(self._data_dir) os.remove(tarball_path) From e99d9d0823537fd3f7511b6049b112eedda3d53d Mon Sep 17 00:00:00 2001 From: noarkhh Date: Wed, 26 Nov 2025 14:54:58 +0100 Subject: [PATCH 11/17] Fix examples --- examples.livemd | 75 +++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/examples.livemd b/examples.livemd index 5c6e68e..653ec83 100644 --- a/examples.livemd +++ b/examples.livemd @@ -11,7 +11,7 @@ System.put_env("PATH", "/opt/homebrew/bin:#{System.get_env("PATH")}") # Examples that don't mention them should still work. # MIX_INSTALL_CONFIG_BEGIN -boombox = {:boombox, github: "membraneframework/boombox"} +boombox = {:boombox, github: "membraneframework/boombox", branch: "refactor-elixir-endpoints"} # This livebook uses boombox from the master branch. If any examples happen to not work, the latest stable version of this livebook # can be found on https://hexdocs.pm/boombox/examples.html or in the latest github release. @@ -574,28 +574,31 @@ reader2 = writer = Boombox.run(input: {:writer, video: :image, audio: false}, output: output) Stream.unfold(%{}, fn _state -> - {result1, packet1} = Boombox.read(reader1) - {result2, packet2} = Boombox.read(reader2) + case {Boombox.read(reader1), Boombox.read(reader2)} do + {:finished, :finished} -> + nil - joined_image = - Vix.Vips.Operation.join!(packet1.payload, packet2.payload, :VIPS_DIRECTION_HORIZONTAL) - - packet = %Boombox.Packet{ - pts: max(packet1.pts, packet2.pts), - payload: joined_image, - kind: :video - } - - Boombox.write(writer, packet) - - if :finished in [result1, result2] do - if result1 == :ok, do: + {{:ok, _packet}, :finished} -> Boombox.close(reader1) - if result2 == :ok, do: + nil + + {:finished, {:ok, _packet}} -> Boombox.close(reader2) - nil - else - {nil, %{}} + nil + + {{:ok, packet1}, {:ok, packet2}} -> + joined_image = + Vix.Vips.Operation.join!(packet1.payload, packet2.payload, :VIPS_DIRECTION_HORIZONTAL) + + packet = %Boombox.Packet{ + pts: max(packet1.pts, packet2.pts), + payload: joined_image, + kind: :video + } + + Boombox.write(writer, packet) + + {nil, %{}} end end) |> Stream.run() @@ -624,29 +627,35 @@ defmodule MyServer do {:ok, %{ - input_boomboxes_states: %{ + bb_states: %{ bb1: %{last_packet: nil, eos: false}, bb2: %{last_packet: nil, eos: false} }, - input_boomboxes: %{bb1 => :bb1, bb2 => :bb2}, + bbs: %{bb1 => :bb1, bb2 => :bb2}, output_writer: output_writer }} end @impl true - def handle_info({:boombox_packet, bb, packet}, state) do - boombox_id = state.input_boomboxes[bb] - state = put_in(state.input_boomboxes_states[boombox_id].last_packet, packet) + def handle_info({:boombox_packet, bb, %Boombox.Packet{} = packet}, state) do + boombox_id = state.bbs[bb] + state = put_in(state.bb_states[boombox_id].last_packet, packet) - if Enum.all?(Map.values(state.input_boomboxes_states), &(&1.last_packet != nil)) do + if Enum.all?(Map.values(state.bb_states), &(&1.last_packet != nil)) do joined_image = Vix.Vips.Operation.join!( - state.input_boomboxes_states.bb1.last_packet.payload, - state.input_boomboxes_states.bb2.last_packet.payload, + state.bb_states.bb1.last_packet.payload, + state.bb_states.bb2.last_packet.payload, :VIPS_DIRECTION_HORIZONTAL ) - packet = %Boombox.Packet{packet | payload: joined_image} + pts = + max( + state.bb_states.bb1.last_packet.pts, + state.bb_states.bb2.last_packet.pts + ) + + packet = %Boombox.Packet{packet | payload: joined_image, pts: pts} Boombox.write(state.output_writer, packet) end @@ -656,10 +665,10 @@ defmodule MyServer do @impl true def handle_info({:boombox_finished, bb}, state) do - boombox_id = state.input_boomboxes[bb] - state = put_in(state.input_boomboxes_states[boombox_id].eos, true) + boombox_id = state.bbs[bb] + state = put_in(state.bb_states[boombox_id].eos, true) - if Enum.all?(Map.values(state.input_boomboxes_states), & &1.eos) do + if Enum.all?(Map.values(state.bb_states), & &1.eos) do Boombox.close(state.output_writer) {:stop, :normal, state} else @@ -672,7 +681,7 @@ input1 = "#{input_dir}/bun.mp4" input2 = "#{input_dir}/ffmpeg-testsrc.mp4" output = "#{out_dir}/index.m3u8" -{:ok, server} = MyServer.start(%{input1: input, input2: input, output: output}) +{:ok, server} = MyServer.start(%{input1: input1, input2: input2, output: output}) monitor = Process.monitor(server) receive do From ce0161b6fbd47bbf4c442c69a5156ece1d33bdb8 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Wed, 26 Nov 2025 16:13:02 +0100 Subject: [PATCH 12/17] Fix naming --- examples.livemd | 20 +++---- lib/boombox.ex | 59 ++++++++++++++------ lib/boombox/internal_bin.ex | 2 +- lib/boombox/internal_bin/elixir_endpoints.ex | 20 +++---- lib/boombox/pipeline.ex | 39 ++----------- lib/boombox/server.ex | 2 +- python/src/boombox/boombox.py | 26 ++++----- 7 files changed, 81 insertions(+), 87 deletions(-) diff --git a/examples.livemd b/examples.livemd index 653ec83..1871af7 100644 --- a/examples.livemd +++ b/examples.livemd @@ -11,7 +11,7 @@ System.put_env("PATH", "/opt/homebrew/bin:#{System.get_env("PATH")}") # Examples that don't mention them should still work. # MIX_INSTALL_CONFIG_BEGIN -boombox = {:boombox, github: "membraneframework/boombox", branch: "refactor-elixir-endpoints"} +boombox = {:boombox, github: "membraneframework/boombox"} # This livebook uses boombox from the master branch. If any examples happen to not work, the latest stable version of this livebook # can be found on https://hexdocs.pm/boombox/examples.html or in the latest github release. @@ -575,27 +575,27 @@ writer = Boombox.run(input: {:writer, video: :image, audio: false}, output: outp Stream.unfold(%{}, fn _state -> case {Boombox.read(reader1), Boombox.read(reader2)} do - {:finished, :finished} -> + {:finished, :finished} -> nil - {{:ok, _packet}, :finished} -> + {{:ok, _packet}, :finished} -> Boombox.close(reader1) nil - - {:finished, {:ok, _packet}} -> + + {:finished, {:ok, _packet}} -> Boombox.close(reader2) nil - - {{:ok, packet1}, {:ok, packet2}} -> + + {{:ok, packet1}, {:ok, packet2}} -> joined_image = Vix.Vips.Operation.join!(packet1.payload, packet2.payload, :VIPS_DIRECTION_HORIZONTAL) - + packet = %Boombox.Packet{ pts: max(packet1.pts, packet2.pts), payload: joined_image, kind: :video } - + Boombox.write(writer, packet) {nil, %{}} @@ -649,7 +649,7 @@ defmodule MyServer do :VIPS_DIRECTION_HORIZONTAL ) - pts = + pts = max( state.bb_states.bb1.last_packet.pts, state.bb_states.bb2.last_packet.pts diff --git a/lib/boombox.ex b/lib/boombox.ex index 344aa48..8de2ed8 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -211,13 +211,13 @@ defmodule Boombox do case opts do %{input: {:stream, _stream_opts}} -> - procs = Pipeline.start_pipeline(opts) - source = Pipeline.await_source_ready() + procs = Pipeline.start(opts) + source = await_source_ready() consume_stream(stream, source, procs) %{output: {:stream, _stream_opts}} -> - procs = Pipeline.start_pipeline(opts) - sink = Pipeline.await_sink_ready() + procs = Pipeline.start(opts) + sink = await_sink_ready() produce_stream(sink, procs) %{input: {:writer, _writer_opts}} -> @@ -236,8 +236,8 @@ defmodule Boombox do opts -> opts - |> Pipeline.start_pipeline() - |> Pipeline.await_pipeline() + |> Pipeline.start() + |> await_pipeline() end end @@ -276,8 +276,8 @@ defmodule Boombox do case opts do %{input: {:stream, _stream_opts}} -> - procs = Pipeline.start_pipeline(opts) - source = Pipeline.await_source_ready() + procs = Pipeline.start(opts) + source = await_source_ready() Task.async(fn -> Process.monitor(procs.supervisor) @@ -285,8 +285,8 @@ defmodule Boombox do end) %{output: {:stream, _stream_opts}} -> - procs = Pipeline.start_pipeline(opts) - sink = Pipeline.await_sink_ready() + procs = Pipeline.start(opts) + sink = await_sink_ready() produce_stream(sink, procs) %{input: {:writer, _writer_opts}} -> @@ -306,23 +306,23 @@ defmodule Boombox do # In case of rtmp, rtmps, rtp, rtsp, we need to wait for the tcp/udp server to be ready # before returning from async/2. %{input: {protocol, _opts}} when protocol in [:rtmp, :rtmps, :rtp, :rtsp, :srt] -> - procs = Pipeline.start_pipeline(opts) + procs = Pipeline.start(opts) task = Task.async(fn -> Process.monitor(procs.supervisor) - Pipeline.await_pipeline(procs) + await_pipeline(procs) end) await_external_resource_ready() task opts -> - procs = Pipeline.start_pipeline(opts) + procs = Pipeline.start(opts) Task.async(fn -> Process.monitor(procs.supervisor) - Pipeline.await_pipeline(procs) + await_pipeline(procs) end) end end @@ -471,7 +471,7 @@ defmodule Boombox do _state -> send(source, {:boombox_eos, self()}) - Pipeline.await_pipeline(procs) + await_pipeline(procs) end end @@ -495,12 +495,39 @@ defmodule Boombox do end end, fn - %{procs: procs} -> Pipeline.terminate_pipeline(procs) + %{procs: procs} -> terminate_pipeline(procs) :eos -> :ok end ) end + @spec terminate_pipeline(Pipeline.procs()) :: :ok + defp terminate_pipeline(procs) do + Membrane.Pipeline.terminate(procs.pipeline) + await_pipeline(procs) + end + + @spec await_pipeline(Pipeline.procs()) :: :ok + defp await_pipeline(%{supervisor: supervisor}) do + receive do + {:DOWN, _monitor, :process, ^supervisor, _reason} -> :ok + end + end + + @spec await_source_ready() :: pid() + defp await_source_ready() do + receive do + {:boombox_elixir_source, source} -> source + end + end + + @spec await_sink_ready() :: pid() + defp await_sink_ready() do + receive do + {:boombox_elixir_sink, sink} -> sink + end + end + # Waits for the external resource to be ready. # This is used to wait for the tcp/udp server to be ready before returning from async/2. # It is used for rtmp, rtmps, rtp, rtsp. diff --git a/lib/boombox/internal_bin.ex b/lib/boombox/internal_bin.ex index 6065c48..96ee563 100644 --- a/lib/boombox/internal_bin.ex +++ b/lib/boombox/internal_bin.ex @@ -336,7 +336,7 @@ defmodule Boombox.InternalBin do end @impl true - def handle_element_end_of_stream(:elixir_stream_sink, Pad.ref(:input, id), _ctx, state) do + def handle_element_end_of_stream(:elixir_sink, Pad.ref(:input, id), _ctx, state) do eos_info = List.delete(state.eos_info, id) state = %{state | eos_info: eos_info} diff --git a/lib/boombox/internal_bin/elixir_endpoints.ex b/lib/boombox/internal_bin/elixir_endpoints.ex index 0bb5d2b..ad06d27 100644 --- a/lib/boombox/internal_bin/elixir_endpoints.ex +++ b/lib/boombox/internal_bin/elixir_endpoints.ex @@ -26,14 +26,14 @@ defmodule Boombox.InternalBin.ElixirEndpoints do |> Map.new(fn :video -> {:video, - get_child(:elixir_stream_source) + get_child(:elixir_source) |> via_out(Pad.ref(:output, :video)) |> child(%SWScale.Converter{format: :I420}) |> child(%Membrane.H264.FFmpeg.Encoder{profile: :baseline, preset: :ultrafast})} :audio -> {:audio, - get_child(:elixir_stream_source) + get_child(:elixir_source) |> via_out(Pad.ref(:output, :audio))} end) @@ -43,7 +43,7 @@ defmodule Boombox.InternalBin.ElixirEndpoints do :pull -> %PullSource{producer: producer} end - spec_builder = child(:elixir_stream_source, source_definition) + spec_builder = child(:elixir_source, source_definition) %Ready{track_builders: builders, spec_builder: spec_builder} end @@ -79,31 +79,31 @@ defmodule Boombox.InternalBin.ElixirEndpoints do spec = [ spec_builder, - child(:elixir_stream_sink, sink_definition), + child(:elixir_sink, sink_definition), Enum.map(track_builders, fn {:audio, builder} -> builder - |> child(:elixir_stream_audio_transcoder, %Membrane.Transcoder{ + |> child(:elixir_audio_transcoder, %Membrane.Transcoder{ output_stream_format: Membrane.RawAudio }) |> maybe_plug_resampler(options) |> maybe_plug_realtimer(:audio, pace_control, is_input_realtime) |> via_in(Pad.ref(:input, :audio)) - |> get_child(:elixir_stream_sink) + |> get_child(:elixir_sink) {:video, builder} -> builder - |> child(:elixir_stream_video_transcoder, %Membrane.Transcoder{ + |> child(:elixir_video_transcoder, %Membrane.Transcoder{ output_stream_format: Membrane.RawVideo }) - |> child(:elixir_stream_rgb_converter, %SWScale.Converter{ + |> child(:elixir_rgb_converter, %SWScale.Converter{ format: :RGB, output_width: options[:video_width], output_height: options[:video_height] }) |> maybe_plug_realtimer(:video, pace_control, is_input_realtime) |> via_in(Pad.ref(:input, :video)) - |> get_child(:elixir_stream_sink) + |> get_child(:elixir_sink) end), Enum.map(to_ignore, fn {_track, builder} -> builder |> child(Membrane.Debug.Sink) end) ] @@ -116,7 +116,7 @@ defmodule Boombox.InternalBin.ElixirEndpoints do defp maybe_plug_realtimer(builder, kind, true, false) do builder |> via_in(:input, toilet_capacity: @realtimer_toilet_capacity) - |> child({:elixir_stream, kind, :realtimer}, Membrane.Realtimer) + |> child({:elixir, kind, :realtimer}, Membrane.Realtimer) end defp maybe_plug_realtimer(builder, _kind, _pace_control, _is_input_realtime), do: builder diff --git a/lib/boombox/pipeline.ex b/lib/boombox/pipeline.ex index 9e05e5d..e79251f 100644 --- a/lib/boombox/pipeline.ex +++ b/lib/boombox/pipeline.ex @@ -9,12 +9,12 @@ defmodule Boombox.Pipeline do } @type procs :: %{pipeline: pid(), supervisor: pid()} - @spec start_pipeline(opts_map()) :: procs() - def start_pipeline(opts) do + @spec start(opts_map()) :: procs() + def start(opts) do opts = opts - |> Map.update!(:input, &resolve_stream_endpoint(&1, self())) - |> Map.update!(:output, &resolve_stream_endpoint(&1, self())) + |> Map.update!(:input, &resolve_elixir_endpoint(&1, self())) + |> Map.update!(:output, &resolve_elixir_endpoint(&1, self())) |> Map.put(:parent, self()) {:ok, supervisor, pipeline} = @@ -24,33 +24,6 @@ defmodule Boombox.Pipeline do %{supervisor: supervisor, pipeline: pipeline} end - @spec terminate_pipeline(procs()) :: :ok - def terminate_pipeline(procs) do - Membrane.Pipeline.terminate(procs.pipeline) - await_pipeline(procs) - end - - @spec await_pipeline(procs()) :: :ok - def await_pipeline(%{supervisor: supervisor}) do - receive do - {:DOWN, _monitor, :process, ^supervisor, _reason} -> :ok - end - end - - @spec await_source_ready() :: pid() - def await_source_ready() do - receive do - {:boombox_elixir_source, source} -> source - end - end - - @spec await_sink_ready() :: pid() - def await_sink_ready() do - receive do - {:boombox_elixir_sink, sink} -> sink - end - end - @impl true def handle_init(_ctx, opts) do spec = @@ -73,9 +46,9 @@ defmodule Boombox.Pipeline do {[terminate: :normal], state} end - defp resolve_stream_endpoint({endpoint_type, opts}, parent) + defp resolve_elixir_endpoint({endpoint_type, opts}, parent) when endpoint_type in @elixir_endpoints, do: {endpoint_type, parent, opts} - defp resolve_stream_endpoint(endpoint, _parent), do: endpoint + defp resolve_elixir_endpoint(endpoint, _parent), do: endpoint end diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index 2e20566..b8ca4d1 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -401,7 +401,7 @@ defmodule Boombox.Server do %{supervisor: pipeline_supervisor, pipeline: pipeline} = boombox_opts |> Map.new() - |> Boombox.Pipeline.start_pipeline() + |> Boombox.Pipeline.start() {:noreply, %State{ diff --git a/python/src/boombox/boombox.py b/python/src/boombox/boombox.py index a0648d2..3749188 100644 --- a/python/src/boombox/boombox.py +++ b/python/src/boombox/boombox.py @@ -168,22 +168,16 @@ def read(self) -> Generator[AudioPacket | VideoPacket, None, None]: RuntimeError If Boombox's output was not defined by an :py:class:`.RawData` endpoint. """ - try: - while True: - match self._call(Atom("produce_packet")): - case (Atom("ok"), packet): - yield self._deserialize_packet(packet) - case Atom("finished"): - return - case (Atom("error"), Atom("incompatible_mode")): - raise RuntimeError( - "Output not defined with an RawData endpoint." - ) - case other: - raise RuntimeError(f"Unknown response: {other}") - finally: - # pass - self.close() + while True: + match self._call(Atom("produce_packet")): + case (Atom("ok"), packet): + yield self._deserialize_packet(packet) + case Atom("finished"): + return + case (Atom("error"), Atom("incompatible_mode")): + raise RuntimeError("Output not defined with an RawData endpoint.") + case other: + raise RuntimeError(f"Unknown response: {other}") def write(self, packet: AudioPacket | VideoPacket) -> bool: """Write packets to Boombox. From bb51710af5c83b5d7531b0284b1d2d0ec94c9d21 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Wed, 3 Dec 2025 15:47:53 +0100 Subject: [PATCH 13/17] Apply reviewers suggestions --- examples.livemd | 58 +++--- lib/boombox.ex | 4 +- lib/boombox/bin.ex | 3 +- lib/boombox/internal_bin.ex | 31 +-- .../elixir_endpoints/pull_sink.ex | 35 ---- .../elixir_endpoints/pull_source.ex | 27 --- .../elixir_endpoints/push_sink.ex | 31 --- .../elixir_endpoints/push_source.ex | 24 --- .../internal_bin/elixir_endpoints/sink.ex | 178 ++++++++++------ .../internal_bin/elixir_endpoints/source.ex | 197 ++++++++++-------- lib/boombox/pipeline.ex | 16 +- lib/boombox/server.ex | 56 +++-- python/src/boombox/boombox.py | 9 +- test/boombox_test.exs | 19 +- 14 files changed, 327 insertions(+), 361 deletions(-) delete mode 100644 lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex delete mode 100644 lib/boombox/internal_bin/elixir_endpoints/pull_source.ex delete mode 100644 lib/boombox/internal_bin/elixir_endpoints/push_sink.ex delete mode 100644 lib/boombox/internal_bin/elixir_endpoints/push_source.ex diff --git a/examples.livemd b/examples.livemd index 1871af7..e575150 100644 --- a/examples.livemd +++ b/examples.livemd @@ -11,7 +11,7 @@ System.put_env("PATH", "/opt/homebrew/bin:#{System.get_env("PATH")}") # Examples that don't mention them should still work. # MIX_INSTALL_CONFIG_BEGIN -boombox = {:boombox, github: "membraneframework/boombox"} +boombox = {:boombox, github: "membraneframework/boombox", branch: "refactor-elixir-endpoints"} # This livebook uses boombox from the master branch. If any examples happen to not work, the latest stable version of this livebook # can be found on https://hexdocs.pm/boombox/examples.html or in the latest github release. @@ -24,7 +24,8 @@ Mix.install([ :exla, :bumblebee, :websockex, - :membrane_simple_rtsp_server + :membrane_simple_rtsp_server, + {:coerce, ">= 1.0.2"} ]) Nx.global_default_backend(EXLA.Backend) @@ -573,19 +574,8 @@ reader2 = writer = Boombox.run(input: {:writer, video: :image, audio: false}, output: output) -Stream.unfold(%{}, fn _state -> +Stream.repeatedly(fn -> case {Boombox.read(reader1), Boombox.read(reader2)} do - {:finished, :finished} -> - nil - - {{:ok, _packet}, :finished} -> - Boombox.close(reader1) - nil - - {:finished, {:ok, _packet}} -> - Boombox.close(reader2) - nil - {{:ok, packet1}, {:ok, packet2}} -> joined_image = Vix.Vips.Operation.join!(packet1.payload, packet2.payload, :VIPS_DIRECTION_HORIZONTAL) @@ -598,12 +588,15 @@ Stream.unfold(%{}, fn _state -> Boombox.write(writer, packet) - {nil, %{}} + _finished -> + :eos end end) -|> Stream.run() +|> Enum.find(& &1 == :eos) Boombox.close(writer) +Boombox.close(reader1) +Boombox.close(reader2) ``` The second cell uses `:message` endpoints, meaning that the server communicates with boomboxes by @@ -620,39 +613,39 @@ defmodule MyServer do @impl true def init(args) do - bb1 = Boombox.run(input: args.input1, output: {:message, video: :image, audio: false}) - bb2 = Boombox.run(input: args.input2, output: {:message, video: :image, audio: false}) + boombox1 = Boombox.run(input: args.input1, output: {:message, video: :image, audio: false}) + boombox2 = Boombox.run(input: args.input2, output: {:message, video: :image, audio: false}) output_writer = Boombox.run(input: {:writer, video: :image, audio: false}, output: args.output) {:ok, %{ - bb_states: %{ - bb1: %{last_packet: nil, eos: false}, - bb2: %{last_packet: nil, eos: false} + boombox_states: %{ + boombox1: %{last_packet: nil, eos: false}, + boombox2: %{last_packet: nil, eos: false} }, - bbs: %{bb1 => :bb1, bb2 => :bb2}, + boomboxes: %{boombox1 => :boombox1, boombox2 => :boombox2}, output_writer: output_writer }} end @impl true def handle_info({:boombox_packet, bb, %Boombox.Packet{} = packet}, state) do - boombox_id = state.bbs[bb] - state = put_in(state.bb_states[boombox_id].last_packet, packet) + boombox_id = state.boomboxes[bb] + state = put_in(state.boombox_states[boombox_id].last_packet, packet) - if Enum.all?(Map.values(state.bb_states), &(&1.last_packet != nil)) do + if Enum.all?(Map.values(state.boombox_states), &(&1.last_packet != nil)) do joined_image = Vix.Vips.Operation.join!( - state.bb_states.bb1.last_packet.payload, - state.bb_states.bb2.last_packet.payload, + state.boombox_states.boombox1.last_packet.payload, + state.boombox_states.boombox2.last_packet.payload, :VIPS_DIRECTION_HORIZONTAL ) pts = max( - state.bb_states.bb1.last_packet.pts, - state.bb_states.bb2.last_packet.pts + state.boombox_states.boombox1.last_packet.pts, + state.boombox_states.boombox2.last_packet.pts ) packet = %Boombox.Packet{packet | payload: joined_image, pts: pts} @@ -665,10 +658,10 @@ defmodule MyServer do @impl true def handle_info({:boombox_finished, bb}, state) do - boombox_id = state.bbs[bb] - state = put_in(state.bb_states[boombox_id].eos, true) + boombox_id = state.boomboxes[bb] + state = put_in(state.boombox_states[boombox_id].eos, true) - if Enum.all?(Map.values(state.bb_states), & &1.eos) do + if Enum.all?(Map.values(state.boombox_states), & &1.eos) do Boombox.close(state.output_writer) {:stop, :normal, state} else @@ -686,7 +679,6 @@ monitor = Process.monitor(server) receive do {:DOWN, ^monitor, :process, ^server, reason} -> - IO.inspect(reason) :ok end ``` diff --git a/lib/boombox.ex b/lib/boombox.ex index 8de2ed8..d1846d0 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -9,9 +9,9 @@ defmodule Boombox do require Membrane.Time require Membrane.Transcoder.{Audio, Video} + alias Boombox.Pipeline alias Membrane.HTTPAdaptiveStream alias Membrane.RTP - alias Boombox.Pipeline @elixir_endpoints [:stream, :message, :writer, :reader] @@ -390,7 +390,7 @@ defmodule Boombox do any more packets with `write/2` and should terminate accordingly. """ - @spec close(Writer.t() | Reader.t()) :: :ok | {:error, :incompatible_mode} + @spec close(Writer.t() | Reader.t()) :: :ok | {:error, :incompatible_mode | :already_finished} def close(%Writer{} = writer) do Boombox.Server.finish_consuming(writer.server_reference) end diff --git a/lib/boombox/bin.ex b/lib/boombox/bin.ex index dc3b74b..8ae73ec 100644 --- a/lib/boombox/bin.ex +++ b/lib/boombox/bin.ex @@ -163,7 +163,8 @@ defmodule Boombox.Bin do spec = child(:boombox, %Boombox.InternalBin{ input: opts.input || :membrane_pad, - output: opts.output || :membrane_pad + output: opts.output || :membrane_pad, + parent: self() }) {[spec: spec], Map.from_struct(opts)} diff --git a/lib/boombox/internal_bin.ex b/lib/boombox/internal_bin.ex index 96ee563..4cd891e 100644 --- a/lib/boombox/internal_bin.ex +++ b/lib/boombox/internal_bin.ex @@ -135,7 +135,8 @@ defmodule Boombox.InternalBin do ] def_options input: [spec: input()], - output: [spec: output()] + output: [spec: output()], + parent: [spec: pid()] @impl true def handle_init(ctx, opts) do @@ -147,8 +148,8 @@ defmodule Boombox.InternalBin do state = %State{ - input: parse_endpoint_opt!(:input, opts.input), - output: parse_endpoint_opt!(:output, opts.output), + input: parse_endpoint_opt!(:input, opts.input, opts.parent), + output: parse_endpoint_opt!(:output, opts.output, opts.parent), status: :init } @@ -730,14 +731,14 @@ defmodule Boombox.InternalBin do Process.sleep(500) end - @spec parse_endpoint_opt!(:input, input()) :: input() - @spec parse_endpoint_opt!(:output, output()) :: output() - defp parse_endpoint_opt!(direction, value) when is_binary(value) do - parse_endpoint_opt!(direction, {value, []}) + @spec parse_endpoint_opt!(:input, input(), pid()) :: input() + @spec parse_endpoint_opt!(:output, output(), pid()) :: output() + defp parse_endpoint_opt!(direction, value, parent) when is_binary(value) do + parse_endpoint_opt!(direction, {value, []}, parent) end # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity - defp parse_endpoint_opt!(direction, {value, opts}) when is_binary(value) do + defp parse_endpoint_opt!(direction, {value, opts}, parent) when is_binary(value) do uri = URI.parse(value) scheme = uri.scheme extension = if uri.path, do: Path.extname(uri.path) @@ -769,16 +770,16 @@ defmodule Boombox.InternalBin do _other -> raise ArgumentError, "Unsupported URI: #{value} for direction: #{direction}" end - |> then(&parse_endpoint_opt!(direction, &1)) + |> then(&parse_endpoint_opt!(direction, &1, parent)) end # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity - defp parse_endpoint_opt!(direction, value) when is_tuple(value) or is_atom(value) do + defp parse_endpoint_opt!(direction, value, parent) when is_tuple(value) or is_atom(value) do case value do {endpoint_type, location} when is_binary(location) and direction == :input and StorageEndpoints.is_storage_endpoint_type(endpoint_type) -> - parse_endpoint_opt!(:input, {endpoint_type, location, []}) + parse_endpoint_opt!(:input, {endpoint_type, location, []}, parent) {endpoint_type, location, opts} when endpoint_type in [:h264, :h265] and is_binary(location) and direction == :input -> @@ -819,7 +820,7 @@ defmodule Boombox.InternalBin do value {:whip, uri} when is_binary(uri) -> - parse_endpoint_opt!(direction, {:whip, uri, []}) + parse_endpoint_opt!(direction, {:whip, uri, []}, parent) {:whip, uri, opts} when is_binary(uri) and is_list(opts) and direction == :input -> if Keyword.keyword?(opts), do: {:webrtc, value} @@ -850,9 +851,9 @@ defmodule Boombox.InternalBin do {:rtp, opts} -> if Keyword.keyword?(opts), do: value - {elixir_endpoint, process, opts} - when is_pid(process) and elixir_endpoint in @elixir_endpoint_types -> - if Keyword.keyword?(opts), do: value + {elixir_endpoint, opts} + when elixir_endpoint in @elixir_endpoint_types -> + if Keyword.keyword?(opts), do: {elixir_endpoint, parent, opts} {:srt, server_awaiting_accept} when direction == :input and is_pid(server_awaiting_accept) -> diff --git a/lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex b/lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex deleted file mode 100644 index 55528af..0000000 --- a/lib/boombox/internal_bin/elixir_endpoints/pull_sink.ex +++ /dev/null @@ -1,35 +0,0 @@ -defmodule Boombox.InternalBin.ElixirEndpoints.PullSink do - @moduledoc false - use Membrane.Sink - - alias Boombox.InternalBin.ElixirEndpoints.Sink - - def_input_pad :input, - accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), - availability: :on_request, - flow_control: :manual, - demand_unit: :buffers - - def_options consumer: [spec: pid()] - - @impl true - defdelegate handle_init(ctx, opts), to: Sink - - @impl true - defdelegate handle_pad_added(pad, ctx, state), to: Sink - - @impl true - defdelegate handle_playing(ctx, state), to: Sink - - @impl true - defdelegate handle_info(info, ctx, state), to: Sink - - @impl true - defdelegate handle_stream_format(pad, stream_format, ctx, state), to: Sink - - @impl true - defdelegate handle_buffer(pad, buffer, ctx, state), to: Sink - - @impl true - defdelegate handle_end_of_stream(pad, ctx, state), to: Sink -end diff --git a/lib/boombox/internal_bin/elixir_endpoints/pull_source.ex b/lib/boombox/internal_bin/elixir_endpoints/pull_source.ex deleted file mode 100644 index 4b3afb7..0000000 --- a/lib/boombox/internal_bin/elixir_endpoints/pull_source.ex +++ /dev/null @@ -1,27 +0,0 @@ -defmodule Boombox.InternalBin.ElixirEndpoints.PullSource do - @moduledoc false - use Membrane.Source - - alias Boombox.InternalBin.ElixirEndpoints.Source - - def_output_pad :output, - accepted_format: any_of(Membrane.RawVideo, Membrane.RawAudio), - availability: :on_request, - flow_control: :manual - - def_options producer: [ - spec: pid() - ] - - @impl true - defdelegate handle_init(ctx, opts), to: Source - - @impl true - defdelegate handle_playing(ctx, state), to: Source - - @impl true - defdelegate handle_demand(pad, size, unit, ctx, state), to: Source - - @impl true - defdelegate handle_info(info, ctx, state), to: Source -end diff --git a/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex b/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex deleted file mode 100644 index 35faba3..0000000 --- a/lib/boombox/internal_bin/elixir_endpoints/push_sink.ex +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Boombox.InternalBin.ElixirEndpoints.PushSink do - @moduledoc false - use Membrane.Sink - - alias Boombox.InternalBin.ElixirEndpoints.Sink - - def_input_pad :input, - accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), - availability: :on_request, - flow_control: :auto - - def_options consumer: [spec: pid()] - - @impl true - defdelegate handle_init(ctx, opts), to: Sink - - @impl true - defdelegate handle_pad_added(pad, ctx, state), to: Sink - - @impl true - defdelegate handle_playing(ctx, state), to: Sink - - @impl true - defdelegate handle_stream_format(pad, stream_format, ctx, state), to: Sink - - @impl true - defdelegate handle_buffer(pad, buffer, ctx, state), to: Sink - - @impl true - defdelegate handle_end_of_stream(pad, ctx, state), to: Sink -end diff --git a/lib/boombox/internal_bin/elixir_endpoints/push_source.ex b/lib/boombox/internal_bin/elixir_endpoints/push_source.ex deleted file mode 100644 index df71d0a..0000000 --- a/lib/boombox/internal_bin/elixir_endpoints/push_source.ex +++ /dev/null @@ -1,24 +0,0 @@ -defmodule Boombox.InternalBin.ElixirEndpoints.PushSource do - @moduledoc false - use Membrane.Source - - alias Boombox.InternalBin.ElixirEndpoints.Source - - def_output_pad :output, - accepted_format: any_of(Membrane.RawVideo, Membrane.RawAudio), - availability: :on_request, - flow_control: :push - - def_options producer: [ - spec: pid() - ] - - @impl true - defdelegate handle_init(ctx, opts), to: Source - - @impl true - defdelegate handle_playing(ctx, state), to: Source - - @impl true - defdelegate handle_info(info, ctx, state), to: Source -end diff --git a/lib/boombox/internal_bin/elixir_endpoints/sink.ex b/lib/boombox/internal_bin/elixir_endpoints/sink.ex index 4cd1772..913c9dd 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/sink.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/sink.ex @@ -1,80 +1,140 @@ -defmodule Boombox.InternalBin.ElixirEndpoints.Sink do - @moduledoc false - alias Membrane.Pad - require Membrane.Pad +[{PushSink, :push}, {PullSink, :manual}] +|> Enum.map(fn {module_name, flow_control} -> + defmodule Module.concat(Boombox.InternalBin.ElixirEndpoints, module_name) do + @moduledoc false + use Membrane.Sink + + case flow_control do + :manual -> + def_input_pad :input, + accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), + availability: :on_request, + flow_control: :manual, + demand_unit: :buffers + + :push -> + def_input_pad :input, + accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), + availability: :on_request, + flow_control: :push + end - def handle_init(_ctx, opts) do - {[], Map.merge(Map.from_struct(opts), %{last_pts: %{}, audio_format: nil})} - end + def_options( + consumer: [ + spec: pid() + ] + ) + + defmodule State do + @moduledoc false + @type t :: %__MODULE__{ + consumer: pid(), + last_pts: %{ + optional(:video) => Membrane.Time.t(), + optional(:audio) => Membrane.Time.t() + }, + audio_format: + %{ + audio_format: Membrane.RawAudio.SampleFormat.t(), + audio_rate: pos_integer(), + audio_channels: pos_integer() + } + | nil + } + + @enforce_keys [:consumer] + + defstruct @enforce_keys ++ + [ + last_pts: %{}, + audio_format: nil + ] + end - def handle_pad_added(Pad.ref(:input, kind), _ctx, state) do - {[], %{state | last_pts: Map.put(state.last_pts, kind, 0)}} - end + @impl true + def handle_init(_ctx, opts) do + {[], %State{consumer: opts.consumer}} + end - def handle_playing(_ctx, state) do - send(state.consumer, {:boombox_elixir_sink, self()}) - {[], state} - end + @impl true + def handle_pad_added(Pad.ref(:input, kind), _ctx, state) do + {[], %{state | last_pts: Map.put(state.last_pts, kind, 0)}} + end - def handle_info({:boombox_demand, consumer}, _ctx, %{consumer: consumer} = state) do - if state.last_pts == %{} do + @impl true + def handle_playing(_ctx, state) do + send(state.consumer, {:boombox_elixir_sink, self()}) {[], state} - else - {kind, _pts} = - Enum.min_by(state.last_pts, fn {_kind, pts} -> pts end) + end - {[demand: Pad.ref(:input, kind)], state} + if flow_control == :manual do + @impl true + def handle_info({:boombox_demand, consumer}, _ctx, %{consumer: consumer} = state) do + if state.last_pts == %{} do + {[], state} + else + {kind, _pts} = + Enum.min_by(state.last_pts, fn {_kind, pts} -> pts end) + + {[demand: Pad.ref(:input, kind)], state} + end + end end - end - def handle_stream_format(Pad.ref(:input, :audio), stream_format, _ctx, state) do - audio_format = %{ - audio_format: stream_format.sample_format, - audio_rate: stream_format.sample_rate, - audio_channels: stream_format.channels - } + @impl true + def handle_stream_format(Pad.ref(:input, :audio), stream_format, _ctx, state) do + audio_format = %{ + audio_format: stream_format.sample_format, + audio_rate: stream_format.sample_rate, + audio_channels: stream_format.channels + } - {[], %{state | audio_format: audio_format}} - end + {[], %{state | audio_format: audio_format}} + end - def handle_stream_format(_pad, _stream_format, _ctx, state) do - {[], state} - end + @impl true + def handle_stream_format(_pad, _stream_format, _ctx, state) do + {[], state} + end - def handle_buffer(Pad.ref(:input, :video), buffer, ctx, state) do - state = %{state | last_pts: %{state.last_pts | video: buffer.pts}} - %{width: width, height: height} = ctx.pads[Pad.ref(:input, :video)].stream_format + @impl true + def handle_buffer(Pad.ref(:input, :video), buffer, ctx, state) do + state = %{state | last_pts: %{state.last_pts | video: buffer.pts}} + %{width: width, height: height} = ctx.pads[Pad.ref(:input, :video)].stream_format - {:ok, image} = - Vix.Vips.Image.new_from_binary(buffer.payload, width, height, 3, :VIPS_FORMAT_UCHAR) + {:ok, image} = + Vix.Vips.Image.new_from_binary(buffer.payload, width, height, 3, :VIPS_FORMAT_UCHAR) - packet = %Boombox.Packet{ - payload: image, - pts: buffer.pts, - kind: :video - } + packet = %Boombox.Packet{ + payload: image, + pts: buffer.pts, + kind: :video + } - send(state.consumer, {:boombox_packet, self(), packet}) + send(state.consumer, {:boombox_packet, self(), packet}) - {[], state} - end + {[], state} + end - def handle_buffer(Pad.ref(:input, :audio), buffer, _ctx, state) do - state = %{state | last_pts: %{state.last_pts | audio: buffer.pts}} + @impl true + def handle_buffer(Pad.ref(:input, :audio), buffer, _ctx, state) do + state = %{state | last_pts: %{state.last_pts | audio: buffer.pts}} - packet = %Boombox.Packet{ - payload: buffer.payload, - pts: buffer.pts, - kind: :audio, - format: state.audio_format - } + packet = %Boombox.Packet{ + payload: buffer.payload, + pts: buffer.pts, + kind: :audio, + format: state.audio_format + } - send(state.consumer, {:boombox_packet, self(), packet}) + send(state.consumer, {:boombox_packet, self(), packet}) - {[], state} - end + {[], state} + end - def handle_end_of_stream(Pad.ref(:input, kind), _ctx, state) do - {[], %{state | last_pts: Map.delete(state.last_pts, kind)}} + @impl true + def handle_end_of_stream(Pad.ref(:input, kind), _ctx, state) do + {[], %{state | last_pts: Map.delete(state.last_pts, kind)}} + end end -end +end) diff --git a/lib/boombox/internal_bin/elixir_endpoints/source.ex b/lib/boombox/internal_bin/elixir_endpoints/source.ex index 6e3f468..1d675b5 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/source.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/source.ex @@ -1,95 +1,128 @@ -defmodule Boombox.InternalBin.ElixirEndpoints.Source do - @moduledoc false - alias Membrane.Pad - require Membrane.Pad - - def handle_init(_ctx, opts) do - state = %{ - producer: opts.producer, - audio_format: nil, - video_dims: nil - } - - {[], state} - end - - def handle_playing(_ctx, state) do - send(state.producer, {:boombox_elixir_source, self()}) - {[], state} - end +[{PushSource, :push}, {PullSource, :manual}] +|> Enum.map(fn {module_name, flow_control} -> + defmodule Module.concat(Boombox.InternalBin.ElixirEndpoints, module_name) do + @moduledoc false + use Membrane.Source + + def_output_pad :output, + accepted_format: any_of(Membrane.RawVideo, Membrane.RawAudio), + availability: :on_request, + flow_control: flow_control + + def_options producer: [ + spec: pid() + ] + + defmodule State do + @moduledoc false + @type t :: %__MODULE__{ + producer: pid(), + audio_format: + %{ + audio_format: Membrane.RawAudio.SampleFormat.t(), + audio_rate: pos_integer(), + audio_channels: pos_integer() + } + | nil, + video_dims: %{width: pos_integer(), height: pos_integer()} | nil + } + + @enforce_keys [:producer] + + defstruct @enforce_keys ++ + [ + audio_format: nil, + video_dims: nil + ] + end - def handle_demand(Pad.ref(:output, _id), _size, _unit, ctx, state) do - demands = Enum.map(ctx.pads, fn {_pad, %{demand: demand}} -> demand end) + @impl true + def handle_init(_ctx, opts) do + {[], %State{producer: opts.producer}} + end - if Enum.all?(demands, &(&1 > 0)) do - send(state.producer, {:boombox_demand, self(), Enum.sum(demands)}) + @impl true + def handle_playing(_ctx, state) do + send(state.producer, {:boombox_elixir_source, self()}) + {[], state} end - {[], state} - end + if flow_control == :manual do + @impl true + def handle_demand(Pad.ref(:output, _id), _size, _unit, ctx, state) do + demands = Enum.map(ctx.pads, fn {_pad, %{demand: demand}} -> demand end) - def handle_info( - {:boombox_packet, producer, %Boombox.Packet{kind: :video} = packet}, - _ctx, - %{producer: producer} = state - ) do - image = packet.payload |> Image.flatten!() |> Image.to_colorspace!(:srgb) - video_dims = %{width: Image.width(image), height: Image.height(image)} - {:ok, payload} = Vix.Vips.Image.write_to_binary(image) - buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} - - if video_dims == state.video_dims do - {[buffer: {Pad.ref(:output, :video), buffer}], state} - else - stream_format = %Membrane.RawVideo{ - width: video_dims.width, - height: video_dims.height, - pixel_format: :RGB, - aligned: true, - framerate: nil - } - - {[ - stream_format: {Pad.ref(:output, :video), stream_format}, - buffer: {Pad.ref(:output, :video), buffer} - ], %{state | video_dims: video_dims}} + if Enum.all?(demands, &(&1 > 0)) do + send(state.producer, {:boombox_demand, self(), Enum.sum(demands)}) + end + + {[], state} + end end - end - def handle_info( - {:boombox_packet, producer, %Boombox.Packet{kind: :audio} = packet}, - _ctx, - %{producer: producer} = state - ) do - %Boombox.Packet{payload: payload, format: format} = packet - buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} - - case format do - empty_format when empty_format == %{} and state.audio_format == nil -> - raise "No audio stream format provided" - - empty_format when empty_format == %{} -> - {[buffer: {Pad.ref(:output, :audio), buffer}], state} - - unchanged_format when unchanged_format == state.audio_format -> - {[buffer: {Pad.ref(:output, :audio), buffer}], state} - - new_format -> - stream_format = %Membrane.RawAudio{ - sample_format: new_format.audio_format, - sample_rate: new_format.audio_rate, - channels: new_format.audio_channels + @impl true + def handle_info( + {:boombox_packet, producer, %Boombox.Packet{kind: :video} = packet}, + _ctx, + %{producer: producer} = state + ) do + image = packet.payload |> Image.flatten!() |> Image.to_colorspace!(:srgb) + video_dims = %{width: Image.width(image), height: Image.height(image)} + {:ok, payload} = Vix.Vips.Image.write_to_binary(image) + buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} + + if video_dims == state.video_dims do + {[buffer: {Pad.ref(:output, :video), buffer}], state} + else + stream_format = %Membrane.RawVideo{ + width: video_dims.width, + height: video_dims.height, + pixel_format: :RGB, + aligned: true, + framerate: nil } {[ - stream_format: {Pad.ref(:output, :audio), stream_format}, - buffer: {Pad.ref(:output, :audio), buffer} - ], %{state | audio_format: format}} + stream_format: {Pad.ref(:output, :video), stream_format}, + buffer: {Pad.ref(:output, :video), buffer} + ], %{state | video_dims: video_dims}} + end end - end - def handle_info({:boombox_eos, producer}, ctx, %{producer: producer} = state) do - actions = Enum.map(ctx.pads, fn {ref, _data} -> {:end_of_stream, ref} end) - {actions, state} + @impl true + def handle_info( + {:boombox_packet, producer, %Boombox.Packet{kind: :audio} = packet}, + _ctx, + %{producer: producer} = state + ) do + %Boombox.Packet{payload: payload, format: format} = packet + buffer = %Membrane.Buffer{payload: payload, pts: packet.pts} + + cond do + format == %{} and state.audio_format == nil -> + raise "No audio stream format provided" + + format == %{} or format == state.audio_format -> + {[buffer: {Pad.ref(:output, :audio), buffer}], state} + + true -> + stream_format = %Membrane.RawAudio{ + sample_format: format.audio_format, + sample_rate: format.audio_rate, + channels: format.audio_channels + } + + {[ + stream_format: {Pad.ref(:output, :audio), stream_format}, + buffer: {Pad.ref(:output, :audio), buffer} + ], %{state | audio_format: format}} + end + end + + @impl true + def handle_info({:boombox_eos, producer}, ctx, %{producer: producer} = state) do + actions = Enum.map(ctx.pads, fn {ref, _data} -> {:end_of_stream, ref} end) + {actions, state} + end end -end +end) diff --git a/lib/boombox/pipeline.ex b/lib/boombox/pipeline.ex index e79251f..ceca4dd 100644 --- a/lib/boombox/pipeline.ex +++ b/lib/boombox/pipeline.ex @@ -1,7 +1,6 @@ defmodule Boombox.Pipeline do @moduledoc false use Membrane.Pipeline - @elixir_endpoints [:stream, :message, :writer, :reader] @type opts_map :: %{ input: Boombox.input() | Boombox.elixir_input(), @@ -11,11 +10,7 @@ defmodule Boombox.Pipeline do @spec start(opts_map()) :: procs() def start(opts) do - opts = - opts - |> Map.update!(:input, &resolve_elixir_endpoint(&1, self())) - |> Map.update!(:output, &resolve_elixir_endpoint(&1, self())) - |> Map.put(:parent, self()) + opts = Map.put(opts, :parent, self()) {:ok, supervisor, pipeline} = Membrane.Pipeline.start_link(Boombox.Pipeline, opts) @@ -29,7 +24,8 @@ defmodule Boombox.Pipeline do spec = child(:boombox, %Boombox.InternalBin{ input: opts.input, - output: opts.output + output: opts.output, + parent: opts.parent }) {[spec: spec], %{parent: opts.parent}} @@ -45,10 +41,4 @@ defmodule Boombox.Pipeline do def handle_child_notification(:processing_finished, :boombox, _ctx, state) do {[terminate: :normal], state} end - - defp resolve_elixir_endpoint({endpoint_type, opts}, parent) - when endpoint_type in @elixir_endpoints, - do: {endpoint_type, parent, opts} - - defp resolve_elixir_endpoint(endpoint, _parent), do: endpoint end diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index b8ca4d1..5e4eeba 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -111,7 +111,7 @@ defmodule Boombox.Server do membrane_source_demand: non_neg_integer(), pipeline_supervisor: pid() | nil, pipeline: pid() | nil, - ghosted_client: GenServer.from() | Process.dest() | nil, + current_client: GenServer.from() | Process.dest() | nil, pipeline_termination_reason: term(), termination_requested: boolean() } @@ -131,7 +131,7 @@ defmodule Boombox.Server do membrane_source_demand: 0, pipeline_supervisor: nil, pipeline: nil, - ghosted_client: nil, + current_client: nil, pipeline_termination_reason: nil, termination_requested: false ] @@ -189,9 +189,13 @@ defmodule Boombox.Server do accordingly. Can be called only when Boombox is in `:consuming` mode. """ - @spec finish_consuming(t()) :: :finished | {:error, :incompatible_mode} + @spec finish_consuming(t()) :: :ok | {:error, :incompatible_mode | :already_finished} def finish_consuming(server) do - GenServer.call(server, :finish_consuming) + if Process.alive?(server) do + GenServer.call(server, :finish_consuming) + else + {:error, :already_finished} + end end @doc """ @@ -212,9 +216,13 @@ defmodule Boombox.Server do Informs Boombox that no more packets will be read and shouldn't be produced. Can be called only when Boombox is in `:producing` mode. """ - @spec finish_producing(t()) :: :finished | {:error, :incompatible_mode} + @spec finish_producing(t()) :: :ok | {:error, :incompatible_mode | :already_finished} def finish_producing(server) do - GenServer.call(server, :finish_producing) + if Process.alive?(server) do + GenServer.call(server, :finish_producing) + else + {:error, :already_finished} + end end @impl true @@ -279,9 +287,9 @@ defmodule Boombox.Server do @impl true def handle_info({boombox_elixir_element, pid}, state) when boombox_elixir_element in [:boombox_elixir_source, :boombox_elixir_sink] do - reply(state.ghosted_client, state.boombox_mode) + reply(state.current_client, state.boombox_mode) - state = %State{state | ghosted_client: nil} + state = %State{state | current_client: nil} state = case boombox_elixir_element do @@ -295,9 +303,9 @@ defmodule Boombox.Server do @impl true def handle_info({:boombox_demand, source, demand}, %State{membrane_source: source} = state) do state = - if state.ghosted_client != nil do - reply(state.ghosted_client, :ok) - %State{state | ghosted_client: nil} + if state.current_client != nil do + reply(state.current_client, :ok) + %State{state | current_client: nil} else state end @@ -310,14 +318,14 @@ defmodule Boombox.Server do {:boombox_packet, sink, packet}, %State{membrane_sink: sink, communication_medium: :calls} = state ) do - if state.ghosted_client != nil do + if state.current_client != nil do packet = if state.packet_serialization, do: serialize_packet(packet), else: packet - reply(state.ghosted_client, {:ok, packet}) - {:noreply, %State{state | ghosted_client: nil}} + reply(state.current_client, {:ok, packet}) + {:noreply, %State{state | current_client: nil}} else {:noreply, state} end @@ -352,8 +360,8 @@ defmodule Boombox.Server do case state.communication_medium do :calls -> cond do - state.ghosted_client != nil -> - reply(state.ghosted_client, :finished) + state.current_client != nil -> + reply(state.current_client, :finished) {:stop, reason, state} state.termination_requested -> @@ -394,7 +402,7 @@ defmodule Boombox.Server do end @spec handle_request({:run, boombox_opts()}, GenServer.from() | Process.dest(), State.t()) :: - {:reply, boombox_mode(), State.t()} + {:noreply, State.t()} defp handle_request({:run, boombox_opts}, from, state) do boombox_mode = get_boombox_mode(boombox_opts) @@ -409,7 +417,7 @@ defmodule Boombox.Server do | boombox_mode: boombox_mode, pipeline_supervisor: pipeline_supervisor, pipeline: pipeline, - ghosted_client: from + current_client: from }} end @@ -449,7 +457,7 @@ defmodule Boombox.Server do {:stop, state.pipeline_termination_reason, :finished, state} state.membrane_source_demand == 0 -> - {:noreply, %State{state | ghosted_client: from}} + {:noreply, %State{state | current_client: from}} true -> {:reply, :ok, state} @@ -489,7 +497,7 @@ defmodule Boombox.Server do {:stop, state.pipeline_termination_reason, :finished, state} else send(state.membrane_sink, {:boombox_demand, self()}) - {:noreply, %State{state | ghosted_client: from}} + {:noreply, %State{state | current_client: from}} end end @@ -500,8 +508,12 @@ defmodule Boombox.Server do @spec handle_request(:finish_producing, GenServer.from() | Process.dest(), State.t()) :: {:reply, :ok | {:error, :incompatible_mode}, State.t()} defp handle_request(:finish_producing, _from, %State{boombox_mode: :producing} = state) do - Membrane.Pipeline.terminate(state.pipeline, asynchronous?: true) - {:reply, :ok, %State{state | termination_requested: true}} + if state.pipeline_termination_reason != nil do + {:stop, state.pipeline_termination_reason, :ok, state} + else + Membrane.Pipeline.terminate(state.pipeline, asynchronous?: true) + {:reply, :ok, %State{state | termination_requested: true}} + end end defp handle_request(:finish_producing, _from, %State{boombox_mode: _other_mode} = state) do diff --git a/python/src/boombox/boombox.py b/python/src/boombox/boombox.py index 3749188..4bbe65b 100644 --- a/python/src/boombox/boombox.py +++ b/python/src/boombox/boombox.py @@ -29,7 +29,7 @@ RELEASES_URL = "https://github.com/membraneframework/boombox/releases" -PACKAGE_NAME = "boomboxlibb" +PACKAGE_NAME = "boomboxlib" class Boombox(process.Process): @@ -139,9 +139,7 @@ def __init__( self._terminated = self.get_node().get_loop().create_future() self._finished = False self._receiver = (self._erlang_node_name, Atom("boombox_server")) - print("a") self._receiver = self._call(Atom("get_pid")) - print("b") self.get_node().monitor_process(self.pid_, self._receiver) boombox_arg = [ @@ -261,7 +259,6 @@ def close(self, wait: bool = True, kill: bool = False) -> None: match self._call(request): case Atom("ok"): - print("uuuu") if kill: self.kill() elif wait: @@ -295,12 +292,8 @@ def handle_one_inbox_message(self, msg: Any) -> None: if not self._response.done(): self._response.set_result(response) case (Atom("DOWN"), _, Atom("process"), _, Atom("normal")): - print(self._boombox_mode) - print("1") self._terminated.set_result(Atom("normal")) case (Atom("DOWN"), _, Atom("process"), _, reason): - print(self._boombox_mode) - print("2") self._terminated.set_result(reason) if not self._response.done(): self._response.set_exception( diff --git a/test/boombox_test.exs b/test/boombox_test.exs index 2fd071c..c25de3c 100644 --- a/test/boombox_test.exs +++ b/test/boombox_test.exs @@ -563,24 +563,25 @@ defmodule BoomboxTest do boombox :reader -> - Stream.unfold(:ok, fn - :ok -> - case Boombox.read(boombox) do - {:ok, packet} -> {packet, :ok} - :finished -> nil - end + Stream.repeatedly(fn -> + case Boombox.read(boombox) do + {:ok, packet} -> packet + :finished -> :eos + end end) + |> Stream.take_while(&(&1 != :eos)) :message -> - Stream.unfold(:ok, fn :ok -> + Stream.repeatedly(fn -> receive do {:boombox_packet, ^boombox, packet} -> - {packet, :ok} + packet {:boombox_finished, ^boombox} -> - nil + :eos end end) + |> Stream.take_while(&(&1 != :eos)) end |> Enum.map_join(& &1.payload) From bac892a63b96acb3b18077bd9bc8567a8b496b69 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Wed, 3 Dec 2025 16:09:45 +0100 Subject: [PATCH 14/17] Add max_instances to elements --- lib/boombox/internal_bin/elixir_endpoints/sink.ex | 2 ++ lib/boombox/internal_bin/elixir_endpoints/source.ex | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/boombox/internal_bin/elixir_endpoints/sink.ex b/lib/boombox/internal_bin/elixir_endpoints/sink.ex index 913c9dd..605a11e 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/sink.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/sink.ex @@ -9,6 +9,7 @@ def_input_pad :input, accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), availability: :on_request, + max_instances: 2, flow_control: :manual, demand_unit: :buffers @@ -16,6 +17,7 @@ def_input_pad :input, accepted_format: any_of(Membrane.RawAudio, Membrane.RawVideo), availability: :on_request, + max_instances: 2, flow_control: :push end diff --git a/lib/boombox/internal_bin/elixir_endpoints/source.ex b/lib/boombox/internal_bin/elixir_endpoints/source.ex index 1d675b5..36fb0c8 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/source.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/source.ex @@ -7,6 +7,7 @@ def_output_pad :output, accepted_format: any_of(Membrane.RawVideo, Membrane.RawAudio), availability: :on_request, + max_instances: 2, flow_control: flow_control def_options producer: [ From 79dc5683c92ee7f9431fc6beff9517100798b4dc Mon Sep 17 00:00:00 2001 From: noarkhh Date: Wed, 3 Dec 2025 16:14:01 +0100 Subject: [PATCH 15/17] Satisfy 1.19 compiler --- lib/boombox/server.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index 5e4eeba..99d523c 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -40,7 +40,8 @@ defmodule Boombox.Server do name: GenServer.name(), packet_serialization: boolean(), stop_application: boolean(), - communication_medium: communication_medium() + communication_medium: communication_medium(), + parent_pid: pid() ] @type boombox_opts :: [ @@ -285,7 +286,7 @@ defmodule Boombox.Server do end @impl true - def handle_info({boombox_elixir_element, pid}, state) + def handle_info({boombox_elixir_element, pid}, %State{} = state) when boombox_elixir_element in [:boombox_elixir_source, :boombox_elixir_sink] do reply(state.current_client, state.boombox_mode) @@ -403,7 +404,7 @@ defmodule Boombox.Server do @spec handle_request({:run, boombox_opts()}, GenServer.from() | Process.dest(), State.t()) :: {:noreply, State.t()} - defp handle_request({:run, boombox_opts}, from, state) do + defp handle_request({:run, boombox_opts}, from, %State{} = state) do boombox_mode = get_boombox_mode(boombox_opts) %{supervisor: pipeline_supervisor, pipeline: pipeline} = From d71885f2ec3fe47885c64897559e1c435f23fe3b Mon Sep 17 00:00:00 2001 From: noarkhh Date: Thu, 11 Dec 2025 12:01:16 +0100 Subject: [PATCH 16/17] Improve documentation of Elixir Sink and Source --- .../internal_bin/elixir_endpoints/sink.ex | 20 +++++++++++++------ .../internal_bin/elixir_endpoints/source.ex | 14 +++++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/boombox/internal_bin/elixir_endpoints/sink.ex b/lib/boombox/internal_bin/elixir_endpoints/sink.ex index 605a11e..0f5b270 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/sink.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/sink.ex @@ -1,4 +1,11 @@ -[{PushSink, :push}, {PullSink, :manual}] +# This generates two variants of the Sink: +# * PullSink - The element has `:manual` flow control on input pads and demands the +# packets from the previous element when receiving `{:boombox_demand, demand}` +# messages from the consumer process. +# * PushSink - The element works has `:push` flow control on input pads and doesn't expect +# demands from the consumer process. + +[{PullSink, :manual}, {PushSink, :push}] |> Enum.map(fn {module_name, flow_control} -> defmodule Module.concat(Boombox.InternalBin.ElixirEndpoints, module_name) do @moduledoc false @@ -21,11 +28,12 @@ flow_control: :push end - def_options( - consumer: [ - spec: pid() - ] - ) + def_options consumer: [ + spec: pid(), + description: """ + PID of a process to which send packets. + """ + ] defmodule State do @moduledoc false diff --git a/lib/boombox/internal_bin/elixir_endpoints/source.ex b/lib/boombox/internal_bin/elixir_endpoints/source.ex index 36fb0c8..e8739ad 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/source.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/source.ex @@ -1,4 +1,11 @@ -[{PushSource, :push}, {PullSource, :manual}] +# This generates two variants of the Source: +# * PullSource - The element has `:manual` flow control on output pads and +# handles demands from subsequent element by demanding packets from the producer +# process with `{:boombox_demand, self(), demand_amount}` messages. +# * PushSource - The element has `:push` flow control on output pads and expects +# the producer process to provide it packets without demanding them. + +[{PullSource, :manual}, {PushSource, :push}] |> Enum.map(fn {module_name, flow_control} -> defmodule Module.concat(Boombox.InternalBin.ElixirEndpoints, module_name) do @moduledoc false @@ -11,7 +18,10 @@ flow_control: flow_control def_options producer: [ - spec: pid() + spec: pid(), + description: """ + PID of a process from which to demand and receive packets. + """ ] defmodule State do From be1bb81265ef8d22ec5fcf2d5cffa62c4704fa21 Mon Sep 17 00:00:00 2001 From: noarkhh Date: Fri, 12 Dec 2025 17:31:49 +0100 Subject: [PATCH 17/17] Add more tests --- examples.livemd | 2 +- lib/boombox.ex | 86 +++++++-- .../internal_bin/elixir_endpoints/source.ex | 11 +- lib/boombox/server.ex | 30 ++-- test/boombox_test.exs | 167 ++++++++++-------- 5 files changed, 189 insertions(+), 107 deletions(-) diff --git a/examples.livemd b/examples.livemd index e575150..e85fa87 100644 --- a/examples.livemd +++ b/examples.livemd @@ -11,7 +11,7 @@ System.put_env("PATH", "/opt/homebrew/bin:#{System.get_env("PATH")}") # Examples that don't mention them should still work. # MIX_INSTALL_CONFIG_BEGIN -boombox = {:boombox, github: "membraneframework/boombox", branch: "refactor-elixir-endpoints"} +boombox = {:boombox, github: "membraneframework/boombox"} # This livebook uses boombox from the master branch. If any examples happen to not work, the latest stable version of this livebook # can be found on https://hexdocs.pm/boombox/examples.html or in the latest github release. diff --git a/lib/boombox.ex b/lib/boombox.ex index d1846d0..4d590d7 100644 --- a/lib/boombox.ex +++ b/lib/boombox.ex @@ -369,7 +369,7 @@ defmodule Boombox do Returns `:ok` if more packets can be provided, and `:finished` when Boombox finished consuming and will not accept any more packets. Returns - synchronously once the packet has been ingested Boombox is ready for more packets. + synchronously once the packet has been ingested and Boombox is ready for more packets. Can be called only when using `:writer` endpoint on input. """ @@ -440,26 +440,92 @@ defmodule Boombox do pid end + # funn = + # fn + # %Boombox.Packet{kind: :video} = packet, %{video_demand: 0} = state -> + # receive do + # {:boombox_demand, ^source, :video, demand} -> + # send(source, {:boombox_packet, self(), packet}) + # {:cont, %{state | video_demand: demand - 1}} + + # {:DOWN, _monitor, :process, supervisor, _reason} + # when supervisor == procs.supervisor -> + # {:halt, :terminated} + # end + + # %Boombox.Packet{kind: :audio} = packet, %{audio_demand: 0} = state -> + # receive do + # {:boombox_demand, ^source, :audio, demand} -> + # send(source, {:boombox_packet, self(), packet}) + # {:cont, %{state | audio_demand: demand - 1}} + + # {:DOWN, _monitor, :process, supervisor, _reason} + # when supervisor == procs.supervisor -> + # {:halt, :terminated} + # end + + # %Boombox.Packet{} = packet, state -> + # audio_demand = + # receive do + # {:boombox_demand, ^source, :audio, value} -> value + # after + # 0 -> state.audio_demand + # end + + # video_demand = + # receive do + # {:boombox_demand, ^source, :video, value} -> value + # after + # 0 -> state.video_demand + # end + + # send(source, {:boombox_packet, self(), packet}) + + # state = + # case packet.kind do + # :video -> + # %{state | video_demand: video_demand - 1} + + # :audio -> + # %{state | audio_demand: audio_demand - 1} + # end + + # {:cont, state} + + # value, _state -> + # raise ArgumentError, "Expected Boombox.Packet.t(), got: #{inspect(value)}" + # end + @spec consume_stream(Enumerable.t(), pid(), Pipeline.procs()) :: term() defp consume_stream(stream, source, procs) do Enum.reduce_while( stream, - %{demand: 0}, + %{demands: %{audio: 0, video: 0}}, fn - %Boombox.Packet{} = packet, %{demand: 0} = state -> + %Boombox.Packet{kind: kind} = packet, state -> + demand_timeout = + if state.demands[kind] == 0, + do: :infinity, + else: 0 + receive do - {:boombox_demand, ^source, demand} -> - send(source, {:boombox_packet, self(), packet}) - {:cont, %{state | demand: demand - 1}} + {:boombox_demand, ^source, ^kind, value} -> + value - 1 {:DOWN, _monitor, :process, supervisor, _reason} when supervisor == procs.supervisor -> - {:halt, :terminated} + nil + after + demand_timeout -> state.demands[kind] - 1 end + |> case do + nil -> + {:halt, :terminated} - %Boombox.Packet{} = packet, %{demand: demand} = state -> - send(source, {:boombox_packet, self(), packet}) - {:cont, %{state | demand: demand - 1}} + new_demand -> + send(source, {:boombox_packet, self(), packet}) + {:cont, put_in(state.demands[kind], new_demand)} + end value, _state -> raise ArgumentError, "Expected Boombox.Packet.t(), got: #{inspect(value)}" diff --git a/lib/boombox/internal_bin/elixir_endpoints/source.ex b/lib/boombox/internal_bin/elixir_endpoints/source.ex index e8739ad..7a4b7b6 100644 --- a/lib/boombox/internal_bin/elixir_endpoints/source.ex +++ b/lib/boombox/internal_bin/elixir_endpoints/source.ex @@ -3,7 +3,7 @@ # handles demands from subsequent element by demanding packets from the producer # process with `{:boombox_demand, self(), demand_amount}` messages. # * PushSource - The element has `:push` flow control on output pads and expects -# the producer process to provide it packets without demanding them. +# the producer process to provide it with packets without demanding them. [{PullSource, :manual}, {PushSource, :push}] |> Enum.map(fn {module_name, flow_control} -> @@ -60,13 +60,8 @@ if flow_control == :manual do @impl true - def handle_demand(Pad.ref(:output, _id), _size, _unit, ctx, state) do - demands = Enum.map(ctx.pads, fn {_pad, %{demand: demand}} -> demand end) - - if Enum.all?(demands, &(&1 > 0)) do - send(state.producer, {:boombox_demand, self(), Enum.sum(demands)}) - end - + def handle_demand(Pad.ref(:output, id), size, _unit, _ctx, state) do + send(state.producer, {:boombox_demand, self(), id, size}) {[], state} end end diff --git a/lib/boombox/server.ex b/lib/boombox/server.ex index 99d523c..fa1dc16 100644 --- a/lib/boombox/server.ex +++ b/lib/boombox/server.ex @@ -109,7 +109,7 @@ defmodule Boombox.Server do parent_pid: pid(), membrane_sink: pid() | nil, membrane_source: pid() | nil, - membrane_source_demand: non_neg_integer(), + membrane_source_demands: %{audio: non_neg_integer(), video: non_neg_integer()}, pipeline_supervisor: pid() | nil, pipeline: pid() | nil, current_client: GenServer.from() | Process.dest() | nil, @@ -129,7 +129,7 @@ defmodule Boombox.Server do boombox_mode: nil, membrane_sink: nil, membrane_source: nil, - membrane_source_demand: 0, + membrane_source_demands: %{audio: 0, video: 0}, pipeline_supervisor: nil, pipeline: nil, current_client: nil, @@ -302,16 +302,20 @@ defmodule Boombox.Server do end @impl true - def handle_info({:boombox_demand, source, demand}, %State{membrane_source: source} = state) do - state = - if state.current_client != nil do - reply(state.current_client, :ok) - %State{state | current_client: nil} - else - state - end + def handle_info( + {:boombox_demand, source, kind, value}, + %State{membrane_source: source} = state + ) do + %State{} = state = put_in(state.membrane_source_demands[kind], value) + + if state.current_client != nil and + Enum.all?(state.membrane_source_demands, fn {_kind, value} -> value > 0 end) do + reply(state.current_client, :ok) - {:noreply, %State{state | membrane_source_demand: state.membrane_source_demand + demand}} + {:noreply, %State{state | current_client: nil}} + else + {:noreply, state} + end end @impl true @@ -451,13 +455,13 @@ defmodule Boombox.Server do else: packet send(state.membrane_source, {:boombox_packet, self(), packet}) - state = %State{state | membrane_source_demand: state.membrane_source_demand - 1} + %State{} = state = update_in(state.membrane_source_demands[packet.kind], &(&1 - 1)) cond do state.pipeline_termination_reason != nil -> {:stop, state.pipeline_termination_reason, :finished, state} - state.membrane_source_demand == 0 -> + state.membrane_source_demands[packet.kind] == 0 -> {:noreply, %State{state | current_client: from}} true -> diff --git a/test/boombox_test.exs b/test/boombox_test.exs index c25de3c..7da7060 100644 --- a/test/boombox_test.exs +++ b/test/boombox_test.exs @@ -456,21 +456,37 @@ defmodule BoomboxTest do Compare.compare(output, "test/fixtures/ref_bun10s_opus_aac.mp4") end - @tag :mp4_elixir_rotate_mp4 - async_test "mp4 -> elixir rotate -> mp4", %{tmp_dir: tmp} do - Boombox.run(input: @bbb_mp4, output: {:stream, video: :image, audio: :binary}) - |> Stream.map(fn - %Boombox.Packet{kind: :video, payload: image} = packet -> - image = Image.rotate!(image, 180) - %Boombox.Packet{packet | payload: image} - - %Boombox.Packet{kind: :audio} = packet -> - packet - end) - |> Boombox.run(input: {:stream, video: :image, audio: :binary}, output: "#{tmp}/output.mp4") + for( + output <- [:stream, :reader, :message], + input <- [:stream, :writer, :message], + do: {output, input} + ) + |> Enum.each(fn {output, input} -> + @tag :mp4_elixir_rotate_mp4 + @tag String.to_atom("mp4_#{output}_rotate_#{input}_mp4") + async_test "mp4 -> #{output} -> rotate -> #{input} -> mp4", %{tmp_dir: tmp} do + produce_packet_stream( + input: @bbb_mp4, + output: {unquote(output), video: :image, audio: :binary} + ) + |> Stream.map(fn + %Boombox.Packet{kind: :video, payload: image} = packet -> + image = Image.rotate!(image, 180) + %Boombox.Packet{packet | payload: image} - Compare.compare("#{tmp}/output.mp4", "test/fixtures/ref_bun_rotated.mp4") - end + %Boombox.Packet{kind: :audio} = packet -> + packet + end) + |> consume_packet_stream( + input: {unquote(input), video: :image, audio: :binary}, + output: "#{tmp}/output.mp4" + ) + + Process.sleep(100) + + Compare.compare("#{tmp}/output.mp4", "test/fixtures/ref_bun_rotated.mp4") + end + end) [:stream, :writer, :message] |> Enum.each(fn elixir_endpoint -> @@ -488,36 +504,6 @@ defmodule BoomboxTest do max_y = Image.height(bg) - Image.height(overlay) fps = 60 - image_sink = fn stream -> - case unquote(elixir_endpoint) do - :stream -> - Boombox.run(stream, - input: {:stream, video: :image, audio: false}, - output: {:webrtc, signaling} - ) - - :writer -> - writer = - Boombox.run( - input: {:writer, video: :image, audio: false}, - output: {:webrtc, signaling} - ) - - Enum.each(stream, &Boombox.write(writer, &1)) - Boombox.close(writer) - - :message -> - server = - Boombox.run( - input: {:message, video: :image, audio: false}, - output: {:webrtc, signaling} - ) - - Enum.each(stream, &send(server, {:boombox_packet, &1})) - send(server, :boombox_close) - end - end - Task.async(fn -> Stream.iterate({_x = 300, _y = 0, _dx = 1, _dy = 2, _pts = 0}, fn {x, y, dx, dy, pts} -> dx = if (x + dx) in 0..max_x, do: dx, else: -dx @@ -530,7 +516,10 @@ defmodule BoomboxTest do %Boombox.Packet{kind: :video, payload: img, pts: pts} end) |> Stream.take(5 * fps) - |> then(image_sink) + |> consume_packet_stream( + input: {unquote(elixir_endpoint), video: :image, audio: false}, + output: {:webrtc, signaling} + ) end) output = Path.join(tmp, "output.mp4") @@ -545,8 +534,8 @@ defmodule BoomboxTest do @tag :mp4_elixir_resampled_pcm @tag String.to_atom("mp4_#{elixir_endpoint}_resampled_pcm") async_test "mp4 -> #{elixir_endpoint} -> resampled PCM" do - boombox = - Boombox.run( + output_pcm = + produce_packet_stream( input: @bbb_mp4, output: {unquote(elixir_endpoint), @@ -556,37 +545,10 @@ defmodule BoomboxTest do audio_channels: 1, audio_format: :s16le} ) - - pcm = - case unquote(elixir_endpoint) do - :stream -> - boombox - - :reader -> - Stream.repeatedly(fn -> - case Boombox.read(boombox) do - {:ok, packet} -> packet - :finished -> :eos - end - end) - |> Stream.take_while(&(&1 != :eos)) - - :message -> - Stream.repeatedly(fn -> - receive do - {:boombox_packet, ^boombox, packet} -> - packet - - {:boombox_finished, ^boombox} -> - :eos - end - end) - |> Stream.take_while(&(&1 != :eos)) - end |> Enum.map_join(& &1.payload) ref = File.read!("test/fixtures/ref_bun.pcm") - assert Compare.samples_min_squared_error(ref, pcm, 16) < 500 + assert Compare.samples_min_squared_error(ref, output_pcm, 16) < 500 end end) @@ -646,6 +608,61 @@ defmodule BoomboxTest do end end) + @spec produce_packet_stream(input: Boombox.input(), output: Boombox.elixir_output()) :: + Enumerable.t() + defp produce_packet_stream([input: _input, output: {:stream, _opts}] = opts) do + Boombox.run(opts) + end + + defp produce_packet_stream([input: _input, output: {:reader, _opts}] = opts) do + boombox = Boombox.run(opts) + + Stream.repeatedly(fn -> + case Boombox.read(boombox) do + {:ok, packet} -> packet + :finished -> :eos + end + end) + |> Stream.take_while(&(&1 != :eos)) + end + + defp produce_packet_stream([input: _input, output: {:message, _opts}] = opts) do + boombox = Boombox.run(opts) + + Stream.repeatedly(fn -> + receive do + {:boombox_packet, ^boombox, packet} -> + packet + + {:boombox_finished, ^boombox} -> + :eos + end + end) + |> Stream.take_while(&(&1 != :eos)) + end + + @spec consume_packet_stream(Enumerable.t(), + input: Boombox.elixir_input(), + output: Boombox.output() + ) :: term() + defp consume_packet_stream(stream, [input: {:stream, _opts}, output: _output] = opts) do + stream |> Boombox.run(opts) + end + + defp consume_packet_stream(stream, [input: {:writer, _opts}, output: _output] = opts) do + writer = Boombox.run(opts) + + Enum.each(stream, &Boombox.write(writer, &1)) + Boombox.close(writer) + end + + defp consume_packet_stream(stream, [input: {:message, _opts}, output: _output] = opts) do + server = Boombox.run(opts) + + Enum.each(stream, &send(server, {:boombox_packet, &1})) + send(server, :boombox_close) + end + defp send_rtmp(url) do p = Testing.Pipeline.start_link_supervised!(