From 0cca3628e80773ad58ff8fc55724188c80927eb3 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 27 Sep 2018 00:54:27 +0200
Subject: [PATCH 01/36] ability to define worker supervisor in pool args,
 register worker supervisor as poolboy_sup

---
 src/poolboy.erl     | 35 ++++++++++++++++++++++++++++++++---
 src/poolboy_sup.erl |  2 +-
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index db20541..505b175 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -36,8 +36,16 @@
 % Copied from gen:start_ret/0
 -type start_ret() :: {'ok', pid()} | 'ignore' | {'error', term()}.
 
+% Copied from supervisor:sup_ref/0
+-type sup_ref() ::
+    (Name :: atom()) |
+    {Name :: atom(), Node :: node()} |
+    {global, Name :: atom()} |
+    {via, Module :: module(), Name :: any()} |
+    pid().
+
 -record(state, {
-    supervisor :: undefined | pid(),
+    supervisor :: undefined | sup_ref(),
     workers :: undefined | pid_queue(),
     waiting :: pid_queue(),
     monitors :: ets:tid(),
@@ -151,9 +159,28 @@ init({PoolArgs, WorkerArgs}) ->
     Monitors = ets:new(monitors, [private]),
     init(PoolArgs, WorkerArgs, #state{waiting = Waiting, monitors = Monitors}).
 
+init([{worker_supervisor, Sup = {Scope, _Name}} | Rest], WorkerArgs, State)
+  when Scope =:= local orelse Scope =:= global ->
+    init(Rest, WorkerArgs, State#state{supervisor=Sup});
+init([{worker_supervisor, Sup = {_Name, Node}} | Rest], WorkerArgs, State) ->
+    (catch erlang:monitor_node(Node, true)),
+    init(Rest, WorkerArgs, State#state{supervisor=Sup});
+init([{worker_supervisor, Sup} | Rest], WorkerArgs, State)
+  when is_pid(Sup) orelse is_atom(Sup) orelse is_tuple(Sup) ->
+    init(Rest, WorkerArgs, State#state{supervisor=Sup});
 init([{worker_module, Mod} | Rest], WorkerArgs, State) when is_atom(Mod) ->
-    {ok, Sup} = poolboy_sup:start_link(Mod, WorkerArgs),
-    init(Rest, WorkerArgs, State#state{supervisor = Sup});
+    {ok, Sup} =
+    case poolboy_sup:start_link(Mod, WorkerArgs) of
+        {ok, _Pid} = Ok -> Ok;
+        {error, {already_started, Pid}} ->
+            MRef = erlang:monitor(process, Pid),
+            receive
+                {'DOWN', MRef, _, _, _} -> ok
+            after ?TIMEOUT -> ok
+            end,
+            poolboy_sup:start_link(Mod, WorkerArgs)
+    end,
+    init(Rest, WorkerArgs, State#state{supervisor=Sup});
 init([{size, Size} | Rest], WorkerArgs, State) when is_integer(Size) ->
     init(Rest, WorkerArgs, State#state{size = Size});
 init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State) when is_integer(MaxOverflow) ->
@@ -275,6 +302,8 @@ handle_info({'EXIT', Pid, _Reason}, State) ->
                     {noreply, State}
             end
     end;
+handle_info({nodedown, Node}, State = #state{supervisor = {_, Node}}) ->
+    {stop, nodedown, State};
 
 handle_info(_Info, State) ->
     {noreply, State}.
diff --git a/src/poolboy_sup.erl b/src/poolboy_sup.erl
index e6485a6..cabe86f 100644
--- a/src/poolboy_sup.erl
+++ b/src/poolboy_sup.erl
@@ -6,7 +6,7 @@
 -export([start_link/2, init/1]).
 
 start_link(Mod, Args) ->
-    supervisor:start_link(?MODULE, {Mod, Args}).
+    supervisor:start_link({local, ?MODULE}, ?MODULE, {Mod, Args}).
 
 init({Mod, Args}) ->
     {ok, {{simple_one_for_one, 0, 1},

From bc69076e587e395322292ee373a78d897a08a1fc Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 27 Sep 2018 10:36:27 +0200
Subject: [PATCH 02/36] register worker supervisor under its module name

---
 src/poolboy_sup.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/poolboy_sup.erl b/src/poolboy_sup.erl
index cabe86f..2c50b6f 100644
--- a/src/poolboy_sup.erl
+++ b/src/poolboy_sup.erl
@@ -6,7 +6,7 @@
 -export([start_link/2, init/1]).
 
 start_link(Mod, Args) ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, {Mod, Args}).
+    supervisor:start_link({local, Mod}, ?MODULE, {Mod, Args}).
 
 init({Mod, Args}) ->
     {ok, {{simple_one_for_one, 0, 1},

From f3dc2938e7e964ae4d41e7a9050142ad19bbb14a Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 28 Sep 2018 16:41:23 +0200
Subject: [PATCH 03/36] fix for stopping a remote supervisor

---
 src/poolboy.erl | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 505b175..63ceb55 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -308,10 +308,12 @@ handle_info({nodedown, Node}, State = #state{supervisor = {_, Node}}) ->
 handle_info(_Info, State) ->
     {noreply, State}.
 
-terminate(_Reason, State) ->
+terminate(_Reason, State = #state{supervisor = Sup}) ->
     Workers = queue:to_list(State#state.workers),
     ok = lists:foreach(fun (W) -> unlink(W) end, Workers),
-    true = exit(State#state.supervisor, shutdown),
+    if is_pid(Sup) -> true = exit(Sup, shutdown);
+       true -> ok = gen_server:stop(Sup)
+    end,
     ok.
 
 code_change(_OldVsn, State, _Extra) ->

From 1e726b062b30b754da036006f9e07a9852c1fefe Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Mon, 8 Oct 2018 11:17:45 +0200
Subject: [PATCH 04/36] stop on supervisor exit

---
 src/poolboy.erl | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 63ceb55..aa5d72c 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -284,9 +284,10 @@ handle_info({'DOWN', MRef, _, _, _}, State) ->
             Waiting = queue:filter(fun ({_, _, R}) -> R =/= MRef end, State#state.waiting),
             {noreply, State#state{waiting = Waiting}}
     end;
-handle_info({'EXIT', Pid, _Reason}, State) ->
+handle_info({'EXIT', Pid, Reason}, State) ->
     #state{supervisor = Sup,
            monitors = Monitors} = State,
+    Next =
     case ets:lookup(Monitors, Pid) of
         [{Pid, _, MRef}] ->
             true = erlang:demonitor(MRef),
@@ -301,19 +302,21 @@ handle_info({'EXIT', Pid, _Reason}, State) ->
                 false ->
                     {noreply, State}
             end
+    end,
+    case {Sup, erlang:node(Pid)} of
+        {{_, Node}, Node} -> {stop, Reason, State#state{supervisor = undefined}};
+        {Pid, _} -> {stop, Reason, State#state{supervisor = undefined}};
+        _ -> Next
     end;
 handle_info({nodedown, Node}, State = #state{supervisor = {_, Node}}) ->
-    {stop, nodedown, State};
-
+    {stop, nodedown, State#state{supervisor = undefined}};
 handle_info(_Info, State) ->
     {noreply, State}.
 
 terminate(_Reason, State = #state{supervisor = Sup}) ->
     Workers = queue:to_list(State#state.workers),
     ok = lists:foreach(fun (W) -> unlink(W) end, Workers),
-    if is_pid(Sup) -> true = exit(Sup, shutdown);
-       true -> ok = gen_server:stop(Sup)
-    end,
+    stop_supervisor(Sup),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->
@@ -406,3 +409,12 @@ state_name(#state{overflow = MaxOverflow, max_overflow = MaxOverflow}) ->
     full;
 state_name(_State) ->
     overflow.
+
+stop_supervisor(undefined) -> ok;
+stop_supervisor(Pid) when is_pid(Pid) ->
+    case erlang:node(Pid) of
+        N when N == node() -> exit(Pid, shutdown);
+        _ -> gen_server:stop(Pid)
+    end;
+stop_supervisor(Tuple) when is_tuple(Tuple) ->
+    gen_server:stop(Tuple).

From 829c58fb813dfde208d253ab7cf6226c1d3c5ba3 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Wed, 10 Oct 2018 14:41:17 +0200
Subject: [PATCH 05/36] do not try to terminate remote superevisor on nodedown

---
 src/poolboy.erl | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index aa5d72c..2bd9ed4 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -313,10 +313,10 @@ handle_info({nodedown, Node}, State = #state{supervisor = {_, Node}}) ->
 handle_info(_Info, State) ->
     {noreply, State}.
 
-terminate(_Reason, State = #state{supervisor = Sup}) ->
+terminate(Reason, State = #state{supervisor = Sup}) ->
     Workers = queue:to_list(State#state.workers),
     ok = lists:foreach(fun (W) -> unlink(W) end, Workers),
-    stop_supervisor(Sup),
+    stop_supervisor(Reason, Sup),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->
@@ -410,11 +410,13 @@ state_name(#state{overflow = MaxOverflow, max_overflow = MaxOverflow}) ->
 state_name(_State) ->
     overflow.
 
-stop_supervisor(undefined) -> ok;
-stop_supervisor(Pid) when is_pid(Pid) ->
+stop_supervisor(_, undefined) -> ok;
+stop_supervisor(Reason, Pid) when is_pid(Pid) ->
     case erlang:node(Pid) of
-        N when N == node() -> exit(Pid, shutdown);
-        _ -> gen_server:stop(Pid)
+        N when N == node() -> exit(Pid, Reason);
+        _ when Reason =/= nodedown -> catch gen_server:stop(Pid, Reason, ?TIMEOUT);
+        _ -> ok
     end;
-stop_supervisor(Tuple) when is_tuple(Tuple) ->
-    gen_server:stop(Tuple).
+stop_supervisor(nodedown, Tuple) when is_tuple(Tuple) -> ok;
+stop_supervisor(Reason, Tuple) when is_tuple(Tuple) ->
+    catch gen_server:stop(Tuple, Reason, ?TIMEOUT).

From fc6b19b2edba15f83acc37debc5c9c7a756237e1 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 29 Nov 2018 14:32:27 +0100
Subject: [PATCH 06/36] add a collection module to encapsulate the data
 structure workers are kept in

---
 src/poolboy.erl            |  93 +++++++----------
 src/poolboy_collection.erl | 209 +++++++++++++++++++++++++++++++++++++
 test/poolboy_tests.erl     |  70 ++++++-------
 3 files changed, 284 insertions(+), 88 deletions(-)
 create mode 100644 src/poolboy_collection.erl

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 2bd9ed4..47735ef 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -12,12 +12,6 @@
 
 -define(TIMEOUT, 5000).
 
--ifdef(pre17).
--type pid_queue() :: queue().
--else.
--type pid_queue() :: queue:queue().
--endif.
-
 -ifdef(OTP_RELEASE). %% this implies 21 or higher
 -define(EXCEPTION(Class, Reason, Stacktrace), Class:Reason:Stacktrace).
 -define(GET_STACK(Stacktrace), Stacktrace).
@@ -46,10 +40,11 @@
 
 -record(state, {
     supervisor :: undefined | sup_ref(),
-    workers :: undefined | pid_queue(),
-    waiting :: pid_queue(),
+    workers :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
+    waiting :: poolboy_collection:pid_queue() | poolboy_collection:pid_queue(tuple()),
     monitors :: ets:tid(),
     size = 5 :: non_neg_integer(),
+    type = list :: list | array | tuple | queue,
     overflow = 0 :: non_neg_integer(),
     max_overflow = 10 :: non_neg_integer(),
     strategy = lifo :: lifo | fifo
@@ -183,6 +178,10 @@ init([{worker_module, Mod} | Rest], WorkerArgs, State) when is_atom(Mod) ->
     init(Rest, WorkerArgs, State#state{supervisor=Sup});
 init([{size, Size} | Rest], WorkerArgs, State) when is_integer(Size) ->
     init(Rest, WorkerArgs, State#state{size = Size});
+init([{type, Type} | Rest], WorkerArgs, State) when Type == list orelse
+                                                    Type == array orelse
+                                                    Type == tuple ->
+    init(Rest, WorkerArgs, State#state{type = Type});
 init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State) when is_integer(MaxOverflow) ->
     init(Rest, WorkerArgs, State#state{max_overflow = MaxOverflow});
 init([{strategy, lifo} | Rest], WorkerArgs, State) ->
@@ -191,9 +190,8 @@ init([{strategy, fifo} | Rest], WorkerArgs, State) ->
     init(Rest, WorkerArgs, State#state{strategy = fifo});
 init([_ | Rest], WorkerArgs, State) ->
     init(Rest, WorkerArgs, State);
-init([], _WorkerArgs, #state{size = Size, supervisor = Sup} = State) ->
-    Workers = prepopulate(Size, Sup),
-    {ok, State#state{workers = Workers}}.
+init([], _WorkerArgs, #state{size = Size, type=Type, supervisor = Sup} = State) ->
+    {ok, State#state{workers = prepopulate(Size, Type, Sup)}}.
 
 handle_cast({checkin, Pid}, State = #state{monitors = Monitors}) ->
     case ets:lookup(Monitors, Pid) of
@@ -232,20 +230,19 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
            workers = Workers,
            monitors = Monitors,
            overflow = Overflow,
-           max_overflow = MaxOverflow,
-           strategy = Strategy} = State,
-    case get_worker_with_strategy(Workers, Strategy) of
-        {{value, Pid},  Left} ->
+           max_overflow = MaxOverflow} = State,
+    case poolboy_collection:hide_head(Workers) of
+        {Pid, Left} when is_pid(Pid) ->
             MRef = erlang:monitor(process, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
             {reply, Pid, State#state{workers = Left}};
-        {empty, _Left} when MaxOverflow > 0, Overflow < MaxOverflow ->
+        empty when MaxOverflow > 0, Overflow < MaxOverflow ->
             {Pid, MRef} = new_worker(Sup, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
             {reply, Pid, State#state{overflow = Overflow + 1}};
-        {empty, _Left} when Block =:= false ->
+        empty when Block =:= false ->
             {reply, full, State};
-        {empty, _Left} ->
+        empty ->
             MRef = erlang:monitor(process, FromPid),
             Waiting = queue:in({From, CRef, MRef}, State#state.waiting),
             {noreply, State#state{waiting = Waiting}}
@@ -256,10 +253,11 @@ handle_call(status, _From, State) ->
            monitors = Monitors,
            overflow = Overflow} = State,
     StateName = state_name(State),
-    {reply, {StateName, queue:len(Workers), Overflow, ets:info(Monitors, size)}, State};
+    {reply, {StateName, poolboy_collection:len(visible, Workers), Overflow, ets:info(Monitors, size)}, State};
 handle_call(get_avail_workers, _From, State) ->
-    Workers = State#state.workers,
-    {reply, Workers, State};
+    {reply, poolboy_collection:all(visible, State#state.workers), State};
+handle_call(get_any_worker, _From, State) ->
+    {reply, poolboy_collection:rand(known, State#state.workers), State};
 handle_call(get_all_workers, _From, State) ->
     Sup = State#state.supervisor,
     WorkerList = supervisor:which_children(Sup),
@@ -295,12 +293,10 @@ handle_info({'EXIT', Pid, Reason}, State) ->
             NewState = handle_worker_exit(Pid, State),
             {noreply, NewState};
         [] ->
-            case queue:member(Pid, State#state.workers) of
-                true ->
-                    W = filter_worker_by_pid(Pid, State#state.workers),
-                    {noreply, State#state{workers = queue:in(new_worker(Sup), W)}};
-                false ->
-                    {noreply, State}
+            try
+                {noreply, replace_worker(Pid, State)}
+            catch ?EXCEPTION(error, enoent, StackTrace) ->
+                {noreply, State}
             end
     end,
     case {Sup, erlang:node(Pid)} of
@@ -314,8 +310,7 @@ handle_info(_Info, State) ->
     {noreply, State}.
 
 terminate(Reason, State = #state{supervisor = Sup}) ->
-    Workers = queue:to_list(State#state.workers),
-    ok = lists:foreach(fun (W) -> unlink(W) end, Workers),
+    ok = poolboy_collection:foreach(fun (W) -> catch unlink(W) end, State#state.workers),
     stop_supervisor(Reason, Sup),
     ok.
 
@@ -340,29 +335,21 @@ new_worker(Sup, FromPid) ->
     Ref = erlang:monitor(process, FromPid),
     {Pid, Ref}.
 
-get_worker_with_strategy(Workers, fifo) ->
-    queue:out(Workers);
-get_worker_with_strategy(Workers, lifo) ->
-    queue:out_r(Workers).
-
 dismiss_worker(Sup, Pid) ->
     true = unlink(Pid),
     supervisor:terminate_child(Sup, Pid).
 
-filter_worker_by_pid(Pid, Workers) ->
-    queue:filter(fun (WPid) -> WPid =/= Pid end, Workers).
-
-prepopulate(N, _Sup) when N < 1 ->
-    queue:new();
-prepopulate(N, Sup) ->
-    prepopulate(N, Sup, queue:new()).
+replace_worker(Pid, State = #state{strategy = Strategy}) ->
+    {NewWorker, Workers} = poolboy_collection:replace(Pid, State#state.workers),
+    case Strategy of
+        lifo -> State#state{workers = poolboy_collection:prepend(NewWorker, Workers)};
+        fifo -> State#state{workers = poolboy_collection:append(NewWorker, Workers)}
+    end.
 
-prepopulate(0, _Sup, Workers) ->
-    Workers;
-prepopulate(N, Sup, Workers) ->
-    prepopulate(N-1, Sup, queue:in(new_worker(Sup), Workers)).
+prepopulate(Size, Type, Sup) ->
+    poolboy_collection:new(Type, Size, fun(_) -> new_worker(Sup) end).
 
-handle_checkin(Pid, State) ->
+handle_checkin(Pid, State = #state{strategy = Strategy}) ->
     #state{supervisor = Sup,
            waiting = Waiting,
            monitors = Monitors,
@@ -376,13 +363,15 @@ handle_checkin(Pid, State) ->
             ok = dismiss_worker(Sup, Pid),
             State#state{waiting = Empty, overflow = Overflow - 1};
         {empty, Empty} ->
-            Workers = queue:in(Pid, State#state.workers),
+            Workers = case Strategy of
+                lifo -> poolboy_collection:prepend(Pid, State#state.workers);
+                fifo -> poolboy_collection:append(Pid, State#state.workers)
+            end,
             State#state{workers = Workers, waiting = Empty, overflow = 0}
     end.
 
 handle_worker_exit(Pid, State) ->
-    #state{supervisor = Sup,
-           monitors = Monitors,
+    #state{monitors = Monitors,
            overflow = Overflow} = State,
     case queue:out(State#state.waiting) of
         {{value, {From, CRef, MRef}}, LeftWaiting} ->
@@ -393,14 +382,12 @@ handle_worker_exit(Pid, State) ->
         {empty, Empty} when Overflow > 0 ->
             State#state{overflow = Overflow - 1, waiting = Empty};
         {empty, Empty} ->
-            W = filter_worker_by_pid(Pid, State#state.workers),
-            Workers = queue:in(new_worker(Sup), W),
-            State#state{workers = Workers, waiting = Empty}
+            replace_worker(Pid, State#state{waiting = Empty})
     end.
 
 state_name(State = #state{overflow = Overflow}) when Overflow < 1 ->
     #state{max_overflow = MaxOverflow, workers = Workers} = State,
-    case queue:len(Workers) == 0 of
+    case poolboy_collection:len(visible, Workers) == 0 of
         true when MaxOverflow < 1 -> full;
         true -> overflow;
         false -> ready
diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
new file mode 100644
index 0000000..89107ea
--- /dev/null
+++ b/src/poolboy_collection.erl
@@ -0,0 +1,209 @@
+-module(poolboy_collection).
+
+-export([new/3,
+         len/2,
+         hide_head/1,
+         replace/2, replace/3,
+         prepend/2,
+         append/2,
+         foreach/2,
+         all/2,
+         rand/2
+        ]).
+
+-ifdef(pre17).
+-type pid_queue() :: queue().
+-type pid_queue(A) :: queue(A).
+-else.
+-type pid_queue() :: queue:queue().
+-type pid_queue(A) :: queue:queue(A).
+-endif.
+-export_type([pid_queue/0, pid_queue/1]).
+
+-type coll_data() :: list()|{}|array:array()|pid_queue().
+-type coll_data(A) :: list(A)|{A}|array:array(A)|pid_queue(A).
+
+-define(Types, 
+        #{list => #{
+            is => fun is_list/1,
+            len => fun length/1,
+            from => fun(L) -> L end,
+            nth => fun lists:nth/2,
+            prep => fun(I, L) -> [I|L] end,
+            app => fun(I, L) -> L ++ [I] end
+           },
+          array => #{
+            is => fun array:is_array/1,
+            len => fun array:size/1,
+            from => fun array:from_list/1,
+            nth => fun(I, A) -> array:get(I-1, A) end,
+            prep => fun(I, A) -> array:foldl(
+                                   fun(Idx, Val, Arr) ->
+                                           array:set(Idx+1, Val, Arr)
+                                   end,
+                                   array:set(0, I, array:new()),
+                                   A)
+                    end,
+            app => fun(I, A) -> array:set(array:size(A), I, A) end
+           },
+          queue => #{
+            is => fun queue:is_queue/1,
+            len => fun queue:len/1,
+            from => fun queue:from_list/1,
+            nth => fun(I, {RL, FL})
+                         when I =< length(FL) ->
+                           lists:nth(I, FL);
+                      (I, {RL, FL}) ->
+                           lists:nth(
+                             length(RL)-(I-length(FL)-1),
+                             RL)
+                   end,
+            prep => fun queue:in_r/2,
+            app => fun queue:in/2
+           },
+          tuple => #{
+            is => fun is_tuple/1,
+            len => fun tuple_size/1,
+            from => fun list_to_tuple/1,
+            nth => fun element/2,
+            prep => fun(I, T) ->  erlang:insert_element(1, T, I) end,
+            app => fun(I, T) -> erlang:append_element(T, I) end
+           }
+         }).
+-define(Colls(T, F), maps:get(F, maps:get(T, ?Types))).
+
+
+-record(coll, {indexes :: [non_neg_integer()],
+               rev_indexes :: #{any()=>non_neg_integer()},
+               data :: coll_data() | coll_data(any()),
+               item_generator :: fun((non_neg_integer()) -> any()) }).
+
+-type coll() :: #coll{rev_indexes :: #{}, data :: coll_data()}.
+-type coll(A) :: #coll{rev_indexes :: #{A=>non_neg_integer()},
+                       data :: coll_data(A),
+                       item_generator :: fun((non_neg_integer()) -> A) }.
+
+-export_type([coll/0, coll/1]).
+
+new(Type, Size, Fun) when is_function(Fun, 1) ->
+    Indexes = lists:seq(1, Size),
+    RevIndexes = maps:from_list([{Fun(I), I} || I <- Indexes]),
+    Data = (from(Type))(maps:keys(RevIndexes)),
+    #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data,
+          item_generator = Fun}.
+
+from(Type) -> ?Colls(Type, ?FUNCTION_NAME).
+
+len(known, #coll{data=Data}) -> len(Data);
+len(visible, #coll{indexes=Indexes}) -> length(Indexes).
+
+len(Data) ->
+    (call(?FUNCTION_NAME, Data))(Data).
+
+call(F, Data) when is_atom(F) ->
+    call(F, Data, false, [none| maps:keys(?Types)]).
+
+call(_, _Data, _, []) -> throw(badarg);
+call(F, _Data, true, [T |_Types]) -> ?Colls(T, F);
+call(F, Data, false, [_ | [T |_] = Types]) ->
+    call(F, Data, ?Colls(T, is), Types);
+call(F, Data, IsType, Types) when is_function(IsType, 1) ->
+    call(F, Data, IsType(Data), Types).
+
+
+hide_head(#coll{indexes = []}) -> empty;
+hide_head(Coll = #coll{indexes = [H|T], data=Data}) ->
+    {nth(H, Data), Coll#coll{indexes = T}}.
+
+nth(Index, Data) ->
+    (call(?FUNCTION_NAME, Data))(Index, Data).
+
+replace(Out, Coll = #coll{item_generator = In}) ->
+    replace(Out, In, Coll).
+
+replace(Out, In, Coll = #coll{data = Data}) ->
+    case maps:take(Out, Coll#coll.rev_indexes) of
+        error -> error(enoent);
+        {OutIndex, RevIndexes} ->
+            NewData = data_replace(OutIndex, Out, In, Data),
+            NewItem = nth(OutIndex, NewData),
+            NewRevIndexes = maps:put(NewItem, OutIndex, RevIndexes),
+            {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
+    end.
+
+data_replace(OutIndex, Out, In, Data) when not is_function(In) ->
+    data_replace(OutIndex, Out, fun(_) -> In end, Data);
+data_replace(OutIndex, Out, In, Data) when is_function(In, 1) andalso is_list(Data) ->
+    {FirstN, [Out | Tail]} = lists:split(OutIndex-1, Data),
+    FirstN ++ [In(OutIndex)| Tail];
+data_replace(OutIndex, Out, In, Data) when is_function(In, 1) andalso is_tuple(Data) ->
+    case array:is_array(Data) of
+        true ->
+            Out = array:get(OutIndex, Data),
+            array:set(OutIndex, In(OutIndex), Data);
+        false when element(OutIndex, Data) == Out ->
+            setelement(OutIndex, Data, In(OutIndex))
+    end.
+
+
+prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
+    case maps:get(In, RevIndexes, undefined) of
+        InIndex when is_integer(InIndex) -> Coll#coll{indexes=[InIndex|Indexes]};
+        undefined ->
+            NewData = prep(In, Data),
+            NewRevIndexes = maps:put(In, 1, maps:map(fun(_, V) -> V + 1 end, RevIndexes)),
+            NewIndexes = [1 | [I+1 || I <- Indexes]],
+            Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
+    end.
+
+prep(In, Data) ->
+    (call(?FUNCTION_NAME, Data))(In, Data).
+
+append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
+    case maps:get(In, RevIndexes, undefined) of
+        InIndex when is_integer(InIndex) -> Coll#coll{indexes=Indexes++[InIndex]};
+        undefined ->
+            NewData = app(In, Data),
+            NewIndex =
+            case {array:is_array(Data), len(Data)} of
+                {true, Len} -> Len - 1;
+                {_, Len} -> Len
+            end,
+            NewRevIndexes = maps:put(In, NewIndex, RevIndexes),
+            NewIndexes = Indexes++[NewIndex],
+            Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
+    end.
+
+app(In, Data) ->
+    (call(?FUNCTION_NAME, Data))(In, Data).
+
+foreach(Fun, #coll{data = Data}) -> data_foreach(Fun, Data).
+
+data_foreach(Fun, Data) when is_function(Fun) andalso is_list(Data) ->
+    lists:foreach(Fun, Data);
+data_foreach(Fun, Data) when is_function(Fun) andalso is_tuple(Data) ->
+    case array:is_array(Data) of
+        true -> array:sparse_map(fun(_, Value) -> Fun(Value) end, Data), ok;
+        false -> tuple_foreach(Fun, Data, erlang:tuple_size(Data))
+    end.
+
+tuple_foreach(_Fun, _Tuple, 0) -> ok;
+tuple_foreach(Fun, Tuple, Index) ->
+    Fun(element(Index, Tuple)),
+    tuple_foreach(Fun, Tuple, Index-1).
+
+
+all(known, #coll{rev_indexes = RevIndexes}) ->
+    maps:keys(RevIndexes);
+all(visible, #coll{indexes = Indexes, data = Data}) ->
+    [nth(I, Data) || I <- Indexes].
+
+
+rand(known, #coll{data = Data}) ->
+    case len(Data) of
+        0 -> empty;
+        L -> nth(rand:uniform(1, L), Data)
+    end;
+rand(visible, #coll{indexes = []}) -> empty;
+rand(visible, #coll{indexes = Indexes, data = Data}) ->
+    nth(lists:nth(rand:uniform(1, length(Indexes)), Indexes), Data).
diff --git a/test/poolboy_tests.erl b/test/poolboy_tests.erl
index 5b27024..552f6e9 100644
--- a/test/poolboy_tests.erl
+++ b/test/poolboy_tests.erl
@@ -127,13 +127,13 @@ transaction_timeout() ->
 pool_startup() ->
     %% Check basic pool operation.
     {ok, Pid} = new_pool(10, 5),
-    ?assertEqual(10, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(10, length(pool_call(Pid, get_avail_workers))),
     poolboy:checkout(Pid),
-    ?assertEqual(9, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(9, length(pool_call(Pid, get_avail_workers))),
     Worker = poolboy:checkout(Pid),
-    ?assertEqual(8, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(8, length(pool_call(Pid, get_avail_workers))),
     checkin_worker(Pid, Worker),
-    ?assertEqual(9, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(9, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(1, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
@@ -141,23 +141,23 @@ pool_overflow() ->
     %% Check that the pool overflows properly.
     {ok, Pid} = new_pool(5, 5),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     [A, B, C, D, E, F, G] = Workers,
     checkin_worker(Pid, A),
     checkin_worker(Pid, B),
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, C),
     checkin_worker(Pid, D),
-    ?assertEqual(2, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(2, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, E),
     checkin_worker(Pid, F),
-    ?assertEqual(4, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(4, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, G),
-    ?assertEqual(5, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
@@ -167,7 +167,7 @@ pool_empty() ->
     %% overflow is enabled.
     {ok, Pid} = new_pool(5, 2),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     [A, B, C, D, E, F, G] = Workers,
     Self = self(),
@@ -192,18 +192,18 @@ pool_empty() ->
     after
         500 -> ?assert(false)
     end,
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, C),
     checkin_worker(Pid, D),
-    ?assertEqual(2, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(2, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, E),
     checkin_worker(Pid, F),
-    ?assertEqual(4, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(4, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, G),
-    ?assertEqual(5, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
@@ -213,7 +213,7 @@ pool_empty_no_overflow() ->
     %% disabled.
     {ok, Pid} = new_pool(5, 0),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 4)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     [A, B, C, D, E] = Workers,
     Self = self(),
@@ -238,14 +238,14 @@ pool_empty_no_overflow() ->
     after
         500 -> ?assert(false)
     end,
-    ?assertEqual(2, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(2, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, C),
     checkin_worker(Pid, D),
-    ?assertEqual(4, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(4, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     checkin_worker(Pid, E),
-    ?assertEqual(5, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
@@ -256,16 +256,16 @@ worker_death() ->
     {ok, Pid} = new_pool(5, 2),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
-    ?assertEqual(5, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
     [A, B, C|_Workers] = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     kill_worker(A),
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(6, length(pool_call(Pid, get_all_workers))),
     kill_worker(B),
     kill_worker(C),
-    ?assertEqual(1, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(1, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(4, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
@@ -277,9 +277,9 @@ worker_death_while_full() ->
     {ok, Pid} = new_pool(5, 2),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
-    ?assertEqual(5, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
     [A, B|_Workers] = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     Self = self(),
     spawn(fun() ->
@@ -306,7 +306,7 @@ worker_death_while_full() ->
         1000 -> ?assert(false)
     end,
     kill_worker(B),
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(6, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(6, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
@@ -318,9 +318,9 @@ worker_death_while_full_no_overflow() ->
     {ok, Pid} = new_pool(5, 0),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
-    ?assertEqual(5, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
     [A, B, C|_Workers] = [poolboy:checkout(Pid) || _ <- lists:seq(0, 4)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     Self = self(),
     spawn(fun() ->
@@ -346,10 +346,10 @@ worker_death_while_full_no_overflow() ->
         1000 -> ?assert(false)
     end,
     kill_worker(B),
-    ?assertEqual(1, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(1, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     kill_worker(C),
-    ?assertEqual(2, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(2, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(3, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
@@ -359,7 +359,7 @@ pool_full_nonblocking_no_overflow() ->
     %% option to use non-blocking checkouts is used.
     {ok, Pid} = new_pool(5, 0),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 4)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(full, poolboy:checkout(Pid, false)),
     ?assertEqual(full, poolboy:checkout(Pid, false)),
@@ -374,7 +374,7 @@ pool_full_nonblocking() ->
     %% option to use non-blocking checkouts is used.
     {ok, Pid} = new_pool(5, 5),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 9)],
-    ?assertEqual(0, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(10, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(full, poolboy:checkout(Pid, false)),
     A = hd(Workers),
@@ -395,17 +395,17 @@ owner_death() ->
         receive after 500 -> exit(normal) end
     end),
     timer:sleep(1000),
-    ?assertEqual(5, queue:len(pool_call(Pid, get_avail_workers))),
+    ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
 checkin_after_exception_in_transaction() ->
     {ok, Pool} = new_pool(2, 0),
-    ?assertEqual(2, queue:len(pool_call(Pool, get_avail_workers))),
+    ?assertEqual(2, length(pool_call(Pool, get_avail_workers))),
     Tx = fun(Worker) ->
         ?assert(is_pid(Worker)),
-        ?assertEqual(1, queue:len(pool_call(Pool, get_avail_workers))),
+        ?assertEqual(1, length(pool_call(Pool, get_avail_workers))),
         throw(it_on_the_ground),
         ?assert(false)
     end,
@@ -414,7 +414,7 @@ checkin_after_exception_in_transaction() ->
     catch
         throw:it_on_the_ground -> ok
     end,
-    ?assertEqual(2, queue:len(pool_call(Pool, get_avail_workers))),
+    ?assertEqual(2, length(pool_call(Pool, get_avail_workers))),
     ok = pool_call(Pool, stop).
 
 pool_returns_status() ->

From 2759f130a478aaa0b2cfc90ab8edd6c5ef69f349 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 29 Nov 2018 16:00:47 +0100
Subject: [PATCH 07/36] extend tests to all collection types; fix replace bug
 for array found by tests

---
 src/poolboy_collection.erl |   5 +-
 test/poolboy_tests.erl     | 147 +++++++++++++++++++++----------------
 2 files changed, 85 insertions(+), 67 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 89107ea..952e5aa 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -139,8 +139,9 @@ data_replace(OutIndex, Out, In, Data) when is_function(In, 1) andalso is_list(Da
 data_replace(OutIndex, Out, In, Data) when is_function(In, 1) andalso is_tuple(Data) ->
     case array:is_array(Data) of
         true ->
-            Out = array:get(OutIndex, Data),
-            array:set(OutIndex, In(OutIndex), Data);
+            ArrIndex = OutIndex-1,
+            Out = array:get(ArrIndex, Data),
+            array:set(ArrIndex, In(OutIndex), Data);
         false when element(OutIndex, Data) == Out ->
             setelement(OutIndex, Data, In(OutIndex))
     end.
diff --git a/test/poolboy_tests.erl b/test/poolboy_tests.erl
index 552f6e9..29ddd54 100644
--- a/test/poolboy_tests.erl
+++ b/test/poolboy_tests.erl
@@ -3,76 +3,83 @@
 -include_lib("eunit/include/eunit.hrl").
 
 pool_test_() ->
-    {foreach,
-        fun() ->
+    {foreachx,
+        fun(_) ->
             error_logger:tty(false)
         end,
-        fun(_) ->
+        fun(_, _) ->
             case whereis(poolboy_test) of
                 undefined -> ok;
                 Pid -> pool_call(Pid, stop)
             end,
             error_logger:tty(true)
         end,
+        [ {Type,
+           fun(T, _) ->
+                   {<<(atom_to_binary(T, latin1))/binary, <<": ">>/binary, Title/binary>>, fun() -> Test(T) end}
+           end} || Type <- [list, array, tuple], {Title, Test} <-
         [
             {<<"Basic pool operations">>,
-                fun pool_startup/0
+                fun pool_startup/1
             },
             {<<"Pool overflow should work">>,
-                fun pool_overflow/0
+                fun pool_overflow/1
             },
             {<<"Pool behaves when empty">>,
-                fun pool_empty/0
+                fun pool_empty/1
             },
             {<<"Pool behaves when empty and oveflow is disabled">>,
-                fun pool_empty_no_overflow/0
+                fun pool_empty_no_overflow/1
             },
             {<<"Pool behaves on worker death">>,
-                fun worker_death/0
+                fun worker_death/1
             },
             {<<"Pool behaves when full and a worker dies">>,
-                fun worker_death_while_full/0
+                fun worker_death_while_full/1
             },
             {<<"Pool behaves when full, a worker dies and overflow disabled">>,
-                fun worker_death_while_full_no_overflow/0
+                fun worker_death_while_full_no_overflow/1
             },
             {<<"Non-blocking pool behaves when full and overflow disabled">>,
-                fun pool_full_nonblocking_no_overflow/0
+                fun pool_full_nonblocking_no_overflow/1
             },
             {<<"Non-blocking pool behaves when full">>,
-                fun pool_full_nonblocking/0
+                fun pool_full_nonblocking/1
             },
             {<<"Pool behaves on owner death">>,
-                fun owner_death/0
+                fun owner_death/1
             },
             {<<"Worker checked-in after an exception in a transaction">>,
-                fun checkin_after_exception_in_transaction/0
+                fun checkin_after_exception_in_transaction/1
             },
             {<<"Pool returns status">>,
-                fun pool_returns_status/0
+                fun pool_returns_status/1
             },
             {<<"Pool demonitors previously waiting processes">>,
-                fun demonitors_previously_waiting_processes/0
+                fun demonitors_previously_waiting_processes/1
             },
             {<<"Pool demonitors when a checkout is cancelled">>,
-                fun demonitors_when_checkout_cancelled/0
+                fun demonitors_when_checkout_cancelled/1
             },
             {<<"Check that LIFO is the default strategy">>,
-                fun default_strategy_lifo/0
+                fun default_strategy_lifo/1
             },
             {<<"Check LIFO strategy">>,
-                fun lifo_strategy/0
+                fun lifo_strategy/1
             },
             {<<"Check FIFO strategy">>,
-                fun fifo_strategy/0
+                fun fifo_strategy/1
             },
             {<<"Pool reuses waiting monitor when a worker exits">>,
-                fun reuses_waiting_monitor_on_worker_exit/0
+                fun reuses_waiting_monitor_on_worker_exit/1
             },
             {<<"Recover from timeout without exit handling">>,
-                fun transaction_timeout_without_exit/0},
+                fun transaction_timeout_without_exit/1
+            },
             {<<"Recover from transaction timeout">>,
-                fun transaction_timeout/0}
+                fun transaction_timeout/1
+            }
+        ]
         ]
     }.
 
@@ -93,8 +100,8 @@ checkin_worker(Pid, Worker) ->
     timer:sleep(500).
 
 
-transaction_timeout_without_exit() ->
-    {ok, Pid} = new_pool(1, 0),
+transaction_timeout_without_exit(Type) ->
+    {ok, Pid} = new_pool(1, 0, lifo, Type),
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)),
     WorkerList = pool_call(Pid, get_all_workers),
     ?assertMatch([_], WorkerList),
@@ -108,8 +115,8 @@ transaction_timeout_without_exit() ->
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)).
 
 
-transaction_timeout() ->
-    {ok, Pid} = new_pool(1, 0),
+transaction_timeout(Type) ->
+    {ok, Pid} = new_pool(1, 0, lifo, Type),
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)),
     WorkerList = pool_call(Pid, get_all_workers),
     ?assertMatch([_], WorkerList),
@@ -124,9 +131,9 @@ transaction_timeout() ->
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)).
 
 
-pool_startup() ->
+pool_startup(Type) ->
     %% Check basic pool operation.
-    {ok, Pid} = new_pool(10, 5),
+    {ok, Pid} = new_pool(10, 5, lifo, Type),
     ?assertEqual(10, length(pool_call(Pid, get_avail_workers))),
     poolboy:checkout(Pid),
     ?assertEqual(9, length(pool_call(Pid, get_avail_workers))),
@@ -137,9 +144,9 @@ pool_startup() ->
     ?assertEqual(1, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_overflow() ->
+pool_overflow(Type) ->
     %% Check that the pool overflows properly.
-    {ok, Pid} = new_pool(5, 5),
+    {ok, Pid} = new_pool(5, 5, lifo, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
@@ -162,10 +169,10 @@ pool_overflow() ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_empty() ->
+pool_empty(Type) ->
     %% Checks that the the pool handles the empty condition correctly when
     %% overflow is enabled.
-    {ok, Pid} = new_pool(5, 2),
+    {ok, Pid} = new_pool(5, 2, lifo, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
@@ -208,10 +215,10 @@ pool_empty() ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_empty_no_overflow() ->
+pool_empty_no_overflow(Type) ->
     %% Checks the pool handles the empty condition properly when overflow is
     %% disabled.
-    {ok, Pid} = new_pool(5, 0),
+    {ok, Pid} = new_pool(5, 0, lifo, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 4)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
@@ -250,10 +257,10 @@ pool_empty_no_overflow() ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-worker_death() ->
+worker_death(Type) ->
     %% Check that dead workers are only restarted when the pool is not full
     %% and the overflow count is 0. Meaning, don't restart overflow workers.
-    {ok, Pid} = new_pool(5, 2),
+    {ok, Pid} = new_pool(5, 2, lifo, Type),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
@@ -270,11 +277,11 @@ worker_death() ->
     ?assertEqual(4, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-worker_death_while_full() ->
+worker_death_while_full(Type) ->
     %% Check that if a worker dies while the pool is full and there is a
     %% queued checkout, a new worker is started and the checkout serviced.
     %% If there are no queued checkouts, a new worker is not started.
-    {ok, Pid} = new_pool(5, 2),
+    {ok, Pid} = new_pool(5, 2, lifo, Type),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
@@ -311,11 +318,11 @@ worker_death_while_full() ->
     ?assertEqual(6, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-worker_death_while_full_no_overflow() ->
+worker_death_while_full_no_overflow(Type) ->
     %% Check that if a worker dies while the pool is full and there's no
     %% overflow, a new worker is started unconditionally and any queued
     %% checkouts are serviced.
-    {ok, Pid} = new_pool(5, 0),
+    {ok, Pid} = new_pool(5, 0, lifo, Type),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
@@ -354,10 +361,10 @@ worker_death_while_full_no_overflow() ->
     ?assertEqual(3, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_full_nonblocking_no_overflow() ->
+pool_full_nonblocking_no_overflow(Type) ->
     %% Check that when the pool is full, checkouts return 'full' when the
     %% option to use non-blocking checkouts is used.
-    {ok, Pid} = new_pool(5, 0),
+    {ok, Pid} = new_pool(5, 0, lifo, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 4)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
@@ -369,10 +376,10 @@ pool_full_nonblocking_no_overflow() ->
     ?assertEqual(5, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_full_nonblocking() ->
+pool_full_nonblocking(Type) ->
     %% Check that when the pool is full, checkouts return 'full' when the
     %% option to use non-blocking checkouts is used.
-    {ok, Pid} = new_pool(5, 5),
+    {ok, Pid} = new_pool(5, 5, lifo, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 9)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(10, length(pool_call(Pid, get_all_workers))),
@@ -386,10 +393,10 @@ pool_full_nonblocking() ->
     ?assertEqual(10, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-owner_death() ->
+owner_death(Type) ->
     %% Check that a dead owner (a process that dies with a worker checked out)
     %% causes the pool to dismiss the worker and prune the state space.
-    {ok, Pid} = new_pool(5, 5),
+    {ok, Pid} = new_pool(5, 5, lifo, Type),
     spawn(fun() ->
         poolboy:checkout(Pid),
         receive after 500 -> exit(normal) end
@@ -400,8 +407,8 @@ owner_death() ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-checkin_after_exception_in_transaction() ->
-    {ok, Pool} = new_pool(2, 0),
+checkin_after_exception_in_transaction(Type) ->
+    {ok, Pool} = new_pool(2, 0, lifo, Type),
     ?assertEqual(2, length(pool_call(Pool, get_avail_workers))),
     Tx = fun(Worker) ->
         ?assert(is_pid(Worker)),
@@ -417,8 +424,8 @@ checkin_after_exception_in_transaction() ->
     ?assertEqual(2, length(pool_call(Pool, get_avail_workers))),
     ok = pool_call(Pool, stop).
 
-pool_returns_status() ->
-    {ok, Pool} = new_pool(2, 0),
+pool_returns_status(Type) ->
+    {ok, Pool} = new_pool(2, 0, lifo, Type),
     ?assertEqual({ready, 2, 0, 0}, poolboy:status(Pool)),
     poolboy:checkout(Pool),
     ?assertEqual({ready, 1, 0, 1}, poolboy:status(Pool)),
@@ -446,8 +453,8 @@ pool_returns_status() ->
     ?assertEqual({full, 0, 0, 0}, poolboy:status(Pool4)),
     ok = pool_call(Pool4, stop).
 
-demonitors_previously_waiting_processes() ->
-    {ok, Pool} = new_pool(1,0),
+demonitors_previously_waiting_processes(Type) ->
+    {ok, Pool} = new_pool(1,0, lifo, Type),
     Self = self(),
     Pid = spawn(fun() ->
         W = poolboy:checkout(Pool),
@@ -465,8 +472,8 @@ demonitors_previously_waiting_processes() ->
     Pid ! ok,
     ok = pool_call(Pool, stop).
 
-demonitors_when_checkout_cancelled() ->
-    {ok, Pool} = new_pool(1,0),
+demonitors_when_checkout_cancelled(Type) ->
+    {ok, Pool} = new_pool(1,0, lifo, Type),
     Self = self(),
     Pid = spawn(fun() ->
         poolboy:checkout(Pool),
@@ -481,23 +488,23 @@ demonitors_when_checkout_cancelled() ->
     Pid ! ok,
     ok = pool_call(Pool, stop).
 
-default_strategy_lifo() ->
+default_strategy_lifo(Type) ->
     %% Default strategy is LIFO
-    {ok, Pid} = new_pool(2, 0),
+    {ok, Pid} = new_pool(2, 0, default, Type),
     Worker1 = poolboy:checkout(Pid),
     ok = poolboy:checkin(Pid, Worker1),
     Worker1 = poolboy:checkout(Pid),
     poolboy:stop(Pid).
 
-lifo_strategy() ->
-    {ok, Pid} = new_pool(2, 0, lifo),
+lifo_strategy(Type) ->
+    {ok, Pid} = new_pool(2, 0, lifo, Type),
     Worker1 = poolboy:checkout(Pid),
     ok = poolboy:checkin(Pid, Worker1),
     Worker1 = poolboy:checkout(Pid),
     poolboy:stop(Pid).
 
-fifo_strategy() ->
-    {ok, Pid} = new_pool(2, 0, fifo),
+fifo_strategy(Type) ->
+    {ok, Pid} = new_pool(2, 0, fifo, Type),
     Worker1 = poolboy:checkout(Pid),
     ok = poolboy:checkin(Pid, Worker1),
     Worker2 = poolboy:checkout(Pid),
@@ -505,8 +512,8 @@ fifo_strategy() ->
     Worker1 = poolboy:checkout(Pid),
     poolboy:stop(Pid).
 
-reuses_waiting_monitor_on_worker_exit() ->
-    {ok, Pool} = new_pool(1,0),
+reuses_waiting_monitor_on_worker_exit(Type) ->
+    {ok, Pool} = new_pool(1,0, lifo, Type),
 
     Self = self(),
     Pid = spawn(fun() ->
@@ -540,11 +547,21 @@ new_pool(Size, MaxOverflow) ->
                         {worker_module, poolboy_test_worker},
                         {size, Size}, {max_overflow, MaxOverflow}]).
 
-new_pool(Size, MaxOverflow, Strategy) ->
+new_pool(Size, MaxOverflow, default, Type) ->
+    poolboy:start_link([{name, {local, poolboy_test}},
+                        {worker_module, poolboy_test_worker},
+                        {size, Size}, {max_overflow, MaxOverflow}, {type, Type}]);
+
+new_pool(Size, MaxOverflow, Strategy, Type) ->
     poolboy:start_link([{name, {local, poolboy_test}},
                         {worker_module, poolboy_test_worker},
-                        {size, Size}, {max_overflow, MaxOverflow},
+                        {size, Size}, {max_overflow, MaxOverflow}, {type, Type},
                         {strategy, Strategy}]).
 
+pool_call(ServerRef, stop) when is_pid(ServerRef) ->
+    case is_process_alive(ServerRef) of
+        true -> gen_server:stop(ServerRef);
+        _ -> ok
+    end;
 pool_call(ServerRef, Request) ->
     gen_server:call(ServerRef, Request).

From 14b54ed551cbad05510d051f1a8a2ca71aa54980 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 30 Nov 2018 10:39:05 +0100
Subject: [PATCH 08/36] fix rand:uniform usage

---
 src/poolboy_collection.erl | 12 ++++++------
 test/poolboy_tests.erl     |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 952e5aa..50fdff2 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -50,7 +50,7 @@
             is => fun queue:is_queue/1,
             len => fun queue:len/1,
             from => fun queue:from_list/1,
-            nth => fun(I, {RL, FL})
+            nth => fun(I, {_RL, FL})
                          when I =< length(FL) ->
                            lists:nth(I, FL);
                       (I, {RL, FL}) ->
@@ -66,11 +66,11 @@
             len => fun tuple_size/1,
             from => fun list_to_tuple/1,
             nth => fun element/2,
-            prep => fun(I, T) ->  erlang:insert_element(1, T, I) end,
-            app => fun(I, T) -> erlang:append_element(T, I) end
+            prep => fun(I, Tu) ->  erlang:insert_element(1, Tu, I) end,
+            app => fun(I, Tu) -> erlang:append_element(Tu, I) end
            }
          }).
--define(Colls(T, F), maps:get(F, maps:get(T, ?Types))).
+-define(Colls(Type, Fun), maps:get(Fun, maps:get(Type, ?Types))).
 
 
 -record(coll, {indexes :: [non_neg_integer()],
@@ -203,8 +203,8 @@ all(visible, #coll{indexes = Indexes, data = Data}) ->
 rand(known, #coll{data = Data}) ->
     case len(Data) of
         0 -> empty;
-        L -> nth(rand:uniform(1, L), Data)
+        L -> nth(rand:uniform(L), Data)
     end;
 rand(visible, #coll{indexes = []}) -> empty;
 rand(visible, #coll{indexes = Indexes, data = Data}) ->
-    nth(lists:nth(rand:uniform(1, length(Indexes)), Indexes), Data).
+    nth(lists:nth(rand:uniform(length(Indexes)), Indexes), Data).
diff --git a/test/poolboy_tests.erl b/test/poolboy_tests.erl
index 29ddd54..d100d86 100644
--- a/test/poolboy_tests.erl
+++ b/test/poolboy_tests.erl
@@ -17,7 +17,7 @@ pool_test_() ->
         [ {Type,
            fun(T, _) ->
                    {<<(atom_to_binary(T, latin1))/binary, <<": ">>/binary, Title/binary>>, fun() -> Test(T) end}
-           end} || Type <- [list, array, tuple], {Title, Test} <-
+           end} || Type <- [list, array, tuple, queue], {Title, Test} <-
         [
             {<<"Basic pool operations">>,
                 fun pool_startup/1

From 59de721c85bff392eb412f4da8c668c14f671741 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Mon, 12 Aug 2019 00:41:16 +0200
Subject: [PATCH 09/36] fix replace for queue; add filter intead of foreach

---
 src/poolboy.erl            |  5 +-
 src/poolboy_collection.erl | 94 ++++++++++++++++++++++++--------------
 2 files changed, 63 insertions(+), 36 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 47735ef..c10ee66 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -180,7 +180,8 @@ init([{size, Size} | Rest], WorkerArgs, State) when is_integer(Size) ->
     init(Rest, WorkerArgs, State#state{size = Size});
 init([{type, Type} | Rest], WorkerArgs, State) when Type == list orelse
                                                     Type == array orelse
-                                                    Type == tuple ->
+                                                    Type == tuple orelse
+                                                    Type == queue ->
     init(Rest, WorkerArgs, State#state{type = Type});
 init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State) when is_integer(MaxOverflow) ->
     init(Rest, WorkerArgs, State#state{max_overflow = MaxOverflow});
@@ -310,7 +311,7 @@ handle_info(_Info, State) ->
     {noreply, State}.
 
 terminate(Reason, State = #state{supervisor = Sup}) ->
-    ok = poolboy_collection:foreach(fun (W) -> catch unlink(W) end, State#state.workers),
+    poolboy_collection:filter(fun (W) -> catch not unlink(W) end, State#state.workers),
     stop_supervisor(Reason, Sup),
     ok.
 
diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 50fdff2..be8a111 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -6,7 +6,7 @@
          replace/2, replace/3,
          prepend/2,
          append/2,
-         foreach/2,
+         filter/2,
          all/2,
          rand/2
         ]).
@@ -23,14 +23,19 @@
 -type coll_data() :: list()|{}|array:array()|pid_queue().
 -type coll_data(A) :: list(A)|{A}|array:array(A)|pid_queue(A).
 
--define(Types, 
+-define(Types,
         #{list => #{
             is => fun is_list/1,
             len => fun length/1,
             from => fun(L) -> L end,
             nth => fun lists:nth/2,
             prep => fun(I, L) -> [I|L] end,
-            app => fun(I, L) -> L ++ [I] end
+            app => fun(I, L) -> L ++ [I] end,
+            filter => fun lists:filter/2,
+            replace => fun(O, X, I, L) ->
+                               {L1, [O | Tl]} = lists:split(X-1, L),
+                               L1 ++ [I| Tl]
+                       end
            },
           array => #{
             is => fun array:is_array/1,
@@ -44,7 +49,21 @@
                                    array:set(0, I, array:new()),
                                    A)
                     end,
-            app => fun(I, A) -> array:set(array:size(A), I, A) end
+            app => fun(I, A) -> array:set(array:size(A), I, A) end,
+            filter => fun(Fun, A) ->
+                              array:sparse_map(
+                                fun(_, V) ->
+                                        case Fun(V) of
+                                            true -> V;
+                                            false -> array:default(A);
+                                            Else -> Else
+                                        end
+                                end, A)
+                      end,
+            replace => fun(O, X, I, A) ->
+                               O = array:get(X-1, A),
+                               array:set(X-1, I, A)
+                       end
            },
           queue => #{
             is => fun queue:is_queue/1,
@@ -59,7 +78,13 @@
                              RL)
                    end,
             prep => fun queue:in_r/2,
-            app => fun queue:in/2
+            app => fun queue:in/2,
+            filter => fun queue:filter/2,
+            replace => fun(O, X, I, Q) ->
+                               {Q1, Q2} = queue:split(X-1, Q),
+                               O = queue:get(Q2),
+                               queue:join(queue:in(I, Q1), queue:drop(Q2))
+                       end
            },
           tuple => #{
             is => fun is_tuple/1,
@@ -67,7 +92,12 @@
             from => fun list_to_tuple/1,
             nth => fun element/2,
             prep => fun(I, Tu) ->  erlang:insert_element(1, Tu, I) end,
-            app => fun(I, Tu) -> erlang:append_element(Tu, I) end
+            app => fun(I, Tu) -> erlang:append_element(Tu, I) end,
+            filter => fun(Fun, Tu) -> tuple_filter(Fun, Tu) end,
+            replace => fun(O, X, I, Tu) ->
+                               O = element(X, Tu),
+                               setelement(X, Tu, I)
+                       end
            }
          }).
 -define(Colls(Type, Fun), maps:get(Fun, maps:get(Type, ?Types))).
@@ -125,26 +155,17 @@ replace(Out, In, Coll = #coll{data = Data}) ->
     case maps:take(Out, Coll#coll.rev_indexes) of
         error -> error(enoent);
         {OutIndex, RevIndexes} ->
-            NewData = data_replace(OutIndex, Out, In, Data),
+            NewData = replace(OutIndex, Out, In, Data),
             NewItem = nth(OutIndex, NewData),
             NewRevIndexes = maps:put(NewItem, OutIndex, RevIndexes),
             {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
     end.
 
-data_replace(OutIndex, Out, In, Data) when not is_function(In) ->
-    data_replace(OutIndex, Out, fun(_) -> In end, Data);
-data_replace(OutIndex, Out, In, Data) when is_function(In, 1) andalso is_list(Data) ->
-    {FirstN, [Out | Tail]} = lists:split(OutIndex-1, Data),
-    FirstN ++ [In(OutIndex)| Tail];
-data_replace(OutIndex, Out, In, Data) when is_function(In, 1) andalso is_tuple(Data) ->
-    case array:is_array(Data) of
-        true ->
-            ArrIndex = OutIndex-1,
-            Out = array:get(ArrIndex, Data),
-            array:set(ArrIndex, In(OutIndex), Data);
-        false when element(OutIndex, Data) == Out ->
-            setelement(OutIndex, Data, In(OutIndex))
-    end.
+
+replace(OutIndex, Out, In, Data) when not is_function(In) ->
+    replace(OutIndex, Out, fun(_) -> In end, Data);
+replace(OutIndex, Out, In, Data)  when is_function(In, 1)  ->
+    (call(?FUNCTION_NAME, Data))(Out, OutIndex, In(OutIndex), Data).
 
 
 prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
@@ -160,13 +181,14 @@ prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Dat
 prep(In, Data) ->
     (call(?FUNCTION_NAME, Data))(In, Data).
 
+
 append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
     case maps:get(In, RevIndexes, undefined) of
         InIndex when is_integer(InIndex) -> Coll#coll{indexes=Indexes++[InIndex]};
         undefined ->
             NewData = app(In, Data),
             NewIndex =
-            case {array:is_array(Data), len(Data)} of
+            case {(?Colls(array, is))(Data), len(Data)} of
                 {true, Len} -> Len - 1;
                 {_, Len} -> Len
             end,
@@ -178,20 +200,24 @@ append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data
 app(In, Data) ->
     (call(?FUNCTION_NAME, Data))(In, Data).
 
-foreach(Fun, #coll{data = Data}) -> data_foreach(Fun, Data).
 
-data_foreach(Fun, Data) when is_function(Fun) andalso is_list(Data) ->
-    lists:foreach(Fun, Data);
-data_foreach(Fun, Data) when is_function(Fun) andalso is_tuple(Data) ->
-    case array:is_array(Data) of
-        true -> array:sparse_map(fun(_, Value) -> Fun(Value) end, Data), ok;
-        false -> tuple_foreach(Fun, Data, erlang:tuple_size(Data))
-    end.
+filter(Fun, #coll{data = Data}) ->
+    (call(?FUNCTION_NAME, Data))(Fun, Data).
+
+
+tuple_filter(Fun, Tuple) ->
+    tuple_filter(Fun, Tuple, erlang:tuple_size(Tuple)).
 
-tuple_foreach(_Fun, _Tuple, 0) -> ok;
-tuple_foreach(Fun, Tuple, Index) ->
-    Fun(element(Index, Tuple)),
-    tuple_foreach(Fun, Tuple, Index-1).
+tuple_filter(_Fun, Tuple, 0) -> Tuple;
+tuple_filter(Fun, Tuple, Index) ->
+    Element = element(Index, Tuple),
+    NewTuple = case Fun(Element) of
+        true -> Tuple;
+        Else when Else == Element -> Tuple;
+        false -> setelement(Index, Tuple, undefined);
+        Else -> setelement(Index, Tuple, Else)
+    end,
+    tuple_filter(Fun, NewTuple, Index-1).
 
 
 all(known, #coll{rev_indexes = RevIndexes}) ->

From ab16e3319c3735c83e8b2efe1f5f2cdb48b3d2f1 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 13 Sep 2019 00:03:40 +0200
Subject: [PATCH 10/36] fix dialyzer warnings; factor out supervisor and worker
 creation from init/3 so that state record can be initialized without the
 'undefined' default

---
 src/poolboy.erl            | 113 ++++++++++++++++++++++++-------------
 src/poolboy_collection.erl |   1 -
 2 files changed, 73 insertions(+), 41 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index c10ee66..ab51569 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -11,6 +11,14 @@
 -export_type([pool/0]).
 
 -define(TIMEOUT, 5000).
+-define(DEFAULT_SIZE, 5).
+-define(DEFAULT_TYPE, list).
+-define(IS_COLLECTION_TYPE(Type), (Type == list orelse
+                                   Type == array orelse
+                                   Type == tuple orelse
+                                   Type == queue)).
+
+
 
 -ifdef(OTP_RELEASE). %% this implies 21 or higher
 -define(EXCEPTION(Class, Reason, Stacktrace), Class:Reason:Stacktrace).
@@ -39,12 +47,11 @@
     pid().
 
 -record(state, {
-    supervisor :: undefined | sup_ref(),
+    supervisor :: sup_ref(),
     workers :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
     waiting :: poolboy_collection:pid_queue() | poolboy_collection:pid_queue(tuple()),
     monitors :: ets:tid(),
-    size = 5 :: non_neg_integer(),
-    type = list :: list | array | tuple | queue,
+    size = ?DEFAULT_SIZE :: non_neg_integer(),
     overflow = 0 :: non_neg_integer(),
     max_overflow = 10 :: non_neg_integer(),
     strategy = lifo :: lifo | fifo
@@ -152,47 +159,76 @@ init({PoolArgs, WorkerArgs}) ->
     process_flag(trap_exit, true),
     Waiting = queue:new(),
     Monitors = ets:new(monitors, [private]),
-    init(PoolArgs, WorkerArgs, #state{waiting = Waiting, monitors = Monitors}).
-
-init([{worker_supervisor, Sup = {Scope, _Name}} | Rest], WorkerArgs, State)
-  when Scope =:= local orelse Scope =:= global ->
-    init(Rest, WorkerArgs, State#state{supervisor=Sup});
-init([{worker_supervisor, Sup = {_Name, Node}} | Rest], WorkerArgs, State) ->
-    (catch erlang:monitor_node(Node, true)),
-    init(Rest, WorkerArgs, State#state{supervisor=Sup});
-init([{worker_supervisor, Sup} | Rest], WorkerArgs, State)
-  when is_pid(Sup) orelse is_atom(Sup) orelse is_tuple(Sup) ->
-    init(Rest, WorkerArgs, State#state{supervisor=Sup});
-init([{worker_module, Mod} | Rest], WorkerArgs, State) when is_atom(Mod) ->
-    {ok, Sup} =
-    case poolboy_sup:start_link(Mod, WorkerArgs) of
-        {ok, _Pid} = Ok -> Ok;
-        {error, {already_started, Pid}} ->
+    Supervisor = ensure_worker_supervisor(PoolArgs, WorkerArgs),
+    Workers = prepopulate(PoolArgs, Supervisor),
+    init(PoolArgs, WorkerArgs,
+         #state{supervisor = Supervisor,
+                workers = Workers,
+                waiting = Waiting,
+                monitors = Monitors}).
+
+ensure_worker_supervisor(PoolArgs, WorkerArgs) ->
+    case proplists:get_value(worker_supervisor, PoolArgs) of
+        undefined ->
+            start_supervisor(
+              proplists:get_value(worker_module, PoolArgs),
+              WorkerArgs);
+        Sup = {Name, Node} when Name =/= local orelse
+                                Name =/= global ->
+            (catch erlang:monitor_node(Node, true)),
+            Sup;
+        Sup when is_pid(Sup) orelse
+                 is_atom(Sup) orelse
+                 is_tuple(Sup) ->
+            Sup
+    end.
+
+start_supervisor(WorkerModule, WorkerArgs) ->
+    start_supervisor(WorkerModule, WorkerArgs, 1).
+
+start_supervisor(undefined, _WorkerArgs, _Retries) ->
+    exit({no_worker_supervisor, {worker_module, undefined}});
+start_supervisor(WorkerModule, WorkerArgs, Retries) ->
+    case poolboy_sup:start_link(WorkerModule, WorkerArgs) of
+        {ok, NewPid} ->
+            NewPid;
+        {error, {already_started, Pid}} when Retries > 0 ->
             MRef = erlang:monitor(process, Pid),
-            receive
-                {'DOWN', MRef, _, _, _} -> ok
+            receive {'DOWN', MRef, _, _, _} -> ok
             after ?TIMEOUT -> ok
             end,
-            poolboy_sup:start_link(Mod, WorkerArgs)
-    end,
-    init(Rest, WorkerArgs, State#state{supervisor=Sup});
+            start_supervisor(WorkerModule, WorkerArgs, Retries - 1);
+        {error, Error} ->
+            exit({no_worker_supervisor, Error})
+    end.
+
+prepopulate(PoolArgs, Supervisor) ->
+    prepopulate(
+      proplists:get_value(size, PoolArgs, ?DEFAULT_SIZE),
+      proplists:get_value(type, PoolArgs, ?DEFAULT_TYPE),
+      Supervisor).
+
+prepopulate(Size, Type, Sup)
+  when is_integer(Size) andalso ?IS_COLLECTION_TYPE(Type) ->
+    poolboy_collection:new(Type, Size, fun(_) -> new_worker(Sup) end);
+prepopulate(Size, Type, Sup) when not is_integer(Size) ->
+    prepopulate(?DEFAULT_SIZE, Type, Sup);
+prepopulate(Size, Type, Sup) when not ?IS_COLLECTION_TYPE(Type) ->
+    prepopulate(Size, ?DEFAULT_TYPE, Sup).
+
 init([{size, Size} | Rest], WorkerArgs, State) when is_integer(Size) ->
     init(Rest, WorkerArgs, State#state{size = Size});
-init([{type, Type} | Rest], WorkerArgs, State) when Type == list orelse
-                                                    Type == array orelse
-                                                    Type == tuple orelse
-                                                    Type == queue ->
-    init(Rest, WorkerArgs, State#state{type = Type});
-init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State) when is_integer(MaxOverflow) ->
+init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State)
+  when is_integer(MaxOverflow) ->
     init(Rest, WorkerArgs, State#state{max_overflow = MaxOverflow});
-init([{strategy, lifo} | Rest], WorkerArgs, State) ->
-    init(Rest, WorkerArgs, State#state{strategy = lifo});
-init([{strategy, fifo} | Rest], WorkerArgs, State) ->
-    init(Rest, WorkerArgs, State#state{strategy = fifo});
+init([{strategy, Strategy} | Rest], WorkerArgs, State)
+  when Strategy == lifo orelse
+       Strategy == fifo ->
+    init(Rest, WorkerArgs, State#state{strategy = Strategy});
 init([_ | Rest], WorkerArgs, State) ->
     init(Rest, WorkerArgs, State);
-init([], _WorkerArgs, #state{size = Size, type=Type, supervisor = Sup} = State) ->
-    {ok, State#state{workers = prepopulate(Size, Type, Sup)}}.
+init([], _WorkerArgs, State) ->
+    {ok, State}.
 
 handle_cast({checkin, Pid}, State = #state{monitors = Monitors}) ->
     case ets:lookup(Monitors, Pid) of
@@ -296,7 +332,7 @@ handle_info({'EXIT', Pid, Reason}, State) ->
         [] ->
             try
                 {noreply, replace_worker(Pid, State)}
-            catch ?EXCEPTION(error, enoent, StackTrace) ->
+            catch ?EXCEPTION(error, enoent, _StackTrace) ->
                 {noreply, State}
             end
     end,
@@ -347,9 +383,6 @@ replace_worker(Pid, State = #state{strategy = Strategy}) ->
         fifo -> State#state{workers = poolboy_collection:append(NewWorker, Workers)}
     end.
 
-prepopulate(Size, Type, Sup) ->
-    poolboy_collection:new(Type, Size, fun(_) -> new_worker(Sup) end).
-
 handle_checkin(Pid, State = #state{strategy = Strategy}) ->
     #state{supervisor = Sup,
            waiting = Waiting,
diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index be8a111..cfe11f1 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -133,7 +133,6 @@ len(Data) ->
 call(F, Data) when is_atom(F) ->
     call(F, Data, false, [none| maps:keys(?Types)]).
 
-call(_, _Data, _, []) -> throw(badarg);
 call(F, _Data, true, [T |_Types]) -> ?Colls(T, F);
 call(F, Data, false, [_ | [T |_] = Types]) ->
     call(F, Data, ?Colls(T, is), Types);

From 8c2c78f970b7a7d2b57da093753af3bdbb2298cb Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 13 Sep 2019 02:07:51 +0200
Subject: [PATCH 11/36] collection: define the type-funs as a record; save this
 in the coll instance, to avoid the need for type-checking, split-out
 multi-line funs from macro

---
 src/poolboy.erl            |   4 +-
 src/poolboy_collection.erl | 317 ++++++++++++++++++++-----------------
 2 files changed, 176 insertions(+), 145 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index ab51569..111b410 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -290,7 +290,7 @@ handle_call(status, _From, State) ->
            monitors = Monitors,
            overflow = Overflow} = State,
     StateName = state_name(State),
-    {reply, {StateName, poolboy_collection:len(visible, Workers), Overflow, ets:info(Monitors, size)}, State};
+    {reply, {StateName, poolboy_collection:length(visible, Workers), Overflow, ets:info(Monitors, size)}, State};
 handle_call(get_avail_workers, _From, State) ->
     {reply, poolboy_collection:all(visible, State#state.workers), State};
 handle_call(get_any_worker, _From, State) ->
@@ -421,7 +421,7 @@ handle_worker_exit(Pid, State) ->
 
 state_name(State = #state{overflow = Overflow}) when Overflow < 1 ->
     #state{max_overflow = MaxOverflow, workers = Workers} = State,
-    case poolboy_collection:len(visible, Workers) == 0 of
+    case poolboy_collection:length(visible, Workers) == 0 of
         true when MaxOverflow < 1 -> full;
         true -> overflow;
         false -> ready
diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index cfe11f1..c39b413 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -1,7 +1,7 @@
 -module(poolboy_collection).
 
 -export([new/3,
-         len/2,
+         length/2,
          hide_head/1,
          replace/2, replace/3,
          prepend/2,
@@ -23,171 +23,150 @@
 -type coll_data() :: list()|{}|array:array()|pid_queue().
 -type coll_data(A) :: list(A)|{A}|array:array(A)|pid_queue(A).
 
--define(Types,
-        #{list => #{
-            is => fun is_list/1,
-            len => fun length/1,
-            from => fun(L) -> L end,
-            nth => fun lists:nth/2,
-            prep => fun(I, L) -> [I|L] end,
-            app => fun(I, L) -> L ++ [I] end,
-            filter => fun lists:filter/2,
-            replace => fun(O, X, I, L) ->
-                               {L1, [O | Tl]} = lists:split(X-1, L),
-                               L1 ++ [I| Tl]
-                       end
-           },
-          array => #{
-            is => fun array:is_array/1,
-            len => fun array:size/1,
-            from => fun array:from_list/1,
-            nth => fun(I, A) -> array:get(I-1, A) end,
-            prep => fun(I, A) -> array:foldl(
-                                   fun(Idx, Val, Arr) ->
-                                           array:set(Idx+1, Val, Arr)
-                                   end,
-                                   array:set(0, I, array:new()),
-                                   A)
-                    end,
-            app => fun(I, A) -> array:set(array:size(A), I, A) end,
-            filter => fun(Fun, A) ->
-                              array:sparse_map(
-                                fun(_, V) ->
-                                        case Fun(V) of
-                                            true -> V;
-                                            false -> array:default(A);
-                                            Else -> Else
-                                        end
-                                end, A)
-                      end,
-            replace => fun(O, X, I, A) ->
-                               O = array:get(X-1, A),
-                               array:set(X-1, I, A)
-                       end
-           },
-          queue => #{
-            is => fun queue:is_queue/1,
-            len => fun queue:len/1,
-            from => fun queue:from_list/1,
-            nth => fun(I, {_RL, FL})
-                         when I =< length(FL) ->
-                           lists:nth(I, FL);
-                      (I, {RL, FL}) ->
-                           lists:nth(
-                             length(RL)-(I-length(FL)-1),
-                             RL)
-                   end,
-            prep => fun queue:in_r/2,
-            app => fun queue:in/2,
-            filter => fun queue:filter/2,
-            replace => fun(O, X, I, Q) ->
-                               {Q1, Q2} = queue:split(X-1, Q),
-                               O = queue:get(Q2),
-                               queue:join(queue:in(I, Q1), queue:drop(Q2))
-                       end
-           },
-          tuple => #{
-            is => fun is_tuple/1,
-            len => fun tuple_size/1,
-            from => fun list_to_tuple/1,
-            nth => fun element/2,
-            prep => fun(I, Tu) ->  erlang:insert_element(1, Tu, I) end,
-            app => fun(I, Tu) -> erlang:append_element(Tu, I) end,
-            filter => fun(Fun, Tu) -> tuple_filter(Fun, Tu) end,
-            replace => fun(O, X, I, Tu) ->
-                               O = element(X, Tu),
-                               setelement(X, Tu, I)
-                       end
-           }
+-record(type, {
+          from :: fun((list(A)) -> coll_data(A)),
+          is :: fun((coll_data() | coll_data(any())) -> boolean()),
+          len :: fun((coll_data() | coll_data(any())) -> non_neg_integer()),
+          nth :: fun((non_neg_integer(), coll_data(A)) -> A),
+          prep :: fun((A, coll_data(A)) -> coll_data(A)),
+          app :: fun((A, coll_data(A)) -> coll_data(A)),
+          filter :: fun((fun((A) -> boolean()), coll_data(A)) -> coll_data(A)),
+          replace :: fun((A, non_neg_integer(), A, coll_data(A)) -> coll_data(A))
          }).
--define(Colls(Type, Fun), maps:get(Fun, maps:get(Type, ?Types))).
+-type type() :: #type{}.
+
+-record(coll, {
+          item_generator :: fun((non_neg_integer()) -> any()),
+          data :: coll_data() | coll_data(any()),
+          type :: type(),
+          indexes :: [non_neg_integer()],
+          rev_indexes :: #{any()=>non_neg_integer()}
+          }).
+
+-type coll() :: #coll{
+                   data :: coll_data(),
+                   rev_indexes :: #{}
+                  }.
+-type coll(A) :: #coll{
+                    item_generator :: fun((non_neg_integer()) -> A),
+                    data :: coll_data(A),
+                    rev_indexes :: #{A=>non_neg_integer()}
+                    }.
 
+-export_type([coll/0, coll/1]).
 
--record(coll, {indexes :: [non_neg_integer()],
-               rev_indexes :: #{any()=>non_neg_integer()},
-               data :: coll_data() | coll_data(any()),
-               item_generator :: fun((non_neg_integer()) -> any()) }).
 
--type coll() :: #coll{rev_indexes :: #{}, data :: coll_data()}.
--type coll(A) :: #coll{rev_indexes :: #{A=>non_neg_integer()},
-                       data :: coll_data(A),
-                       item_generator :: fun((non_neg_integer()) -> A) }.
+-define(TYPES, #{
+          list => #type{
+                     from = fun(L) -> L end,
+                     is = fun is_list/1,
+                     len = fun length/1,
+                     nth = fun lists:nth/2,
+                     prep = fun(I, L) -> [I|L] end,
+                     app = fun(I, L) -> L ++ [I] end,
+                     filter = fun lists:filter/2,
+                     replace = fun list_replace/4
+                    },
+          array => #type{
+                      from = fun array:from_list/1,
+                      is = fun array:is_array/1,
+                      len = fun array:size/1,
+                      nth = fun(I, A) -> array:get(I-1, A) end,
+                      prep = fun array_prep/2,
+                      app = fun(I, A) -> array:set(array:size(A), I, A) end,
+                      filter = fun array_filter/2,
+                      replace = fun array_replace/4
+                     },
+          queue => #type{
+                      from = fun queue:from_list/1,
+                      is = fun queue:is_queue/1,
+                      len = fun queue:len/1,
+                      nth = fun queue_nth/2,
+                      prep = fun queue:in_r/2,
+                      app = fun queue:in/2,
+                      filter = fun queue:filter/2,
+                      replace = fun queue_replace/4
+                     },
+          tuple => #type{
+                      from = fun list_to_tuple/1,
+                      is = fun is_tuple/1,
+                      len = fun tuple_size/1,
+                      nth = fun element/2,
+                      prep = fun(I, Tu) ->  erlang:insert_element(1, Tu, I) end,
+                      app = fun(I, Tu) -> erlang:append_element(Tu, I) end,
+                      filter = fun tuple_filter/2,
+                      replace = fun tuple_replace/4
+                     }
+         }).
+
+from(Data, T) -> (T#type.?FUNCTION_NAME)(Data).
+is(Data, T) -> (T#type.?FUNCTION_NAME)(Data).
+len(Data, T) -> (T#type.?FUNCTION_NAME)(Data).
+nth(Index, Data, T) -> (T#type.?FUNCTION_NAME)(Index, Data).
+prep(In, Data, T) -> (T#type.?FUNCTION_NAME)(In, Data).
+app(In, Data, T) -> (T#type.?FUNCTION_NAME)(In, Data).
+filter(Fun, Data, T) -> (T#type.?FUNCTION_NAME)(Fun, Data).
+replace(In, Index, Out, Data, T) -> (T#type.?FUNCTION_NAME)(In, Index, Out, Data).
 
--export_type([coll/0, coll/1]).
 
 new(Type, Size, Fun) when is_function(Fun, 1) ->
+    CollType = maps:get(Type, ?TYPES),
     Indexes = lists:seq(1, Size),
     RevIndexes = maps:from_list([{Fun(I), I} || I <- Indexes]),
-    Data = (from(Type))(maps:keys(RevIndexes)),
-    #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data,
-          item_generator = Fun}.
-
-from(Type) -> ?Colls(Type, ?FUNCTION_NAME).
-
-len(known, #coll{data=Data}) -> len(Data);
-len(visible, #coll{indexes=Indexes}) -> length(Indexes).
+    #coll{
+       item_generator = Fun,
+       data = from(maps:keys(RevIndexes), CollType),
+       type = CollType,
+       indexes = Indexes,
+       rev_indexes = RevIndexes
+      }.
 
-len(Data) ->
-    (call(?FUNCTION_NAME, Data))(Data).
 
-call(F, Data) when is_atom(F) ->
-    call(F, Data, false, [none| maps:keys(?Types)]).
-
-call(F, _Data, true, [T |_Types]) -> ?Colls(T, F);
-call(F, Data, false, [_ | [T |_] = Types]) ->
-    call(F, Data, ?Colls(T, is), Types);
-call(F, Data, IsType, Types) when is_function(IsType, 1) ->
-    call(F, Data, IsType(Data), Types).
+length(known, #coll{data=Data, type=T}) -> len(Data, T);
+length(visible, #coll{indexes=Indexes}) -> length(Indexes).
 
 
 hide_head(#coll{indexes = []}) -> empty;
-hide_head(Coll = #coll{indexes = [H|T], data=Data}) ->
-    {nth(H, Data), Coll#coll{indexes = T}}.
+hide_head(Coll = #coll{indexes = [Hd|Tl], data=Data, type=T}) ->
+    {nth(Hd, Data, T), Coll#coll{indexes = Tl}}.
 
-nth(Index, Data) ->
-    (call(?FUNCTION_NAME, Data))(Index, Data).
 
 replace(Out, Coll = #coll{item_generator = In}) ->
     replace(Out, In, Coll).
 
-replace(Out, In, Coll = #coll{data = Data}) ->
+replace(Out, In, Coll) when not is_function(In, 1) ->
+    replace(Out, fun(_) -> In end, Coll);
+replace(Out, In, Coll = #coll{data = Data, type = T}) ->
     case maps:take(Out, Coll#coll.rev_indexes) of
         error -> error(enoent);
         {OutIndex, RevIndexes} ->
-            NewData = replace(OutIndex, Out, In, Data),
-            NewItem = nth(OutIndex, NewData),
+            NewData = replace(Out, OutIndex, In(OutIndex), Data, T),
+            NewItem = nth(OutIndex, NewData, T),
             NewRevIndexes = maps:put(NewItem, OutIndex, RevIndexes),
             {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
     end.
 
 
-replace(OutIndex, Out, In, Data) when not is_function(In) ->
-    replace(OutIndex, Out, fun(_) -> In end, Data);
-replace(OutIndex, Out, In, Data)  when is_function(In, 1)  ->
-    (call(?FUNCTION_NAME, Data))(Out, OutIndex, In(OutIndex), Data).
-
-
-prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
+prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data, type = T}) ->
     case maps:get(In, RevIndexes, undefined) of
         InIndex when is_integer(InIndex) -> Coll#coll{indexes=[InIndex|Indexes]};
         undefined ->
-            NewData = prep(In, Data),
+            NewData = prep(In, Data, T),
             NewRevIndexes = maps:put(In, 1, maps:map(fun(_, V) -> V + 1 end, RevIndexes)),
             NewIndexes = [1 | [I+1 || I <- Indexes]],
             Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
     end.
 
-prep(In, Data) ->
-    (call(?FUNCTION_NAME, Data))(In, Data).
 
 
-append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
+append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data, type = T}) ->
     case maps:get(In, RevIndexes, undefined) of
         InIndex when is_integer(InIndex) -> Coll#coll{indexes=Indexes++[InIndex]};
         undefined ->
-            NewData = app(In, Data),
+            NewData = app(In, Data, T),
+            ArrayT = maps:get(array, ?TYPES),
             NewIndex =
-            case {(?Colls(array, is))(Data), len(Data)} of
+            case {is(Data, ArrayT), len(Data, T)} of
                 {true, Len} -> Len - 1;
                 {_, Len} -> Len
             end,
@@ -196,12 +175,75 @@ append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data
             Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
     end.
 
-app(In, Data) ->
-    (call(?FUNCTION_NAME, Data))(In, Data).
+
+filter(Fun, #coll{data = Data, type = T}) ->
+    filter(Fun, Data, T).
+
+
+all(known, #coll{rev_indexes = RevIndexes}) ->
+    maps:keys(RevIndexes);
+all(visible, #coll{indexes = Indexes, data = Data, type = T}) ->
+    [nth(I, Data, T) || I <- Indexes].
+
+
+rand(known, #coll{data = Data, type = T}) ->
+    case len(Data, T) of
+        0 -> empty;
+        L -> nth(rand:uniform(L), Data, T)
+    end;
+rand(visible, #coll{indexes = []}) -> empty;
+rand(visible, #coll{indexes = Indexes, data = Data, type = T}) ->
+    nth(lists:nth(rand:uniform(length(Indexes)), Indexes), Data, T).
+
+
 
 
-filter(Fun, #coll{data = Data}) ->
-    (call(?FUNCTION_NAME, Data))(Fun, Data).
+
+list_replace(O, X, I, L) ->
+    {L1, [O | Tl]} = lists:split(X-1, L),
+    L1 ++ [I| Tl].
+
+
+
+array_prep(I, A) ->
+    array:foldl(
+      fun(Idx, Val, Arr) ->
+              array:set(Idx+1, Val, Arr)
+      end,
+      array:set(0, I, array:new()),
+      A).
+
+
+array_filter(F, A) ->
+    array:sparse_map(
+      fun(_, V) ->
+              case F(V) of
+                  true -> V;
+                  false -> array:default(A);
+                  Else -> Else
+              end
+      end,
+      A).
+
+
+array_replace(O, X, I, A) ->
+    O = array:get(X-1, A),
+    array:set(X-1, I, A).
+
+
+
+queue_nth(I, {_RL, FL}) when I =< length(FL) ->
+    lists:nth(I, FL);
+queue_nth(I, {RL, FL}) ->
+    J = length(RL)-(I-length(FL)-1),
+    lists:nth(J, RL).
+
+
+queue_replace(O, X, I, Q) ->
+    {Q1, Q2} = queue:split(X-1, Q),
+    O = queue:get(Q2),
+    queue:join(queue:in(I, Q1), queue:drop(Q2)).
+
 
 
 tuple_filter(Fun, Tuple) ->
@@ -219,17 +261,6 @@ tuple_filter(Fun, Tuple, Index) ->
     tuple_filter(Fun, NewTuple, Index-1).
 
 
-all(known, #coll{rev_indexes = RevIndexes}) ->
-    maps:keys(RevIndexes);
-all(visible, #coll{indexes = Indexes, data = Data}) ->
-    [nth(I, Data) || I <- Indexes].
-
-
-rand(known, #coll{data = Data}) ->
-    case len(Data) of
-        0 -> empty;
-        L -> nth(rand:uniform(L), Data)
-    end;
-rand(visible, #coll{indexes = []}) -> empty;
-rand(visible, #coll{indexes = Indexes, data = Data}) ->
-    nth(lists:nth(rand:uniform(length(Indexes)), Indexes), Data).
+tuple_replace(O, X, I, Tu) ->
+    O = element(X, Tu),
+    setelement(X, Tu, I).

From 5976142787d88c6a60af2ab7be815c6a096da4ea Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 13 Feb 2020 12:07:54 +0100
Subject: [PATCH 12/36] Add worker supervisor behaviour with a start_child
 callback

---
 src/poolboy.erl                   | 6 +++++-
 src/poolboy_worker_supervisor.erl | 6 ++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 src/poolboy_worker_supervisor.erl

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 111b410..76d0d49 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -363,7 +363,11 @@ start_pool(StartFun, PoolArgs, WorkerArgs) ->
     end.
 
 new_worker(Sup) ->
-    {ok, Pid} = supervisor:start_child(Sup, []),
+    {ok, Pid} =
+    case is_atom(Sup) andalso erlang:function_exported(Sup, start_child, 0) of
+        true -> Sup:start_child();
+        false -> supervisor:start_child(Sup, [])
+    end,
     true = link(Pid),
     Pid.
 
diff --git a/src/poolboy_worker_supervisor.erl b/src/poolboy_worker_supervisor.erl
new file mode 100644
index 0000000..2243cab
--- /dev/null
+++ b/src/poolboy_worker_supervisor.erl
@@ -0,0 +1,6 @@
+-module(poolboy_worker_supervisor).
+
+-callback start_child() -> {ok, Pid} |
+                           {error, Reason} when
+    Pid    :: pid(),
+    Reason :: term().

From 8ad3365ce4acff66321132ad5d93a3ec22dc6bcb Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Mon, 17 Feb 2020 01:49:35 +0100
Subject: [PATCH 13/36] Stop supervisor given by name

---
 src/poolboy.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 76d0d49..f978be7 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -436,6 +436,8 @@ state_name(_State) ->
     overflow.
 
 stop_supervisor(_, undefined) -> ok;
+stop_supervisor(Reason, Atom) when is_atom(Atom) ->
+    stop_supervisor(Reason, whereis(Atom));
 stop_supervisor(Reason, Pid) when is_pid(Pid) ->
     case erlang:node(Pid) of
         N when N == node() -> exit(Pid, Reason);

From 5eafeceb4130be9365624e9fbf703fd85e2974f6 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 2 Apr 2020 22:48:05 +0200
Subject: [PATCH 14/36] Fix: Do-not rely on maps' key-order for the item-order
 of the worker collection

---
 src/poolboy_collection.erl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index c39b413..2c452e5 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -112,10 +112,11 @@ replace(In, Index, Out, Data, T) -> (T#type.?FUNCTION_NAME)(In, Index, Out, Data
 new(Type, Size, Fun) when is_function(Fun, 1) ->
     CollType = maps:get(Type, ?TYPES),
     Indexes = lists:seq(1, Size),
-    RevIndexes = maps:from_list([{Fun(I), I} || I <- Indexes]),
+    Items = [Fun(I) || I <- Indexes],
+    RevIndexes = maps:from_list(lists:zip(Items, Indexes)),
     #coll{
        item_generator = Fun,
-       data = from(maps:keys(RevIndexes), CollType),
+       data = from(Items, CollType),
        type = CollType,
        indexes = Indexes,
        rev_indexes = RevIndexes

From 7a0703b98014a74b38a00a136182889d5c2a6279 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 2 Apr 2020 23:03:59 +0200
Subject: [PATCH 15/36] Add status_map/1 that includes waiting processes too

---
 src/poolboy.erl | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index f978be7..bb31565 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -5,7 +5,7 @@
 
 -export([checkout/1, checkout/2, checkout/3, checkin/2, transaction/2,
          transaction/3, child_spec/2, child_spec/3, child_spec/4, start/1,
-         start/2, start_link/1, start_link/2, stop/1, status/1]).
+         start/2, start_link/1, start_link/2, stop/1, status_map/1, status/1]).
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
          code_change/3]).
 -export_type([pool/0]).
@@ -57,6 +57,15 @@
     strategy = lifo :: lifo | fifo
 }).
 
+-type status_key() ::
+      state | available | overflow | monitored | waiting.
+
+-type state_name() ::
+      full | overflow | ready.
+
+-type status_map() ::
+      #{status_key() := integer() | state_name()}.
+
 -spec checkout(Pool :: pool()) -> pid().
 checkout(Pool) ->
     checkout(Pool, true).
@@ -151,6 +160,10 @@ start_link(PoolArgs, WorkerArgs)  ->
 stop(Pool) ->
     gen_server:call(Pool, stop).
 
+-spec status_map(Pool :: pool()) -> status_map().
+status_map(Pool) ->
+    gen_server:call(Pool, status_map).
+
 -spec status(Pool :: pool()) -> {atom(), integer(), integer(), integer()}.
 status(Pool) ->
     gen_server:call(Pool, status).
@@ -285,6 +298,16 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
             {noreply, State#state{waiting = Waiting}}
     end;
 
+handle_call(status_map, _From, State) ->
+    #state{workers = Workers,
+           monitors = Monitors,
+           overflow = Overflow} = State,
+    StateName = state_name(State),
+    {reply, #{state => StateName,
+              available => poolboy_collection:length(visible, Workers),
+              overflow => Overflow,
+              monitored => ets:info(Monitors, size),
+              waiting => queue:len(State#state.waiting)}, State};
 handle_call(status, _From, State) ->
     #state{workers = Workers,
            monitors = Monitors,

From 13ef0996203dcd575fe76a74778317bb03baeaf5 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 2 Apr 2020 23:15:54 +0200
Subject: [PATCH 16/36] Remove FUNCTION_NAME macro for backward compatibility

---
 src/poolboy_collection.erl | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 2c452e5..9ae21ce 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -99,14 +99,14 @@
                      }
          }).
 
-from(Data, T) -> (T#type.?FUNCTION_NAME)(Data).
-is(Data, T) -> (T#type.?FUNCTION_NAME)(Data).
-len(Data, T) -> (T#type.?FUNCTION_NAME)(Data).
-nth(Index, Data, T) -> (T#type.?FUNCTION_NAME)(Index, Data).
-prep(In, Data, T) -> (T#type.?FUNCTION_NAME)(In, Data).
-app(In, Data, T) -> (T#type.?FUNCTION_NAME)(In, Data).
-filter(Fun, Data, T) -> (T#type.?FUNCTION_NAME)(Fun, Data).
-replace(In, Index, Out, Data, T) -> (T#type.?FUNCTION_NAME)(In, Index, Out, Data).
+from(Data, T) -> (T#type.from)(Data).
+is(Data, T) -> (T#type.is)(Data).
+len(Data, T) -> (T#type.len)(Data).
+nth(Index, Data, T) -> (T#type.nth)(Index, Data).
+prep(In, Data, T) -> (T#type.prep)(In, Data).
+app(In, Data, T) -> (T#type.app)(In, Data).
+filter(Fun, Data, T) -> (T#type.filter)(Fun, Data).
+replace(In, Index, Out, Data, T) -> (T#type.replace)(In, Index, Out, Data).
 
 
 new(Type, Size, Fun) when is_function(Fun, 1) ->

From 029e2ccf64f86b08e3bb9f85d383ba6a58e326b6 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Tue, 21 Apr 2020 11:12:44 +0200
Subject: [PATCH 17/36] Spare item-lookup during replace; Fix Arg.-order in
 replace type-wrapper

---
 src/poolboy_collection.erl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 9ae21ce..2504801 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -106,7 +106,7 @@ nth(Index, Data, T) -> (T#type.nth)(Index, Data).
 prep(In, Data, T) -> (T#type.prep)(In, Data).
 app(In, Data, T) -> (T#type.app)(In, Data).
 filter(Fun, Data, T) -> (T#type.filter)(Fun, Data).
-replace(In, Index, Out, Data, T) -> (T#type.replace)(In, Index, Out, Data).
+replace(Out, Index, In, Data, T) -> (T#type.replace)(Out, Index, In, Data).
 
 
 new(Type, Size, Fun) when is_function(Fun, 1) ->
@@ -141,8 +141,8 @@ replace(Out, In, Coll = #coll{data = Data, type = T}) ->
     case maps:take(Out, Coll#coll.rev_indexes) of
         error -> error(enoent);
         {OutIndex, RevIndexes} ->
-            NewData = replace(Out, OutIndex, In(OutIndex), Data, T),
-            NewItem = nth(OutIndex, NewData, T),
+            NewItem = In(OutIndex),
+            NewData = replace(Out, OutIndex, NewItem, Data, T),
             NewRevIndexes = maps:put(NewItem, OutIndex, RevIndexes),
             {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
     end.

From cfb4ba3b33f63b8c0965130c856a3686c60a1358 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Wed, 22 Apr 2020 13:05:53 +0200
Subject: [PATCH 18/36] Use pool-collection for Overflow; propagete the
 Pool-Index of the Worker to the Worker-process

---
 src/poolboy.erl        | 183 ++++++++++++++++++++++++++++-------------
 test/poolboy_tests.erl |  15 ++--
 2 files changed, 135 insertions(+), 63 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index bb31565..79e6361 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -48,11 +48,12 @@
 
 -record(state, {
     supervisor :: sup_ref(),
+    worker_module :: atom(),
     workers :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
     waiting :: poolboy_collection:pid_queue() | poolboy_collection:pid_queue(tuple()),
     monitors :: ets:tid(),
     size = ?DEFAULT_SIZE :: non_neg_integer(),
-    overflow = 0 :: non_neg_integer(),
+    overflow :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
     max_overflow = 10 :: non_neg_integer(),
     strategy = lifo :: lifo | fifo
 }).
@@ -172,13 +173,18 @@ init({PoolArgs, WorkerArgs}) ->
     process_flag(trap_exit, true),
     Waiting = queue:new(),
     Monitors = ets:new(monitors, [private]),
+    WorkerModule = proplists:get_value(worker_module, PoolArgs),
     Supervisor = ensure_worker_supervisor(PoolArgs, WorkerArgs),
-    Workers = prepopulate(PoolArgs, Supervisor),
+    Workers = prepopulate(PoolArgs, Supervisor, WorkerModule),
+    Overflow = prepopulate(0, ?DEFAULT_TYPE, Supervisor, WorkerModule),
     init(PoolArgs, WorkerArgs,
          #state{supervisor = Supervisor,
+                worker_module = WorkerModule,
                 workers = Workers,
                 waiting = Waiting,
-                monitors = Monitors}).
+                monitors = Monitors,
+                overflow = Overflow
+               }).
 
 ensure_worker_supervisor(PoolArgs, WorkerArgs) ->
     case proplists:get_value(worker_supervisor, PoolArgs) of
@@ -215,25 +221,29 @@ start_supervisor(WorkerModule, WorkerArgs, Retries) ->
             exit({no_worker_supervisor, Error})
     end.
 
-prepopulate(PoolArgs, Supervisor) ->
+prepopulate(PoolArgs, Supervisor, WorkerModule) ->
     prepopulate(
       proplists:get_value(size, PoolArgs, ?DEFAULT_SIZE),
       proplists:get_value(type, PoolArgs, ?DEFAULT_TYPE),
-      Supervisor).
+      Supervisor,
+      WorkerModule).
 
-prepopulate(Size, Type, Sup)
+prepopulate(Size, Type, Sup, Mod)
   when is_integer(Size) andalso ?IS_COLLECTION_TYPE(Type) ->
-    poolboy_collection:new(Type, Size, fun(_) -> new_worker(Sup) end);
-prepopulate(Size, Type, Sup) when not is_integer(Size) ->
-    prepopulate(?DEFAULT_SIZE, Type, Sup);
-prepopulate(Size, Type, Sup) when not ?IS_COLLECTION_TYPE(Type) ->
-    prepopulate(Size, ?DEFAULT_TYPE, Sup).
+    poolboy_collection:new(Type, Size, fun(Idx) -> new_worker(Sup, Mod, Idx) end);
+prepopulate(Size, Type, Sup, Mod) when not is_integer(Size) ->
+    prepopulate(?DEFAULT_SIZE, Type, Sup, Mod);
+prepopulate(Size, Type, Sup, Mod) when not ?IS_COLLECTION_TYPE(Type) ->
+    prepopulate(Size, ?DEFAULT_TYPE, Sup, Mod).
 
 init([{size, Size} | Rest], WorkerArgs, State) when is_integer(Size) ->
     init(Rest, WorkerArgs, State#state{size = Size});
-init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State)
+init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State = #state{size = Size})
   when is_integer(MaxOverflow) ->
-    init(Rest, WorkerArgs, State#state{max_overflow = MaxOverflow});
+    Overflow = poolboy_collection:new(?DEFAULT_TYPE, MaxOverflow,
+                                      fun(Idx) -> Size + Idx end),
+    init(Rest, WorkerArgs, State#state{max_overflow = MaxOverflow,
+                                       overflow = Overflow});
 init([{strategy, Strategy} | Rest], WorkerArgs, State)
   when Strategy == lifo orelse
        Strategy == fifo ->
@@ -277,19 +287,24 @@ handle_cast(_Msg, State) ->
 
 handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
     #state{supervisor = Sup,
+           worker_module = Mod,
            workers = Workers,
            monitors = Monitors,
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
+    OverflowLeft = poolboy_collection:length(visible, Overflow),
     case poolboy_collection:hide_head(Workers) of
         {Pid, Left} when is_pid(Pid) ->
             MRef = erlang:monitor(process, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
             {reply, Pid, State#state{workers = Left}};
-        empty when MaxOverflow > 0, Overflow < MaxOverflow ->
-            {Pid, MRef} = new_worker(Sup, FromPid),
+        empty when MaxOverflow > 0, OverflowLeft > 0 ->
+            MRef = erlang:monitor(process, FromPid),
+            {NextIdx, NewOverflow} = poolboy_collection:hide_head(Overflow),
+            Pid = new_worker(Sup, Mod, NextIdx),
+            {Pid, NewerOverflow} = poolboy_collection:replace(NextIdx, Pid, NewOverflow),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
-            {reply, Pid, State#state{overflow = Overflow + 1}};
+            {reply, Pid, State#state{overflow = NewerOverflow}};
         empty when Block =:= false ->
             {reply, full, State};
         empty ->
@@ -301,19 +316,27 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
 handle_call(status_map, _From, State) ->
     #state{workers = Workers,
            monitors = Monitors,
-           overflow = Overflow} = State,
+           overflow = Overflow,
+           max_overflow = MaxOverflow} = State,
     StateName = state_name(State),
+    OverflowLeft = poolboy_collection:length(visible, Overflow),
+    OverflowLevel = MaxOverflow - OverflowLeft,
     {reply, #{state => StateName,
               available => poolboy_collection:length(visible, Workers),
-              overflow => Overflow,
+              overflow => OverflowLevel,
               monitored => ets:info(Monitors, size),
               waiting => queue:len(State#state.waiting)}, State};
 handle_call(status, _From, State) ->
     #state{workers = Workers,
            monitors = Monitors,
-           overflow = Overflow} = State,
+           overflow = Overflow,
+           max_overflow = MaxOverflow} = State,
     StateName = state_name(State),
-    {reply, {StateName, poolboy_collection:length(visible, Workers), Overflow, ets:info(Monitors, size)}, State};
+    VisibleWorkers = poolboy_collection:length(visible, Workers),
+    OverflowLeft = poolboy_collection:length(visible, Overflow),
+    OverflowLevel = MaxOverflow - OverflowLeft,
+    MonitorSize = ets:info(Monitors, size),
+    {reply, {StateName, VisibleWorkers, OverflowLevel, MonitorSize}, State};
 handle_call(get_avail_workers, _From, State) ->
     {reply, poolboy_collection:all(visible, State#state.workers), State};
 handle_call(get_any_worker, _From, State) ->
@@ -385,19 +408,31 @@ start_pool(StartFun, PoolArgs, WorkerArgs) ->
             gen_server:StartFun(Name, ?MODULE, {PoolArgs, WorkerArgs}, [])
     end.
 
-new_worker(Sup) ->
+new_worker(Sup, Mod, Index) when is_atom(Sup) ->
+    IsStart1 = erlang:function_exported(Sup, start_child, 1),
+    IsStart0 = erlang:function_exported(Sup, start_child, 0),
     {ok, Pid} =
-    case is_atom(Sup) andalso erlang:function_exported(Sup, start_child, 0) of
-        true -> Sup:start_child();
-        false -> supervisor:start_child(Sup, [])
+    case {IsStart1, IsStart0} of
+        {true, _} -> Sup:start_child(Index);
+        {_, true} -> Sup:start_child();
+        _ -> supervisor:start_child(Sup, child_args(Sup, Mod, Index))
     end,
     true = link(Pid),
+    Pid;
+new_worker(Sup, Mod, Index) ->
+    {ok, Pid} = supervisor:start_child(Sup, child_args(Sup, Mod, Index)),
+    true = link(Pid),
     Pid.
 
-new_worker(Sup, FromPid) ->
-    Pid = new_worker(Sup),
-    Ref = erlang:monitor(process, FromPid),
-    {Pid, Ref}.
+child_args(Sup, Mod, Index) ->
+    case supervisor:get_childspec(Sup, Mod) of
+        {ok, #{start := {M,F,A}}} ->
+            case erlang:function_exported(M, F, length(A) + 1) of
+                true -> [Index];
+                _ -> []
+            end;
+        _ -> []
+    end.
 
 dismiss_worker(Sup, Pid) ->
     true = unlink(Pid),
@@ -410,53 +445,87 @@ replace_worker(Pid, State = #state{strategy = Strategy}) ->
         fifo -> State#state{workers = poolboy_collection:append(NewWorker, Workers)}
     end.
 
-handle_checkin(Pid, State = #state{strategy = Strategy}) ->
+handle_checkin(Pid, State) ->
     #state{supervisor = Sup,
            waiting = Waiting,
            monitors = Monitors,
+           size = Size,
            overflow = Overflow} = State,
     case queue:out(Waiting) of
         {{value, {From, CRef, MRef}}, Left} ->
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
             gen_server:reply(From, Pid),
             State#state{waiting = Left};
-        {empty, Empty} when Overflow > 0 ->
-            ok = dismiss_worker(Sup, Pid),
-            State#state{waiting = Empty, overflow = Overflow - 1};
         {empty, Empty} ->
-            Workers = case Strategy of
-                lifo -> poolboy_collection:prepend(Pid, State#state.workers);
-                fifo -> poolboy_collection:append(Pid, State#state.workers)
-            end,
-            State#state{workers = Workers, waiting = Empty, overflow = 0}
+            try poolboy_collection:replace(Pid, fun(Idx) -> Size + Idx end, Overflow) of
+                {NewIdx, NewOverflow} ->
+                    ok = dismiss_worker(Sup, Pid),
+                    NewerOverflow = poolboy_collection:prepend(NewIdx, NewOverflow),
+                    State#state{waiting = Empty, overflow = NewerOverflow}
+            catch
+                error:enoent ->
+                    Workers =
+                    case State#state.strategy of
+                        lifo -> poolboy_collection:prepend(Pid, State#state.workers);
+                        fifo -> poolboy_collection:append(Pid, State#state.workers)
+                    end,
+                    State#state{waiting = Empty, workers = Workers}
+            end
     end.
 
 handle_worker_exit(Pid, State) ->
-    #state{monitors = Monitors,
-           overflow = Overflow} = State,
+    #state{supervisor = Sup,
+           worker_module = Mod,
+           monitors = Monitors,
+           size = Size,
+           strategy = Strategy,
+           overflow = Overflow,
+           max_overflow = MaxOverflow} = State,
+    {NewWorker, Workers} =
+    try poolboy_collection:replace(Pid, State#state.workers)
+    catch error:enoent -> {enoent, State#state.workers}
+    end,
+    OverflowLeft = poolboy_collection:length(visible, Overflow),
     case queue:out(State#state.waiting) of
-        {{value, {From, CRef, MRef}}, LeftWaiting} ->
-            NewWorker = new_worker(State#state.supervisor),
+        {{value, {From, CRef, MRef}}, LeftWaiting} when is_pid(NewWorker) ->
             true = ets:insert(Monitors, {NewWorker, CRef, MRef}),
             gen_server:reply(From, NewWorker),
-            State#state{waiting = LeftWaiting};
-        {empty, Empty} when Overflow > 0 ->
-            State#state{overflow = Overflow - 1, waiting = Empty};
-        {empty, Empty} ->
-            replace_worker(Pid, State#state{waiting = Empty})
+            State#state{waiting = LeftWaiting, workers = Workers};
+        {{value, {From, CRef, MRef}}, LeftWaiting} when MaxOverflow > OverflowLeft ->
+            try
+                NewFun = fun(Idx) -> new_worker(Sup, Mod, Size + Idx) end,
+                {NewPid, NewOverflow} = poolboy_collection:replace(Pid, NewFun, Overflow),
+                true = ets:insert(Monitors, {NewPid, CRef, MRef}),
+                gen_server:reply(From, NewPid),
+                State#state{waiting = LeftWaiting, overflow = NewOverflow}
+            catch error:enoent ->
+                State
+            end;
+        {empty, Empty} when is_pid(NewWorker), Strategy == lifo ->
+            State#state{waiting = Empty,
+                        workers = poolboy_collection:prepend(NewWorker, Workers)};
+        {empty, Empty} when is_pid(NewWorker), Strategy == fifo ->
+            State#state{waiting = Empty,
+                        workers = poolboy_collection:append(NewWorker, Workers)};
+        {empty, Empty} when MaxOverflow > 0 ->
+            {Idx, NewOverflow} = poolboy_collection:replace(Pid, Overflow),
+            NewerOverflow = poolboy_collection:prepend(Idx, NewOverflow),
+            State#state{waiting = Empty, overflow = NewerOverflow}
     end.
 
-state_name(State = #state{overflow = Overflow}) when Overflow < 1 ->
-    #state{max_overflow = MaxOverflow, workers = Workers} = State,
-    case poolboy_collection:length(visible, Workers) == 0 of
-        true when MaxOverflow < 1 -> full;
-        true -> overflow;
-        false -> ready
-    end;
-state_name(#state{overflow = MaxOverflow, max_overflow = MaxOverflow}) ->
-    full;
-state_name(_State) ->
-    overflow.
+state_name(State) ->
+    #state{workers = Workers,
+           overflow = Overflow,
+           max_overflow = MaxOverflow} = State,
+    case poolboy_collection:length(visible, Workers) of
+        0 when MaxOverflow < 1 -> full;
+        0 ->
+            case poolboy_collection:length(visible, Overflow) of
+                0 -> full;
+                _ -> overflow
+            end;
+        _ -> ready
+    end.
 
 stop_supervisor(_, undefined) -> ok;
 stop_supervisor(Reason, Atom) when is_atom(Atom) ->
diff --git a/test/poolboy_tests.erl b/test/poolboy_tests.erl
index d100d86..c997798 100644
--- a/test/poolboy_tests.erl
+++ b/test/poolboy_tests.erl
@@ -147,7 +147,7 @@ pool_startup(Type) ->
 pool_overflow(Type) ->
     %% Check that the pool overflows properly.
     {ok, Pid} = new_pool(5, 5, lifo, Type),
-    Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
+    Workers = lists:reverse([poolboy:checkout(Pid) || _ <- lists:seq(0, 6)]),
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     [A, B, C, D, E, F, G] = Workers,
@@ -173,7 +173,7 @@ pool_empty(Type) ->
     %% Checks that the the pool handles the empty condition correctly when
     %% overflow is enabled.
     {ok, Pid} = new_pool(5, 2, lifo, Type),
-    Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
+    Workers = lists:reverse([poolboy:checkout(Pid) || _ <- lists:seq(0, 6)]),
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     [A, B, C, D, E, F, G] = Workers,
@@ -264,7 +264,7 @@ worker_death(Type) ->
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
-    [A, B, C|_Workers] = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
+    [A, B, C|_Workers] = lists:reverse([poolboy:checkout(Pid) || _ <- lists:seq(0, 6)]),
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     kill_worker(A),
@@ -285,7 +285,7 @@ worker_death_while_full(Type) ->
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
-    [A, B|_Workers] = [poolboy:checkout(Pid) || _ <- lists:seq(0, 6)],
+    [A, B|_Workers] = lists:reverse([poolboy:checkout(Pid) || _ <- lists:seq(0, 6)]),
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
     Self = self(),
@@ -384,7 +384,7 @@ pool_full_nonblocking(Type) ->
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(10, length(pool_call(Pid, get_all_workers))),
     ?assertEqual(full, poolboy:checkout(Pid, false)),
-    A = hd(Workers),
+    A = lists:last(Workers),
     checkin_worker(Pid, A),
     NewWorker = poolboy:checkout(Pid, false),
     ?assertEqual(false, is_process_alive(A)), %% Overflow workers get shutdown
@@ -560,7 +560,10 @@ new_pool(Size, MaxOverflow, Strategy, Type) ->
 
 pool_call(ServerRef, stop) when is_pid(ServerRef) ->
     case is_process_alive(ServerRef) of
-        true -> gen_server:stop(ServerRef);
+        true ->
+            try gen_server:stop(ServerRef)
+            catch exit:noproc -> ok
+            end;
         _ -> ok
     end;
 pool_call(ServerRef, Request) ->

From 093e9dfda9a2f8aa8ebe9fdba84a4947bb992b92 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 24 Apr 2020 12:44:54 +0200
Subject: [PATCH 19/36] Adjust worker-supervisor callbacks

---
 src/poolboy_worker_supervisor.erl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/poolboy_worker_supervisor.erl b/src/poolboy_worker_supervisor.erl
index 2243cab..b5a7ac0 100644
--- a/src/poolboy_worker_supervisor.erl
+++ b/src/poolboy_worker_supervisor.erl
@@ -4,3 +4,10 @@
                            {error, Reason} when
     Pid    :: pid(),
     Reason :: term().
+
+-callback start_child(integer()) -> {ok, Pid} |
+                                    {error, Reason} when
+    Pid    :: pid(),
+    Reason :: term().
+
+-optional_callbacks([start_child/0, start_child/1]).

From 60b235c85430504a6beec5ccca8be0ec09fb24ac Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 7 May 2020 00:44:41 +0200
Subject: [PATCH 20/36] Add lookup-tables for monitor and checkout references

---
 src/poolboy.erl | 60 ++++++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 79e6361..33b9263 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -52,6 +52,8 @@
     workers :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
     waiting :: poolboy_collection:pid_queue() | poolboy_collection:pid_queue(tuple()),
     monitors :: ets:tid(),
+    mrefs :: ets:tid(),
+    crefs :: ets:tid(),
     size = ?DEFAULT_SIZE :: non_neg_integer(),
     overflow :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
     max_overflow = 10 :: non_neg_integer(),
@@ -173,6 +175,8 @@ init({PoolArgs, WorkerArgs}) ->
     process_flag(trap_exit, true),
     Waiting = queue:new(),
     Monitors = ets:new(monitors, [private]),
+    MRefs = ets:new(mrefs, [private]),
+    CRefs = ets:new(crefs, [private]),
     WorkerModule = proplists:get_value(worker_module, PoolArgs),
     Supervisor = ensure_worker_supervisor(PoolArgs, WorkerArgs),
     Workers = prepopulate(PoolArgs, Supervisor, WorkerModule),
@@ -183,6 +187,8 @@ init({PoolArgs, WorkerArgs}) ->
                 workers = Workers,
                 waiting = Waiting,
                 monitors = Monitors,
+                mrefs = MRefs,
+                crefs = CRefs,
                 overflow = Overflow
                }).
 
@@ -253,11 +259,14 @@ init([_ | Rest], WorkerArgs, State) ->
 init([], _WorkerArgs, State) ->
     {ok, State}.
 
-handle_cast({checkin, Pid}, State = #state{monitors = Monitors}) ->
+handle_cast({checkin, Pid}, State) ->
+    #state{monitors = Monitors, mrefs = MRefs, crefs = CRefs} = State,
     case ets:lookup(Monitors, Pid) of
-        [{Pid, _, MRef}] ->
-            true = erlang:demonitor(MRef),
+        [{Pid, CRef, MRef}] ->
+            true = erlang:demonitor(MRef, [flush]),
             true = ets:delete(Monitors, Pid),
+            true = ets:delete(MRefs, MRef),
+            true = ets:delete(CRefs, CRef),
             NewState = handle_checkin(Pid, State),
             {noreply, NewState};
         [] ->
@@ -265,12 +274,9 @@ handle_cast({checkin, Pid}, State = #state{monitors = Monitors}) ->
     end;
 
 handle_cast({cancel_waiting, CRef}, State) ->
-    case ets:match(State#state.monitors, {'$1', CRef, '$2'}) of
-        [[Pid, MRef]] ->
-            demonitor(MRef, [flush]),
-            true = ets:delete(State#state.monitors, Pid),
-            NewState = handle_checkin(Pid, State),
-            {noreply, NewState};
+    case ets:lookup(State#state.crefs, CRef) of
+        [{CRef, Pid}] ->
+            handle_cast({checkin, Pid}, State);
         [] ->
             Cancel = fun({_, Ref, MRef}) when Ref =:= CRef ->
                              demonitor(MRef, [flush]),
@@ -290,6 +296,8 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
            worker_module = Mod,
            workers = Workers,
            monitors = Monitors,
+           mrefs = MRefs,
+           crefs = CRefs,
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
     OverflowLeft = poolboy_collection:length(visible, Overflow),
@@ -297,13 +305,17 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
         {Pid, Left} when is_pid(Pid) ->
             MRef = erlang:monitor(process, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
+            true = ets:insert(MRefs, {MRef, Pid}),
+            true = ets:insert(CRefs, {CRef, Pid}),
             {reply, Pid, State#state{workers = Left}};
         empty when MaxOverflow > 0, OverflowLeft > 0 ->
-            MRef = erlang:monitor(process, FromPid),
             {NextIdx, NewOverflow} = poolboy_collection:hide_head(Overflow),
             Pid = new_worker(Sup, Mod, NextIdx),
             {Pid, NewerOverflow} = poolboy_collection:replace(NextIdx, Pid, NewOverflow),
+            MRef = erlang:monitor(process, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
+            true = ets:insert(MRefs, {MRef, Pid}),
+            true = ets:insert(CRefs, {CRef, Pid}),
             {reply, Pid, State#state{overflow = NewerOverflow}};
         empty when Block =:= false ->
             {reply, full, State};
@@ -356,23 +368,25 @@ handle_call(_Msg, _From, State) ->
     {reply, Reply, State}.
 
 handle_info({'DOWN', MRef, _, _, _}, State) ->
-    case ets:match(State#state.monitors, {'$1', '_', MRef}) of
-        [[Pid]] ->
-            true = ets:delete(State#state.monitors, Pid),
-            NewState = handle_checkin(Pid, State),
-            {noreply, NewState};
+    case ets:lookup(State#state.mrefs, MRef) of
+        [{MRef, Pid}] ->
+            handle_cast({checkin, Pid}, State);
         [] ->
             Waiting = queue:filter(fun ({_, _, R}) -> R =/= MRef end, State#state.waiting),
             {noreply, State#state{waiting = Waiting}}
     end;
 handle_info({'EXIT', Pid, Reason}, State) ->
     #state{supervisor = Sup,
-           monitors = Monitors} = State,
+           monitors = Monitors,
+           mrefs = MRefs,
+           crefs = CRefs} = State,
     Next =
     case ets:lookup(Monitors, Pid) of
-        [{Pid, _, MRef}] ->
-            true = erlang:demonitor(MRef),
+        [{Pid, CRef, MRef}] ->
+            true = erlang:demonitor(MRef, [flush]),
             true = ets:delete(Monitors, Pid),
+            true = ets:delete(MRefs, MRef),
+            true = ets:delete(CRefs, CRef),
             NewState = handle_worker_exit(Pid, State),
             {noreply, NewState};
         [] ->
@@ -449,11 +463,15 @@ handle_checkin(Pid, State) ->
     #state{supervisor = Sup,
            waiting = Waiting,
            monitors = Monitors,
+           mrefs = MRefs,
+           crefs = CRefs,
            size = Size,
            overflow = Overflow} = State,
     case queue:out(Waiting) of
         {{value, {From, CRef, MRef}}, Left} ->
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
+            true = ets:insert(MRefs, {MRef, Pid}),
+            true = ets:insert(CRefs, {CRef, Pid}),
             gen_server:reply(From, Pid),
             State#state{waiting = Left};
         {empty, Empty} ->
@@ -477,6 +495,8 @@ handle_worker_exit(Pid, State) ->
     #state{supervisor = Sup,
            worker_module = Mod,
            monitors = Monitors,
+           mrefs = MRefs,
+           crefs = CRefs,
            size = Size,
            strategy = Strategy,
            overflow = Overflow,
@@ -489,6 +509,8 @@ handle_worker_exit(Pid, State) ->
     case queue:out(State#state.waiting) of
         {{value, {From, CRef, MRef}}, LeftWaiting} when is_pid(NewWorker) ->
             true = ets:insert(Monitors, {NewWorker, CRef, MRef}),
+            true = ets:insert(MRefs, {MRef, Pid}),
+            true = ets:insert(CRefs, {CRef, Pid}),
             gen_server:reply(From, NewWorker),
             State#state{waiting = LeftWaiting, workers = Workers};
         {{value, {From, CRef, MRef}}, LeftWaiting} when MaxOverflow > OverflowLeft ->
@@ -496,6 +518,8 @@ handle_worker_exit(Pid, State) ->
                 NewFun = fun(Idx) -> new_worker(Sup, Mod, Size + Idx) end,
                 {NewPid, NewOverflow} = poolboy_collection:replace(Pid, NewFun, Overflow),
                 true = ets:insert(Monitors, {NewPid, CRef, MRef}),
+                true = ets:insert(MRefs, {MRef, Pid}),
+                true = ets:insert(CRefs, {CRef, Pid}),
                 gen_server:reply(From, NewPid),
                 State#state{waiting = LeftWaiting, overflow = NewOverflow}
             catch error:enoent ->

From 9bb29762291b783da111c8b78237a68861f7129b Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Sun, 24 May 2020 00:15:01 +0200
Subject: [PATCH 21/36] Same type for Overflow as for Pool

---
 src/poolboy.erl | 144 +++++++++++++++++++++++++-----------------------
 1 file changed, 74 insertions(+), 70 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 33b9263..6c64831 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -13,11 +13,8 @@
 -define(TIMEOUT, 5000).
 -define(DEFAULT_SIZE, 5).
 -define(DEFAULT_TYPE, list).
--define(IS_COLLECTION_TYPE(Type), (Type == list orelse
-                                   Type == array orelse
-                                   Type == tuple orelse
-                                   Type == queue)).
-
+-define(DEFAULT_STRATEGY, lifo).
+-define(DEFAULT_OVERFLOW, 10).
 
 
 -ifdef(OTP_RELEASE). %% this implies 21 or higher
@@ -56,8 +53,8 @@
     crefs :: ets:tid(),
     size = ?DEFAULT_SIZE :: non_neg_integer(),
     overflow :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
-    max_overflow = 10 :: non_neg_integer(),
-    strategy = lifo :: lifo | fifo
+    max_overflow = ?DEFAULT_OVERFLOW :: non_neg_integer(),
+    strategy = ?DEFAULT_STRATEGY :: lifo | fifo
 }).
 
 -type status_key() ::
@@ -173,46 +170,51 @@ status(Pool) ->
 
 init({PoolArgs, WorkerArgs}) ->
     process_flag(trap_exit, true),
+
+    WorkerModule = worker_module(PoolArgs),
+    WorkerSup = worker_supervisor(PoolArgs),
+    (undefined == WorkerModule) andalso (undefined == WorkerSup)
+    andalso error({badarg, "worker_module or worker_supervisor is required"}),
+    Supervisor = ensure_supervisor(WorkerSup, WorkerModule, WorkerArgs),
+    Size = pool_size(PoolArgs),
+    Type = pool_type(PoolArgs),
+    Workers = init_workers(Supervisor, WorkerModule, Size, Type),
+
+    MaxOverflow = max_overflow(PoolArgs),
+    Overflow = init_overflow(Size, MaxOverflow, Type),
+
     Waiting = queue:new(),
     Monitors = ets:new(monitors, [private]),
     MRefs = ets:new(mrefs, [private]),
     CRefs = ets:new(crefs, [private]),
-    WorkerModule = proplists:get_value(worker_module, PoolArgs),
-    Supervisor = ensure_worker_supervisor(PoolArgs, WorkerArgs),
-    Workers = prepopulate(PoolArgs, Supervisor, WorkerModule),
-    Overflow = prepopulate(0, ?DEFAULT_TYPE, Supervisor, WorkerModule),
-    init(PoolArgs, WorkerArgs,
-         #state{supervisor = Supervisor,
-                worker_module = WorkerModule,
-                workers = Workers,
-                waiting = Waiting,
-                monitors = Monitors,
-                mrefs = MRefs,
-                crefs = CRefs,
-                overflow = Overflow
-               }).
-
-ensure_worker_supervisor(PoolArgs, WorkerArgs) ->
-    case proplists:get_value(worker_supervisor, PoolArgs) of
-        undefined ->
-            start_supervisor(
-              proplists:get_value(worker_module, PoolArgs),
-              WorkerArgs);
-        Sup = {Name, Node} when Name =/= local orelse
-                                Name =/= global ->
-            (catch erlang:monitor_node(Node, true)),
-            Sup;
-        Sup when is_pid(Sup) orelse
-                 is_atom(Sup) orelse
-                 is_tuple(Sup) ->
-            Sup
-    end.
+    {ok, #state{
+            supervisor = Supervisor,
+            worker_module = WorkerModule,
+            workers = Workers,
+            waiting = Waiting,
+            monitors = Monitors,
+            mrefs = MRefs,
+            crefs = CRefs,
+            size = Size,
+            overflow = Overflow,
+            max_overflow = MaxOverflow,
+            strategy = strategy(PoolArgs)
+           }}.
+
+ensure_supervisor(undefined, WorkerModule, WorkerArgs) ->
+    start_supervisor(WorkerModule, WorkerArgs);
+ensure_supervisor(Sup = {Name, Node}, _, _) when Name =/= local orelse
+                                                 Name =/= global ->
+    (catch erlang:monitor_node(Node, true)),
+    Sup;
+ensure_supervisor(Sup, _, _) when is_pid(Sup) orelse
+                                  is_atom(Sup) orelse
+                                  is_tuple(Sup) ->
+    Sup.
 
 start_supervisor(WorkerModule, WorkerArgs) ->
     start_supervisor(WorkerModule, WorkerArgs, 1).
 
-start_supervisor(undefined, _WorkerArgs, _Retries) ->
-    exit({no_worker_supervisor, {worker_module, undefined}});
 start_supervisor(WorkerModule, WorkerArgs, Retries) ->
     case poolboy_sup:start_link(WorkerModule, WorkerArgs) of
         {ok, NewPid} ->
@@ -227,37 +229,39 @@ start_supervisor(WorkerModule, WorkerArgs, Retries) ->
             exit({no_worker_supervisor, Error})
     end.
 
-prepopulate(PoolArgs, Supervisor, WorkerModule) ->
-    prepopulate(
-      proplists:get_value(size, PoolArgs, ?DEFAULT_SIZE),
-      proplists:get_value(type, PoolArgs, ?DEFAULT_TYPE),
-      Supervisor,
-      WorkerModule).
-
-prepopulate(Size, Type, Sup, Mod)
-  when is_integer(Size) andalso ?IS_COLLECTION_TYPE(Type) ->
-    poolboy_collection:new(Type, Size, fun(Idx) -> new_worker(Sup, Mod, Idx) end);
-prepopulate(Size, Type, Sup, Mod) when not is_integer(Size) ->
-    prepopulate(?DEFAULT_SIZE, Type, Sup, Mod);
-prepopulate(Size, Type, Sup, Mod) when not ?IS_COLLECTION_TYPE(Type) ->
-    prepopulate(Size, ?DEFAULT_TYPE, Sup, Mod).
-
-init([{size, Size} | Rest], WorkerArgs, State) when is_integer(Size) ->
-    init(Rest, WorkerArgs, State#state{size = Size});
-init([{max_overflow, MaxOverflow} | Rest], WorkerArgs, State = #state{size = Size})
-  when is_integer(MaxOverflow) ->
-    Overflow = poolboy_collection:new(?DEFAULT_TYPE, MaxOverflow,
-                                      fun(Idx) -> Size + Idx end),
-    init(Rest, WorkerArgs, State#state{max_overflow = MaxOverflow,
-                                       overflow = Overflow});
-init([{strategy, Strategy} | Rest], WorkerArgs, State)
-  when Strategy == lifo orelse
-       Strategy == fifo ->
-    init(Rest, WorkerArgs, State#state{strategy = Strategy});
-init([_ | Rest], WorkerArgs, State) ->
-    init(Rest, WorkerArgs, State);
-init([], _WorkerArgs, State) ->
-    {ok, State}.
+init_workers(Sup, Mod, Size, Type) ->
+    Fun = fun(Idx) -> new_worker(Sup, Mod, Idx) end,
+    poolboy_collection:new(Type, Size, Fun).
+
+init_overflow(Size, MaxOverflow, Type) ->
+    Fun = fun(Idx) -> Size + Idx end,
+    poolboy_collection:new(Type, MaxOverflow, Fun).
+
+worker_module(PoolArgs) ->
+    Is = is_atom(V = proplists:get_value(worker_module, PoolArgs)),
+    if not Is -> undefined; true -> V end.
+
+worker_supervisor(PoolArgs) ->
+    Is = is_atom(V = proplists:get_value(worker_supervisor, PoolArgs)),
+    if not Is -> undefined; true -> V end.
+
+pool_size(PoolArgs) ->
+    Is = is_integer(V = proplists:get_value(size, PoolArgs)),
+    if not Is -> ?DEFAULT_SIZE; true -> V end.
+
+-define(IS_COLLECTION_TYPE(T), lists:member(T, [list,array,tuple,queue])).
+pool_type(PoolArgs) ->
+    Is = ?IS_COLLECTION_TYPE(V = proplists:get_value(type, PoolArgs)),
+    if not Is -> ?DEFAULT_TYPE; true -> V end.
+
+max_overflow(PoolArgs) ->
+    Is = is_integer(V = proplists:get_value(max_overflow, PoolArgs)),
+    if not Is -> ?DEFAULT_OVERFLOW; true -> V end.
+
+-define(IS_STRATEGY(S), lists:member(S, [lifo, fifo])).
+strategy(PoolArgs) ->
+    Is = ?IS_STRATEGY(V = proplists:get_value(strategy, PoolArgs)),
+    if not Is -> ?DEFAULT_STRATEGY; true -> V end.
 
 handle_cast({checkin, Pid}, State) ->
     #state{monitors = Monitors, mrefs = MRefs, crefs = CRefs} = State,

From 008f26bfc5ad0c174a45910d1d263977a748b713 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Sun, 24 May 2020 00:18:35 +0200
Subject: [PATCH 22/36] Default to tuple collection

---
 src/poolboy.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 6c64831..d3e48a6 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -12,7 +12,7 @@
 
 -define(TIMEOUT, 5000).
 -define(DEFAULT_SIZE, 5).
--define(DEFAULT_TYPE, list).
+-define(DEFAULT_TYPE, tuple).
 -define(DEFAULT_STRATEGY, lifo).
 -define(DEFAULT_OVERFLOW, 10).
 

From 5cdad3ce48d7401ba4d4fe8f791ae333cbe668e3 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Sat, 30 May 2020 15:11:33 +0200
Subject: [PATCH 23/36] Terminate on supervisor EXIT; Use PID of supervisor

---
 src/poolboy.erl | 140 +++++++++++++++++++++++++-----------------------
 1 file changed, 72 insertions(+), 68 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index d3e48a6..ebc55b3 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -35,16 +35,8 @@
 % Copied from gen:start_ret/0
 -type start_ret() :: {'ok', pid()} | 'ignore' | {'error', term()}.
 
-% Copied from supervisor:sup_ref/0
--type sup_ref() ::
-    (Name :: atom()) |
-    {Name :: atom(), Node :: node()} |
-    {global, Name :: atom()} |
-    {via, Module :: module(), Name :: any()} |
-    pid().
-
 -record(state, {
-    supervisor :: sup_ref(),
+    supervisor :: pid(),
     worker_module :: atom(),
     workers :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
     waiting :: poolboy_collection:pid_queue() | poolboy_collection:pid_queue(tuple()),
@@ -172,10 +164,14 @@ init({PoolArgs, WorkerArgs}) ->
     process_flag(trap_exit, true),
 
     WorkerModule = worker_module(PoolArgs),
-    WorkerSup = worker_supervisor(PoolArgs),
-    (undefined == WorkerModule) andalso (undefined == WorkerSup)
-    andalso error({badarg, "worker_module or worker_supervisor is required"}),
-    Supervisor = ensure_supervisor(WorkerSup, WorkerModule, WorkerArgs),
+    Supervisor =
+    case worker_supervisor(PoolArgs) of
+        undefined ->
+            start_supervisor(WorkerModule, WorkerArgs);
+        Sup when is_pid(Sup) ->
+            true = link(Sup),
+            Sup
+    end,
     Size = pool_size(PoolArgs),
     Type = pool_type(PoolArgs),
     Workers = init_workers(Supervisor, WorkerModule, Size, Type),
@@ -201,17 +197,8 @@ init({PoolArgs, WorkerArgs}) ->
             strategy = strategy(PoolArgs)
            }}.
 
-ensure_supervisor(undefined, WorkerModule, WorkerArgs) ->
-    start_supervisor(WorkerModule, WorkerArgs);
-ensure_supervisor(Sup = {Name, Node}, _, _) when Name =/= local orelse
-                                                 Name =/= global ->
-    (catch erlang:monitor_node(Node, true)),
-    Sup;
-ensure_supervisor(Sup, _, _) when is_pid(Sup) orelse
-                                  is_atom(Sup) orelse
-                                  is_tuple(Sup) ->
-    Sup.
-
+start_supervisor(undefined, _WorkerArgs) ->
+    error({badarg, "worker_module or worker_supervisor is required"});
 start_supervisor(WorkerModule, WorkerArgs) ->
     start_supervisor(WorkerModule, WorkerArgs, 1).
 
@@ -242,8 +229,22 @@ worker_module(PoolArgs) ->
     if not Is -> undefined; true -> V end.
 
 worker_supervisor(PoolArgs) ->
-    Is = is_atom(V = proplists:get_value(worker_supervisor, PoolArgs)),
-    if not Is -> undefined; true -> V end.
+    Is = is_pid(Res = find_pid(V = proplists:get_value(worker_supervisor, PoolArgs))),
+    if not Is andalso Res =/= V -> exit({not_found, V, Res}); true -> Res end.
+
+find_pid(undefined) ->
+    undefined;
+find_pid(Name) when is_atom(Name) ->
+    find_pid({local, Name});
+find_pid({local, Name}) ->
+    whereis(Name);
+find_pid({global, Name}) ->
+    find_pid({via, global, Name});
+find_pid({via, Registry, Name}) ->
+    Registry:whereis_name(Name);
+find_pid({Name, Node}) ->
+    (catch erlang:monitor_node(Node, true)),
+    rpc:call(Node, erlang, whereis, [Name], ?TIMEOUT).
 
 pool_size(PoolArgs) ->
     Is = is_integer(V = proplists:get_value(size, PoolArgs)),
@@ -379,34 +380,26 @@ handle_info({'DOWN', MRef, _, _, _}, State) ->
             Waiting = queue:filter(fun ({_, _, R}) -> R =/= MRef end, State#state.waiting),
             {noreply, State#state{waiting = Waiting}}
     end;
-handle_info({'EXIT', Pid, Reason}, State) ->
-    #state{supervisor = Sup,
-           monitors = Monitors,
+handle_info({'EXIT', Pid, Reason}, State = #state{supervisor = Pid}) ->
+    {stop, Reason, State};
+handle_info({'EXIT', Pid, _Reason}, State) ->
+    #state{monitors = Monitors,
            mrefs = MRefs,
            crefs = CRefs} = State,
-    Next =
     case ets:lookup(Monitors, Pid) of
         [{Pid, CRef, MRef}] ->
             true = erlang:demonitor(MRef, [flush]),
             true = ets:delete(Monitors, Pid),
             true = ets:delete(MRefs, MRef),
-            true = ets:delete(CRefs, CRef),
-            NewState = handle_worker_exit(Pid, State),
-            {noreply, NewState};
+            true = ets:delete(CRefs, CRef);
         [] ->
-            try
-                {noreply, replace_worker(Pid, State)}
-            catch ?EXCEPTION(error, enoent, _StackTrace) ->
-                {noreply, State}
-            end
+            ok
     end,
-    case {Sup, erlang:node(Pid)} of
-        {{_, Node}, Node} -> {stop, Reason, State#state{supervisor = undefined}};
-        {Pid, _} -> {stop, Reason, State#state{supervisor = undefined}};
-        _ -> Next
-    end;
-handle_info({nodedown, Node}, State = #state{supervisor = {_, Node}}) ->
-    {stop, nodedown, State#state{supervisor = undefined}};
+    NewState = handle_worker_exit(Pid, State),
+    {noreply, NewState};
+handle_info({nodedown, Node}, State = #state{supervisor = Sup})
+  when Node == erlang:node(Sup) ->
+    {stop, nodedown, State};
 handle_info(_Info, State) ->
     {noreply, State}.
 
@@ -426,32 +419,47 @@ start_pool(StartFun, PoolArgs, WorkerArgs) ->
             gen_server:StartFun(Name, ?MODULE, {PoolArgs, WorkerArgs}, [])
     end.
 
-new_worker(Sup, Mod, Index) when is_atom(Sup) ->
-    IsStart1 = erlang:function_exported(Sup, start_child, 1),
-    IsStart0 = erlang:function_exported(Sup, start_child, 0),
+new_worker(Sup, Mod, Index)  ->
+    Node = erlang:node(Sup),
     {ok, Pid} =
-    case {IsStart1, IsStart0} of
-        {true, _} -> Sup:start_child(Index);
-        {_, true} -> Sup:start_child();
-        _ -> supervisor:start_child(Sup, child_args(Sup, Mod, Index))
+    case rpc:pinfo(Sup, registered_name) of
+        {registered_name, Name} ->
+            case function_exported(Node, Name, start_child, 1) of
+                true -> rpc:call(Node, Name, start_child, [Index]);
+                false ->
+                    case function_exported(Node, Name, start_child, 0) of
+                        true -> rpc:call(Node, Name, start_child, []);
+                        false ->
+                            Args = child_args(Sup, Mod, Index),
+                            supervisor:start_child(Sup, Args)
+                    end
+            end;
+        R when R == undefined; R == [] ->
+            Args = child_args(Sup, Mod, Index),
+            supervisor:start_child(Sup, Args)
     end,
     true = link(Pid),
-    Pid;
-new_worker(Sup, Mod, Index) ->
-    {ok, Pid} = supervisor:start_child(Sup, child_args(Sup, Mod, Index)),
-    true = link(Pid),
     Pid.
 
 child_args(Sup, Mod, Index) ->
+    Node = erlang:node(Sup),
     case supervisor:get_childspec(Sup, Mod) of
         {ok, #{start := {M,F,A}}} ->
-            case erlang:function_exported(M, F, length(A) + 1) of
+            case function_exported(Node, M, F, length(A) + 1) of
                 true -> [Index];
-                _ -> []
+                false -> []
+            end;
+        {ok, {_Id, {M,F,A}, _R, _SD, _T, _M}} ->
+            case function_exported(Node, M, F, length(A) + 1) of
+                true -> [Index];
+                false -> []
             end;
         _ -> []
     end.
 
+function_exported(Node, Module, Name, Arity) ->
+    rpc:call(Node, erlang, function_exported, [Module, Name, Arity]).
+
 dismiss_worker(Sup, Pid) ->
     true = unlink(Pid),
     supervisor:terminate_child(Sup, Pid).
@@ -479,7 +487,7 @@ handle_checkin(Pid, State) ->
             gen_server:reply(From, Pid),
             State#state{waiting = Left};
         {empty, Empty} ->
-            try poolboy_collection:replace(Pid, fun(Idx) -> Size + Idx end, Overflow) of
+            try poolboy_collection:replace(Pid, Overflow) of
                 {NewIdx, NewOverflow} ->
                     ok = dismiss_worker(Sup, Pid),
                     NewerOverflow = poolboy_collection:prepend(NewIdx, NewOverflow),
@@ -555,15 +563,11 @@ state_name(State) ->
         _ -> ready
     end.
 
-stop_supervisor(_, undefined) -> ok;
-stop_supervisor(Reason, Atom) when is_atom(Atom) ->
-    stop_supervisor(Reason, whereis(Atom));
 stop_supervisor(Reason, Pid) when is_pid(Pid) ->
     case erlang:node(Pid) of
-        N when N == node() -> exit(Pid, Reason);
-        _ when Reason =/= nodedown -> catch gen_server:stop(Pid, Reason, ?TIMEOUT);
+        N when N == node() ->
+            exit(Pid, Reason);
+        _ when Reason =/= nodedown ->
+            catch gen_server:stop(Pid, Reason, ?TIMEOUT);
         _ -> ok
-    end;
-stop_supervisor(nodedown, Tuple) when is_tuple(Tuple) -> ok;
-stop_supervisor(Reason, Tuple) when is_tuple(Tuple) ->
-    catch gen_server:stop(Tuple, Reason, ?TIMEOUT).
+    end.

From 478abcd6003f07405e31533a0899ee0ee5504dd2 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Sat, 30 May 2020 15:12:49 +0200
Subject: [PATCH 24/36] Strategy alias for prepend and append

---
 src/poolboy.erl            | 24 +++++-------------------
 src/poolboy_collection.erl |  6 ++++--
 2 files changed, 9 insertions(+), 21 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index ebc55b3..57f12fe 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -464,20 +464,13 @@ dismiss_worker(Sup, Pid) ->
     true = unlink(Pid),
     supervisor:terminate_child(Sup, Pid).
 
-replace_worker(Pid, State = #state{strategy = Strategy}) ->
-    {NewWorker, Workers} = poolboy_collection:replace(Pid, State#state.workers),
-    case Strategy of
-        lifo -> State#state{workers = poolboy_collection:prepend(NewWorker, Workers)};
-        fifo -> State#state{workers = poolboy_collection:append(NewWorker, Workers)}
-    end.
-
 handle_checkin(Pid, State) ->
     #state{supervisor = Sup,
            waiting = Waiting,
            monitors = Monitors,
            mrefs = MRefs,
            crefs = CRefs,
-           size = Size,
+           strategy = Strategy,
            overflow = Overflow} = State,
     case queue:out(Waiting) of
         {{value, {From, CRef, MRef}}, Left} ->
@@ -490,15 +483,11 @@ handle_checkin(Pid, State) ->
             try poolboy_collection:replace(Pid, Overflow) of
                 {NewIdx, NewOverflow} ->
                     ok = dismiss_worker(Sup, Pid),
-                    NewerOverflow = poolboy_collection:prepend(NewIdx, NewOverflow),
+                    NewerOverflow = poolboy_collection:Strategy(NewIdx, NewOverflow),
                     State#state{waiting = Empty, overflow = NewerOverflow}
             catch
                 error:enoent ->
-                    Workers =
-                    case State#state.strategy of
-                        lifo -> poolboy_collection:prepend(Pid, State#state.workers);
-                        fifo -> poolboy_collection:append(Pid, State#state.workers)
-                    end,
+                    Workers = poolboy_collection:Strategy(Pid, State#state.workers),
                     State#state{waiting = Empty, workers = Workers}
             end
     end.
@@ -537,12 +526,9 @@ handle_worker_exit(Pid, State) ->
             catch error:enoent ->
                 State
             end;
-        {empty, Empty} when is_pid(NewWorker), Strategy == lifo ->
-            State#state{waiting = Empty,
-                        workers = poolboy_collection:prepend(NewWorker, Workers)};
-        {empty, Empty} when is_pid(NewWorker), Strategy == fifo ->
+        {empty, Empty} when is_pid(NewWorker) ->
             State#state{waiting = Empty,
-                        workers = poolboy_collection:append(NewWorker, Workers)};
+                        workers = poolboy_collection:Strategy(NewWorker, Workers)};
         {empty, Empty} when MaxOverflow > 0 ->
             {Idx, NewOverflow} = poolboy_collection:replace(Pid, Overflow),
             NewerOverflow = poolboy_collection:prepend(Idx, NewOverflow),
diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 2504801..e8935d8 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -4,8 +4,8 @@
          length/2,
          hide_head/1,
          replace/2, replace/3,
-         prepend/2,
-         append/2,
+         lifo/2, prepend/2,
+         fifo/2, append/2,
          filter/2,
          all/2,
          rand/2
@@ -147,6 +147,7 @@ replace(Out, In, Coll = #coll{data = Data, type = T}) ->
             {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
     end.
 
+lifo(In, Coll) -> prepend(In, Coll).
 
 prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data, type = T}) ->
     case maps:get(In, RevIndexes, undefined) of
@@ -159,6 +160,7 @@ prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Dat
     end.
 
 
+fifo(In, Coll) -> append(In, Coll).
 
 append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data, type = T}) ->
     case maps:get(In, RevIndexes, undefined) of

From b209ccc95925d6cf663a5337f69cb8103ca999eb Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Mon, 1 Jun 2020 14:22:08 +0200
Subject: [PATCH 25/36] Monitor external supervisor instead of linking it

---
 src/poolboy.erl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 57f12fe..476b433 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -169,7 +169,7 @@ init({PoolArgs, WorkerArgs}) ->
         undefined ->
             start_supervisor(WorkerModule, WorkerArgs);
         Sup when is_pid(Sup) ->
-            true = link(Sup),
+            monitor(process, Sup),
             Sup
     end,
     Size = pool_size(PoolArgs),
@@ -372,6 +372,8 @@ handle_call(_Msg, _From, State) ->
     Reply = {error, invalid_message},
     {reply, Reply, State}.
 
+handle_info({'DOWN', _, process, Pid, Reason}, State = #state{supervisor = Pid}) ->
+    {stop, Reason, State};
 handle_info({'DOWN', MRef, _, _, _}, State) ->
     case ets:lookup(State#state.mrefs, MRef) of
         [{MRef, Pid}] ->

From 7da8fcafa40d60923bc80efc89c4147f402a289b Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Thu, 18 Jun 2020 00:06:52 +0200
Subject: [PATCH 26/36] Move type from state into data; Change Map of types to
 case-stmt.

---
 src/poolboy_collection.erl | 136 +++++++++++++++++++------------------
 1 file changed, 71 insertions(+), 65 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index e8935d8..0df54c2 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -23,9 +23,14 @@
 -type coll_data() :: list()|{}|array:array()|pid_queue().
 -type coll_data(A) :: list(A)|{A}|array:array(A)|pid_queue(A).
 
+-record(typed_data, {
+          type :: 'list' |'array' |'queue' |'tuple',
+          data :: coll_data() | coll_data(any())
+         }).
+-type typed_data() :: #typed_data{data :: coll_data()}.
+-type typed_data(A) :: #typed_data{data :: coll_data(A)}.
+
 -record(type, {
-          from :: fun((list(A)) -> coll_data(A)),
-          is :: fun((coll_data() | coll_data(any())) -> boolean()),
           len :: fun((coll_data() | coll_data(any())) -> non_neg_integer()),
           nth :: fun((non_neg_integer(), coll_data(A)) -> A),
           prep :: fun((A, coll_data(A)) -> coll_data(A)),
@@ -33,63 +38,54 @@
           filter :: fun((fun((A) -> boolean()), coll_data(A)) -> coll_data(A)),
           replace :: fun((A, non_neg_integer(), A, coll_data(A)) -> coll_data(A))
          }).
--type type() :: #type{}.
 
 -record(coll, {
           item_generator :: fun((non_neg_integer()) -> any()),
-          data :: coll_data() | coll_data(any()),
-          type :: type(),
+          data :: typed_data() | typed_data(any()),
           indexes :: [non_neg_integer()],
           rev_indexes :: #{any()=>non_neg_integer()}
           }).
 
 -type coll() :: #coll{
-                   data :: coll_data(),
+                   data :: typed_data(),
                    rev_indexes :: #{}
                   }.
 -type coll(A) :: #coll{
                     item_generator :: fun((non_neg_integer()) -> A),
-                    data :: coll_data(A),
+                    data :: typed_data(A),
                     rev_indexes :: #{A=>non_neg_integer()}
                     }.
 
 -export_type([coll/0, coll/1]).
 
 
--define(TYPES, #{
-          list => #type{
-                     from = fun(L) -> L end,
-                     is = fun is_list/1,
+-define(TYPES(T),
+        case T of
+          list -> #type{
                      len = fun length/1,
                      nth = fun lists:nth/2,
                      prep = fun(I, L) -> [I|L] end,
                      app = fun(I, L) -> L ++ [I] end,
                      filter = fun lists:filter/2,
                      replace = fun list_replace/4
-                    },
-          array => #type{
-                      from = fun array:from_list/1,
-                      is = fun array:is_array/1,
+                    };
+          array -> #type{
                       len = fun array:size/1,
                       nth = fun(I, A) -> array:get(I-1, A) end,
                       prep = fun array_prep/2,
                       app = fun(I, A) -> array:set(array:size(A), I, A) end,
                       filter = fun array_filter/2,
                       replace = fun array_replace/4
-                     },
-          queue => #type{
-                      from = fun queue:from_list/1,
-                      is = fun queue:is_queue/1,
+                     };
+          queue -> #type{
                       len = fun queue:len/1,
                       nth = fun queue_nth/2,
                       prep = fun queue:in_r/2,
                       app = fun queue:in/2,
                       filter = fun queue:filter/2,
                       replace = fun queue_replace/4
-                     },
-          tuple => #type{
-                      from = fun list_to_tuple/1,
-                      is = fun is_tuple/1,
+                     };
+          tuple -> #type{
                       len = fun tuple_size/1,
                       nth = fun element/2,
                       prep = fun(I, Tu) ->  erlang:insert_element(1, Tu, I) end,
@@ -97,39 +93,52 @@
                       filter = fun tuple_filter/2,
                       replace = fun tuple_replace/4
                      }
-         }).
-
-from(Data, T) -> (T#type.from)(Data).
-is(Data, T) -> (T#type.is)(Data).
-len(Data, T) -> (T#type.len)(Data).
-nth(Index, Data, T) -> (T#type.nth)(Index, Data).
-prep(In, Data, T) -> (T#type.prep)(In, Data).
-app(In, Data, T) -> (T#type.app)(In, Data).
-filter(Fun, Data, T) -> (T#type.filter)(Fun, Data).
-replace(Out, Index, In, Data, T) -> (T#type.replace)(Out, Index, In, Data).
+        end).
+
+from(List, T) when T == list ->
+    #typed_data{type = T, data = List};
+from(List, T) when T == queue ->
+    #typed_data{type = T, data = queue:from_list(List)};
+from(List, T) when T == array ->
+    #typed_data{type = T, data = array:from_list(List)};
+from(List, T) when T == tuple ->
+    #typed_data{type = T, data = list_to_tuple(List)}.
+
+
+len(#typed_data{type = T, data = Data}) ->
+    (?TYPES(T)#type.len)(Data).
+nth(Index, #typed_data{type = T, data = Data}) ->
+    (?TYPES(T)#type.nth)(Index, Data).
+prep(In, TD = #typed_data{type = T, data = Data}) ->
+    TD#typed_data{data = (?TYPES(T)#type.prep)(In, Data)}.
+app(In, TD = #typed_data{type = T, data = Data}) ->
+    TD#typed_data{data = (?TYPES(T)#type.app)(In, Data)}.
+filter(Fun, #coll{data = Data}) -> filter(Fun, Data);
+filter(Fun, TD = #typed_data{type = T, data = Data}) ->
+    TD#typed_data{data = (?TYPES(T)#type.filter)(Fun, Data)}.
+replace(Out, Index, In, TD = #typed_data{type = T, data = Data}) ->
+    TD#typed_data{data = (?TYPES(T)#type.replace)(Out, Index, In, Data)}.
 
 
 new(Type, Size, Fun) when is_function(Fun, 1) ->
-    CollType = maps:get(Type, ?TYPES),
     Indexes = lists:seq(1, Size),
     Items = [Fun(I) || I <- Indexes],
     RevIndexes = maps:from_list(lists:zip(Items, Indexes)),
     #coll{
        item_generator = Fun,
-       data = from(Items, CollType),
-       type = CollType,
+       data = from(Items, Type),
        indexes = Indexes,
        rev_indexes = RevIndexes
       }.
 
 
-length(known, #coll{data=Data, type=T}) -> len(Data, T);
+length(known, #coll{data=Data}) -> len(Data);
 length(visible, #coll{indexes=Indexes}) -> length(Indexes).
 
 
 hide_head(#coll{indexes = []}) -> empty;
-hide_head(Coll = #coll{indexes = [Hd|Tl], data=Data, type=T}) ->
-    {nth(Hd, Data, T), Coll#coll{indexes = Tl}}.
+hide_head(Coll = #coll{indexes = [Hd|Tl], data=Data}) ->
+    {nth(Hd, Data), Coll#coll{indexes = Tl}}.
 
 
 replace(Out, Coll = #coll{item_generator = In}) ->
@@ -137,23 +146,23 @@ replace(Out, Coll = #coll{item_generator = In}) ->
 
 replace(Out, In, Coll) when not is_function(In, 1) ->
     replace(Out, fun(_) -> In end, Coll);
-replace(Out, In, Coll = #coll{data = Data, type = T}) ->
+replace(Out, In, Coll = #coll{data = Data}) ->
     case maps:take(Out, Coll#coll.rev_indexes) of
         error -> error(enoent);
         {OutIndex, RevIndexes} ->
             NewItem = In(OutIndex),
-            NewData = replace(Out, OutIndex, NewItem, Data, T),
+            NewData = replace(Out, OutIndex, NewItem, Data),
             NewRevIndexes = maps:put(NewItem, OutIndex, RevIndexes),
             {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
     end.
 
 lifo(In, Coll) -> prepend(In, Coll).
 
-prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data, type = T}) ->
+prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
     case maps:get(In, RevIndexes, undefined) of
         InIndex when is_integer(InIndex) -> Coll#coll{indexes=[InIndex|Indexes]};
         undefined ->
-            NewData = prep(In, Data, T),
+            NewData = prep(In, Data),
             NewRevIndexes = maps:put(In, 1, maps:map(fun(_, V) -> V + 1 end, RevIndexes)),
             NewIndexes = [1 | [I+1 || I <- Indexes]],
             Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
@@ -162,15 +171,14 @@ prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Dat
 
 fifo(In, Coll) -> append(In, Coll).
 
-append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data, type = T}) ->
+append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
     case maps:get(In, RevIndexes, undefined) of
         InIndex when is_integer(InIndex) -> Coll#coll{indexes=Indexes++[InIndex]};
         undefined ->
-            NewData = app(In, Data, T),
-            ArrayT = maps:get(array, ?TYPES),
+            NewData = app(In, Data),
             NewIndex =
-            case {is(Data, ArrayT), len(Data, T)} of
-                {true, Len} -> Len - 1;
+            case {Data#typed_data.type, len(Data)} of
+                {array, Len} -> Len - 1;
                 {_, Len} -> Len
             end,
             NewRevIndexes = maps:put(In, NewIndex, RevIndexes),
@@ -179,24 +187,20 @@ append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data
     end.
 
 
-filter(Fun, #coll{data = Data, type = T}) ->
-    filter(Fun, Data, T).
-
-
 all(known, #coll{rev_indexes = RevIndexes}) ->
     maps:keys(RevIndexes);
-all(visible, #coll{indexes = Indexes, data = Data, type = T}) ->
-    [nth(I, Data, T) || I <- Indexes].
+all(visible, #coll{indexes = Indexes, data = Data}) ->
+    [nth(I, Data) || I <- Indexes].
 
 
-rand(known, #coll{data = Data, type = T}) ->
-    case len(Data, T) of
+rand(known, #coll{data = Data}) ->
+    case len(Data) of
         0 -> empty;
-        L -> nth(rand:uniform(L), Data, T)
+        L -> nth(rand:uniform(L), Data)
     end;
 rand(visible, #coll{indexes = []}) -> empty;
-rand(visible, #coll{indexes = Indexes, data = Data, type = T}) ->
-    nth(lists:nth(rand:uniform(length(Indexes)), Indexes), Data, T).
+rand(visible, #coll{indexes = Indexes, data = Data}) ->
+    nth(lists:nth(rand:uniform(length(Indexes)), Indexes), Data).
 
 
 
@@ -235,12 +239,14 @@ array_replace(O, X, I, A) ->
 
 
 
-queue_nth(I, {_RL, FL}) when I =< length(FL) ->
-    lists:nth(I, FL);
-queue_nth(I, {RL, FL}) ->
-    J = length(RL)-(I-length(FL)-1),
-    lists:nth(J, RL).
-
+queue_nth(I, Q) ->
+    case queue:is_queue(Q) of
+        true ->
+            {Q1, _Q2} = queue:split(I, Q),
+            queue:last(Q1);
+        _ ->
+            throw(badarg)
+    end.
 
 queue_replace(O, X, I, Q) ->
     {Q1, Q2} = queue:split(X-1, Q),

From d1d873f3df674cf598472b5dfef2d1fa468f69ba Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 19 Jun 2020 00:38:40 +0200
Subject: [PATCH 27/36] Use for typed_data for Indexes too

---
 src/poolboy_collection.erl | 88 ++++++++++++++++++++++++++------------
 1 file changed, 61 insertions(+), 27 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 0df54c2..21b050c 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -31,18 +31,20 @@
 -type typed_data(A) :: #typed_data{data :: coll_data(A)}.
 
 -record(type, {
+          out :: fun((coll_data() | coll_data(any())) -> {{'value', any()}, coll_data(any())} | {'empty', coll_data()}),
           len :: fun((coll_data() | coll_data(any())) -> non_neg_integer()),
           nth :: fun((non_neg_integer(), coll_data(A)) -> A),
           prep :: fun((A, coll_data(A)) -> coll_data(A)),
           app :: fun((A, coll_data(A)) -> coll_data(A)),
           filter :: fun((fun((A) -> boolean()), coll_data(A)) -> coll_data(A)),
-          replace :: fun((A, non_neg_integer(), A, coll_data(A)) -> coll_data(A))
+          replace :: fun((A, non_neg_integer(), A, coll_data(A)) -> coll_data(A)),
+          to :: fun((coll_data() | coll_data(any())) -> [any()])
          }).
 
 -record(coll, {
           item_generator :: fun((non_neg_integer()) -> any()),
           data :: typed_data() | typed_data(any()),
-          indexes :: [non_neg_integer()],
+          indexes :: typed_data() | typed_data(any()),
           rev_indexes :: #{any()=>non_neg_integer()}
           }).
 
@@ -62,36 +64,57 @@
 -define(TYPES(T),
         case T of
           list -> #type{
-                     len = fun length/1,
-                     nth = fun lists:nth/2,
-                     prep = fun(I, L) -> [I|L] end,
-                     app = fun(I, L) -> L ++ [I] end,
-                     filter = fun lists:filter/2,
-                     replace = fun list_replace/4
-                    };
+                      out = fun([] = L) -> {empty, L};
+                               ([Hd|Tl]) -> {{value, Hd}, Tl} end,
+                      len = fun length/1,
+                      nth = fun lists:nth/2,
+                      prep = fun(I, L) -> [I|L] end,
+                      app = fun(I, L) -> L ++ [I] end,
+                      filter = fun lists:filter/2,
+                      replace = fun list_replace/4,
+                      to = fun(L) -> L end
+                     };
           array -> #type{
+                      out = fun(A) ->
+                                case array:size(A) of
+                                    0 -> {empty, A};
+                                    _ -> {{value, array:get(0, A)},
+                                          array:reset(0, A)}
+                                end
+                            end,
                       len = fun array:size/1,
                       nth = fun(I, A) -> array:get(I-1, A) end,
                       prep = fun array_prep/2,
                       app = fun(I, A) -> array:set(array:size(A), I, A) end,
                       filter = fun array_filter/2,
-                      replace = fun array_replace/4
+                      replace = fun array_replace/4,
+                      to = fun array:to_list/1
                      };
           queue -> #type{
+                      out = fun queue:out/1,
                       len = fun queue:len/1,
                       nth = fun queue_nth/2,
                       prep = fun queue:in_r/2,
                       app = fun queue:in/2,
                       filter = fun queue:filter/2,
-                      replace = fun queue_replace/4
+                      replace = fun queue_replace/4,
+                      to = fun queue:to_list/1
                      };
           tuple -> #type{
+                      out = fun(Tu) ->
+                                case erlang:tuple_size(Tu) of
+                                    0 -> {empty, Tu};
+                                    _ -> {{value, element(1, Tu)},
+                                            erlang:delete_element(1, Tu)}
+                                end
+                            end,
                       len = fun tuple_size/1,
                       nth = fun element/2,
                       prep = fun(I, Tu) ->  erlang:insert_element(1, Tu, I) end,
                       app = fun(I, Tu) -> erlang:append_element(Tu, I) end,
                       filter = fun tuple_filter/2,
-                      replace = fun tuple_replace/4
+                      replace = fun tuple_replace/4,
+                      to = fun erlang:tuple_to_list/1
                      }
         end).
 
@@ -105,6 +128,9 @@ from(List, T) when T == tuple ->
     #typed_data{type = T, data = list_to_tuple(List)}.
 
 
+out(TD = #typed_data{type = T, data = Data}) ->
+    {V, D} = (?TYPES(T)#type.out)(Data),
+    {V, TD#typed_data{data = D}}.
 len(#typed_data{type = T, data = Data}) ->
     (?TYPES(T)#type.len)(Data).
 nth(Index, #typed_data{type = T, data = Data}) ->
@@ -118,6 +144,8 @@ filter(Fun, TD = #typed_data{type = T, data = Data}) ->
     TD#typed_data{data = (?TYPES(T)#type.filter)(Fun, Data)}.
 replace(Out, Index, In, TD = #typed_data{type = T, data = Data}) ->
     TD#typed_data{data = (?TYPES(T)#type.replace)(Out, Index, In, Data)}.
+to(#typed_data{type = T, data = Data}) ->
+    (?TYPES(T)#type.to)(Data).
 
 
 new(Type, Size, Fun) when is_function(Fun, 1) ->
@@ -127,18 +155,22 @@ new(Type, Size, Fun) when is_function(Fun, 1) ->
     #coll{
        item_generator = Fun,
        data = from(Items, Type),
-       indexes = Indexes,
+       indexes = from(Indexes, queue),
        rev_indexes = RevIndexes
       }.
 
 
 length(known, #coll{data=Data}) -> len(Data);
-length(visible, #coll{indexes=Indexes}) -> length(Indexes).
+length(visible, #coll{indexes=Indexes}) -> len(Indexes).
+
 
+hide_head(Coll = #coll{indexes = Indexes, data=Data}) ->
+    case out(Indexes) of
+        {empty, _} -> empty;
+        {{value, Hd}, Tl} ->
+            {nth(Hd, Data), Coll#coll{indexes = Tl}}
+    end.
 
-hide_head(#coll{indexes = []}) -> empty;
-hide_head(Coll = #coll{indexes = [Hd|Tl], data=Data}) ->
-    {nth(Hd, Data), Coll#coll{indexes = Tl}}.
 
 
 replace(Out, Coll = #coll{item_generator = In}) ->
@@ -160,11 +192,11 @@ lifo(In, Coll) -> prepend(In, Coll).
 
 prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
     case maps:get(In, RevIndexes, undefined) of
-        InIndex when is_integer(InIndex) -> Coll#coll{indexes=[InIndex|Indexes]};
+        InIndex when is_integer(InIndex) -> Coll#coll{indexes = prep(InIndex, Indexes)};
         undefined ->
             NewData = prep(In, Data),
             NewRevIndexes = maps:put(In, 1, maps:map(fun(_, V) -> V + 1 end, RevIndexes)),
-            NewIndexes = [1 | [I+1 || I <- Indexes]],
+            NewIndexes = prep(1, filter(fun(I) -> [I + 1] end, Indexes)),
             Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
     end.
 
@@ -173,16 +205,16 @@ fifo(In, Coll) -> append(In, Coll).
 
 append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
     case maps:get(In, RevIndexes, undefined) of
-        InIndex when is_integer(InIndex) -> Coll#coll{indexes=Indexes++[InIndex]};
+        InIndex when is_integer(InIndex) -> Coll#coll{indexes = app(InIndex, Indexes)};
         undefined ->
             NewData = app(In, Data),
-            NewIndex =
+            InIndex =
             case {Data#typed_data.type, len(Data)} of
                 {array, Len} -> Len - 1;
                 {_, Len} -> Len
             end,
-            NewRevIndexes = maps:put(In, NewIndex, RevIndexes),
-            NewIndexes = Indexes++[NewIndex],
+            NewRevIndexes = maps:put(In, InIndex, RevIndexes),
+            NewIndexes = app(InIndex, Indexes),
             Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
     end.
 
@@ -190,7 +222,7 @@ append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data
 all(known, #coll{rev_indexes = RevIndexes}) ->
     maps:keys(RevIndexes);
 all(visible, #coll{indexes = Indexes, data = Data}) ->
-    [nth(I, Data) || I <- Indexes].
+    to(filter(fun(I) -> [nth(I, Data)] end, Indexes)).
 
 
 rand(known, #coll{data = Data}) ->
@@ -198,9 +230,11 @@ rand(known, #coll{data = Data}) ->
         0 -> empty;
         L -> nth(rand:uniform(L), Data)
     end;
-rand(visible, #coll{indexes = []}) -> empty;
 rand(visible, #coll{indexes = Indexes, data = Data}) ->
-    nth(lists:nth(rand:uniform(length(Indexes)), Indexes), Data).
+    case len(Indexes) of
+        0 -> empty;
+        L -> nth(nth(rand:uniform(L), Indexes), Data)
+    end.
 
 
 
@@ -256,7 +290,7 @@ queue_replace(O, X, I, Q) ->
 
 
 tuple_filter(Fun, Tuple) ->
-    tuple_filter(Fun, Tuple, erlang:tuple_size(Tuple)).
+    tuple_filter(Fun, Tuple, tuple_size(Tuple)).
 
 tuple_filter(_Fun, Tuple, 0) -> Tuple;
 tuple_filter(Fun, Tuple, Index) ->

From b9ba9bcbb091e906b30838fe9227c2ece32adc81 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 19 Jun 2020 01:07:28 +0200
Subject: [PATCH 28/36] Split typed-data and worker collections into separate
 modules

---
 src/poolboy.erl                   |  52 ++++++-------
 src/poolboy_collection.erl        | 123 +-----------------------------
 src/poolboy_worker_collection.erl | 108 ++++++++++++++++++++++++++
 3 files changed, 136 insertions(+), 147 deletions(-)
 create mode 100644 src/poolboy_worker_collection.erl

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 476b433..679b260 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -38,13 +38,13 @@
 -record(state, {
     supervisor :: pid(),
     worker_module :: atom(),
-    workers :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
+    workers :: poolboy_worker_collection:coll() | poolboy_worker_collection:coll(pid()),
     waiting :: poolboy_collection:pid_queue() | poolboy_collection:pid_queue(tuple()),
     monitors :: ets:tid(),
     mrefs :: ets:tid(),
     crefs :: ets:tid(),
     size = ?DEFAULT_SIZE :: non_neg_integer(),
-    overflow :: poolboy_collection:coll() | poolboy_collection:coll(pid()),
+    overflow :: poolboy_worker_collection:coll() | poolboy_worker_collection:coll(pid()),
     max_overflow = ?DEFAULT_OVERFLOW :: non_neg_integer(),
     strategy = ?DEFAULT_STRATEGY :: lifo | fifo
 }).
@@ -218,11 +218,11 @@ start_supervisor(WorkerModule, WorkerArgs, Retries) ->
 
 init_workers(Sup, Mod, Size, Type) ->
     Fun = fun(Idx) -> new_worker(Sup, Mod, Idx) end,
-    poolboy_collection:new(Type, Size, Fun).
+    poolboy_worker_collection:new(Type, Size, Fun).
 
 init_overflow(Size, MaxOverflow, Type) ->
     Fun = fun(Idx) -> Size + Idx end,
-    poolboy_collection:new(Type, MaxOverflow, Fun).
+    poolboy_worker_collection:new(Type, MaxOverflow, Fun).
 
 worker_module(PoolArgs) ->
     Is = is_atom(V = proplists:get_value(worker_module, PoolArgs)),
@@ -305,8 +305,8 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
            crefs = CRefs,
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
-    OverflowLeft = poolboy_collection:length(visible, Overflow),
-    case poolboy_collection:hide_head(Workers) of
+    OverflowLeft = poolboy_worker_collection:length(visible, Overflow),
+    case poolboy_worker_collection:hide_head(Workers) of
         {Pid, Left} when is_pid(Pid) ->
             MRef = erlang:monitor(process, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
@@ -314,9 +314,9 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
             true = ets:insert(CRefs, {CRef, Pid}),
             {reply, Pid, State#state{workers = Left}};
         empty when MaxOverflow > 0, OverflowLeft > 0 ->
-            {NextIdx, NewOverflow} = poolboy_collection:hide_head(Overflow),
+            {NextIdx, NewOverflow} = poolboy_worker_collection:hide_head(Overflow),
             Pid = new_worker(Sup, Mod, NextIdx),
-            {Pid, NewerOverflow} = poolboy_collection:replace(NextIdx, Pid, NewOverflow),
+            {Pid, NewerOverflow} = poolboy_worker_collection:replace(NextIdx, Pid, NewOverflow),
             MRef = erlang:monitor(process, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
             true = ets:insert(MRefs, {MRef, Pid}),
@@ -336,10 +336,10 @@ handle_call(status_map, _From, State) ->
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
     StateName = state_name(State),
-    OverflowLeft = poolboy_collection:length(visible, Overflow),
+    OverflowLeft = poolboy_worker_collection:length(visible, Overflow),
     OverflowLevel = MaxOverflow - OverflowLeft,
     {reply, #{state => StateName,
-              available => poolboy_collection:length(visible, Workers),
+              available => poolboy_worker_collection:length(visible, Workers),
               overflow => OverflowLevel,
               monitored => ets:info(Monitors, size),
               waiting => queue:len(State#state.waiting)}, State};
@@ -349,15 +349,15 @@ handle_call(status, _From, State) ->
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
     StateName = state_name(State),
-    VisibleWorkers = poolboy_collection:length(visible, Workers),
-    OverflowLeft = poolboy_collection:length(visible, Overflow),
+    VisibleWorkers = poolboy_worker_collection:length(visible, Workers),
+    OverflowLeft = poolboy_worker_collection:length(visible, Overflow),
     OverflowLevel = MaxOverflow - OverflowLeft,
     MonitorSize = ets:info(Monitors, size),
     {reply, {StateName, VisibleWorkers, OverflowLevel, MonitorSize}, State};
 handle_call(get_avail_workers, _From, State) ->
-    {reply, poolboy_collection:all(visible, State#state.workers), State};
+    {reply, poolboy_worker_collection:all(visible, State#state.workers), State};
 handle_call(get_any_worker, _From, State) ->
-    {reply, poolboy_collection:rand(known, State#state.workers), State};
+    {reply, poolboy_worker_collection:rand(known, State#state.workers), State};
 handle_call(get_all_workers, _From, State) ->
     Sup = State#state.supervisor,
     WorkerList = supervisor:which_children(Sup),
@@ -406,7 +406,7 @@ handle_info(_Info, State) ->
     {noreply, State}.
 
 terminate(Reason, State = #state{supervisor = Sup}) ->
-    poolboy_collection:filter(fun (W) -> catch not unlink(W) end, State#state.workers),
+    poolboy_worker_collection:filter(fun (W) -> catch not unlink(W) end, State#state.workers),
     stop_supervisor(Reason, Sup),
     ok.
 
@@ -482,14 +482,14 @@ handle_checkin(Pid, State) ->
             gen_server:reply(From, Pid),
             State#state{waiting = Left};
         {empty, Empty} ->
-            try poolboy_collection:replace(Pid, Overflow) of
+            try poolboy_worker_collection:replace(Pid, Overflow) of
                 {NewIdx, NewOverflow} ->
                     ok = dismiss_worker(Sup, Pid),
-                    NewerOverflow = poolboy_collection:Strategy(NewIdx, NewOverflow),
+                    NewerOverflow = poolboy_worker_collection:Strategy(NewIdx, NewOverflow),
                     State#state{waiting = Empty, overflow = NewerOverflow}
             catch
                 error:enoent ->
-                    Workers = poolboy_collection:Strategy(Pid, State#state.workers),
+                    Workers = poolboy_worker_collection:Strategy(Pid, State#state.workers),
                     State#state{waiting = Empty, workers = Workers}
             end
     end.
@@ -505,10 +505,10 @@ handle_worker_exit(Pid, State) ->
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
     {NewWorker, Workers} =
-    try poolboy_collection:replace(Pid, State#state.workers)
+    try poolboy_worker_collection:replace(Pid, State#state.workers)
     catch error:enoent -> {enoent, State#state.workers}
     end,
-    OverflowLeft = poolboy_collection:length(visible, Overflow),
+    OverflowLeft = poolboy_worker_collection:length(visible, Overflow),
     case queue:out(State#state.waiting) of
         {{value, {From, CRef, MRef}}, LeftWaiting} when is_pid(NewWorker) ->
             true = ets:insert(Monitors, {NewWorker, CRef, MRef}),
@@ -519,7 +519,7 @@ handle_worker_exit(Pid, State) ->
         {{value, {From, CRef, MRef}}, LeftWaiting} when MaxOverflow > OverflowLeft ->
             try
                 NewFun = fun(Idx) -> new_worker(Sup, Mod, Size + Idx) end,
-                {NewPid, NewOverflow} = poolboy_collection:replace(Pid, NewFun, Overflow),
+                {NewPid, NewOverflow} = poolboy_worker_collection:replace(Pid, NewFun, Overflow),
                 true = ets:insert(Monitors, {NewPid, CRef, MRef}),
                 true = ets:insert(MRefs, {MRef, Pid}),
                 true = ets:insert(CRefs, {CRef, Pid}),
@@ -530,10 +530,10 @@ handle_worker_exit(Pid, State) ->
             end;
         {empty, Empty} when is_pid(NewWorker) ->
             State#state{waiting = Empty,
-                        workers = poolboy_collection:Strategy(NewWorker, Workers)};
+                        workers = poolboy_worker_collection:Strategy(NewWorker, Workers)};
         {empty, Empty} when MaxOverflow > 0 ->
-            {Idx, NewOverflow} = poolboy_collection:replace(Pid, Overflow),
-            NewerOverflow = poolboy_collection:prepend(Idx, NewOverflow),
+            {Idx, NewOverflow} = poolboy_worker_collection:replace(Pid, Overflow),
+            NewerOverflow = poolboy_worker_collection:prepend(Idx, NewOverflow),
             State#state{waiting = Empty, overflow = NewerOverflow}
     end.
 
@@ -541,10 +541,10 @@ state_name(State) ->
     #state{workers = Workers,
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
-    case poolboy_collection:length(visible, Workers) of
+    case poolboy_worker_collection:length(visible, Workers) of
         0 when MaxOverflow < 1 -> full;
         0 ->
-            case poolboy_collection:length(visible, Overflow) of
+            case poolboy_worker_collection:length(visible, Overflow) of
                 0 -> full;
                 _ -> overflow
             end;
diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 21b050c..59fa4b2 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -1,15 +1,6 @@
 -module(poolboy_collection).
 
--export([new/3,
-         length/2,
-         hide_head/1,
-         replace/2, replace/3,
-         lifo/2, prepend/2,
-         fifo/2, append/2,
-         filter/2,
-         all/2,
-         rand/2
-        ]).
+-export([from/2, out/1, len/1, nth/2, prep/2, app/2, filter/2, replace/4, to/1]).
 
 -ifdef(pre17).
 -type pid_queue() :: queue().
@@ -29,6 +20,7 @@
          }).
 -type typed_data() :: #typed_data{data :: coll_data()}.
 -type typed_data(A) :: #typed_data{data :: coll_data(A)}.
+-export_type([typed_data/0, typed_data/1]).
 
 -record(type, {
           out :: fun((coll_data() | coll_data(any())) -> {{'value', any()}, coll_data(any())} | {'empty', coll_data()}),
@@ -41,26 +33,6 @@
           to :: fun((coll_data() | coll_data(any())) -> [any()])
          }).
 
--record(coll, {
-          item_generator :: fun((non_neg_integer()) -> any()),
-          data :: typed_data() | typed_data(any()),
-          indexes :: typed_data() | typed_data(any()),
-          rev_indexes :: #{any()=>non_neg_integer()}
-          }).
-
--type coll() :: #coll{
-                   data :: typed_data(),
-                   rev_indexes :: #{}
-                  }.
--type coll(A) :: #coll{
-                    item_generator :: fun((non_neg_integer()) -> A),
-                    data :: typed_data(A),
-                    rev_indexes :: #{A=>non_neg_integer()}
-                    }.
-
--export_type([coll/0, coll/1]).
-
-
 -define(TYPES(T),
         case T of
           list -> #type{
@@ -139,7 +111,6 @@ prep(In, TD = #typed_data{type = T, data = Data}) ->
     TD#typed_data{data = (?TYPES(T)#type.prep)(In, Data)}.
 app(In, TD = #typed_data{type = T, data = Data}) ->
     TD#typed_data{data = (?TYPES(T)#type.app)(In, Data)}.
-filter(Fun, #coll{data = Data}) -> filter(Fun, Data);
 filter(Fun, TD = #typed_data{type = T, data = Data}) ->
     TD#typed_data{data = (?TYPES(T)#type.filter)(Fun, Data)}.
 replace(Out, Index, In, TD = #typed_data{type = T, data = Data}) ->
@@ -148,96 +119,6 @@ to(#typed_data{type = T, data = Data}) ->
     (?TYPES(T)#type.to)(Data).
 
 
-new(Type, Size, Fun) when is_function(Fun, 1) ->
-    Indexes = lists:seq(1, Size),
-    Items = [Fun(I) || I <- Indexes],
-    RevIndexes = maps:from_list(lists:zip(Items, Indexes)),
-    #coll{
-       item_generator = Fun,
-       data = from(Items, Type),
-       indexes = from(Indexes, queue),
-       rev_indexes = RevIndexes
-      }.
-
-
-length(known, #coll{data=Data}) -> len(Data);
-length(visible, #coll{indexes=Indexes}) -> len(Indexes).
-
-
-hide_head(Coll = #coll{indexes = Indexes, data=Data}) ->
-    case out(Indexes) of
-        {empty, _} -> empty;
-        {{value, Hd}, Tl} ->
-            {nth(Hd, Data), Coll#coll{indexes = Tl}}
-    end.
-
-
-
-replace(Out, Coll = #coll{item_generator = In}) ->
-    replace(Out, In, Coll).
-
-replace(Out, In, Coll) when not is_function(In, 1) ->
-    replace(Out, fun(_) -> In end, Coll);
-replace(Out, In, Coll = #coll{data = Data}) ->
-    case maps:take(Out, Coll#coll.rev_indexes) of
-        error -> error(enoent);
-        {OutIndex, RevIndexes} ->
-            NewItem = In(OutIndex),
-            NewData = replace(Out, OutIndex, NewItem, Data),
-            NewRevIndexes = maps:put(NewItem, OutIndex, RevIndexes),
-            {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
-    end.
-
-lifo(In, Coll) -> prepend(In, Coll).
-
-prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
-    case maps:get(In, RevIndexes, undefined) of
-        InIndex when is_integer(InIndex) -> Coll#coll{indexes = prep(InIndex, Indexes)};
-        undefined ->
-            NewData = prep(In, Data),
-            NewRevIndexes = maps:put(In, 1, maps:map(fun(_, V) -> V + 1 end, RevIndexes)),
-            NewIndexes = prep(1, filter(fun(I) -> [I + 1] end, Indexes)),
-            Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
-    end.
-
-
-fifo(In, Coll) -> append(In, Coll).
-
-append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = Data}) ->
-    case maps:get(In, RevIndexes, undefined) of
-        InIndex when is_integer(InIndex) -> Coll#coll{indexes = app(InIndex, Indexes)};
-        undefined ->
-            NewData = app(In, Data),
-            InIndex =
-            case {Data#typed_data.type, len(Data)} of
-                {array, Len} -> Len - 1;
-                {_, Len} -> Len
-            end,
-            NewRevIndexes = maps:put(In, InIndex, RevIndexes),
-            NewIndexes = app(InIndex, Indexes),
-            Coll#coll{indexes = NewIndexes, rev_indexes = NewRevIndexes, data = NewData}
-    end.
-
-
-all(known, #coll{rev_indexes = RevIndexes}) ->
-    maps:keys(RevIndexes);
-all(visible, #coll{indexes = Indexes, data = Data}) ->
-    to(filter(fun(I) -> [nth(I, Data)] end, Indexes)).
-
-
-rand(known, #coll{data = Data}) ->
-    case len(Data) of
-        0 -> empty;
-        L -> nth(rand:uniform(L), Data)
-    end;
-rand(visible, #coll{indexes = Indexes, data = Data}) ->
-    case len(Indexes) of
-        0 -> empty;
-        L -> nth(nth(rand:uniform(L), Indexes), Data)
-    end.
-
-
-
 
 
 list_replace(O, X, I, L) ->
diff --git a/src/poolboy_worker_collection.erl b/src/poolboy_worker_collection.erl
new file mode 100644
index 0000000..591ef27
--- /dev/null
+++ b/src/poolboy_worker_collection.erl
@@ -0,0 +1,108 @@
+-module(poolboy_worker_collection).
+
+-export([new/3,
+         length/2,
+         hide_head/1,
+         replace/2, replace/3,
+         lifo/2, prepend/2,
+         fifo/2, append/2,
+         filter/2,
+         all/2,
+         rand/2
+        ]).
+
+
+-record(coll, {
+          item_generator :: fun((non_neg_integer()) -> any()),
+          data :: poolboy_collection:typed_data() | poolboy_collection:typed_data(any()),
+          indexes :: poolboy_collection:typed_data() | poolboy_collection:typed_data(any()),
+          rev_indexes :: #{any()=>non_neg_integer()}
+          }).
+
+-type coll() :: #coll{
+                   data :: poolboy_collection:typed_data(),
+                   rev_indexes :: #{}
+                  }.
+-type coll(A) :: #coll{
+                    item_generator :: fun((non_neg_integer()) -> A),
+                    data :: poolboy_collection:typed_data(A),
+                    rev_indexes :: #{A=>non_neg_integer()}
+                    }.
+
+-export_type([coll/0, coll/1]).
+
+
+new(Type, Size, Fun) when is_function(Fun, 1) ->
+    Indexes = lists:seq(1, Size),
+    Items = [Fun(I) || I <- Indexes],
+    RevIndexes = maps:from_list(lists:zip(Items, Indexes)),
+    #coll{
+       item_generator = Fun,
+       data = poolboy_collection:from(Items, Type),
+       indexes = poolboy_collection:from(Indexes, queue),
+       rev_indexes = RevIndexes
+      }.
+
+
+length(known, #coll{data=Data}) -> poolboy_collection:len(Data);
+length(visible, #coll{indexes=Indexes}) -> poolboy_collection:len(Indexes).
+
+
+hide_head(Coll = #coll{indexes = Indexes, data=Data}) ->
+    case poolboy_collection:out(Indexes) of
+        {empty, _} -> empty;
+        {{value, Hd}, Tl} ->
+            {poolboy_collection:nth(Hd, Data), Coll#coll{indexes = Tl}}
+    end.
+
+
+
+replace(Out, Coll = #coll{item_generator = In}) ->
+    replace(Out, In, Coll).
+
+replace(Out, In, Coll) when not is_function(In, 1) ->
+    replace(Out, fun(_) -> In end, Coll);
+replace(Out, In, Coll = #coll{data = Data}) ->
+    case maps:take(Out, Coll#coll.rev_indexes) of
+        error -> error(enoent);
+        {OutIndex, RevIndexes} ->
+            NewItem = In(OutIndex),
+            NewData = poolboy_collection:replace(Out, OutIndex, NewItem, Data),
+            NewRevIndexes = maps:put(NewItem, OutIndex, RevIndexes),
+            {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
+    end.
+
+lifo(In, Coll) -> prepend(In, Coll).
+
+prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = _Data}) ->
+    case maps:get(In, RevIndexes, undefined) of
+        InIndex when is_integer(InIndex) -> Coll#coll{indexes = poolboy_collection:prep(InIndex, Indexes)}
+    end.
+
+
+fifo(In, Coll) -> append(In, Coll).
+
+append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = _Data}) ->
+    case maps:get(In, RevIndexes, undefined) of
+        InIndex when is_integer(InIndex) -> Coll#coll{indexes = poolboy_collection:app(InIndex, Indexes)}
+    end.
+
+filter(Fun, #coll{data = Data}) ->
+    poolboy_collection:filter(Fun, Data).
+
+all(known, #coll{rev_indexes = RevIndexes}) ->
+    maps:keys(RevIndexes);
+all(visible, #coll{indexes = Indexes, data = Data}) ->
+    poolboy_collection:to(poolboy_collection:filter(fun(I) -> [poolboy_collection:nth(I, Data)] end, Indexes)).
+
+
+rand(known, #coll{data = Data}) ->
+    case poolboy_collection:len(Data) of
+        0 -> empty;
+        L -> poolboy_collection:nth(rand:uniform(L), Data)
+    end;
+rand(visible, #coll{indexes = Indexes, data = Data}) ->
+    case poolboy_collection:len(Indexes) of
+        0 -> empty;
+        L -> poolboy_collection:nth(poolboy_collection:nth(rand:uniform(L), Indexes), Data)
+    end.

From e5fa06569b33fecdf5c0ebba3248520083eb314f Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Sun, 5 Jul 2020 10:58:22 +0200
Subject: [PATCH 29/36] Move TYPES macro and record to direct application in
 function-clauses

---
 src/poolboy_collection.erl | 185 +++++++++++++++++++------------------
 1 file changed, 96 insertions(+), 89 deletions(-)

diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 59fa4b2..8364516 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -22,105 +22,112 @@
 -type typed_data(A) :: #typed_data{data :: coll_data(A)}.
 -export_type([typed_data/0, typed_data/1]).
 
--record(type, {
-          out :: fun((coll_data() | coll_data(any())) -> {{'value', any()}, coll_data(any())} | {'empty', coll_data()}),
-          len :: fun((coll_data() | coll_data(any())) -> non_neg_integer()),
-          nth :: fun((non_neg_integer(), coll_data(A)) -> A),
-          prep :: fun((A, coll_data(A)) -> coll_data(A)),
-          app :: fun((A, coll_data(A)) -> coll_data(A)),
-          filter :: fun((fun((A) -> boolean()), coll_data(A)) -> coll_data(A)),
-          replace :: fun((A, non_neg_integer(), A, coll_data(A)) -> coll_data(A)),
-          to :: fun((coll_data() | coll_data(any())) -> [any()])
-         }).
 
--define(TYPES(T),
-        case T of
-          list -> #type{
-                      out = fun([] = L) -> {empty, L};
-                               ([Hd|Tl]) -> {{value, Hd}, Tl} end,
-                      len = fun length/1,
-                      nth = fun lists:nth/2,
-                      prep = fun(I, L) -> [I|L] end,
-                      app = fun(I, L) -> L ++ [I] end,
-                      filter = fun lists:filter/2,
-                      replace = fun list_replace/4,
-                      to = fun(L) -> L end
-                     };
-          array -> #type{
-                      out = fun(A) ->
-                                case array:size(A) of
-                                    0 -> {empty, A};
-                                    _ -> {{value, array:get(0, A)},
-                                          array:reset(0, A)}
-                                end
-                            end,
-                      len = fun array:size/1,
-                      nth = fun(I, A) -> array:get(I-1, A) end,
-                      prep = fun array_prep/2,
-                      app = fun(I, A) -> array:set(array:size(A), I, A) end,
-                      filter = fun array_filter/2,
-                      replace = fun array_replace/4,
-                      to = fun array:to_list/1
-                     };
-          queue -> #type{
-                      out = fun queue:out/1,
-                      len = fun queue:len/1,
-                      nth = fun queue_nth/2,
-                      prep = fun queue:in_r/2,
-                      app = fun queue:in/2,
-                      filter = fun queue:filter/2,
-                      replace = fun queue_replace/4,
-                      to = fun queue:to_list/1
-                     };
-          tuple -> #type{
-                      out = fun(Tu) ->
-                                case erlang:tuple_size(Tu) of
-                                    0 -> {empty, Tu};
-                                    _ -> {{value, element(1, Tu)},
-                                            erlang:delete_element(1, Tu)}
-                                end
-                            end,
-                      len = fun tuple_size/1,
-                      nth = fun element/2,
-                      prep = fun(I, Tu) ->  erlang:insert_element(1, Tu, I) end,
-                      app = fun(I, Tu) -> erlang:append_element(Tu, I) end,
-                      filter = fun tuple_filter/2,
-                      replace = fun tuple_replace/4,
-                      to = fun erlang:tuple_to_list/1
-                     }
-        end).
-
-from(List, T) when T == list ->
+from(List, T = list) ->
     #typed_data{type = T, data = List};
-from(List, T) when T == queue ->
+from(List, T = queue) ->
     #typed_data{type = T, data = queue:from_list(List)};
-from(List, T) when T == array ->
+from(List, T = array) ->
     #typed_data{type = T, data = array:from_list(List)};
-from(List, T) when T == tuple ->
+from(List, T = tuple) ->
     #typed_data{type = T, data = list_to_tuple(List)}.
 
 
-out(TD = #typed_data{type = T, data = Data}) ->
-    {V, D} = (?TYPES(T)#type.out)(Data),
-    {V, TD#typed_data{data = D}}.
-len(#typed_data{type = T, data = Data}) ->
-    (?TYPES(T)#type.len)(Data).
-nth(Index, #typed_data{type = T, data = Data}) ->
-    (?TYPES(T)#type.nth)(Index, Data).
-prep(In, TD = #typed_data{type = T, data = Data}) ->
-    TD#typed_data{data = (?TYPES(T)#type.prep)(In, Data)}.
-app(In, TD = #typed_data{type = T, data = Data}) ->
-    TD#typed_data{data = (?TYPES(T)#type.app)(In, Data)}.
-filter(Fun, TD = #typed_data{type = T, data = Data}) ->
-    TD#typed_data{data = (?TYPES(T)#type.filter)(Fun, Data)}.
-replace(Out, Index, In, TD = #typed_data{type = T, data = Data}) ->
-    TD#typed_data{data = (?TYPES(T)#type.replace)(Out, Index, In, Data)}.
-to(#typed_data{type = T, data = Data}) ->
-    (?TYPES(T)#type.to)(Data).
+out(TD = #typed_data{data = Data, type = list}) ->
+    case Data of
+        [] = L -> out(TD, {empty, L});
+        [H|T] -> out(TD, {{value, H}, T})
+    end;
+out(TD = #typed_data{data = Data, type = queue}) ->
+    out(TD, queue:out(Data));
+out(TD = #typed_data{data = Data, type = array}) ->
+    case array:sparse_size(Data) of
+        0 -> out(TD, {empty, Data});
+        _ -> out(TD, {{value, array:get(0, Data)}, array:reset(0, Data)})
+    end;
+out(TD = #typed_data{data = Data, type = tuple}) ->
+    case erlang:tuple_size(Data) of
+        0 -> out(TD, {empty, Data});
+        _ -> out(TD, {{value, element(1, Data)}, erlang:delete_element(1, Data)})
+    end.
+
+out(TD, {V, D}) -> {V, data(TD, D)}.
+
+
+len(#typed_data{data = Data, type = list}) -> length(Data);
+len(#typed_data{data = Data, type = queue}) -> queue:len(Data);
+len(#typed_data{data = Data, type = array}) -> array:sparse_size(Data);
+len(#typed_data{data = Data, type = tuple}) -> tuple_size(Data).
+
+
+nth(Index, #typed_data{data = Data, type = list}) -> lists:nth(Index, Data);
+nth(Index, #typed_data{data = Data, type = queue}) -> queue_nth(Index, Data);
+nth(Index, #typed_data{data = Data, type = array}) -> array:get(Index-1, Data);
+nth(Index, #typed_data{data = Data, type = tuple}) -> element(Index, Data).
+
+
+prep(In, TD = #typed_data{data = Data, type = list}) ->
+    data(TD, [In | Data]);
+prep(In, TD = #typed_data{data = Data, type = queue}) ->
+    data(TD, queue:cons(In, Data));
+prep(In, TD = #typed_data{data = Data, type = array}) ->
+    data(TD, array_prep(In, Data));
+prep(In, TD = #typed_data{data = Data, type = tuple}) ->
+    data(TD, erlang:insert_element(1, Data, In)).
+
+
+app(In, TD = #typed_data{data = Data, type = list}) ->
+    data(TD, Data ++ [In]);
+app(In, TD = #typed_data{data = Data, type = queue}) ->
+    data(TD, queue:in(In, Data));
+app(In, TD = #typed_data{data = Data, type = array}) ->
+    data(TD, array:set(array:size(Data), In, Data));
+app(In, TD = #typed_data{data = Data, type = tuple}) ->
+    data(TD, erlang:append_element(Data, In)).
 
 
+filter(Fun, TD = #typed_data{data = Data, type = list}) ->
+    data(TD, list_filter(Fun, Data));
+filter(Fun, TD = #typed_data{data = Data, type = queue}) ->
+    data(TD, queue:filter(Fun, Data));
+filter(Fun, TD = #typed_data{data = Data, type = array}) ->
+    data(TD, array_filter(Fun, Data));
+filter(Fun, TD = #typed_data{data = Data, type = tuple}) ->
+    data(TD, tuple_filter(Fun, Data)).
 
 
+replace(Out, Index, In, TD = #typed_data{data = Data, type = list}) ->
+    data(TD, list_replace(Out, Index, In, Data));
+replace(Out, Index, In, TD = #typed_data{data = Data, type = queue}) ->
+    data(TD, queue_replace(Out, Index, In, Data));
+replace(Out, Index, In, TD = #typed_data{data = Data, type = array}) ->
+    data(TD, array_replace(Out, Index, In, Data));
+replace(Out, Index, In, TD = #typed_data{data = Data, type = tuple}) ->
+    data(TD, tuple_replace(Out, Index, In, Data)).
+
+
+to(#typed_data{data = Data, type = list}) -> Data;
+to(#typed_data{data = Data, type = queue}) -> queue:to_list(Data);
+to(#typed_data{data = Data, type = array}) -> array:to_list(Data);
+to(#typed_data{data = Data, type = tuple}) -> tuple_to_list(Data).
+
+
+data(TD, Data) -> TD#typed_data{data = Data}.
+
+
+
+list_filter(Fun, L) ->
+    list_filter(Fun, L, []).
+
+list_filter(_Fun, [], Acc) -> lists:reverse(Acc);
+list_filter(Fun, [H | T], Acc) ->
+    case Fun(H) of
+        true -> list_filter(Fun, T, [H | Acc]);
+        false -> list_filter(Fun, T, Acc);
+        H -> list_filter(Fun, T, [H | Acc]);
+        Else -> list_filter(Fun, T, [Else | Acc])
+    end.
+
 list_replace(O, X, I, L) ->
     {L1, [O | Tl]} = lists:split(X-1, L),
     L1 ++ [I| Tl].
@@ -179,7 +186,7 @@ tuple_filter(Fun, Tuple, Index) ->
     NewTuple = case Fun(Element) of
         true -> Tuple;
         Else when Else == Element -> Tuple;
-        false -> setelement(Index, Tuple, undefined);
+        false -> erlang:delete_element(Index, Tuple);
         Else -> setelement(Index, Tuple, Else)
     end,
     tuple_filter(Fun, NewTuple, Index-1).

From 7ea1ad1a6c765d4f0b622b6c43623030ad993972 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Tue, 4 Aug 2020 22:28:43 +0200
Subject: [PATCH 30/36] Choose data-type for indexes based on Strategy

---
 src/poolboy.erl                   | 15 ++++++++-------
 src/poolboy_worker_collection.erl |  8 +++++---
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 679b260..6bea712 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -174,10 +174,11 @@ init({PoolArgs, WorkerArgs}) ->
     end,
     Size = pool_size(PoolArgs),
     Type = pool_type(PoolArgs),
-    Workers = init_workers(Supervisor, WorkerModule, Size, Type),
+    Strategy = strategy(PoolArgs),
+    Workers = init_workers(Supervisor, WorkerModule, Size, Type, Strategy),
 
     MaxOverflow = max_overflow(PoolArgs),
-    Overflow = init_overflow(Size, MaxOverflow, Type),
+    Overflow = init_overflow(Size, MaxOverflow, Type, Strategy),
 
     Waiting = queue:new(),
     Monitors = ets:new(monitors, [private]),
@@ -194,7 +195,7 @@ init({PoolArgs, WorkerArgs}) ->
             size = Size,
             overflow = Overflow,
             max_overflow = MaxOverflow,
-            strategy = strategy(PoolArgs)
+            strategy = Strategy
            }}.
 
 start_supervisor(undefined, _WorkerArgs) ->
@@ -216,13 +217,13 @@ start_supervisor(WorkerModule, WorkerArgs, Retries) ->
             exit({no_worker_supervisor, Error})
     end.
 
-init_workers(Sup, Mod, Size, Type) ->
+init_workers(Sup, Mod, Size, Type, Strategy) ->
     Fun = fun(Idx) -> new_worker(Sup, Mod, Idx) end,
-    poolboy_worker_collection:new(Type, Size, Fun).
+    poolboy_worker_collection:new(Type, Size, Strategy, Fun).
 
-init_overflow(Size, MaxOverflow, Type) ->
+init_overflow(Size, MaxOverflow, Type, Strategy) ->
     Fun = fun(Idx) -> Size + Idx end,
-    poolboy_worker_collection:new(Type, MaxOverflow, Fun).
+    poolboy_worker_collection:new(Type, MaxOverflow, Strategy, Fun).
 
 worker_module(PoolArgs) ->
     Is = is_atom(V = proplists:get_value(worker_module, PoolArgs)),
diff --git a/src/poolboy_worker_collection.erl b/src/poolboy_worker_collection.erl
index 591ef27..78c3eb5 100644
--- a/src/poolboy_worker_collection.erl
+++ b/src/poolboy_worker_collection.erl
@@ -1,6 +1,6 @@
 -module(poolboy_worker_collection).
 
--export([new/3,
+-export([new/4,
          length/2,
          hide_head/1,
          replace/2, replace/3,
@@ -32,14 +32,16 @@
 -export_type([coll/0, coll/1]).
 
 
-new(Type, Size, Fun) when is_function(Fun, 1) ->
+new(Type, Size, lifo, Fun) -> new(Type, Size, list, Fun);
+new(Type, Size, fifo, Fun) -> new(Type, Size, queue, Fun);
+new(Type, Size, IndexesType, Fun) when is_function(Fun, 1) ->
     Indexes = lists:seq(1, Size),
     Items = [Fun(I) || I <- Indexes],
     RevIndexes = maps:from_list(lists:zip(Items, Indexes)),
     #coll{
        item_generator = Fun,
        data = poolboy_collection:from(Items, Type),
-       indexes = poolboy_collection:from(Indexes, queue),
+       indexes = poolboy_collection:from(Indexes, IndexesType),
        rev_indexes = RevIndexes
       }.
 

From 0cf9df52b2694f98ef06f888cf34e0e4df75c150 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Tue, 4 Aug 2020 23:30:15 +0200
Subject: [PATCH 31/36] Move Strategy into the worker collection

---
 src/poolboy.erl                   | 22 ++++++++----------
 src/poolboy_worker_collection.erl | 37 ++++++++++++++-----------------
 2 files changed, 26 insertions(+), 33 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 6bea712..4cb5bb5 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -45,8 +45,7 @@
     crefs :: ets:tid(),
     size = ?DEFAULT_SIZE :: non_neg_integer(),
     overflow :: poolboy_worker_collection:coll() | poolboy_worker_collection:coll(pid()),
-    max_overflow = ?DEFAULT_OVERFLOW :: non_neg_integer(),
-    strategy = ?DEFAULT_STRATEGY :: lifo | fifo
+    max_overflow = ?DEFAULT_OVERFLOW :: non_neg_integer()
 }).
 
 -type status_key() ::
@@ -194,8 +193,7 @@ init({PoolArgs, WorkerArgs}) ->
             crefs = CRefs,
             size = Size,
             overflow = Overflow,
-            max_overflow = MaxOverflow,
-            strategy = Strategy
+            max_overflow = MaxOverflow
            }}.
 
 start_supervisor(undefined, _WorkerArgs) ->
@@ -307,7 +305,7 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
     OverflowLeft = poolboy_worker_collection:length(visible, Overflow),
-    case poolboy_worker_collection:hide_head(Workers) of
+    case poolboy_worker_collection:checkout(Workers) of
         {Pid, Left} when is_pid(Pid) ->
             MRef = erlang:monitor(process, FromPid),
             true = ets:insert(Monitors, {Pid, CRef, MRef}),
@@ -315,7 +313,7 @@ handle_call({checkout, CRef, Block}, {FromPid, _} = From, State) ->
             true = ets:insert(CRefs, {CRef, Pid}),
             {reply, Pid, State#state{workers = Left}};
         empty when MaxOverflow > 0, OverflowLeft > 0 ->
-            {NextIdx, NewOverflow} = poolboy_worker_collection:hide_head(Overflow),
+            {NextIdx, NewOverflow} = poolboy_worker_collection:checkout(Overflow),
             Pid = new_worker(Sup, Mod, NextIdx),
             {Pid, NewerOverflow} = poolboy_worker_collection:replace(NextIdx, Pid, NewOverflow),
             MRef = erlang:monitor(process, FromPid),
@@ -473,7 +471,6 @@ handle_checkin(Pid, State) ->
            monitors = Monitors,
            mrefs = MRefs,
            crefs = CRefs,
-           strategy = Strategy,
            overflow = Overflow} = State,
     case queue:out(Waiting) of
         {{value, {From, CRef, MRef}}, Left} ->
@@ -486,11 +483,11 @@ handle_checkin(Pid, State) ->
             try poolboy_worker_collection:replace(Pid, Overflow) of
                 {NewIdx, NewOverflow} ->
                     ok = dismiss_worker(Sup, Pid),
-                    NewerOverflow = poolboy_worker_collection:Strategy(NewIdx, NewOverflow),
+                    NewerOverflow = poolboy_worker_collection:checkin(NewIdx, NewOverflow),
                     State#state{waiting = Empty, overflow = NewerOverflow}
             catch
                 error:enoent ->
-                    Workers = poolboy_worker_collection:Strategy(Pid, State#state.workers),
+                    Workers = poolboy_worker_collection:checkin(Pid, State#state.workers),
                     State#state{waiting = Empty, workers = Workers}
             end
     end.
@@ -502,7 +499,6 @@ handle_worker_exit(Pid, State) ->
            mrefs = MRefs,
            crefs = CRefs,
            size = Size,
-           strategy = Strategy,
            overflow = Overflow,
            max_overflow = MaxOverflow} = State,
     {NewWorker, Workers} =
@@ -530,11 +526,11 @@ handle_worker_exit(Pid, State) ->
                 State
             end;
         {empty, Empty} when is_pid(NewWorker) ->
-            State#state{waiting = Empty,
-                        workers = poolboy_worker_collection:Strategy(NewWorker, Workers)};
+            NewWorkers = poolboy_worker_collection:checkin(NewWorker, Workers),
+            State#state{waiting = Empty, workers = NewWorkers};
         {empty, Empty} when MaxOverflow > 0 ->
             {Idx, NewOverflow} = poolboy_worker_collection:replace(Pid, Overflow),
-            NewerOverflow = poolboy_worker_collection:prepend(Idx, NewOverflow),
+            NewerOverflow = poolboy_worker_collection:checkin(Idx, NewOverflow),
             State#state{waiting = Empty, overflow = NewerOverflow}
     end.
 
diff --git a/src/poolboy_worker_collection.erl b/src/poolboy_worker_collection.erl
index 78c3eb5..fb4ba5c 100644
--- a/src/poolboy_worker_collection.erl
+++ b/src/poolboy_worker_collection.erl
@@ -2,19 +2,20 @@
 
 -export([new/4,
          length/2,
-         hide_head/1,
+         checkout/1,
          replace/2, replace/3,
-         lifo/2, prepend/2,
-         fifo/2, append/2,
+         checkin/2,
          filter/2,
          all/2,
          rand/2
         ]).
 
+-type strategy() :: lifo | fifo.
 
 -record(coll, {
           item_generator :: fun((non_neg_integer()) -> any()),
           data :: poolboy_collection:typed_data() | poolboy_collection:typed_data(any()),
+          strategy :: strategy(),
           indexes :: poolboy_collection:typed_data() | poolboy_collection:typed_data(any()),
           rev_indexes :: #{any()=>non_neg_integer()}
           }).
@@ -32,15 +33,15 @@
 -export_type([coll/0, coll/1]).
 
 
-new(Type, Size, lifo, Fun) -> new(Type, Size, list, Fun);
-new(Type, Size, fifo, Fun) -> new(Type, Size, queue, Fun);
-new(Type, Size, IndexesType, Fun) when is_function(Fun, 1) ->
+new(Type, Size, Strategy, Fun) when is_function(Fun, 1) ->
     Indexes = lists:seq(1, Size),
     Items = [Fun(I) || I <- Indexes],
     RevIndexes = maps:from_list(lists:zip(Items, Indexes)),
+    IndexesType = case Strategy of lifo -> list; fifo -> queue end,
     #coll{
        item_generator = Fun,
        data = poolboy_collection:from(Items, Type),
+       strategy = Strategy,
        indexes = poolboy_collection:from(Indexes, IndexesType),
        rev_indexes = RevIndexes
       }.
@@ -50,7 +51,7 @@ length(known, #coll{data=Data}) -> poolboy_collection:len(Data);
 length(visible, #coll{indexes=Indexes}) -> poolboy_collection:len(Indexes).
 
 
-hide_head(Coll = #coll{indexes = Indexes, data=Data}) ->
+checkout(Coll = #coll{indexes = Indexes, data=Data}) ->
     case poolboy_collection:out(Indexes) of
         {empty, _} -> empty;
         {{value, Hd}, Tl} ->
@@ -58,7 +59,6 @@ hide_head(Coll = #coll{indexes = Indexes, data=Data}) ->
     end.
 
 
-
 replace(Out, Coll = #coll{item_generator = In}) ->
     replace(Out, In, Coll).
 
@@ -74,24 +74,21 @@ replace(Out, In, Coll = #coll{data = Data}) ->
             {NewItem, Coll#coll{rev_indexes = NewRevIndexes, data = NewData}}
     end.
 
-lifo(In, Coll) -> prepend(In, Coll).
-
-prepend(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = _Data}) ->
-    case maps:get(In, RevIndexes, undefined) of
-        InIndex when is_integer(InIndex) -> Coll#coll{indexes = poolboy_collection:prep(InIndex, Indexes)}
-    end.
-
 
-fifo(In, Coll) -> append(In, Coll).
+checkin(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes}) ->
+    InIndex = maps:get(In, RevIndexes),
+    NewIndexes =
+    case Coll#coll.strategy of
+        lifo -> poolboy_collection:prep(InIndex, Indexes);
+        fifo -> poolboy_collection:app(InIndex, Indexes)
+    end,
+    Coll#coll{indexes = NewIndexes}.
 
-append(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes, data = _Data}) ->
-    case maps:get(In, RevIndexes, undefined) of
-        InIndex when is_integer(InIndex) -> Coll#coll{indexes = poolboy_collection:app(InIndex, Indexes)}
-    end.
 
 filter(Fun, #coll{data = Data}) ->
     poolboy_collection:filter(Fun, Data).
 
+
 all(known, #coll{rev_indexes = RevIndexes}) ->
     maps:keys(RevIndexes);
 all(visible, #coll{indexes = Indexes, data = Data}) ->

From 6fd7101aebe7444e332083bfa8e97c5b8d8e0021 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Mon, 17 Aug 2020 23:19:16 +0200
Subject: [PATCH 32/36] Introduce rand strategy

* Add ets (ordered_set) to collection types
* Implement strategies via out/1 of collection-types
* Use only add/2 instead of prep/2 and app/2
* Run tests with all available strategies
---
 src/poolboy.erl                   |   2 +-
 src/poolboy_collection.erl        | 114 +++++++++++++++++---------
 src/poolboy_worker_collection.erl |  18 ++--
 test/poolboy_tests.erl            | 132 +++++++++++++++++++-----------
 4 files changed, 170 insertions(+), 96 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 4cb5bb5..b22dc60 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -258,7 +258,7 @@ max_overflow(PoolArgs) ->
     Is = is_integer(V = proplists:get_value(max_overflow, PoolArgs)),
     if not Is -> ?DEFAULT_OVERFLOW; true -> V end.
 
--define(IS_STRATEGY(S), lists:member(S, [lifo, fifo])).
+-define(IS_STRATEGY(S), lists:member(S, [lifo, fifo, rand])).
 strategy(PoolArgs) ->
     Is = ?IS_STRATEGY(V = proplists:get_value(strategy, PoolArgs)),
     if not Is -> ?DEFAULT_STRATEGY; true -> V end.
diff --git a/src/poolboy_collection.erl b/src/poolboy_collection.erl
index 8364516..8c8260a 100644
--- a/src/poolboy_collection.erl
+++ b/src/poolboy_collection.erl
@@ -1,6 +1,8 @@
 -module(poolboy_collection).
 
--export([from/2, out/1, len/1, nth/2, prep/2, app/2, filter/2, replace/4, to/1]).
+-include_lib("stdlib/include/ms_transform.hrl").
+
+-export([from/2, out/1, len/1, nth/2, add/2, filter/2, replace/4, to/1]).
 
 -ifdef(pre17).
 -type pid_queue() :: queue().
@@ -11,11 +13,11 @@
 -endif.
 -export_type([pid_queue/0, pid_queue/1]).
 
--type coll_data() :: list()|{}|array:array()|pid_queue().
--type coll_data(A) :: list(A)|{A}|array:array(A)|pid_queue(A).
+-type coll_data() :: list()|{}|array:array()|pid_queue()|ets:tid().
+-type coll_data(A) :: list(A)|{A}|array:array(A)|pid_queue(A)|ets:tid().
 
 -record(typed_data, {
-          type :: 'list' |'array' |'queue' |'tuple',
+          type :: 'list' |'array' |'queue' |'tuple' |'ets',
           data :: coll_data() | coll_data(any())
          }).
 -type typed_data() :: #typed_data{data :: coll_data()}.
@@ -30,7 +32,11 @@ from(List, T = queue) ->
 from(List, T = array) ->
     #typed_data{type = T, data = array:from_list(List)};
 from(List, T = tuple) ->
-    #typed_data{type = T, data = list_to_tuple(List)}.
+    #typed_data{type = T, data = list_to_tuple(List)};
+from(List, T = ets) ->
+    Tab = ets:new(table, [ordered_set]),
+    true = ets:insert_new(Tab, [{I} || I <- List]),
+    #typed_data{type = T, data = Tab}.
 
 
 out(TD = #typed_data{data = Data, type = list}) ->
@@ -49,6 +55,15 @@ out(TD = #typed_data{data = Data, type = tuple}) ->
     case erlang:tuple_size(Data) of
         0 -> out(TD, {empty, Data});
         _ -> out(TD, {{value, element(1, Data)}, erlang:delete_element(1, Data)})
+    end;
+out(TD = #typed_data{data = Data, type = ets}) ->
+    case ets:info(Data, size) of
+        0 -> {empty, TD};
+        Size ->
+            Index = rand:uniform(Size),
+            [{Value}] = ets:slot(Data, Index-1),
+            true = ets:delete(Data, Value),
+            {{value, Value}, TD}
     end.
 
 out(TD, {V, D}) -> {V, data(TD, D)}.
@@ -57,33 +72,30 @@ out(TD, {V, D}) -> {V, data(TD, D)}.
 len(#typed_data{data = Data, type = list}) -> length(Data);
 len(#typed_data{data = Data, type = queue}) -> queue:len(Data);
 len(#typed_data{data = Data, type = array}) -> array:sparse_size(Data);
-len(#typed_data{data = Data, type = tuple}) -> tuple_size(Data).
+len(#typed_data{data = Data, type = tuple}) -> tuple_size(Data);
+len(#typed_data{data = Data, type = ets}) -> ets:info(Data, size).
 
 
 nth(Index, #typed_data{data = Data, type = list}) -> lists:nth(Index, Data);
 nth(Index, #typed_data{data = Data, type = queue}) -> queue_nth(Index, Data);
 nth(Index, #typed_data{data = Data, type = array}) -> array:get(Index-1, Data);
-nth(Index, #typed_data{data = Data, type = tuple}) -> element(Index, Data).
+nth(Index, #typed_data{data = Data, type = tuple}) -> element(Index, Data);
+nth(Index, #typed_data{data = Data, type = ets}) ->
+    [{Value}] = ets:slot(Data, Index-1),
+    Value.
 
 
-prep(In, TD = #typed_data{data = Data, type = list}) ->
+add(In, TD = #typed_data{data = Data, type = list}) ->
     data(TD, [In | Data]);
-prep(In, TD = #typed_data{data = Data, type = queue}) ->
-    data(TD, queue:cons(In, Data));
-prep(In, TD = #typed_data{data = Data, type = array}) ->
-    data(TD, array_prep(In, Data));
-prep(In, TD = #typed_data{data = Data, type = tuple}) ->
-    data(TD, erlang:insert_element(1, Data, In)).
-
-
-app(In, TD = #typed_data{data = Data, type = list}) ->
-    data(TD, Data ++ [In]);
-app(In, TD = #typed_data{data = Data, type = queue}) ->
+add(In, TD = #typed_data{data = Data, type = queue}) ->
     data(TD, queue:in(In, Data));
-app(In, TD = #typed_data{data = Data, type = array}) ->
+add(In, TD = #typed_data{data = Data, type = array}) ->
     data(TD, array:set(array:size(Data), In, Data));
-app(In, TD = #typed_data{data = Data, type = tuple}) ->
-    data(TD, erlang:append_element(Data, In)).
+add(In, TD = #typed_data{data = Data, type = tuple}) ->
+    data(TD, erlang:append_element(Data, In));
+add(In, TD = #typed_data{data = Data, type = ets}) ->
+    true = ets:insert_new(Data, {In}),
+    TD.
 
 
 filter(Fun, TD = #typed_data{data = Data, type = list}) ->
@@ -93,7 +105,9 @@ filter(Fun, TD = #typed_data{data = Data, type = queue}) ->
 filter(Fun, TD = #typed_data{data = Data, type = array}) ->
     data(TD, array_filter(Fun, Data));
 filter(Fun, TD = #typed_data{data = Data, type = tuple}) ->
-    data(TD, tuple_filter(Fun, Data)).
+    data(TD, tuple_filter(Fun, Data));
+filter(Fun, TD = #typed_data{data = Data, type = ets}) ->
+    data(TD, ets_filter(Fun, Data)).
 
 
 replace(Out, Index, In, TD = #typed_data{data = Data, type = list}) ->
@@ -103,13 +117,20 @@ replace(Out, Index, In, TD = #typed_data{data = Data, type = queue}) ->
 replace(Out, Index, In, TD = #typed_data{data = Data, type = array}) ->
     data(TD, array_replace(Out, Index, In, Data));
 replace(Out, Index, In, TD = #typed_data{data = Data, type = tuple}) ->
-    data(TD, tuple_replace(Out, Index, In, Data)).
+    data(TD, tuple_replace(Out, Index, In, Data));
+replace(Out, Index, In, TD = #typed_data{data = Data, type = ets}) ->
+    [{Out}] = ets:slot(Data, Index),
+    [{Out}] = ets:take(Data, Out),
+    true = ets:insert_new(Data, {In}),
+    TD.
 
 
 to(#typed_data{data = Data, type = list}) -> Data;
 to(#typed_data{data = Data, type = queue}) -> queue:to_list(Data);
 to(#typed_data{data = Data, type = array}) -> array:to_list(Data);
-to(#typed_data{data = Data, type = tuple}) -> tuple_to_list(Data).
+to(#typed_data{data = Data, type = tuple}) -> tuple_to_list(Data);
+to(#typed_data{data = Data, type = ets}) ->
+    ets:select(Data, ets:fun2ms(fun({I}) -> I end)).
 
 
 data(TD, Data) -> TD#typed_data{data = Data}.
@@ -123,9 +144,10 @@ list_filter(_Fun, [], Acc) -> lists:reverse(Acc);
 list_filter(Fun, [H | T], Acc) ->
     case Fun(H) of
         true -> list_filter(Fun, T, [H | Acc]);
+        [Else] when Else == H -> list_filter(Fun, T, [H | Acc]);
         false -> list_filter(Fun, T, Acc);
-        H -> list_filter(Fun, T, [H | Acc]);
-        Else -> list_filter(Fun, T, [Else | Acc])
+        [] -> list_filter(Fun, T, Acc);
+        [Else] -> list_filter(Fun, T, [Else | Acc])
     end.
 
 list_replace(O, X, I, L) ->
@@ -134,22 +156,15 @@ list_replace(O, X, I, L) ->
 
 
 
-array_prep(I, A) ->
-    array:foldl(
-      fun(Idx, Val, Arr) ->
-              array:set(Idx+1, Val, Arr)
-      end,
-      array:set(0, I, array:new()),
-      A).
-
-
 array_filter(F, A) ->
     array:sparse_map(
       fun(_, V) ->
               case F(V) of
                   true -> V;
+                  [Else] when Else == V -> V;
                   false -> array:default(A);
-                  Else -> Else
+                  [] -> array:default(A);
+                  [Else] -> Else
               end
       end,
       A).
@@ -185,9 +200,10 @@ tuple_filter(Fun, Tuple, Index) ->
     Element = element(Index, Tuple),
     NewTuple = case Fun(Element) of
         true -> Tuple;
-        Else when Else == Element -> Tuple;
+        [Else] when Else == Element -> Tuple;
         false -> erlang:delete_element(Index, Tuple);
-        Else -> setelement(Index, Tuple, Else)
+        [] -> erlang:delete_element(Index, Tuple);
+        [Else] -> setelement(Index, Tuple, Else)
     end,
     tuple_filter(Fun, NewTuple, Index-1).
 
@@ -195,3 +211,23 @@ tuple_filter(Fun, Tuple, Index) ->
 tuple_replace(O, X, I, Tu) ->
     O = element(X, Tu),
     setelement(X, Tu, I).
+
+
+
+ets_filter(Fun, Tab) ->
+    {Ins, Outs} =
+    ets:foldl(
+      fun({Item}, {In, Out} = Acc) ->
+              case Fun(Item) of
+                  true -> Acc;
+                  [Else] when Else == Item -> Acc;
+                  false -> {In, [Item | Out]};
+                  [] -> {In, [Item | Out]};
+                  [Else] -> {[Else | In], [Item | Out]}
+              end
+      end,
+      {[], []},
+      Tab),
+    true = lists:min([true | [ets:delete(Tab, O) || O <- Outs]]),
+    true = ets:insert_new(Tab, Ins),
+    Tab.
diff --git a/src/poolboy_worker_collection.erl b/src/poolboy_worker_collection.erl
index fb4ba5c..3fe4f84 100644
--- a/src/poolboy_worker_collection.erl
+++ b/src/poolboy_worker_collection.erl
@@ -10,7 +10,7 @@
          rand/2
         ]).
 
--type strategy() :: lifo | fifo.
+-type strategy() :: lifo | fifo | rand.
 
 -record(coll, {
           item_generator :: fun((non_neg_integer()) -> any()),
@@ -37,7 +37,12 @@ new(Type, Size, Strategy, Fun) when is_function(Fun, 1) ->
     Indexes = lists:seq(1, Size),
     Items = [Fun(I) || I <- Indexes],
     RevIndexes = maps:from_list(lists:zip(Items, Indexes)),
-    IndexesType = case Strategy of lifo -> list; fifo -> queue end,
+    IndexesType =
+    case Strategy of
+        lifo -> list;
+        fifo -> queue;
+        rand -> ets
+    end,
     #coll{
        item_generator = Fun,
        data = poolboy_collection:from(Items, Type),
@@ -77,11 +82,7 @@ replace(Out, In, Coll = #coll{data = Data}) ->
 
 checkin(In, Coll = #coll{indexes = Indexes, rev_indexes = RevIndexes}) ->
     InIndex = maps:get(In, RevIndexes),
-    NewIndexes =
-    case Coll#coll.strategy of
-        lifo -> poolboy_collection:prep(InIndex, Indexes);
-        fifo -> poolboy_collection:app(InIndex, Indexes)
-    end,
+    NewIndexes = poolboy_collection:add(InIndex, Indexes),
     Coll#coll{indexes = NewIndexes}.
 
 
@@ -92,7 +93,8 @@ filter(Fun, #coll{data = Data}) ->
 all(known, #coll{rev_indexes = RevIndexes}) ->
     maps:keys(RevIndexes);
 all(visible, #coll{indexes = Indexes, data = Data}) ->
-    poolboy_collection:to(poolboy_collection:filter(fun(I) -> [poolboy_collection:nth(I, Data)] end, Indexes)).
+    [ poolboy_collection:nth(I, Data)
+      || I <- poolboy_collection:to(Indexes) ].
 
 
 rand(known, #coll{data = Data}) ->
diff --git a/test/poolboy_tests.erl b/test/poolboy_tests.erl
index c997798..1897f84 100644
--- a/test/poolboy_tests.erl
+++ b/test/poolboy_tests.erl
@@ -14,10 +14,14 @@ pool_test_() ->
             end,
             error_logger:tty(true)
         end,
-        [ {Type,
-           fun(T, _) ->
-                   {<<(atom_to_binary(T, latin1))/binary, <<": ">>/binary, Title/binary>>, fun() -> Test(T) end}
-           end} || Type <- [list, array, tuple, queue], {Title, Test} <-
+        [ {{Type, Strategy},
+           fun({T, S}, _) ->
+                   {<<(atom_to_binary(T, latin1))/binary, <<"-">>/binary,
+                      (atom_to_binary(S, latin1))/binary, <<": ">>/binary,
+                      Title/binary>>, fun() -> Test({T, S}) end}
+           end} || Type <- [list, array, tuple, queue, ets],
+                   Strategy <- [lifo, fifo, rand],
+                   {Title, Test} <-
         [
             {<<"Basic pool operations">>,
                 fun pool_startup/1
@@ -70,6 +74,9 @@ pool_test_() ->
             {<<"Check FIFO strategy">>,
                 fun fifo_strategy/1
             },
+            {<<"Check RAND strategy">>,
+                fun rand_strategy/1
+            },
             {<<"Pool reuses waiting monitor when a worker exits">>,
                 fun reuses_waiting_monitor_on_worker_exit/1
             },
@@ -100,8 +107,8 @@ checkin_worker(Pid, Worker) ->
     timer:sleep(500).
 
 
-transaction_timeout_without_exit(Type) ->
-    {ok, Pid} = new_pool(1, 0, lifo, Type),
+transaction_timeout_without_exit({Type, Strategy}) ->
+    {ok, Pid} = new_pool(1, 0, Strategy, Type),
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)),
     WorkerList = pool_call(Pid, get_all_workers),
     ?assertMatch([_], WorkerList),
@@ -115,8 +122,8 @@ transaction_timeout_without_exit(Type) ->
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)).
 
 
-transaction_timeout(Type) ->
-    {ok, Pid} = new_pool(1, 0, lifo, Type),
+transaction_timeout({Type, Strategy}) ->
+    {ok, Pid} = new_pool(1, 0, Strategy, Type),
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)),
     WorkerList = pool_call(Pid, get_all_workers),
     ?assertMatch([_], WorkerList),
@@ -131,9 +138,9 @@ transaction_timeout(Type) ->
     ?assertEqual({ready,1,0,0}, pool_call(Pid, status)).
 
 
-pool_startup(Type) ->
+pool_startup({Type, Strategy}) ->
     %% Check basic pool operation.
-    {ok, Pid} = new_pool(10, 5, lifo, Type),
+    {ok, Pid} = new_pool(10, 5, Strategy, Type),
     ?assertEqual(10, length(pool_call(Pid, get_avail_workers))),
     poolboy:checkout(Pid),
     ?assertEqual(9, length(pool_call(Pid, get_avail_workers))),
@@ -144,9 +151,9 @@ pool_startup(Type) ->
     ?assertEqual(1, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_overflow(Type) ->
+pool_overflow({Type, Strategy}) ->
     %% Check that the pool overflows properly.
-    {ok, Pid} = new_pool(5, 5, lifo, Type),
+    {ok, Pid} = new_pool(5, 5, Strategy, Type),
     Workers = lists:reverse([poolboy:checkout(Pid) || _ <- lists:seq(0, 6)]),
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
@@ -169,10 +176,10 @@ pool_overflow(Type) ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_empty(Type) ->
+pool_empty({Type, Strategy}) ->
     %% Checks that the the pool handles the empty condition correctly when
     %% overflow is enabled.
-    {ok, Pid} = new_pool(5, 2, lifo, Type),
+    {ok, Pid} = new_pool(5, 2, Strategy, Type),
     Workers = lists:reverse([poolboy:checkout(Pid) || _ <- lists:seq(0, 6)]),
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(7, length(pool_call(Pid, get_all_workers))),
@@ -215,10 +222,10 @@ pool_empty(Type) ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_empty_no_overflow(Type) ->
+pool_empty_no_overflow({Type, Strategy}) ->
     %% Checks the pool handles the empty condition properly when overflow is
     %% disabled.
-    {ok, Pid} = new_pool(5, 0, lifo, Type),
+    {ok, Pid} = new_pool(5, 0, Strategy, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 4)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
@@ -257,10 +264,10 @@ pool_empty_no_overflow(Type) ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-worker_death(Type) ->
+worker_death({Type, Strategy}) ->
     %% Check that dead workers are only restarted when the pool is not full
     %% and the overflow count is 0. Meaning, don't restart overflow workers.
-    {ok, Pid} = new_pool(5, 2, lifo, Type),
+    {ok, Pid} = new_pool(5, 2, Strategy, Type),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
@@ -277,11 +284,11 @@ worker_death(Type) ->
     ?assertEqual(4, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-worker_death_while_full(Type) ->
+worker_death_while_full({Type, Strategy}) ->
     %% Check that if a worker dies while the pool is full and there is a
     %% queued checkout, a new worker is started and the checkout serviced.
     %% If there are no queued checkouts, a new worker is not started.
-    {ok, Pid} = new_pool(5, 2, lifo, Type),
+    {ok, Pid} = new_pool(5, 2, Strategy, Type),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
@@ -318,11 +325,11 @@ worker_death_while_full(Type) ->
     ?assertEqual(6, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-worker_death_while_full_no_overflow(Type) ->
+worker_death_while_full_no_overflow({Type, Strategy}) ->
     %% Check that if a worker dies while the pool is full and there's no
     %% overflow, a new worker is started unconditionally and any queued
     %% checkouts are serviced.
-    {ok, Pid} = new_pool(5, 0, lifo, Type),
+    {ok, Pid} = new_pool(5, 0, Strategy, Type),
     Worker = poolboy:checkout(Pid),
     kill_worker(Worker),
     ?assertEqual(5, length(pool_call(Pid, get_avail_workers))),
@@ -361,10 +368,10 @@ worker_death_while_full_no_overflow(Type) ->
     ?assertEqual(3, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_full_nonblocking_no_overflow(Type) ->
+pool_full_nonblocking_no_overflow({Type, Strategy}) ->
     %% Check that when the pool is full, checkouts return 'full' when the
     %% option to use non-blocking checkouts is used.
-    {ok, Pid} = new_pool(5, 0, lifo, Type),
+    {ok, Pid} = new_pool(5, 0, Strategy, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 4)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(5, length(pool_call(Pid, get_all_workers))),
@@ -376,10 +383,10 @@ pool_full_nonblocking_no_overflow(Type) ->
     ?assertEqual(5, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-pool_full_nonblocking(Type) ->
+pool_full_nonblocking({Type, Strategy}) ->
     %% Check that when the pool is full, checkouts return 'full' when the
     %% option to use non-blocking checkouts is used.
-    {ok, Pid} = new_pool(5, 5, lifo, Type),
+    {ok, Pid} = new_pool(5, 5, Strategy, Type),
     Workers = [poolboy:checkout(Pid) || _ <- lists:seq(0, 9)],
     ?assertEqual(0, length(pool_call(Pid, get_avail_workers))),
     ?assertEqual(10, length(pool_call(Pid, get_all_workers))),
@@ -393,10 +400,10 @@ pool_full_nonblocking(Type) ->
     ?assertEqual(10, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-owner_death(Type) ->
+owner_death({Type, Strategy}) ->
     %% Check that a dead owner (a process that dies with a worker checked out)
     %% causes the pool to dismiss the worker and prune the state space.
-    {ok, Pid} = new_pool(5, 5, lifo, Type),
+    {ok, Pid} = new_pool(5, 5, Strategy, Type),
     spawn(fun() ->
         poolboy:checkout(Pid),
         receive after 500 -> exit(normal) end
@@ -407,8 +414,8 @@ owner_death(Type) ->
     ?assertEqual(0, length(pool_call(Pid, get_all_monitors))),
     ok = pool_call(Pid, stop).
 
-checkin_after_exception_in_transaction(Type) ->
-    {ok, Pool} = new_pool(2, 0, lifo, Type),
+checkin_after_exception_in_transaction({Type, Strategy}) ->
+    {ok, Pool} = new_pool(2, 0, Strategy, Type),
     ?assertEqual(2, length(pool_call(Pool, get_avail_workers))),
     Tx = fun(Worker) ->
         ?assert(is_pid(Worker)),
@@ -424,8 +431,8 @@ checkin_after_exception_in_transaction(Type) ->
     ?assertEqual(2, length(pool_call(Pool, get_avail_workers))),
     ok = pool_call(Pool, stop).
 
-pool_returns_status(Type) ->
-    {ok, Pool} = new_pool(2, 0, lifo, Type),
+pool_returns_status({Type, Strategy}) ->
+    {ok, Pool} = new_pool(2, 0, Strategy, Type),
     ?assertEqual({ready, 2, 0, 0}, poolboy:status(Pool)),
     poolboy:checkout(Pool),
     ?assertEqual({ready, 1, 0, 1}, poolboy:status(Pool)),
@@ -433,7 +440,7 @@ pool_returns_status(Type) ->
     ?assertEqual({full, 0, 0, 2}, poolboy:status(Pool)),
     ok = pool_call(Pool, stop),
 
-    {ok, Pool2} = new_pool(1, 1),
+    {ok, Pool2} = new_pool(1, 1, Strategy, Type),
     ?assertEqual({ready, 1, 0, 0}, poolboy:status(Pool2)),
     poolboy:checkout(Pool2),
     ?assertEqual({overflow, 0, 0, 1}, poolboy:status(Pool2)),
@@ -441,7 +448,7 @@ pool_returns_status(Type) ->
     ?assertEqual({full, 0, 1, 2}, poolboy:status(Pool2)),
     ok = pool_call(Pool2, stop),
 
-    {ok, Pool3} = new_pool(0, 2),
+    {ok, Pool3} = new_pool(0, 2, Strategy, Type),
     ?assertEqual({overflow, 0, 0, 0}, poolboy:status(Pool3)),
     poolboy:checkout(Pool3),
     ?assertEqual({overflow, 0, 1, 1}, poolboy:status(Pool3)),
@@ -449,12 +456,12 @@ pool_returns_status(Type) ->
     ?assertEqual({full, 0, 2, 2}, poolboy:status(Pool3)),
     ok = pool_call(Pool3, stop),
 
-    {ok, Pool4} = new_pool(0, 0),
+    {ok, Pool4} = new_pool(0, 0, Strategy, Type),
     ?assertEqual({full, 0, 0, 0}, poolboy:status(Pool4)),
     ok = pool_call(Pool4, stop).
 
-demonitors_previously_waiting_processes(Type) ->
-    {ok, Pool} = new_pool(1,0, lifo, Type),
+demonitors_previously_waiting_processes({Type, Strategy}) ->
+    {ok, Pool} = new_pool(1,0, Strategy, Type),
     Self = self(),
     Pid = spawn(fun() ->
         W = poolboy:checkout(Pool),
@@ -472,8 +479,8 @@ demonitors_previously_waiting_processes(Type) ->
     Pid ! ok,
     ok = pool_call(Pool, stop).
 
-demonitors_when_checkout_cancelled(Type) ->
-    {ok, Pool} = new_pool(1,0, lifo, Type),
+demonitors_when_checkout_cancelled({Type, Strategy}) ->
+    {ok, Pool} = new_pool(1, 0, Strategy, Type),
     Self = self(),
     Pid = spawn(fun() ->
         poolboy:checkout(Pool),
@@ -488,32 +495,61 @@ demonitors_when_checkout_cancelled(Type) ->
     Pid ! ok,
     ok = pool_call(Pool, stop).
 
-default_strategy_lifo(Type) ->
+default_strategy_lifo({Type, lifo}) ->
     %% Default strategy is LIFO
     {ok, Pid} = new_pool(2, 0, default, Type),
     Worker1 = poolboy:checkout(Pid),
     ok = poolboy:checkin(Pid, Worker1),
     Worker1 = poolboy:checkout(Pid),
-    poolboy:stop(Pid).
+    poolboy:stop(Pid);
+default_strategy_lifo({_Type, _Strategy}) ->
+    ok.
 
-lifo_strategy(Type) ->
+lifo_strategy({Type, lifo}) ->
     {ok, Pid} = new_pool(2, 0, lifo, Type),
     Worker1 = poolboy:checkout(Pid),
     ok = poolboy:checkin(Pid, Worker1),
     Worker1 = poolboy:checkout(Pid),
-    poolboy:stop(Pid).
+    poolboy:stop(Pid);
+lifo_strategy({_Type, _Strategy}) ->
+    ok.
 
-fifo_strategy(Type) ->
+fifo_strategy({Type, fifo}) ->
     {ok, Pid} = new_pool(2, 0, fifo, Type),
     Worker1 = poolboy:checkout(Pid),
     ok = poolboy:checkin(Pid, Worker1),
     Worker2 = poolboy:checkout(Pid),
     ?assert(Worker1 =/= Worker2),
     Worker1 = poolboy:checkout(Pid),
-    poolboy:stop(Pid).
-
-reuses_waiting_monitor_on_worker_exit(Type) ->
-    {ok, Pool} = new_pool(1,0, lifo, Type),
+    poolboy:stop(Pid);
+fifo_strategy({_Type, _Strategy}) ->
+    ok.
+
+rand_strategy({Type, rand}) ->
+    {ok, Pid} = new_pool(3, 0, rand, Type),
+    Workers1 =
+    [ begin
+          Worker = poolboy:checkout(Pid),
+          ok = poolboy:checkin(Pid, Worker),
+          Worker
+      end || _ <- lists:seq(1,5) ],
+    Workers2 =
+    [ begin
+          Worker = poolboy:checkout(Pid),
+          ok = poolboy:checkin(Pid, Worker),
+          Worker
+      end || _ <- lists:seq(1,5) ],
+    ?assertNotEqual(Workers1, Workers2),
+    Workers = [ poolboy:checkout(Pid)
+                || _ <- lists:seq(1,3) ],
+    ?assertEqual(lists:usort(Workers),
+                 lists:usort(Workers1 ++ Workers2)),
+    poolboy:stop(Pid);
+rand_strategy({_Type, _Strategy}) ->
+    ok.
+
+reuses_waiting_monitor_on_worker_exit({Type, Strategy}) ->
+    {ok, Pool} = new_pool(1,0, Strategy, Type),
 
     Self = self(),
     Pid = spawn(fun() ->

From 8a049d6ab9a7917646f8ac51f6d0183062217ba1 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Wed, 19 Aug 2020 11:19:39 +0200
Subject: [PATCH 33/36] Handle badrpc when looking for Supervisor Pid, Turn
 undefined into noproc if Pid is not found

---
 src/poolboy.erl | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index b22dc60..886fa9c 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -228,8 +228,12 @@ worker_module(PoolArgs) ->
     if not Is -> undefined; true -> V end.
 
 worker_supervisor(PoolArgs) ->
-    Is = is_pid(Res = find_pid(V = proplists:get_value(worker_supervisor, PoolArgs))),
-    if not Is andalso Res =/= V -> exit({not_found, V, Res}); true -> Res end.
+    case find_pid(V = proplists:get_value(worker_supervisor, PoolArgs)) of
+        Res = undefined when Res =:= V -> Res;
+        Res when is_pid(Res) -> Res;
+        Res = undefined when Res =/= V -> exit({noproc, V});
+        Res -> exit({Res, V})
+    end.
 
 find_pid(undefined) ->
     undefined;
@@ -243,7 +247,13 @@ find_pid({via, Registry, Name}) ->
     Registry:whereis_name(Name);
 find_pid({Name, Node}) ->
     (catch erlang:monitor_node(Node, true)),
-    rpc:call(Node, erlang, whereis, [Name], ?TIMEOUT).
+    try rpc:call(Node, erlang, whereis, [Name], ?TIMEOUT) of
+        {badrpc, Reason} -> Reason;
+        Result -> Result
+    catch
+        _:Reason -> Reason
+    end.
+
 
 pool_size(PoolArgs) ->
     Is = is_integer(V = proplists:get_value(size, PoolArgs)),

From 36e72ca4c846e1b21c4c6ce0efda86bef7a982b3 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Wed, 19 Aug 2020 11:20:19 +0200
Subject: [PATCH 34/36] Increase accuracy of rand strategy test

---
 test/poolboy_tests.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/poolboy_tests.erl b/test/poolboy_tests.erl
index 1897f84..f434999 100644
--- a/test/poolboy_tests.erl
+++ b/test/poolboy_tests.erl
@@ -532,13 +532,13 @@ rand_strategy({Type, rand}) ->
           Worker = poolboy:checkout(Pid),
           ok = poolboy:checkin(Pid, Worker),
           Worker
-      end || _ <- lists:seq(1,5) ],
+      end || _ <- lists:seq(1,9) ],
     Workers2 =
     [ begin
           Worker = poolboy:checkout(Pid),
           ok = poolboy:checkin(Pid, Worker),
           Worker
-      end || _ <- lists:seq(1,5) ],
+      end || _ <- lists:seq(1,9) ],
     ?assertNotEqual(Workers1, Workers2),
     Workers = [ poolboy:checkout(Pid)
                 || _ <- lists:seq(1,3) ],

From 11375320d5cddcceabb064c4c64a4ac6f8f0a57c Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Tue, 25 Aug 2020 16:38:42 +0200
Subject: [PATCH 35/36] Unify rpc error-handling

---
 src/poolboy.erl | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 886fa9c..1e936ce 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -247,13 +247,15 @@ find_pid({via, Registry, Name}) ->
     Registry:whereis_name(Name);
 find_pid({Name, Node}) ->
     (catch erlang:monitor_node(Node, true)),
-    try rpc:call(Node, erlang, whereis, [Name], ?TIMEOUT) of
-        {badrpc, Reason} -> Reason;
-        Result -> Result
-    catch
-        _:Reason -> Reason
+    try rpc_call(Node, erlang, whereis, [Name], ?TIMEOUT)
+    catch _:Reason -> Reason
     end.
 
+rpc_call(Node, Mod, Fun, Args, Timeout) ->
+    case rpc:call(Node, Mod, Fun, Args, Timeout) of
+        {badrpc, Reason} -> exit({Reason, {Node, {Mod, Fun, Args}}});
+        Result -> Result
+    end.
 
 pool_size(PoolArgs) ->
     Is = is_integer(V = proplists:get_value(size, PoolArgs)),
@@ -433,13 +435,13 @@ start_pool(StartFun, PoolArgs, WorkerArgs) ->
 new_worker(Sup, Mod, Index)  ->
     Node = erlang:node(Sup),
     {ok, Pid} =
-    case rpc:pinfo(Sup, registered_name) of
+    case rpc_call(Node, erlang, process_info, [Sup, registered_name], ?TIMEOUT) of
         {registered_name, Name} ->
             case function_exported(Node, Name, start_child, 1) of
-                true -> rpc:call(Node, Name, start_child, [Index]);
+                true -> rpc_call(Node, Name, start_child, [Index], ?TIMEOUT);
                 false ->
                     case function_exported(Node, Name, start_child, 0) of
-                        true -> rpc:call(Node, Name, start_child, []);
+                        true -> rpc_call(Node, Name, start_child, [], ?TIMEOUT);
                         false ->
                             Args = child_args(Sup, Mod, Index),
                             supervisor:start_child(Sup, Args)
@@ -469,7 +471,7 @@ child_args(Sup, Mod, Index) ->
     end.
 
 function_exported(Node, Module, Name, Arity) ->
-    rpc:call(Node, erlang, function_exported, [Module, Name, Arity]).
+    rpc_call(Node, erlang, function_exported, [Module, Name, Arity], ?TIMEOUT).
 
 dismiss_worker(Sup, Pid) ->
     true = unlink(Pid),

From e2da6d3506b3ec4c306169a4f538105fbab0e025 Mon Sep 17 00:00:00 2001
From: Matyas Markovics <matyas@scrapinghub.com>
Date: Fri, 4 Dec 2020 11:22:53 +0100
Subject: [PATCH 36/36] Short-circuit process-termination when workers exit
 with reason: noconnection

---
 src/poolboy.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/poolboy.erl b/src/poolboy.erl
index 1e936ce..bbeead2 100644
--- a/src/poolboy.erl
+++ b/src/poolboy.erl
@@ -393,6 +393,8 @@ handle_info({'DOWN', MRef, _, _, _}, State) ->
             Waiting = queue:filter(fun ({_, _, R}) -> R =/= MRef end, State#state.waiting),
             {noreply, State#state{waiting = Waiting}}
     end;
+handle_info({'EXIT', _Pid, noconnection = Reason}, State) ->
+    {stop, Reason, State};
 handle_info({'EXIT', Pid, Reason}, State = #state{supervisor = Pid}) ->
     {stop, Reason, State};
 handle_info({'EXIT', Pid, _Reason}, State) ->