From a7aa39cfe4436dbe211d340891f9aaac6201edaf Mon Sep 17 00:00:00 2001 From: Alex S Date: Sat, 24 Aug 2019 17:44:34 +0300 Subject: [PATCH] calculated crf for closing connections don't close conn where are waiting pids --- lib/pleroma/gun/conn.ex | 9 +- lib/pleroma/gun/connections.ex | 39 +++++++-- test/gun/connections_test.exs | 149 +++++++++++++++++++++++++-------- 3 files changed, 154 insertions(+), 43 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 20ddec64c..9e5c2b184 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -10,8 +10,13 @@ defmodule Pleroma.Gun.Conn do conn: pid(), state: atom(), waiting_pids: [pid()], - used: pos_integer() + last_reference: pos_integer(), + crf: float() } - defstruct conn: nil, state: :open, waiting_pids: [], used: 0 + defstruct conn: nil, + state: :open, + waiting_pids: [], + last_reference: :os.system_time(:second), + crf: 1 end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 5b1c75d78..73d54e94d 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -73,8 +73,20 @@ defmodule Pleroma.Gun.Connections do key = compose_key(uri) case state.conns[key] do - %{conn: conn, state: conn_state, used: used} when conn_state == :up -> - state = put_in(state.conns[key].used, used + 1) + %{conn: conn, state: conn_state, last_reference: reference, crf: last_crf} = current_conn + when conn_state == :up -> + time = current_time() + last_reference = time - reference + + current_crf = crf(last_reference, 100, last_crf) + + state = + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf + }) + {:reply, conn, state} %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> @@ -87,7 +99,12 @@ defmodule Pleroma.Gun.Connections do if Enum.count(state.conns) < max_connections do open_conn(key, uri, from, state, opts) else - [{close_key, least_used} | _conns] = Enum.sort_by(state.conns, fn {_k, v} -> v.used end) + [{close_key, least_used} | _conns] = + state.conns + |> Enum.filter(fn {_k, v} -> v.waiting_pids == [] end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf < y.crf and x.last_reference < y.last_reference + end) :ok = API.close(least_used.conn) @@ -114,12 +131,17 @@ defmodule Pleroma.Gun.Connections do Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) # Update state of the current connection and set waiting_pids to empty list + time = current_time() + last_reference = time - conn.last_reference + current_crf = crf(last_reference, 100, conn.crf) + state = put_in(state.conns[key], %{ conn | state: :up, waiting_pids: [], - used: conn.used + length(conn.waiting_pids) + last_reference: time, + crf: current_crf }) {:noreply, state} @@ -180,7 +202,6 @@ defmodule Pleroma.Gun.Connections do put_in(state.conns[key], %Conn{ conn: conn, waiting_pids: [], - used: 1, state: :up }) @@ -210,4 +231,12 @@ defmodule Pleroma.Gun.Connections do {:reply, nil, state} end end + + defp current_time do + :os.system_time(:second) + end + + def crf(current, steps, crf) do + 1 + :math.pow(0.5, current / steps) * crf + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 4d84821a0..cc3675ad9 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -48,8 +48,7 @@ defmodule Gun.ConnectionsTest do "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -112,8 +111,7 @@ defmodule Gun.ConnectionsTest do "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :down, - waiting_pids: _, - used: 0 + waiting_pids: _ } } } = Connections.get_state(name) @@ -128,8 +126,7 @@ defmodule Gun.ConnectionsTest do "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -157,8 +154,7 @@ defmodule Gun.ConnectionsTest do "http:some-domain.com:80" => %Conn{ conn: conn, state: :up, - waiting_pids: [], - used: 5 + waiting_pids: [] } } } = Connections.get_state(name) @@ -178,14 +174,12 @@ defmodule Gun.ConnectionsTest do "http:some-domain.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] }, "https:some-domain.com:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -198,14 +192,12 @@ defmodule Gun.ConnectionsTest do "http:another-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] }, "http:some-domain.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -233,8 +225,7 @@ defmodule Gun.ConnectionsTest do "http:httpbin.org:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -258,8 +249,7 @@ defmodule Gun.ConnectionsTest do "https:httpbin.org:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -281,14 +271,12 @@ defmodule Gun.ConnectionsTest do "https:httpbin.org:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] }, "https:www.google.com:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -301,14 +289,60 @@ defmodule Gun.ConnectionsTest do "http:httpbin.org:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] }, "https:httpbin.org:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + + test "remove earlier used", %{name: name, pid: pid} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + + Process.sleep(1_000) + Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] + }, + "https:www.google.com:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + Process.sleep(1_000) + conn = Connections.get_conn("http://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -330,8 +364,7 @@ defmodule Gun.ConnectionsTest do "http:proxy_string.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -360,8 +393,7 @@ defmodule Gun.ConnectionsTest do "http:proxy_tuple_atom.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -390,8 +422,7 @@ defmodule Gun.ConnectionsTest do "https:proxy_string.com:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -420,8 +451,7 @@ defmodule Gun.ConnectionsTest do "https:proxy_tuple_atom.com:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -437,4 +467,51 @@ defmodule Gun.ConnectionsTest do assert reused_conn == conn end end + + describe "crf/3" do + setup do + crf = Connections.crf(1, 10, 1) + {:ok, crf: crf} + end + + test "more used will have crf higher", %{crf: crf} do + # used 3 times + crf1 = Connections.crf(1, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 > crf2 + end + + test "recently used will have crf higher on equal references", %{crf: crf} do + # used 4 sec ago + crf1 = Connections.crf(3, 10, crf) + + # used 3 sec ago + crf2 = Connections.crf(4, 10, crf) + + assert crf1 > crf2 + end + + test "equal crf on equal reference and time", %{crf: crf} do + # used 2 times + crf1 = Connections.crf(1, 10, crf) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 == crf2 + end + + test "recently used will have higher crf", %{crf: crf} do + crf1 = Connections.crf(2, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + crf2 = Connections.crf(3, 10, crf) + crf2 = Connections.crf(4, 10, crf2) + assert crf1 > crf2 + end + end end