sjw
/
pleroma
mirror of https://git.pleroma.social/sjw/pleroma.git
1
0
Fork 0
Code Issues Releases Wiki Activity

calculated crf for closing connections 

don't close conn where are waiting pids
This commit is contained in:
Alex S 2019-08-24 17:44:34 +03:00
parent 26a98737a8
commit a7aa39cfe4
3 changed files with 154 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/pleroma/gun/conn.ex
View File

@ -10,8 +10,13 @@ defmodule Pleroma.Gun.Conn do
conn: pid(),
state: atom(),
waiting_pids: [pid()],
used: pos_integer()
last_reference: pos_integer(),
crf: float()
}
defstruct conn: nil, state: :open, waiting_pids: [], used: 0
defstruct conn: nil,
state: :open,
waiting_pids: [],
last_reference: :os.system_time(:second),
crf: 1
end

39
lib/pleroma/gun/connections.ex
View File

@ -73,8 +73,20 @@ defmodule Pleroma.Gun.Connections do
key = compose_key(uri)
case state.conns[key] do
%{conn: conn, state: conn_state, used: used} when conn_state == :up ->
state = put_in(state.conns[key].used, used + 1)
%{conn: conn, state: conn_state, last_reference: reference, crf: last_crf} = current_conn
when conn_state == :up ->
time = current_time()
last_reference = time - reference
current_crf = crf(last_reference, 100, last_crf)
state =
put_in(state.conns[key], %{
current_conn
| last_reference: time,
crf: current_crf
})
{:reply, conn, state}
%{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] ->
@ -87,7 +99,12 @@ defmodule Pleroma.Gun.Connections do
if Enum.count(state.conns) < max_connections do
open_conn(key, uri, from, state, opts)
else
[{close_key, least_used} | _conns] = Enum.sort_by(state.conns, fn {_k, v} -> v.used end)
[{close_key, least_used} | _conns] =
state.conns
|> Enum.filter(fn {_k, v} -> v.waiting_pids == [] end)
|> Enum.sort(fn {_x_k, x}, {_y_k, y} ->
x.crf < y.crf and x.last_reference < y.last_reference
end)
:ok = API.close(least_used.conn)
@ -114,12 +131,17 @@ defmodule Pleroma.Gun.Connections do
Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end)
# Update state of the current connection and set waiting_pids to empty list
time = current_time()
last_reference = time - conn.last_reference
current_crf = crf(last_reference, 100, conn.crf)
state =
put_in(state.conns[key], %{
conn
| state: :up,
waiting_pids: [],
used: conn.used + length(conn.waiting_pids)
last_reference: time,
crf: current_crf
})
{:noreply, state}
@ -180,7 +202,6 @@ defmodule Pleroma.Gun.Connections do
put_in(state.conns[key], %Conn{
conn: conn,
waiting_pids: [],
used: 1,
state: :up
})
@ -210,4 +231,12 @@ defmodule Pleroma.Gun.Connections do
{:reply, nil, state}
end
end
defp current_time do
:os.system_time(:second)
end
def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf
end
end

149
test/gun/connections_test.exs
View File

@ -48,8 +48,7 @@ defmodule Gun.ConnectionsTest do
"http:some-domain.com:80" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 2
waiting_pids: []
}
}
} = Connections.get_state(name)
@ -112,8 +111,7 @@ defmodule Gun.ConnectionsTest do
"http:gun_down_and_up.com:80" => %Conn{
conn: _,
state: :down,
waiting_pids: _,
used: 0
waiting_pids: _
}
}
} = Connections.get_state(name)
@ -128,8 +126,7 @@ defmodule Gun.ConnectionsTest do
"http:gun_down_and_up.com:80" => %Conn{
conn: _,
state: :up,
waiting_pids: [],
used: 2
waiting_pids: []
}
}
} = Connections.get_state(name)
@ -157,8 +154,7 @@ defmodule Gun.ConnectionsTest do
"http:some-domain.com:80" => %Conn{
conn: conn,
state: :up,
waiting_pids: [],
used: 5
waiting_pids: []
}
}
} = Connections.get_state(name)
@ -178,14 +174,12 @@ defmodule Gun.ConnectionsTest do
"http:some-domain.com:80" => %Conn{
conn: _,
state: :up,
waiting_pids: [],
used: 4
waiting_pids: []
},
"https:some-domain.com:443" => %Conn{
conn: _,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -198,14 +192,12 @@ defmodule Gun.ConnectionsTest do
"http:another-domain.com:80" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
},
"http:some-domain.com:80" => %Conn{
conn: _,
state: :up,
waiting_pids: [],
used: 4
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -233,8 +225,7 @@ defmodule Gun.ConnectionsTest do
"http:httpbin.org:80" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 2
waiting_pids: []
}
}
} = Connections.get_state(name)
@ -258,8 +249,7 @@ defmodule Gun.ConnectionsTest do
"https:httpbin.org:443" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 2
waiting_pids: []
}
}
} = Connections.get_state(name)
@ -281,14 +271,12 @@ defmodule Gun.ConnectionsTest do
"https:httpbin.org:443" => %Conn{
conn: _,
state: :up,
waiting_pids: [],
used: 4
waiting_pids: []
},
"https:www.google.com:443" => %Conn{
conn: _,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -301,14 +289,60 @@ defmodule Gun.ConnectionsTest do
"http:httpbin.org:80" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
},
"https:httpbin.org:443" => %Conn{
conn: _,
state: :up,
waiting_pids: [],
used: 4
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
end
test "remove earlier used", %{name: name, pid: pid} do
api = Pleroma.Config.get([API])
Pleroma.Config.put([API], API.Gun)
on_exit(fn -> Pleroma.Config.put([API], api) end)
Connections.get_conn("https://www.google.com", [genserver_pid: pid], name)
Connections.get_conn("https://www.google.com", [genserver_pid: pid], name)
Process.sleep(1_000)
Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name)
Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name)
%Connections{
conns: %{
"https:httpbin.org:443" => %Conn{
conn: _,
state: :up,
waiting_pids: []
},
"https:www.google.com:443" => %Conn{
conn: _,
state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
} = Connections.get_state(name)
Process.sleep(1_000)
conn = Connections.get_conn("http://httpbin.org", [genserver_pid: pid], name)
%Connections{
conns: %{
"http:httpbin.org:80" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: []
},
"https:httpbin.org:443" => %Conn{
conn: _,
state: :up,
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -330,8 +364,7 @@ defmodule Gun.ConnectionsTest do
"http:proxy_string.com:80" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -360,8 +393,7 @@ defmodule Gun.ConnectionsTest do
"http:proxy_tuple_atom.com:80" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -390,8 +422,7 @@ defmodule Gun.ConnectionsTest do
"https:proxy_string.com:443" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -420,8 +451,7 @@ defmodule Gun.ConnectionsTest do
"https:proxy_tuple_atom.com:443" => %Conn{
conn: ^conn,
state: :up,
waiting_pids: [],
used: 1
waiting_pids: []
}
},
opts: [max_connections: 2, timeout: 10]
@ -437,4 +467,51 @@ defmodule Gun.ConnectionsTest do
assert reused_conn == conn
end
end
describe "crf/3" do
setup do
crf = Connections.crf(1, 10, 1)
{:ok, crf: crf}
end
test "more used will have crf higher", %{crf: crf} do
# used 3 times
crf1 = Connections.crf(1, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 > crf2
end
test "recently used will have crf higher on equal references", %{crf: crf} do
# used 4 sec ago
crf1 = Connections.crf(3, 10, crf)
# used 3 sec ago
crf2 = Connections.crf(4, 10, crf)
assert crf1 > crf2
end
test "equal crf on equal reference and time", %{crf: crf} do
# used 2 times
crf1 = Connections.crf(1, 10, crf)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 == crf2
end
test "recently used will have higher crf", %{crf: crf} do
crf1 = Connections.crf(2, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
crf2 = Connections.crf(3, 10, crf)
crf2 = Connections.crf(4, 10, crf2)
assert crf1 > crf2
end
end
end