From ca16ea05400cb6e327b63fcbdffa78c6bc3333ca Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 18:00:43 +0300 Subject: [PATCH 001/102] adding gun adapter --- config/config.exs | 2 +- mix.exs | 5 ++++- mix.lock | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 758661120..63162d594 100644 --- a/config/config.exs +++ b/config/config.exs @@ -186,7 +186,7 @@ config :mime, :types, %{ "application/ld+json" => ["activity+json"] } -config :tesla, adapter: Tesla.Adapter.Hackney +config :tesla, adapter: Tesla.Adapter.Gun # Configures http settings, upstream proxy etc. config :pleroma, :http, diff --git a/mix.exs b/mix.exs index 3170d6f2d..9aab71e90 100644 --- a/mix.exs +++ b/mix.exs @@ -111,7 +111,10 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, "~> 1.2"}, + {:tesla, + github: "teamon/tesla", ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822", override: true}, + {:cowlib, "~> 2.6.0", override: true}, + {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 2639e96e9..bf3d4f01a 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, + "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, @@ -37,6 +37,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, + "gun": {:hex, :gun, "1.3.0", "18e5d269649c987af95aec309f68a27ffc3930531dd227a6eaa0884d6684286e", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -84,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, + "tesla": {:git, "https://github.com/teamon/tesla.git", "97950754a77cde4e162ada31fb9d8cf4fd6ab822", [ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 33d04c1874d5c35acf1dabbf99a6d74c0b9b662f Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 20:04:37 +0300 Subject: [PATCH 002/102] namings --- lib/pleroma/http/connection.ex | 13 ++++++++----- lib/pleroma/http/http.ex | 2 +- lib/pleroma/reverse_proxy/reverse_proxy.ex | 2 +- .../activity_pub/mrf/mediaproxy_warming_policy.ex | 4 ++-- lib/pleroma/web/rel_me.ex | 4 ++-- lib/pleroma/web/rich_media/parser.ex | 4 ++-- mix.exs | 4 +++- mix.lock | 2 +- 8 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 7e2c6f5e8..8caf989a7 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -7,8 +7,10 @@ defmodule Pleroma.HTTP.Connection do Connection for http-requests. """ - @hackney_options [ + @options [ connect_timeout: 10_000, + protocols: [:http], + timeout: 20_000, recv_timeout: 20_000, follow_redirect: true, force_redirect: true, @@ -25,17 +27,18 @@ defmodule Pleroma.HTTP.Connection do """ @spec new(Keyword.t()) :: Tesla.Env.client() def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + middleware = [Tesla.Middleware.FollowRedirects] + Tesla.client(middleware, {@adapter, options(opts)}) end - # fetch Hackney options + # fetch http options # - def hackney_options(opts) do + def options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - @hackney_options + @options |> Keyword.merge(adapter_options) |> Keyword.merge(options) |> Keyword.merge(proxy: proxy_url) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index dec24458a..6d7934841 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,7 +65,7 @@ defmodule Pleroma.HTTP do end def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) + Keyword.merge(Pleroma.HTTP.Connection.options([]), options) end @doc """ diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 1f98f215c..7a7559165 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -94,7 +94,7 @@ defmodule Pleroma.ReverseProxy do def call(conn = %{method: method}, url, opts) when method in @methods do hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) + Pleroma.HTTP.Connection.options([]) |> Keyword.merge(@default_hackney_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index a179dd54d..52ef0167c 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ + @options [ pool: :media, recv_timeout: 10_000 ] @@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: @options) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index d376e2069..947234aa2 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -25,7 +25,7 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) data = Floki.attribute(html, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index f5f9e358c..ade4ac891 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -78,7 +78,7 @@ defmodule Pleroma.Web.RichMedia.Parser do defp parse_url(url) do try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) html |> maybe_parse() diff --git a/mix.exs b/mix.exs index 9aab71e90..84d58f5a6 100644 --- a/mix.exs +++ b/mix.exs @@ -112,7 +112,9 @@ defmodule Pleroma.Mixfile do {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, {:tesla, - github: "teamon/tesla", ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822", override: true}, + github: "alex-strizhakov/tesla", + ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", + override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index bf3d4f01a..d79bb9c80 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/teamon/tesla.git", "97950754a77cde4e162ada31fb9d8cf4fd6ab822", [ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", [ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From e7e11d52d34ffa3da57534bcc49601f4372d6c12 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 20:06:43 +0300 Subject: [PATCH 003/102] headers standardisation for tesla --- lib/pleroma/http/request_builder.ex | 2 +- lib/pleroma/object/fetcher.ex | 6 +- lib/pleroma/web/ostatus/ostatus.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- test/http/request_builder_test.exs | 2 +- test/support/http_request_mock.ex | 84 +++++++++++-------- test/web/twitter_api/util_controller_test.exs | 16 ++-- .../web_finger/web_finger_controller_test.exs | 13 +-- 8 files changed, 72 insertions(+), 55 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index e23457999..4e77870bd 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -48,7 +48,7 @@ defmodule Pleroma.HTTP.RequestBuilder do def headers(request, header_list) do header_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + header_list ++ [{"user-agent", Pleroma.Application.user_agent()}] else header_list end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 8d79ddb1f..46531a13a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -108,7 +108,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -122,7 +122,7 @@ defmodule Pleroma.Object.Fetcher do |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 331cbc0b7..1c400d9e4 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -373,7 +373,7 @@ defmodule Pleroma.Web.OStatus do {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( url, - [{:Accept, "application/atom+xml"}] + [{"accept", "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") handle_incoming(body, options) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index ecb39ee50..624ee5ef7 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -217,7 +217,7 @@ defmodule Pleroma.Web.WebFinger do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 7febe84c5..5a645d4f3 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -20,7 +20,7 @@ defmodule Pleroma.HTTP.RequestBuilderTest do end) assert RequestBuilder.headers(%{}, []) == %{ - headers: [{"User-Agent", Pleroma.Application.user_agent()}] + headers: [{"user-agent", Pleroma.Application.user_agent()}] } end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 3adb5ba3b..c4862eb1c 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -88,7 +88,7 @@ defmodule HttpRequestMock do "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -101,7 +101,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -122,7 +122,7 @@ defmodule HttpRequestMock do "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -148,7 +148,7 @@ defmodule HttpRequestMock do "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -169,7 +169,7 @@ defmodule HttpRequestMock do "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -182,7 +182,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -195,7 +195,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd/?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -208,7 +208,7 @@ defmodule HttpRequestMock do "https://mst3k.interlinked.me/users/luciferMysticus", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -229,7 +229,7 @@ defmodule HttpRequestMock do "https://hubzilla.example.org/channel/kaniini", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -238,7 +238,7 @@ defmodule HttpRequestMock do }} end - def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -246,7 +246,7 @@ defmodule HttpRequestMock do }} end - def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -265,7 +265,7 @@ defmodule HttpRequestMock do }} end - def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do + def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -321,7 +321,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/admin", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -329,7 +331,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/gargron", _, _, [ + {"accept", "application/activity+json"} + ]) do {:error, :nxdomain} end @@ -337,7 +341,7 @@ defmodule HttpRequestMock do "http://mastodon.example.org/@admin/99541947525187367", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -354,7 +358,7 @@ defmodule HttpRequestMock do }} end - def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do + def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -366,7 +370,7 @@ defmodule HttpRequestMock do "https://mstdn.io/users/mayuutann/statuses/99568293732299394", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -386,7 +390,7 @@ defmodule HttpRequestMock do }} end - def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json") + def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}]) when url in [ "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain", "https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain" @@ -413,7 +417,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -457,7 +461,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -510,7 +514,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -523,7 +527,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", _, _, - Accept: "application/atom+xml" + [{"accept", "application/atom+xml"}] ) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} end @@ -540,7 +544,7 @@ defmodule HttpRequestMock do "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -562,7 +566,7 @@ defmodule HttpRequestMock do "http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -576,7 +580,7 @@ defmodule HttpRequestMock do "http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{status: 406, body: ""}} end @@ -612,7 +616,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -625,7 +629,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -638,7 +642,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{status: 200, body: ""}} end @@ -655,7 +659,7 @@ defmodule HttpRequestMock do "http://framatube.org/main/xrd?uri=framasoft@framatube.org", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -677,7 +681,7 @@ defmodule HttpRequestMock do "http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -714,7 +718,7 @@ defmodule HttpRequestMock do "https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -772,7 +776,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}} end - def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do + def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)} end @@ -889,7 +893,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -902,7 +906,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -924,7 +928,9 @@ defmodule HttpRequestMock do }} end - def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -936,7 +942,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity2.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -948,7 +956,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity3.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 640579c09..b55656ae9 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Web.CommonAPI import Pleroma.Factory import Mock + import ExUnit.CaptureLog setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -377,15 +378,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert html_response(response, 200) =~ "Remote follow" end - test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do + test "show follow page with error when user cannot fetch by `acct` link", %{conn: conn} do user = insert(:user) - response = - conn - |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + assert capture_log(fn -> + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") - assert html_response(response, 200) =~ "Error fetching user" + assert html_response(response, 200) =~ "Error fetching user" + end) =~ + "Could not decode user at fetch https://mastodon.social/users/not_found, {:error, \"Object has been deleted\"}" end end diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index 7d861cbf5..d9c24b8d5 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do import Pleroma.Factory import Tesla.Mock + import ExUnit.CaptureLog setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -77,11 +78,13 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do test "Sends a 404 when invalid format" do user = insert(:user) - assert_raise Phoenix.NotAcceptableError, fn -> - build_conn() - |> put_req_header("accept", "text/html") - |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") - end + assert capture_log(fn -> + assert_raise Phoenix.NotAcceptableError, fn -> + build_conn() + |> put_req_header("accept", "text/html") + |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") + end + end) =~ "Internal server error:" end test "Sends a 400 when resource param is missing" do From 1f6b2a3d4378abb486aec938037162774ccb3f2b Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 8 Aug 2019 18:42:50 +0300 Subject: [PATCH 004/102] added tesla client for reverse proxy --- lib/pleroma/http/connection.ex | 6 +- lib/pleroma/reverse_proxy/client.ex | 17 +++-- lib/pleroma/reverse_proxy/client/hackney.ex | 17 +++++ lib/pleroma/reverse_proxy/client/tesla.ex | 48 +++++++++++++ lib/pleroma/reverse_proxy/reverse_proxy.ex | 10 +-- mix.exs | 2 +- mix.lock | 2 +- test/reverse_proxy_test.exs | 76 ++++++++++----------- 8 files changed, 122 insertions(+), 56 deletions(-) create mode 100644 lib/pleroma/reverse_proxy/client/hackney.ex create mode 100644 lib/pleroma/reverse_proxy/client/tesla.ex diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 8caf989a7..4ebe16e18 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -10,11 +10,7 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, protocols: [:http], - timeout: 20_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation + timeout: 20_000 ] @adapter Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 776c4794c..42f2ff13b 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,9 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} @@ -14,8 +19,8 @@ defmodule Pleroma.ReverseProxy.Client do @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) end end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex new file mode 100644 index 000000000..e6293646a --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.ReverseProxy.Client.Hackney do + @behaviour Pleroma.ReverseProxy.Client + + def request(method, url, headers, body, opts \\ []) do + :hackney.request(method, url, headers, body, opts) + end + + def stream_body(ref) do + case :hackney.stream_body(ref) do + :done -> :done + {:ok, data} -> {:ok, data, ref} + {:error, error} -> {:error, error} + end + end + + def close(ref), do: :hackney.close(ref) +end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..d2944b9dc --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,48 @@ +defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + + @adapters [Tesla.Adapter.Gun] + alias Pleroma.HTTP + + def request(method, url, headers, body, opts \\ []) do + adapter_opts = + Keyword.get(opts, :adapter, []) + |> Keyword.put(:chunks_response, true) + + with {:ok, response} <- + HTTP.request(method, url, body, headers, Keyword.put(opts, :adapter, adapter_opts)) do + {:ok, response.status, response.headers, response.body} + else + {:error, error} -> {:error, error} + end + end + + def stream_body(%{fin: true}), do: :done + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> {:ok, body, Map.put(client, :fin, true)} + {:nofin, part} -> {:ok, part, client} + end + end + + defp read_chunk!(client) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter.read_chunk(client) + end + + def close(client) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support closing connection" + end + + adapter.close(client) + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 7a7559165..c3ef412a4 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -93,9 +93,9 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = + client_opts = Pleroma.HTTP.Connection.options([]) - |> Keyword.merge(@default_hackney_options) + |> Keyword.merge(@default_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() @@ -108,7 +108,7 @@ defmodule Pleroma.ReverseProxy do opts end - with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + with {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length)) do response(conn, client, url, code, headers, opts) else @@ -197,7 +197,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), diff --git a/mix.exs b/mix.exs index 84d58f5a6..f1fbdc6b3 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", + ref: "beb8927358dfaa66ecd458df607befde12dd56e0", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index d79bb9c80..635f20185 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", [ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "beb8927358dfaa66ecd458df607befde12dd56e0", [ref: "beb8927358dfaa66ecd458df607befde12dd56e0"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs index f4b7d6add..df559cb66 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy_test.exs @@ -29,11 +29,11 @@ defmodule Pleroma.ReverseProxyTest do {"content-length", byte_size(json) |> to_string()} ], %{url: url}} end) - |> expect(:stream_body, invokes, fn %{url: url} -> + |> expect(:stream_body, invokes, fn %{url: url} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) - {:ok, json} + {:ok, json, client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -66,6 +66,38 @@ defmodule Pleroma.ReverseProxyTest do assert conn.halted end + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789", client} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + describe "max_body " do test "length returns error if content-length more than option", %{conn: conn} do user_agent_mock("hackney/1.15.1", 0) @@ -76,38 +108,6 @@ defmodule Pleroma.ReverseProxyTest do "[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large" end - defp stream_mock(invokes, with_close? \\ false) do - ClientMock - |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) - - {:ok, 200, [{"content-type", "application/octet-stream"}], - %{url: "/stream-bytes/" <> length}} - end) - |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> - max = String.to_integer(length) - - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do - [{_, current}] when current < max -> - Registry.update_value( - Pleroma.ReverseProxy.ClientMock, - "/stream-bytes/" <> length, - &(&1 + 10) - ) - - {:ok, "0123456789"} - - [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) - :done - end - end) - - if with_close? do - expect(ClientMock, :close, fn _ -> :ok end) - end - end - test "max_body_size returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) @@ -179,12 +179,12 @@ defmodule Pleroma.ReverseProxyTest do Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} end) - |> expect(:stream_body, 2, fn %{url: url, headers: headers} -> + |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} - {:ok, Jason.encode!(%{headers: headers})} + {:ok, Jason.encode!(%{headers: headers}), client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -261,11 +261,11 @@ defmodule Pleroma.ReverseProxyTest do {:ok, 200, headers, %{url: "/disposition"}} end) - |> expect(:stream_body, 2, fn %{url: "/disposition"} -> + |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) - {:ok, ""} + {:ok, "", client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") From 33e7657fccee9cccf04c59212b674cd1f0d8135a Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 8 Aug 2019 20:08:10 +0300 Subject: [PATCH 005/102] fix --- lib/pleroma/reverse_proxy/client/tesla.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index d2944b9dc..9b0be5bda 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -26,14 +26,14 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do end end - defp read_chunk!(client) do + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do adapter = Application.get_env(:tesla, :adapter) unless adapter in @adapters do raise "#{adapter} doesn't support reading body in chunks" end - adapter.read_chunk(client) + adapter.read_chunk(pid, stream, opts) end def close(client) do From b0eefe235f93814a621dfd14e45ccae54fd71a07 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 9 Aug 2019 12:24:43 +0300 Subject: [PATCH 006/102] possibility to set tesla adapter in runtime --- lib/pleroma/http/connection.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 4ebe16e18..ff03a3ee3 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -12,7 +12,6 @@ defmodule Pleroma.HTTP.Connection do protocols: [:http], timeout: 20_000 ] - @adapter Application.get_env(:tesla, :adapter) @doc """ Configure a client connection @@ -24,7 +23,8 @@ defmodule Pleroma.HTTP.Connection do @spec new(Keyword.t()) :: Tesla.Env.client() def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] - Tesla.client(middleware, {@adapter, options(opts)}) + adapter = Application.get_env(:tesla, :adapter) + Tesla.client(middleware, {adapter, options(opts)}) end # fetch http options From 26691b1b35da6d192263fbd0938eefe2657cf25d Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 9 Aug 2019 12:25:58 +0300 Subject: [PATCH 007/102] reverse proxy clients integration tests tesla reverse proxy client fixes --- lib/pleroma/reverse_proxy/client/tesla.ex | 14 +++- test/reverse_proxy/client/hackney_test.exs | 7 ++ test/reverse_proxy/client/tesla_test.exs | 9 +++ .../reverse_proxy_test.exs | 4 + test/support/reverse_proxy_client_case.ex | 76 +++++++++++++++++++ 5 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 test/reverse_proxy/client/hackney_test.exs create mode 100644 test/reverse_proxy/client/tesla_test.exs rename test/{ => reverse_proxy}/reverse_proxy_test.exs (99%) create mode 100644 test/support/reverse_proxy_client_case.ex diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 9b0be5bda..b1498a5a4 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -2,7 +2,6 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client @adapters [Tesla.Adapter.Gun] - alias Pleroma.HTTP def request(method, url, headers, body, opts \\ []) do adapter_opts = @@ -10,8 +9,16 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do |> Keyword.put(:chunks_response, true) with {:ok, response} <- - HTTP.request(method, url, body, headers, Keyword.put(opts, :adapter, adapter_opts)) do - {:ok, response.status, response.headers, response.body} + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, adapter_opts) + ) do + if is_map(response.body), + do: {:ok, response.status, response.headers, response.body}, + else: {:ok, response.status, response.headers} else {:error, error} -> {:error, error} end @@ -23,6 +30,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do case read_chunk!(client) do {:fin, body} -> {:ok, body, Map.put(client, :fin, true)} {:nofin, part} -> {:ok, part, client} + {:error, error} -> {:error, error} end end diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs new file mode 100644 index 000000000..577e0b0b2 --- /dev/null +++ b/test/reverse_proxy/client/hackney_test.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.ReverseProxy.Client.HackneyTest do + use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Hackney + + defp check_ref(ref) do + assert is_reference(ref) + end +end diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs new file mode 100644 index 000000000..029a25d0f --- /dev/null +++ b/test/reverse_proxy/client/tesla_test.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.ReverseProxy.Client.TeslaTest do + use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla + + defp check_ref(%{pid: pid, stream: stream} = ref) do + assert is_pid(pid) + assert is_reference(stream) + assert ref[:fin] + end +end diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs similarity index 99% rename from test/reverse_proxy_test.exs rename to test/reverse_proxy/reverse_proxy_test.exs index df559cb66..96667ab04 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -297,4 +297,8 @@ defmodule Pleroma.ReverseProxyTest do assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers end end + + describe "integration tests" do + @describetag :integration + end end diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex new file mode 100644 index 000000000..40cd59ea2 --- /dev/null +++ b/test/support/reverse_proxy_client_case.ex @@ -0,0 +1,76 @@ +defmodule Pleroma.ReverseProxyClientCase do + defmacro __using__(client: client) do + quote do + use ExUnit.Case + @moduletag :integration + @client unquote(client) + + setup do + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, Tesla.Mock) end) + end + + test "get response body stream" do + {:ok, status, headers, ref} = + @client.request( + :get, + "http://httpbin.org/stream-bytes/10", + [{"accept", "application/octet-stream"}], + "", + [] + ) + + assert status == 200 + assert headers != [] + + {:ok, response, ref} = @client.stream_body(ref) + check_ref(ref) + assert is_binary(response) + assert byte_size(response) == 10 + + assert :done == @client.stream_body(ref) + end + + test "head response" do + {:ok, status, headers} = @client.request(:head, "http://httpbin.org/get", [], "", []) + + assert status == 200 + assert headers != [] + end + + test "get error response" do + case @client.request( + :get, + "http://httpbin.org/status/500", + [], + "", + [] + ) do + {:ok, status, headers, ref} -> + assert status == 500 + assert headers != [] + check_ref(ref) + + assert :ok = @client.close(ref) + + {:ok, status, headers} -> + assert headers != [] + end + end + + test "head error response" do + {:ok, status, headers} = + @client.request( + :head, + "http://httpbin.org/status/500", + [], + "", + [] + ) + + assert status == 500 + assert headers != [] + end + end + end +end From 246906165776874ca7e4064b197ddf1237af2d1c Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 13 Aug 2019 14:37:19 +0300 Subject: [PATCH 008/102] added gun connections genserver --- config/test.exs | 2 + lib/pleroma/gun/api/api.ex | 15 ++ lib/pleroma/gun/api/mock.ex | 40 +++++ lib/pleroma/gun/conn.ex | 17 +++ lib/pleroma/gun/connections.ex | 104 +++++++++++++ lib/pleroma/http/connection.ex | 1 - lib/pleroma/reverse_proxy/client.ex | 2 +- lib/pleroma/reverse_proxy/client/tesla.ex | 10 +- mix.exs | 2 +- mix.lock | 2 +- test/gun/connections_test.exs | 172 ++++++++++++++++++++++ test/reverse_proxy/reverse_proxy_test.exs | 33 +++++ 12 files changed, 393 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/gun/api/api.ex create mode 100644 lib/pleroma/gun/api/mock.ex create mode 100644 lib/pleroma/gun/conn.ex create mode 100644 lib/pleroma/gun/connections.ex create mode 100644 test/gun/connections_test.exs diff --git a/config/test.exs b/config/test.exs index 6f75f39b5..ca916d59e 100644 --- a/config/test.exs +++ b/config/test.exs @@ -85,6 +85,8 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock +config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock + try do import_config "test.secret.exs" rescue diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex new file mode 100644 index 000000000..c69ab890e --- /dev/null +++ b/lib/pleroma/gun/api/api.ex @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + + def open(host, port, opts) do + api().open(host, port, opts) + end + + defp api do + Pleroma.Config.get([Pleroma.Gun.API], :gun) + end +end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex new file mode 100644 index 000000000..3348715c4 --- /dev/null +++ b/lib/pleroma/gun/api/mock.ex @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Mock do + @behaviour Pleroma.Gun.API + @impl Pleroma.Gun.API + def open('some-domain.com', 80, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_up, conn_pid, :http}) + {:ok, conn_pid} + end + + def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_up, conn_pid, :https}) + {:ok, conn_pid} + end + + @impl Pleroma.Gun.API + def open('gun_down.com', _port, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + {:ok, conn_pid} + end + + @impl Pleroma.Gun.API + def open('gun_down_and_up.com', _port, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + + {:ok, _} = + Task.start_link(fn -> + Process.sleep(500) + send(genserver_pid, {:gun_up, conn_pid, :http}) + end) + + {:ok, conn_pid} + end +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..62ef146a1 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + @type t :: %__MODULE__{ + conn: pid(), + state: atom(), + waiting_pids: [pid()], + protocol: atom() + } + + defstruct conn: nil, state: :open, waiting_pids: [], protocol: :http +end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex new file mode 100644 index 000000000..60ec68d89 --- /dev/null +++ b/lib/pleroma/gun/connections.ex @@ -0,0 +1,104 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Connections do + use GenServer + + @type domain :: String.t() + @type conn :: Gun.Conn.t() + @type t :: %__MODULE__{ + conns: %{domain() => conn()} + } + + defstruct conns: %{} + + def start_link(name \\ __MODULE__) do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + GenServer.start_link(__MODULE__, [], name: name) + else + :ignore + end + end + + @impl true + def init(_) do + {:ok, %__MODULE__{conns: %{}}} + end + + @spec get_conn(atom(), String.t(), keyword()) :: pid() + def get_conn(name \\ __MODULE__, url, opts \\ []) do + opts = Enum.into(opts, %{}) + uri = URI.parse(url) + + opts = if uri.scheme == "https", do: Map.put(opts, :transport, :tls), else: opts + + GenServer.call( + name, + {:conn, %{opts: opts, uri: uri}} + ) + end + + @spec get_state(atom()) :: t() + def get_state(name \\ __MODULE__) do + GenServer.call(name, {:state}) + end + + @impl true + def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do + key = compose_key(uri) + + case state.conns[key] do + %{conn: conn, state: conn_state} when conn_state == :up -> + {:reply, conn, state} + + %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> + state = put_in(state.conns[key].waiting_pids, [from | pids]) + {:noreply, state} + + nil -> + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + + state = + put_in(state.conns[key], %Pleroma.Gun.Conn{ + conn: conn, + waiting_pids: [from], + protocol: String.to_atom(uri.scheme) + }) + + {:noreply, state} + end + end + + @impl true + def handle_call({:state}, _from, state), do: {:reply, state, state} + + @impl true + def handle_info({:gun_up, conn_pid, protocol}, state) do + {key, conn} = find_conn(state.conns, conn_pid, protocol) + + # Send to all waiting processes connection pid + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) + + # Update state of the current connection and set waiting_pids to empty list + state = put_in(state.conns[key], %{conn | state: :up, waiting_pids: []}) + {:noreply, state} + end + + @impl true + # Do we need to do something with killed & unprocessed references? + def handle_info({:gun_down, conn_pid, protocol, _reason, _killed, _unprocessed}, state) do + {key, conn} = find_conn(state.conns, conn_pid, protocol) + + # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) + + state = put_in(state.conns[key].state, :down) + {:noreply, state} + end + + defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) + + defp find_conn(conns, conn_pid, protocol), + do: Enum.find(conns, fn {_, conn} -> conn.conn == conn_pid and conn.protocol == protocol end) +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index ff03a3ee3..6cb26c0fe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,6 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - protocols: [:http], timeout: 20_000 ] diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 42f2ff13b..71c2b2911 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -28,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) + Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) end end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index b1498a5a4..fad577ec1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-onl + defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client @@ -6,7 +10,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do adapter_opts = Keyword.get(opts, :adapter, []) - |> Keyword.put(:chunks_response, true) + |> Keyword.put(:body_as, :chunks) with {:ok, response} <- Pleroma.HTTP.request( @@ -44,13 +48,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do adapter.read_chunk(pid, stream, opts) end - def close(client) do + def close(pid) do adapter = Application.get_env(:tesla, :adapter) unless adapter in @adapters do raise "#{adapter} doesn't support closing connection" end - adapter.close(client) + adapter.close(pid) end end diff --git a/mix.exs b/mix.exs index f1fbdc6b3..9d04b5771 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "beb8927358dfaa66ecd458df607befde12dd56e0", + ref: "c29a7fd030fa6decbf7091152f563fe322e2b589", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 635f20185..ef5ebda6c 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "beb8927358dfaa66ecd458df607befde12dd56e0", [ref: "beb8927358dfaa66ecd458df607befde12dd56e0"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "c29a7fd030fa6decbf7091152f563fe322e2b589", [ref: "c29a7fd030fa6decbf7091152f563fe322e2b589"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs new file mode 100644 index 000000000..2ec8f3993 --- /dev/null +++ b/test/gun/connections_test.exs @@ -0,0 +1,172 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Gun.ConnectionsTest do + use ExUnit.Case, async: true + alias Pleroma.Gun.{Connections, Conn, API} + + setup do + name = :test_gun_connections + {:ok, pid} = Connections.start_link(name) + + {:ok, name: name, pid: pid} + end + + test "opens connection and reuse it on next request", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "reuses connection based on protocol", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + assert is_pid(conn) + assert Process.alive?(conn) + + https_conn = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + + refute conn == https_conn + + reused_https = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + + refute conn == reused_https + + assert reused_https == https_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + }, + "some-domain.com:443" => %Conn{ + conn: ^https_conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "process gun_down message", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://gun_down.com", genserver_pid: pid) + + refute conn + + %Connections{ + conns: %{ + "gun_down.com:80" => %Conn{ + conn: _, + state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + end + + test "process gun_down message and then gun_up", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + + refute conn + + %Connections{ + conns: %{ + "gun_down_and_up.com:80" => %Conn{ + conn: _, + state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + + conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + + assert is_pid(conn) + assert Process.alive?(conn) + + %Connections{ + conns: %{ + "gun_down_and_up.com:80" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "async processes get same conn for same domain", %{name: name, pid: pid} do + tasks = + for _ <- 1..5 do + Task.async(fn -> + Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + end) + end + + tasks_with_results = Task.yield_many(tasks) + + results = + Enum.map(tasks_with_results, fn {task, res} -> + res || Task.shutdown(task, :brutal_kill) + end) + + conns = for {:ok, value} <- results, do: value + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + + assert Enum.all?(conns, fn res -> res == conn end) + end + + describe "integration test" do + @describetag :integration + + test "opens connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], :gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.get_conn(name, "http://httpbin.org") + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn(name, "http://httpbin.org") + + assert conn == reused_conn + + %Connections{ + conns: %{ + "httpbin.org:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + end +end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 96667ab04..c2430a2b8 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -300,5 +300,38 @@ defmodule Pleroma.ReverseProxyTest do describe "integration tests" do @describetag :integration + + test "with hackney client", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + end) + + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + end + + test "with tesla client with gun adapter", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + end) + end end end From aa0ab31b5ba24f9d482847193a309037907bb71d Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 13:37:27 +0300 Subject: [PATCH 009/102] added gun connections holder genserver --- lib/pleroma/application.ex | 3 +- lib/pleroma/gun/connections.ex | 25 +++++++++- lib/pleroma/http/http.ex | 23 +++++++++ test/gun/connections_test.exs | 57 ++++++++++++++++++----- test/reverse_proxy/reverse_proxy_test.exs | 7 ++- 5 files changed, 99 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 25e56b9e2..f755a1355 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,8 @@ defmodule Pleroma.Application do Pleroma.Emoji, Pleroma.Captcha, Pleroma.FlakeId, - Pleroma.ScheduledActivityWorker + Pleroma.ScheduledActivityWorker, + Pleroma.Gun.Connections ] ++ cachex_children() ++ hackney_pool_children() ++ diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 60ec68d89..d5a9d8607 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -26,8 +26,8 @@ defmodule Pleroma.Gun.Connections do {:ok, %__MODULE__{conns: %{}}} end - @spec get_conn(atom(), String.t(), keyword()) :: pid() - def get_conn(name \\ __MODULE__, url, opts \\ []) do + @spec get_conn(String.t(), keyword(), atom()) :: pid() + def get_conn(url, opts \\ [], name \\ __MODULE__) do opts = Enum.into(opts, %{}) uri = URI.parse(url) @@ -39,6 +39,27 @@ defmodule Pleroma.Gun.Connections do ) end + # TODO: only for testing, add this parameter to the config + @spec try_to_get_gun_conn(String.t(), keyword(), atom()) :: nil | pid() + def try_to_get_gun_conn(url, opts \\ [], name \\ __MODULE__), + do: try_to_get_gun_conn(url, opts, name, 0) + + @spec try_to_get_gun_conn(String.t(), keyword(), atom(), pos_integer()) :: nil | pid() + def try_to_get_gun_conn(_url, _, _, 3), do: nil + + def try_to_get_gun_conn(url, opts, name, acc) do + case Pleroma.Gun.Connections.get_conn(url, opts, name) do + nil -> try_to_get_gun_conn(url, acc + 1) + conn -> conn + end + end + + @spec alive?(atom()) :: boolean() + def alive?(name \\ __MODULE__) do + pid = Process.whereis(name) + if pid, do: Process.alive?(pid), else: false + end + @spec get_state(atom()) :: t() def get_state(name \\ __MODULE__) do GenServer.call(name, {:state}) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 6d7934841..1846749c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -32,6 +32,15 @@ defmodule Pleroma.HTTP do process_request_options(options) |> process_sni_options(url) + adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun + + options = + if adapter_gun? and Pleroma.Gun.Connections.alive?() do + get_conn_for_gun(url, options) + else + options + end + params = Keyword.get(options, :params, []) %{} @@ -52,6 +61,20 @@ defmodule Pleroma.HTTP do end end + defp get_conn_for_gun(url, options) do + case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + nil -> + options + + conn -> + adapter_opts = + Keyword.get(options, :adapter, []) + |> Keyword.put(:conn, conn) + + Keyword.put(options, :adapter, adapter_opts) + end + end + defp process_sni_options(options, nil), do: options defp process_sni_options(options, url) do diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 2ec8f3993..42354d330 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -3,23 +3,56 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Gun.ConnectionsTest do - use ExUnit.Case, async: true + use ExUnit.Case alias Pleroma.Gun.{Connections, Conn, API} setup do name = :test_gun_connections + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) {:ok, pid} = Connections.start_link(name) {:ok, name: name, pid: pid} end + describe "alive?/2" do + test "is alive", %{name: name} do + assert Connections.alive?(name) + end + + test "returns false if not started" do + refute Connections.alive?(:some_random_name) + end + end + + test "try_to_get_gun_conn/1 returns conn", %{name: name, pid: pid} do + conn = Connections.try_to_get_gun_conn("http://some-domain.com", [genserver_pid: pid], name) + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + test "opens connection and reuse it on next request", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert conn == reused_conn @@ -35,15 +68,15 @@ defmodule Gun.ConnectionsTest do end test "reuses connection based on protocol", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - https_conn = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + https_conn = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) refute conn == https_conn - reused_https = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + reused_https = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) refute conn == reused_https @@ -66,7 +99,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://gun_down.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down.com", [genserver_pid: pid], name) refute conn @@ -82,7 +115,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message and then gun_up", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) refute conn @@ -96,7 +129,7 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) - conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) @@ -116,7 +149,7 @@ defmodule Gun.ConnectionsTest do tasks = for _ <- 1..5 do Task.async(fn -> - Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) end) end @@ -149,12 +182,12 @@ defmodule Gun.ConnectionsTest do api = Pleroma.Config.get([API]) Pleroma.Config.put([API], :gun) on_exit(fn -> Pleroma.Config.put([API], api) end) - conn = Connections.get_conn(name, "http://httpbin.org") + conn = Connections.get_conn("http://httpbin.org", [], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn(name, "http://httpbin.org") + reused_conn = Connections.get_conn("http://httpbin.org", [], name) assert conn == reused_conn diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index c2430a2b8..91cf5c1c8 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxyTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase import ExUnit.CaptureLog import Mox alias Pleroma.ReverseProxy @@ -322,6 +322,10 @@ defmodule Pleroma.ReverseProxyTest do adapter = Application.get_env(:tesla, :adapter) Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") assert byte_size(conn.resp_body) == 10 @@ -331,6 +335,7 @@ defmodule Pleroma.ReverseProxyTest do on_exit(fn -> Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) end) end end From c4dc8a800983efe33981cc9d5f345db77aebdf3e Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:08:10 +0300 Subject: [PATCH 010/102] some debug --- lib/pleroma/gun/connections.ex | 5 +++++ lib/pleroma/http/http.ex | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index d5a9d8607..82579604a 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -69,6 +69,10 @@ defmodule Pleroma.Gun.Connections do def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do key = compose_key(uri) + IO.inspect(state) + + IO.inspect(key) + case state.conns[key] do %{conn: conn, state: conn_state} when conn_state == :up -> {:reply, conn, state} @@ -87,6 +91,7 @@ defmodule Pleroma.Gun.Connections do protocol: String.to_atom(uri.scheme) }) + IO.inspect(state) {:noreply, state} end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 1846749c0..3ad891d11 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + case Pleroma.Gun.Connections.get_conn(url) do nil -> options From 407390eee75982bcf1a4d14a0507c4d7669f81bf Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:27:55 +0300 Subject: [PATCH 011/102] ssl fixes --- lib/pleroma/gun/api/mock.ex | 2 +- lib/pleroma/gun/connections.ex | 16 +++++++++------ test/gun/connections_test.exs | 24 +++++++++++++++++++++++ test/reverse_proxy/reverse_proxy_test.exs | 23 ++++++++++++++++++++++ 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index 3348715c4..ff9e13a74 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Gun.API.Mock do def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - send(genserver_pid, {:gun_up, conn_pid, :https}) + send(genserver_pid, {:gun_up, conn_pid, :http2}) {:ok, conn_pid} end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 82579604a..6fcf4332a 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -100,8 +100,8 @@ defmodule Pleroma.Gun.Connections do def handle_call({:state}, _from, state), do: {:reply, state, state} @impl true - def handle_info({:gun_up, conn_pid, protocol}, state) do - {key, conn} = find_conn(state.conns, conn_pid, protocol) + def handle_info({:gun_up, conn_pid, _protocol}, state) do + {key, conn} = find_conn(state.conns, conn_pid) # Send to all waiting processes connection pid Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) @@ -113,8 +113,8 @@ defmodule Pleroma.Gun.Connections do @impl true # Do we need to do something with killed & unprocessed references? - def handle_info({:gun_down, conn_pid, protocol, _reason, _killed, _unprocessed}, state) do - {key, conn} = find_conn(state.conns, conn_pid, protocol) + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do + {key, conn} = find_conn(state.conns, conn_pid) # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) @@ -125,6 +125,10 @@ defmodule Pleroma.Gun.Connections do defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) - defp find_conn(conns, conn_pid, protocol), - do: Enum.find(conns, fn {_, conn} -> conn.conn == conn_pid and conn.protocol == protocol end) + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {key, conn} -> + protocol = if String.ends_with?(key, ":443"), do: :https, else: :http + conn.conn == conn_pid and conn.protocol == protocol + end) + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 42354d330..aad70a644 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -201,5 +201,29 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) end + + test "opens ssl connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], :gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.get_conn("https://httpbin.org", [], name) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn("https://httpbin.org", [], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "httpbin.org:443" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end end end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 91cf5c1c8..1e9967278 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -338,5 +338,28 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], api) end) end + + test "with tesla client with gun adapter with ssl", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end end end From d04da53ad68701a8a8ecc9b2d05949f1c07e5fa7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:34:49 +0300 Subject: [PATCH 012/102] removing debug --- lib/pleroma/gun/connections.ex | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 6fcf4332a..e06cad276 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -69,10 +69,6 @@ defmodule Pleroma.Gun.Connections do def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do key = compose_key(uri) - IO.inspect(state) - - IO.inspect(key) - case state.conns[key] do %{conn: conn, state: conn_state} when conn_state == :up -> {:reply, conn, state} @@ -91,7 +87,6 @@ defmodule Pleroma.Gun.Connections do protocol: String.to_atom(uri.scheme) }) - IO.inspect(state) {:noreply, state} end end From 2ca6d54437f326756f659bd52a658625a022e24a Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:38:13 +0300 Subject: [PATCH 013/102] some debug --- lib/pleroma/http/http.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3ad891d11..a6c325017 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -41,6 +41,8 @@ defmodule Pleroma.HTTP do options end + IO.inspect(options) + params = Keyword.get(options, :params, []) %{} @@ -62,7 +64,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url) do + case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do nil -> options From 8b26870809b37b52c782e0de85dd91cc0255ee8d Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:41:08 +0300 Subject: [PATCH 014/102] tesla adapter options debug --- lib/pleroma/http/http.ex | 2 -- lib/pleroma/reverse_proxy/client/tesla.ex | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a6c325017..1846749c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -41,8 +41,6 @@ defmodule Pleroma.HTTP do options end - IO.inspect(options) - params = Keyword.get(options, :params, []) %{} diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index fad577ec1..282f9dcb7 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -12,6 +12,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do Keyword.get(opts, :adapter, []) |> Keyword.put(:body_as, :chunks) + IO.inspect(adapter_opts) + with {:ok, response} <- Pleroma.HTTP.request( method, From 9f2e2dea65ebc05662b1918939324f32f6179307 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:45:21 +0300 Subject: [PATCH 015/102] another debug --- lib/pleroma/http/connection.ex | 4 +++- lib/pleroma/reverse_proxy/client/tesla.ex | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 6cb26c0fe..20c83c572 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -23,7 +23,9 @@ defmodule Pleroma.HTTP.Connection do def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) - Tesla.client(middleware, {adapter, options(opts)}) + options = options(opts) + IO.inspect(options) + Tesla.client(middleware, {adapter, options}) end # fetch http options diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 282f9dcb7..fad577ec1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -12,8 +12,6 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do Keyword.get(opts, :adapter, []) |> Keyword.put(:body_as, :chunks) - IO.inspect(adapter_opts) - with {:ok, response} <- Pleroma.HTTP.request( method, From 0c68cf435725a252be54988dfb6a914e3276b766 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 15:54:25 +0300 Subject: [PATCH 016/102] like this --- lib/pleroma/http/connection.ex | 1 - lib/pleroma/http/http.ex | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 20c83c572..222a8b6aa 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -24,7 +24,6 @@ defmodule Pleroma.HTTP.Connection do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) options = options(opts) - IO.inspect(options) Tesla.client(middleware, {adapter, options}) end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 1846749c0..3ad891d11 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + case Pleroma.Gun.Connections.get_conn(url) do nil -> options From beab94d6898f8985e19e30dc24416d311c3f04c7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 16:31:13 +0300 Subject: [PATCH 017/102] only http --- lib/pleroma/http/connection.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 222a8b6aa..ef1c1cabd 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,8 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000 + timeout: 20_000, + protocols: [:http] ] @doc """ From 44e2136191a579cafd91dbe8fd91255e080f21f2 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 17:48:22 +0300 Subject: [PATCH 018/102] don't close connection in gun adapter on reusing --- lib/pleroma/http/http.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3ad891d11..d493a8b4a 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -70,6 +70,7 @@ defmodule Pleroma.HTTP do adapter_opts = Keyword.get(options, :adapter, []) |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) Keyword.put(options, :adapter, adapter_opts) end From a577fa3c1a39f7dd79ad7c5ae3928c23c6be2da2 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 17:56:16 +0300 Subject: [PATCH 019/102] tesla update --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 9d04b5771..f5200d15d 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "c29a7fd030fa6decbf7091152f563fe322e2b589", + ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index ef5ebda6c..f09b7fece 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "c29a7fd030fa6decbf7091152f563fe322e2b589", [ref: "c29a7fd030fa6decbf7091152f563fe322e2b589"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "99856638d0b8f382eb22e704b76bc30b4c4c379d", [ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 5aa1b8283da91aa4b6c6fe35236ffd3b7ee4ef58 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 20:08:15 +0300 Subject: [PATCH 020/102] adding host header for gun adapter --- lib/pleroma/http/connection.ex | 6 ++---- lib/pleroma/http/http.ex | 2 +- lib/pleroma/http/request_builder.ex | 9 +++++++++ test/http/request_builder_test.exs | 13 ++++++++++++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index ef1c1cabd..6cb26c0fe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,8 +9,7 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000, - protocols: [:http] + timeout: 20_000 ] @doc """ @@ -24,8 +23,7 @@ defmodule Pleroma.HTTP.Connection do def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) - options = options(opts) - Tesla.client(middleware, {adapter, options}) + Tesla.client(middleware, {adapter, options(opts)}) end # fetch http options diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index d493a8b4a..21c057f4b 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -45,9 +45,9 @@ defmodule Pleroma.HTTP do %{} |> Builder.method(method) + |> Builder.url(url) |> Builder.headers(headers) |> Builder.opts(options) - |> Builder.url(url) |> Builder.add_param(:body, :body, body) |> Builder.add_param(:query, :query, params) |> Enum.into([]) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 4e77870bd..6edf15bbe 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -53,6 +53,15 @@ defmodule Pleroma.HTTP.RequestBuilder do header_list end + # TODO: maybe we need this header for all adapters, so we won't fail on adapter change + header_list = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + %{host: host} = URI.parse(request.url) + header_list ++ [{"host", host}] + else + header_list + end + Map.put_new(request, :headers, header_list) end diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 5a645d4f3..73eaac273 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.RequestBuilderTest do - use ExUnit.Case, async: true + use ExUnit.Case alias Pleroma.HTTP.RequestBuilder describe "headers/2" do @@ -23,6 +23,17 @@ defmodule Pleroma.HTTP.RequestBuilderTest do headers: [{"user-agent", Pleroma.Application.user_agent()}] } end + + test "it adds host header for gun adapter" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) + + assert RequestBuilder.headers(%{url: "https://example.com"}, []) == %{ + headers: [{"host", "example.com"}], + url: "https://example.com" + } + end end describe "add_optional_params/3" do From b5e35dfaa231d413c812eb0e229f8b10b3e3c378 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 15 Aug 2019 15:49:32 +0300 Subject: [PATCH 021/102] receive adapter messages in adapter --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f5200d15d..7c10ab75a 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d", + ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index f09b7fece..26be6d663 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "99856638d0b8f382eb22e704b76bc30b4c4c379d", [ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "82dd3ace650b93d6699cf379b1a238d891f45cfb", [ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From d53514d111dba6dba65c68d1b5db99f00c451f60 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 15 Aug 2019 20:43:15 +0300 Subject: [PATCH 022/102] updating tesla --- lib/pleroma/http/http.ex | 3 +++ mix.exs | 10 ++++++---- mix.lock | 2 +- test/reverse_proxy/reverse_proxy_test.exs | 23 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 21c057f4b..6a09b8260 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -67,10 +67,13 @@ defmodule Pleroma.HTTP do options conn -> + %{host: host, port: port} = URI.parse(url) + adapter_opts = Keyword.get(options, :adapter, []) |> Keyword.put(:conn, conn) |> Keyword.put(:close_conn, false) + |> Keyword.put(:original, "#{host}:#{port}") Keyword.put(options, :adapter, adapter_opts) end diff --git a/mix.exs b/mix.exs index 7c10ab75a..d156b9e36 100644 --- a/mix.exs +++ b/mix.exs @@ -111,10 +111,12 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, - github: "alex-strizhakov/tesla", - ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb", - override: true}, + { + :tesla, + github: "alex-strizhakov/tesla", + ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", + override: true + }, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 26be6d663..606dbfa61 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "82dd3ace650b93d6699cf379b1a238d891f45cfb", [ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", [ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 1e9967278..e3c26c676 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -361,5 +361,28 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], api) end) end + + test "tesla client with gun client follow redirects", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/7") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end end end From f7fea15d55bfe25ad8efca0ee1aa2df62cb9af9e Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:35:47 +0300 Subject: [PATCH 023/102] updated deps --- lib/pleroma/http/request_builder.ex | 9 --------- mix.exs | 7 ++++--- mix.lock | 6 +++--- test/http/request_builder_test.exs | 11 ----------- test/reverse_proxy/reverse_proxy_test.exs | 3 +-- 5 files changed, 8 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 6edf15bbe..4e77870bd 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -53,15 +53,6 @@ defmodule Pleroma.HTTP.RequestBuilder do header_list end - # TODO: maybe we need this header for all adapters, so we won't fail on adapter change - header_list = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - %{host: host} = URI.parse(request.url) - header_list ++ [{"host", host}] - else - header_list - end - Map.put_new(request, :headers, header_list) end diff --git a/mix.exs b/mix.exs index d156b9e36..29f1c90c5 100644 --- a/mix.exs +++ b/mix.exs @@ -114,11 +114,12 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", + ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", override: true }, - {:cowlib, "~> 2.6.0", override: true}, - {:gun, "~> 1.3"}, + {:cowlib, "~> 2.7.3", override: true}, + {:gun, + github: "ninenines/gun", ref: "491ddf58c0e14824a741852fdc522b390b306ae2", override: true}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 606dbfa61..1d4532fed 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, + "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, @@ -37,7 +37,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, - "gun": {:hex, :gun, "1.3.0", "18e5d269649c987af95aec309f68a27ffc3930531dd227a6eaa0884d6684286e", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm"}, + "gun": {:git, "https://github.com/ninenines/gun.git", "491ddf58c0e14824a741852fdc522b390b306ae2", [ref: "491ddf58c0e14824a741852fdc522b390b306ae2"]}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", [ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", [ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 73eaac273..331399bd2 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -23,17 +23,6 @@ defmodule Pleroma.HTTP.RequestBuilderTest do headers: [{"user-agent", Pleroma.Application.user_agent()}] } end - - test "it adds host header for gun adapter" do - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - - assert RequestBuilder.headers(%{url: "https://example.com"}, []) == %{ - headers: [{"host", "example.com"}], - url: "https://example.com" - } - end end describe "add_optional_params/3" do diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index e3c26c676..87ee639df 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -372,9 +372,8 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], :gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) - conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/7") + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") - assert byte_size(conn.resp_body) == 10 assert conn.state == :chunked assert conn.status == 200 From a7c668c4d9495f4b0c938edc895240743fb3b14d Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:44:00 +0300 Subject: [PATCH 024/102] fix --- lib/pleroma/gun/connections.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index e06cad276..a3f1b0351 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -31,7 +31,10 @@ defmodule Pleroma.Gun.Connections do opts = Enum.into(opts, %{}) uri = URI.parse(url) - opts = if uri.scheme == "https", do: Map.put(opts, :transport, :tls), else: opts + opts = + if uri.scheme == "https" and uri.port != 443, + do: Map.put(opts, :transport, :tls), + else: opts GenServer.call( name, From 8e91a7731745f1799abcfcba529946f00e79642e Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:51:10 +0300 Subject: [PATCH 025/102] debug for media proxy --- lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 8403850ff..7d285a549 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -13,6 +13,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + IO.inspect(sig64), + IO.inspect(url64), + IO.inspect(url), :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else From 806febba607fc59accd8598c88023ec64ab50301 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:23:48 +0300 Subject: [PATCH 026/102] passing options for gun --- lib/pleroma/gun/connections.ex | 17 ++++++++++++++++- lib/pleroma/http/http.ex | 13 ++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index a3f1b0351..b8cf2f9b5 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -11,6 +11,20 @@ defmodule Pleroma.Gun.Connections do conns: %{domain() => conn()} } + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :ws_opts + ] + defstruct conns: %{} def start_link(name \\ __MODULE__) do @@ -81,7 +95,8 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + {:ok, conn} = + Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, Map.take(opts, @gun_keys)) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 6a09b8260..a7f42d0c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url) do + case Pleroma.Gun.Connections.get_conn(url, options) do nil -> options @@ -86,8 +86,15 @@ defmodule Pleroma.HTTP do host = uri.host |> to_charlist() case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options + "https" -> + tls_opts = + Keyword.get(options, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) + + Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] + + _ -> + options end end From b11eeb48c4f6a491057bebc1404ea5fe295935c5 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:33:23 +0300 Subject: [PATCH 027/102] test fixes --- lib/pleroma/gun/api/api.ex | 2 +- lib/pleroma/gun/api/gun.ex | 22 ++++++++++++++++++++++ lib/pleroma/gun/connections.ex | 17 +---------------- mix.exs | 2 +- mix.lock | 2 +- 5 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 lib/pleroma/gun/api/gun.ex diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex index c69ab890e..19adc1bf0 100644 --- a/lib/pleroma/gun/api/api.ex +++ b/lib/pleroma/gun/api/api.ex @@ -10,6 +10,6 @@ defmodule Pleroma.Gun.API do end defp api do - Pleroma.Config.get([Pleroma.Gun.API], :gun) + Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) end end diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex new file mode 100644 index 000000000..14a4b7275 --- /dev/null +++ b/lib/pleroma/gun/api/gun.ex @@ -0,0 +1,22 @@ +defmodule Pleroma.Gun.API.Gun do + @behaviour Pleroma.Gun.API + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :ws_opts + ] + + @impl Pleroma.Gun.API + def open(host, port, opts) do + :gun.open(host, port, Map.take(opts, @gun_keys)) + end +end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index b8cf2f9b5..a3f1b0351 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -11,20 +11,6 @@ defmodule Pleroma.Gun.Connections do conns: %{domain() => conn()} } - @gun_keys [ - :connect_timeout, - :http_opts, - :http2_opts, - :protocols, - :retry, - :retry_timeout, - :trace, - :transport, - :tls_opts, - :tcp_opts, - :ws_opts - ] - defstruct conns: %{} def start_link(name \\ __MODULE__) do @@ -95,8 +81,7 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = - Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, Map.take(opts, @gun_keys)) + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/mix.exs b/mix.exs index 29f1c90c5..64c56c4e3 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", + ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 1d4532fed..554b81fa8 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", [ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", [ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 5177d589d7a05489fb1bc086934b9b6907d30bf0 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:45:50 +0300 Subject: [PATCH 028/102] more fixes --- lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 --- test/gun/connections_test.exs | 4 ++-- test/reverse_proxy/reverse_proxy_test.exs | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 7d285a549..8403850ff 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -13,9 +13,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), - IO.inspect(sig64), - IO.inspect(url64), - IO.inspect(url), :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index aad70a644..a63c8eaf9 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -180,7 +180,7 @@ defmodule Gun.ConnectionsTest do test "opens connection and reuse it on next request", %{name: name} do api = Pleroma.Config.get([API]) - Pleroma.Config.put([API], :gun) + Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) conn = Connections.get_conn("http://httpbin.org", [], name) @@ -204,7 +204,7 @@ defmodule Gun.ConnectionsTest do test "opens ssl connection and reuse it on next request", %{name: name} do api = Pleroma.Config.get([API]) - Pleroma.Config.put([API], :gun) + Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) conn = Connections.get_conn("https://httpbin.org", [], name) diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 87ee639df..2a967697b 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -323,7 +323,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") @@ -346,7 +346,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") @@ -369,7 +369,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") From 722f97b347b70f67eb1164452100a8fba3c89e94 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 19:21:02 +0300 Subject: [PATCH 029/102] tesla update --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 64c56c4e3..d51b1488d 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", + ref: "81433bd4011217323a1f443661252fdab23f5e59", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 554b81fa8..daec58b4f 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", [ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "81433bd4011217323a1f443661252fdab23f5e59", [ref: "81433bd4011217323a1f443661252fdab23f5e59"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From a6101f0991f63b82951853c56b43f0b6383cdaf9 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:04:13 +0300 Subject: [PATCH 030/102] tls versions for gun --- lib/pleroma/http/http.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a7f42d0c0..9339532af 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,6 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) + |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', 'tlsv1']) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 2370394a4e90e2e728e847a06de47aae8812a888 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:07:08 +0300 Subject: [PATCH 031/102] like this --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 9339532af..b6e338aab 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', 'tlsv1']) + |> Keyword.put(:versions, [:tlsv1, :"tlsv1.1", :"tlsv1.2"]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 93a01b3496eb677066b79de0daffa2811c443531 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:47:24 +0300 Subject: [PATCH 032/102] like thiis --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b6e338aab..40b3e1a29 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, [:tlsv1, :"tlsv1.1", :"tlsv1.2"]) + |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 4a0d82f72740de385a6b895bb86ee10d758af07a Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:52:00 +0300 Subject: [PATCH 033/102] order --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 40b3e1a29..3e29b7b6c 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', :tlsv1]) + |> Keyword.put(:versions, [:"tlsv1.2", :"tlsv1.1", :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 2121e82da6e115764bbfa0c5b3cc6283fc9812cb Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 18 Aug 2019 22:21:54 +0000 Subject: [PATCH 034/102] test: gun: fix aliases --- test/gun/connections_test.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index a63c8eaf9..e5e0e8db1 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -4,7 +4,9 @@ defmodule Gun.ConnectionsTest do use ExUnit.Case - alias Pleroma.Gun.{Connections, Conn, API} + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + alias Pleroma.Gun.Connections setup do name = :test_gun_connections From 967d6732d436c9106f0b86927a2d511fe3ba086f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 18 Aug 2019 22:42:55 +0000 Subject: [PATCH 035/102] gun: connection: fix up a comment pointed out by credo --- lib/pleroma/gun/connections.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index a3f1b0351..336306ce8 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -114,7 +114,9 @@ defmodule Pleroma.Gun.Connections do def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do {key, conn} = find_conn(state.conns, conn_pid) - # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up + # We don't want to block requests to GenServer. + # If gun sends a down message, return nil, so we can make some + # retries, while the connection is not up. Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) state = put_in(state.conns[key].state, :down) From f7eaf9f763e4c2f482ac26f60c7657b3cdce10a1 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 18 Aug 2019 23:14:45 +0000 Subject: [PATCH 036/102] gun connection pool worker: fix up start_link() --- lib/pleroma/gun/connections.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 336306ce8..dc96b876c 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -13,9 +13,9 @@ defmodule Pleroma.Gun.Connections do defstruct conns: %{} - def start_link(name \\ __MODULE__) do + def start_link(_) do if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - GenServer.start_link(__MODULE__, [], name: name) + GenServer.start_link(__MODULE__, []) else :ignore end From a53a450550c49841b54637aff7aafc0e2d261707 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 18 Aug 2019 23:34:28 +0000 Subject: [PATCH 037/102] gun: connection pool worker: start up when in the test environment --- lib/pleroma/gun/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index dc96b876c..78fe0bce5 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Gun.Connections do defstruct conns: %{} def start_link(_) do - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun || Mix.env() == :test do GenServer.start_link(__MODULE__, []) else :ignore From 337340c3a1ff188dd1b8382cb3597191b50461b5 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Mon, 19 Aug 2019 00:00:22 +0000 Subject: [PATCH 038/102] gun: connection pool: allow the connection pool to be started with a name when appropriate --- lib/pleroma/gun/connections.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 78fe0bce5..695c3c93e 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -13,8 +13,14 @@ defmodule Pleroma.Gun.Connections do defstruct conns: %{} + def start_link(name \\ __MODULE__) + + def start_link(name) when is_atom(name) do + GenServer.start_link(__MODULE__, [], name: name) + end + def start_link(_) do - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun || Mix.env() == :test do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do GenServer.start_link(__MODULE__, []) else :ignore From 4f41634ccc1bde180dadf0a6462e4e5edeb486a5 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 18:00:43 +0300 Subject: [PATCH 039/102] adding gun adapter --- config/config.exs | 2 +- mix.exs | 5 ++++- mix.lock | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 758661120..63162d594 100644 --- a/config/config.exs +++ b/config/config.exs @@ -186,7 +186,7 @@ config :mime, :types, %{ "application/ld+json" => ["activity+json"] } -config :tesla, adapter: Tesla.Adapter.Hackney +config :tesla, adapter: Tesla.Adapter.Gun # Configures http settings, upstream proxy etc. config :pleroma, :http, diff --git a/mix.exs b/mix.exs index 3170d6f2d..9aab71e90 100644 --- a/mix.exs +++ b/mix.exs @@ -111,7 +111,10 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, "~> 1.2"}, + {:tesla, + github: "teamon/tesla", ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822", override: true}, + {:cowlib, "~> 2.6.0", override: true}, + {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 2639e96e9..bf3d4f01a 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, + "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, @@ -37,6 +37,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, + "gun": {:hex, :gun, "1.3.0", "18e5d269649c987af95aec309f68a27ffc3930531dd227a6eaa0884d6684286e", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -84,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, + "tesla": {:git, "https://github.com/teamon/tesla.git", "97950754a77cde4e162ada31fb9d8cf4fd6ab822", [ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From b233875ef5b7c1cb7b6c7fea1d30604fece2a659 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 20:04:37 +0300 Subject: [PATCH 040/102] namings --- lib/pleroma/http/connection.ex | 13 ++++++++----- lib/pleroma/http/http.ex | 2 +- lib/pleroma/reverse_proxy/reverse_proxy.ex | 2 +- .../activity_pub/mrf/mediaproxy_warming_policy.ex | 4 ++-- lib/pleroma/web/rel_me.ex | 4 ++-- lib/pleroma/web/rich_media/parser.ex | 4 ++-- mix.exs | 4 +++- mix.lock | 2 +- 8 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 7e2c6f5e8..8caf989a7 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -7,8 +7,10 @@ defmodule Pleroma.HTTP.Connection do Connection for http-requests. """ - @hackney_options [ + @options [ connect_timeout: 10_000, + protocols: [:http], + timeout: 20_000, recv_timeout: 20_000, follow_redirect: true, force_redirect: true, @@ -25,17 +27,18 @@ defmodule Pleroma.HTTP.Connection do """ @spec new(Keyword.t()) :: Tesla.Env.client() def new(opts \\ []) do - Tesla.client([], {@adapter, hackney_options(opts)}) + middleware = [Tesla.Middleware.FollowRedirects] + Tesla.client(middleware, {@adapter, options(opts)}) end - # fetch Hackney options + # fetch http options # - def hackney_options(opts) do + def options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - @hackney_options + @options |> Keyword.merge(adapter_options) |> Keyword.merge(options) |> Keyword.merge(proxy: proxy_url) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index dec24458a..6d7934841 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,7 +65,7 @@ defmodule Pleroma.HTTP do end def process_request_options(options) do - Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) + Keyword.merge(Pleroma.HTTP.Connection.options([]), options) end @doc """ diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 03efad30a..3212bf90d 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -94,7 +94,7 @@ defmodule Pleroma.ReverseProxy do def call(conn = %{method: method}, url, opts) when method in @methods do hackney_opts = - Pleroma.HTTP.Connection.hackney_options([]) + Pleroma.HTTP.Connection.options([]) |> Keyword.merge(@default_hackney_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index a179dd54d..52ef0167c 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @hackney_options [ + @options [ pool: :media, recv_timeout: 10_000 ] @@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> MediaProxy.url() - |> HTTP.get([], adapter: @hackney_options) + |> HTTP.get([], adapter: @options) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index d376e2069..947234aa2 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RelMe do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -25,7 +25,7 @@ defmodule Pleroma.Web.RelMe do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) data = Floki.attribute(html, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index f5f9e358c..ade4ac891 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do - @hackney_options [ + @options [ pool: :media, recv_timeout: 2_000, max_body: 2_000_000, @@ -78,7 +78,7 @@ defmodule Pleroma.Web.RichMedia.Parser do defp parse_url(url) do try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @options) html |> maybe_parse() diff --git a/mix.exs b/mix.exs index 9aab71e90..84d58f5a6 100644 --- a/mix.exs +++ b/mix.exs @@ -112,7 +112,9 @@ defmodule Pleroma.Mixfile do {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, {:tesla, - github: "teamon/tesla", ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822", override: true}, + github: "alex-strizhakov/tesla", + ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", + override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index bf3d4f01a..d79bb9c80 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/teamon/tesla.git", "97950754a77cde4e162ada31fb9d8cf4fd6ab822", [ref: "97950754a77cde4e162ada31fb9d8cf4fd6ab822"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", [ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From b383d85b9b8229751dd538c66e02a18f2de45835 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 7 Aug 2019 20:06:43 +0300 Subject: [PATCH 041/102] headers standardisation for tesla --- lib/pleroma/http/request_builder.ex | 2 +- lib/pleroma/object/fetcher.ex | 6 +- lib/pleroma/web/ostatus/ostatus.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- test/http/request_builder_test.exs | 2 +- test/support/http_request_mock.ex | 84 +++++++++++-------- test/web/twitter_api/util_controller_test.exs | 16 ++-- .../web_finger/web_finger_controller_test.exs | 13 +-- 8 files changed, 72 insertions(+), 55 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index e23457999..4e77870bd 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -48,7 +48,7 @@ defmodule Pleroma.HTTP.RequestBuilder do def headers(request, header_list) do header_list = if Pleroma.Config.get([:http, :send_user_agent]) do - header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + header_list ++ [{"user-agent", Pleroma.Application.user_agent()}] else header_list end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 8d79ddb1f..46531a13a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Object.Fetcher do date: date }) - [{:Signature, signature}] + [{"signature", signature}] end defp sign_fetch(headers, id, date) do @@ -108,7 +108,7 @@ defmodule Pleroma.Object.Fetcher do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{:Date, date}] + headers ++ [{"date", date}] else headers end @@ -122,7 +122,7 @@ defmodule Pleroma.Object.Fetcher do |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") headers = - [{:Accept, "application/activity+json"}] + [{"accept", "application/activity+json"}] |> maybe_date_fetch(date) |> sign_fetch(id, date) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 331cbc0b7..1c400d9e4 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -373,7 +373,7 @@ defmodule Pleroma.Web.OStatus do {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( url, - [{:Accept, "application/atom+xml"}] + [{"accept", "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") handle_incoming(body, options) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index ecb39ee50..624ee5ef7 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -217,7 +217,7 @@ defmodule Pleroma.Web.WebFinger do with response <- HTTP.get( address, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ), {:ok, %{status: status, body: body}} when status in 200..299 <- response do doc = XML.parse_document(body) diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 170ca916f..ea4dd087e 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -18,7 +18,7 @@ defmodule Pleroma.HTTP.RequestBuilderTest do Pleroma.Config.put([:http, :send_user_agent], true) assert RequestBuilder.headers(%{}, []) == %{ - headers: [{"User-Agent", Pleroma.Application.user_agent()}] + headers: [{"user-agent", Pleroma.Application.user_agent()}] } end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 3adb5ba3b..c4862eb1c 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -88,7 +88,7 @@ defmodule HttpRequestMock do "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -101,7 +101,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -122,7 +122,7 @@ defmodule HttpRequestMock do "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -148,7 +148,7 @@ defmodule HttpRequestMock do "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -169,7 +169,7 @@ defmodule HttpRequestMock do "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -182,7 +182,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -195,7 +195,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd/?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -208,7 +208,7 @@ defmodule HttpRequestMock do "https://mst3k.interlinked.me/users/luciferMysticus", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -229,7 +229,7 @@ defmodule HttpRequestMock do "https://hubzilla.example.org/channel/kaniini", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -238,7 +238,7 @@ defmodule HttpRequestMock do }} end - def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -246,7 +246,7 @@ defmodule HttpRequestMock do }} end - def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do + def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -265,7 +265,7 @@ defmodule HttpRequestMock do }} end - def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do + def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -321,7 +321,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/admin", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -329,7 +331,9 @@ defmodule HttpRequestMock do }} end - def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do + def get("http://mastodon.example.org/users/gargron", _, _, [ + {"accept", "application/activity+json"} + ]) do {:error, :nxdomain} end @@ -337,7 +341,7 @@ defmodule HttpRequestMock do "http://mastodon.example.org/@admin/99541947525187367", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -354,7 +358,7 @@ defmodule HttpRequestMock do }} end - def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do + def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do {:ok, %Tesla.Env{ status: 200, @@ -366,7 +370,7 @@ defmodule HttpRequestMock do "https://mstdn.io/users/mayuutann/statuses/99568293732299394", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{ @@ -386,7 +390,7 @@ defmodule HttpRequestMock do }} end - def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json") + def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}]) when url in [ "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain", "https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain" @@ -413,7 +417,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -457,7 +461,7 @@ defmodule HttpRequestMock do "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -510,7 +514,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -523,7 +527,7 @@ defmodule HttpRequestMock do "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", _, _, - Accept: "application/atom+xml" + [{"accept", "application/atom+xml"}] ) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} end @@ -540,7 +544,7 @@ defmodule HttpRequestMock do "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -562,7 +566,7 @@ defmodule HttpRequestMock do "http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -576,7 +580,7 @@ defmodule HttpRequestMock do "http://gs.example.org:4040/index.php/user/1", _, _, - Accept: "application/activity+json" + [{"accept", "application/activity+json"}] ) do {:ok, %Tesla.Env{status: 406, body: ""}} end @@ -612,7 +616,7 @@ defmodule HttpRequestMock do "https://squeet.me/xrd?uri=lain@squeet.me", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -625,7 +629,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -638,7 +642,7 @@ defmodule HttpRequestMock do "https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{status: 200, body: ""}} end @@ -655,7 +659,7 @@ defmodule HttpRequestMock do "http://framatube.org/main/xrd?uri=framasoft@framatube.org", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -677,7 +681,7 @@ defmodule HttpRequestMock do "http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -714,7 +718,7 @@ defmodule HttpRequestMock do "https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -772,7 +776,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}} end - def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do + def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)} end @@ -889,7 +893,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -902,7 +906,7 @@ defmodule HttpRequestMock do "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain", _, _, - Accept: "application/xrd+xml,application/jrd+json" + [{"accept", "application/xrd+xml,application/jrd+json"}] ) do {:ok, %Tesla.Env{ @@ -924,7 +928,9 @@ defmodule HttpRequestMock do }} end - def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -936,7 +942,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity2.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, @@ -948,7 +956,9 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 404, body: ""}} end - def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do + def get("https://info.pleroma.site/activity3.json", _, _, [ + {"accept", "application/activity+json"} + ]) do {:ok, %Tesla.Env{ status: 200, diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index fe4ffdb59..1061403f4 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Web.CommonAPI import Pleroma.Factory import Mock + import ExUnit.CaptureLog setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -367,15 +368,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert html_response(response, 200) =~ "Remote follow" end - test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do + test "show follow page with error when user cannot fetch by `acct` link", %{conn: conn} do user = insert(:user) - response = - conn - |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + assert capture_log(fn -> + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") - assert html_response(response, 200) =~ "Error fetching user" + assert html_response(response, 200) =~ "Error fetching user" + end) =~ + "Could not decode user at fetch https://mastodon.social/users/not_found, {:error, \"Object has been deleted\"}" end end diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index e23086b2a..f940f95b6 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do import Pleroma.Factory import Tesla.Mock + import ExUnit.CaptureLog setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -75,11 +76,13 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do test "Sends a 404 when invalid format" do user = insert(:user) - assert_raise Phoenix.NotAcceptableError, fn -> - build_conn() - |> put_req_header("accept", "text/html") - |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") - end + assert capture_log(fn -> + assert_raise Phoenix.NotAcceptableError, fn -> + build_conn() + |> put_req_header("accept", "text/html") + |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") + end + end) =~ "Internal server error:" end test "Sends a 400 when resource param is missing" do From 4e9d9209c3c598532e1eaacb35e276357906afb4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 8 Aug 2019 18:42:50 +0300 Subject: [PATCH 042/102] added tesla client for reverse proxy --- lib/pleroma/http/connection.ex | 6 +-- lib/pleroma/reverse_proxy/client.ex | 17 +++++--- lib/pleroma/reverse_proxy/client/hackney.ex | 17 ++++++++ lib/pleroma/reverse_proxy/client/tesla.ex | 48 +++++++++++++++++++++ lib/pleroma/reverse_proxy/reverse_proxy.ex | 10 ++--- mix.exs | 2 +- mix.lock | 2 +- test/reverse_proxy_test.exs | 44 ++++++++++++++++--- 8 files changed, 122 insertions(+), 24 deletions(-) create mode 100644 lib/pleroma/reverse_proxy/client/hackney.ex create mode 100644 lib/pleroma/reverse_proxy/client/tesla.ex diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 8caf989a7..4ebe16e18 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -10,11 +10,7 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, protocols: [:http], - timeout: 20_000, - recv_timeout: 20_000, - follow_redirect: true, - force_redirect: true, - pool: :federation + timeout: 20_000 ] @adapter Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 776c4794c..42f2ff13b 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -3,9 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy.Client do - @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: - {:ok, pos_integer(), [tuple()], reference() | map()} - | {:ok, pos_integer(), [tuple()]} + @type status :: pos_integer() + @type header_name :: String.t() + @type header_value :: String.t() + @type headers :: [{header_name(), header_value()}] + + @callback request(atom(), String.t(), headers(), String.t(), list()) :: + {:ok, status(), headers(), reference() | map()} + | {:ok, status(), headers()} | {:ok, reference()} | {:error, term()} @@ -14,8 +19,8 @@ defmodule Pleroma.ReverseProxy.Client do @callback close(reference() | pid() | map()) :: :ok - def request(method, url, headers, "", opts \\ []) do - client().request(method, url, headers, "", opts) + def request(method, url, headers, body \\ "", opts \\ []) do + client().request(method, url, headers, body, opts) end def stream_body(ref), do: client().stream_body(ref) @@ -23,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) end end diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex new file mode 100644 index 000000000..e6293646a --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.ReverseProxy.Client.Hackney do + @behaviour Pleroma.ReverseProxy.Client + + def request(method, url, headers, body, opts \\ []) do + :hackney.request(method, url, headers, body, opts) + end + + def stream_body(ref) do + case :hackney.stream_body(ref) do + :done -> :done + {:ok, data} -> {:ok, data, ref} + {:error, error} -> {:error, error} + end + end + + def close(ref), do: :hackney.close(ref) +end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex new file mode 100644 index 000000000..d2944b9dc --- /dev/null +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -0,0 +1,48 @@ +defmodule Pleroma.ReverseProxy.Client.Tesla do + @behaviour Pleroma.ReverseProxy.Client + + @adapters [Tesla.Adapter.Gun] + alias Pleroma.HTTP + + def request(method, url, headers, body, opts \\ []) do + adapter_opts = + Keyword.get(opts, :adapter, []) + |> Keyword.put(:chunks_response, true) + + with {:ok, response} <- + HTTP.request(method, url, body, headers, Keyword.put(opts, :adapter, adapter_opts)) do + {:ok, response.status, response.headers, response.body} + else + {:error, error} -> {:error, error} + end + end + + def stream_body(%{fin: true}), do: :done + + def stream_body(client) do + case read_chunk!(client) do + {:fin, body} -> {:ok, body, Map.put(client, :fin, true)} + {:nofin, part} -> {:ok, part, client} + end + end + + defp read_chunk!(client) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support reading body in chunks" + end + + adapter.read_chunk(client) + end + + def close(client) do + adapter = Application.get_env(:tesla, :adapter) + + unless adapter in @adapters do + raise "#{adapter} doesn't support closing connection" + end + + adapter.close(client) + end +end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 3212bf90d..a2cdcf393 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @default_hackney_options [pool: :media] + @default_options [pool: :media] @inline_content_types [ "image/gif", @@ -93,9 +93,9 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - hackney_opts = + client_opts = Pleroma.HTTP.Connection.options([]) - |> Keyword.merge(@default_hackney_options) + |> Keyword.merge(@default_options) |> Keyword.merge(Keyword.get(opts, :http, [])) |> HTTP.process_request_options() @@ -108,7 +108,7 @@ defmodule Pleroma.ReverseProxy do opts end - with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), + with {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), :ok <- header_length_constraint( headers, @@ -201,7 +201,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- client().stream_body(client), + {:ok, data, client} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- diff --git a/mix.exs b/mix.exs index 84d58f5a6..f1fbdc6b3 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", + ref: "beb8927358dfaa66ecd458df607befde12dd56e0", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index d79bb9c80..635f20185 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9ad792fb630bdfc2266ed13b830c28b6552fb3f9", [ref: "9ad792fb630bdfc2266ed13b830c28b6552fb3f9"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "beb8927358dfaa66ecd458df607befde12dd56e0", [ref: "beb8927358dfaa66ecd458df607befde12dd56e0"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs index 3a83c4c48..0644d1ed9 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy_test.exs @@ -29,11 +29,11 @@ defmodule Pleroma.ReverseProxyTest do {"content-length", byte_size(json) |> to_string()} ], %{url: url}} end) - |> expect(:stream_body, invokes, fn %{url: url} -> + |> expect(:stream_body, invokes, fn %{url: url} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) - {:ok, json} + {:ok, json, client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -66,6 +66,38 @@ defmodule Pleroma.ReverseProxyTest do assert conn.halted end + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789", client} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + describe "max_body " do test "length returns error if content-length more than option", %{conn: conn} do user_agent_mock("hackney/1.15.1", 0) @@ -179,12 +211,12 @@ defmodule Pleroma.ReverseProxyTest do Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} end) - |> expect(:stream_body, 2, fn %{url: url, headers: headers} -> + |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} - {:ok, Jason.encode!(%{headers: headers})} + {:ok, Jason.encode!(%{headers: headers}), client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) @@ -261,11 +293,11 @@ defmodule Pleroma.ReverseProxyTest do {:ok, 200, headers, %{url: "/disposition"}} end) - |> expect(:stream_body, 2, fn %{url: "/disposition"} -> + |> expect(:stream_body, 2, fn %{url: "/disposition"} = client -> case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do [{_, 0}] -> Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) - {:ok, ""} + {:ok, "", client} [{_, 1}] -> Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") From 80ddc2428199e4ea5638ae9f080faceacd5c15fd Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 8 Aug 2019 20:08:10 +0300 Subject: [PATCH 043/102] fix --- lib/pleroma/reverse_proxy/client/tesla.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index d2944b9dc..9b0be5bda 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -26,14 +26,14 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do end end - defp read_chunk!(client) do + defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do adapter = Application.get_env(:tesla, :adapter) unless adapter in @adapters do raise "#{adapter} doesn't support reading body in chunks" end - adapter.read_chunk(client) + adapter.read_chunk(pid, stream, opts) end def close(client) do From 02daf21d786c8823efcfeec37420f6b3a1790335 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 9 Aug 2019 12:24:43 +0300 Subject: [PATCH 044/102] possibility to set tesla adapter in runtime --- lib/pleroma/http/connection.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 4ebe16e18..ff03a3ee3 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -12,7 +12,6 @@ defmodule Pleroma.HTTP.Connection do protocols: [:http], timeout: 20_000 ] - @adapter Application.get_env(:tesla, :adapter) @doc """ Configure a client connection @@ -24,7 +23,8 @@ defmodule Pleroma.HTTP.Connection do @spec new(Keyword.t()) :: Tesla.Env.client() def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] - Tesla.client(middleware, {@adapter, options(opts)}) + adapter = Application.get_env(:tesla, :adapter) + Tesla.client(middleware, {adapter, options(opts)}) end # fetch http options From c51aa48e60103307307c7c40c2d046719def7054 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 9 Aug 2019 12:25:58 +0300 Subject: [PATCH 045/102] reverse proxy clients integration tests tesla reverse proxy client fixes --- lib/pleroma/reverse_proxy/client/tesla.ex | 14 +++- test/reverse_proxy/client/hackney_test.exs | 7 ++ test/reverse_proxy/client/tesla_test.exs | 9 +++ .../reverse_proxy_test.exs | 4 + test/support/reverse_proxy_client_case.ex | 76 +++++++++++++++++++ 5 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 test/reverse_proxy/client/hackney_test.exs create mode 100644 test/reverse_proxy/client/tesla_test.exs rename test/{ => reverse_proxy}/reverse_proxy_test.exs (99%) create mode 100644 test/support/reverse_proxy_client_case.ex diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 9b0be5bda..b1498a5a4 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -2,7 +2,6 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client @adapters [Tesla.Adapter.Gun] - alias Pleroma.HTTP def request(method, url, headers, body, opts \\ []) do adapter_opts = @@ -10,8 +9,16 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do |> Keyword.put(:chunks_response, true) with {:ok, response} <- - HTTP.request(method, url, body, headers, Keyword.put(opts, :adapter, adapter_opts)) do - {:ok, response.status, response.headers, response.body} + Pleroma.HTTP.request( + method, + url, + body, + headers, + Keyword.put(opts, :adapter, adapter_opts) + ) do + if is_map(response.body), + do: {:ok, response.status, response.headers, response.body}, + else: {:ok, response.status, response.headers} else {:error, error} -> {:error, error} end @@ -23,6 +30,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do case read_chunk!(client) do {:fin, body} -> {:ok, body, Map.put(client, :fin, true)} {:nofin, part} -> {:ok, part, client} + {:error, error} -> {:error, error} end end diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs new file mode 100644 index 000000000..577e0b0b2 --- /dev/null +++ b/test/reverse_proxy/client/hackney_test.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.ReverseProxy.Client.HackneyTest do + use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Hackney + + defp check_ref(ref) do + assert is_reference(ref) + end +end diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs new file mode 100644 index 000000000..029a25d0f --- /dev/null +++ b/test/reverse_proxy/client/tesla_test.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.ReverseProxy.Client.TeslaTest do + use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla + + defp check_ref(%{pid: pid, stream: stream} = ref) do + assert is_pid(pid) + assert is_reference(stream) + assert ref[:fin] + end +end diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs similarity index 99% rename from test/reverse_proxy_test.exs rename to test/reverse_proxy/reverse_proxy_test.exs index 0644d1ed9..95adae666 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -329,4 +329,8 @@ defmodule Pleroma.ReverseProxyTest do assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers end end + + describe "integration tests" do + @describetag :integration + end end diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex new file mode 100644 index 000000000..40cd59ea2 --- /dev/null +++ b/test/support/reverse_proxy_client_case.ex @@ -0,0 +1,76 @@ +defmodule Pleroma.ReverseProxyClientCase do + defmacro __using__(client: client) do + quote do + use ExUnit.Case + @moduletag :integration + @client unquote(client) + + setup do + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, Tesla.Mock) end) + end + + test "get response body stream" do + {:ok, status, headers, ref} = + @client.request( + :get, + "http://httpbin.org/stream-bytes/10", + [{"accept", "application/octet-stream"}], + "", + [] + ) + + assert status == 200 + assert headers != [] + + {:ok, response, ref} = @client.stream_body(ref) + check_ref(ref) + assert is_binary(response) + assert byte_size(response) == 10 + + assert :done == @client.stream_body(ref) + end + + test "head response" do + {:ok, status, headers} = @client.request(:head, "http://httpbin.org/get", [], "", []) + + assert status == 200 + assert headers != [] + end + + test "get error response" do + case @client.request( + :get, + "http://httpbin.org/status/500", + [], + "", + [] + ) do + {:ok, status, headers, ref} -> + assert status == 500 + assert headers != [] + check_ref(ref) + + assert :ok = @client.close(ref) + + {:ok, status, headers} -> + assert headers != [] + end + end + + test "head error response" do + {:ok, status, headers} = + @client.request( + :head, + "http://httpbin.org/status/500", + [], + "", + [] + ) + + assert status == 500 + assert headers != [] + end + end + end +end From 2caf9ad95424b0a4f47c1e22ce9d57a29fcf9fbb Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 13 Aug 2019 14:37:19 +0300 Subject: [PATCH 046/102] added gun connections genserver --- config/test.exs | 2 + lib/pleroma/gun/api/api.ex | 15 ++ lib/pleroma/gun/api/mock.ex | 40 +++++ lib/pleroma/gun/conn.ex | 17 +++ lib/pleroma/gun/connections.ex | 104 +++++++++++++ lib/pleroma/http/connection.ex | 1 - lib/pleroma/reverse_proxy/client.ex | 2 +- lib/pleroma/reverse_proxy/client/tesla.ex | 10 +- mix.exs | 2 +- mix.lock | 2 +- test/gun/connections_test.exs | 172 ++++++++++++++++++++++ test/reverse_proxy/reverse_proxy_test.exs | 33 +++++ 12 files changed, 393 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/gun/api/api.ex create mode 100644 lib/pleroma/gun/api/mock.ex create mode 100644 lib/pleroma/gun/conn.ex create mode 100644 lib/pleroma/gun/connections.ex create mode 100644 test/gun/connections_test.exs diff --git a/config/test.exs b/config/test.exs index 6f75f39b5..ca916d59e 100644 --- a/config/test.exs +++ b/config/test.exs @@ -85,6 +85,8 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock +config :pleroma, Pleroma.Gun.API, Pleroma.Gun.API.Mock + try do import_config "test.secret.exs" rescue diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex new file mode 100644 index 000000000..c69ab890e --- /dev/null +++ b/lib/pleroma/gun/api/api.ex @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API do + @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + + def open(host, port, opts) do + api().open(host, port, opts) + end + + defp api do + Pleroma.Config.get([Pleroma.Gun.API], :gun) + end +end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex new file mode 100644 index 000000000..3348715c4 --- /dev/null +++ b/lib/pleroma/gun/api/mock.ex @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.API.Mock do + @behaviour Pleroma.Gun.API + @impl Pleroma.Gun.API + def open('some-domain.com', 80, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_up, conn_pid, :http}) + {:ok, conn_pid} + end + + def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_up, conn_pid, :https}) + {:ok, conn_pid} + end + + @impl Pleroma.Gun.API + def open('gun_down.com', _port, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + {:ok, conn_pid} + end + + @impl Pleroma.Gun.API + def open('gun_down_and_up.com', _port, %{genserver_pid: genserver_pid}) do + {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) + + {:ok, _} = + Task.start_link(fn -> + Process.sleep(500) + send(genserver_pid, {:gun_up, conn_pid, :http}) + end) + + {:ok, conn_pid} + end +end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex new file mode 100644 index 000000000..62ef146a1 --- /dev/null +++ b/lib/pleroma/gun/conn.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Conn do + @moduledoc """ + Struct for gun connection data + """ + @type t :: %__MODULE__{ + conn: pid(), + state: atom(), + waiting_pids: [pid()], + protocol: atom() + } + + defstruct conn: nil, state: :open, waiting_pids: [], protocol: :http +end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex new file mode 100644 index 000000000..60ec68d89 --- /dev/null +++ b/lib/pleroma/gun/connections.ex @@ -0,0 +1,104 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Gun.Connections do + use GenServer + + @type domain :: String.t() + @type conn :: Gun.Conn.t() + @type t :: %__MODULE__{ + conns: %{domain() => conn()} + } + + defstruct conns: %{} + + def start_link(name \\ __MODULE__) do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + GenServer.start_link(__MODULE__, [], name: name) + else + :ignore + end + end + + @impl true + def init(_) do + {:ok, %__MODULE__{conns: %{}}} + end + + @spec get_conn(atom(), String.t(), keyword()) :: pid() + def get_conn(name \\ __MODULE__, url, opts \\ []) do + opts = Enum.into(opts, %{}) + uri = URI.parse(url) + + opts = if uri.scheme == "https", do: Map.put(opts, :transport, :tls), else: opts + + GenServer.call( + name, + {:conn, %{opts: opts, uri: uri}} + ) + end + + @spec get_state(atom()) :: t() + def get_state(name \\ __MODULE__) do + GenServer.call(name, {:state}) + end + + @impl true + def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do + key = compose_key(uri) + + case state.conns[key] do + %{conn: conn, state: conn_state} when conn_state == :up -> + {:reply, conn, state} + + %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> + state = put_in(state.conns[key].waiting_pids, [from | pids]) + {:noreply, state} + + nil -> + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + + state = + put_in(state.conns[key], %Pleroma.Gun.Conn{ + conn: conn, + waiting_pids: [from], + protocol: String.to_atom(uri.scheme) + }) + + {:noreply, state} + end + end + + @impl true + def handle_call({:state}, _from, state), do: {:reply, state, state} + + @impl true + def handle_info({:gun_up, conn_pid, protocol}, state) do + {key, conn} = find_conn(state.conns, conn_pid, protocol) + + # Send to all waiting processes connection pid + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) + + # Update state of the current connection and set waiting_pids to empty list + state = put_in(state.conns[key], %{conn | state: :up, waiting_pids: []}) + {:noreply, state} + end + + @impl true + # Do we need to do something with killed & unprocessed references? + def handle_info({:gun_down, conn_pid, protocol, _reason, _killed, _unprocessed}, state) do + {key, conn} = find_conn(state.conns, conn_pid, protocol) + + # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) + + state = put_in(state.conns[key].state, :down) + {:noreply, state} + end + + defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) + + defp find_conn(conns, conn_pid, protocol), + do: Enum.find(conns, fn {_, conn} -> conn.conn == conn_pid and conn.protocol == protocol end) +end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index ff03a3ee3..6cb26c0fe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,6 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - protocols: [:http], timeout: 20_000 ] diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex index 42f2ff13b..71c2b2911 100644 --- a/lib/pleroma/reverse_proxy/client.ex +++ b/lib/pleroma/reverse_proxy/client.ex @@ -28,6 +28,6 @@ defmodule Pleroma.ReverseProxy.Client do def close(ref), do: client().close(ref) defp client do - Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) + Pleroma.Config.get([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) end end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index b1498a5a4..fad577ec1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-onl + defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client @@ -6,7 +10,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do adapter_opts = Keyword.get(opts, :adapter, []) - |> Keyword.put(:chunks_response, true) + |> Keyword.put(:body_as, :chunks) with {:ok, response} <- Pleroma.HTTP.request( @@ -44,13 +48,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do adapter.read_chunk(pid, stream, opts) end - def close(client) do + def close(pid) do adapter = Application.get_env(:tesla, :adapter) unless adapter in @adapters do raise "#{adapter} doesn't support closing connection" end - adapter.close(client) + adapter.close(pid) end end diff --git a/mix.exs b/mix.exs index f1fbdc6b3..9d04b5771 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "beb8927358dfaa66ecd458df607befde12dd56e0", + ref: "c29a7fd030fa6decbf7091152f563fe322e2b589", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 635f20185..ef5ebda6c 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "beb8927358dfaa66ecd458df607befde12dd56e0", [ref: "beb8927358dfaa66ecd458df607befde12dd56e0"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "c29a7fd030fa6decbf7091152f563fe322e2b589", [ref: "c29a7fd030fa6decbf7091152f563fe322e2b589"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs new file mode 100644 index 000000000..2ec8f3993 --- /dev/null +++ b/test/gun/connections_test.exs @@ -0,0 +1,172 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Gun.ConnectionsTest do + use ExUnit.Case, async: true + alias Pleroma.Gun.{Connections, Conn, API} + + setup do + name = :test_gun_connections + {:ok, pid} = Connections.start_link(name) + + {:ok, name: name, pid: pid} + end + + test "opens connection and reuse it on next request", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "reuses connection based on protocol", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + assert is_pid(conn) + assert Process.alive?(conn) + + https_conn = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + + refute conn == https_conn + + reused_https = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + + refute conn == reused_https + + assert reused_https == https_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + }, + "some-domain.com:443" => %Conn{ + conn: ^https_conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "process gun_down message", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://gun_down.com", genserver_pid: pid) + + refute conn + + %Connections{ + conns: %{ + "gun_down.com:80" => %Conn{ + conn: _, + state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + end + + test "process gun_down message and then gun_up", %{name: name, pid: pid} do + conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + + refute conn + + %Connections{ + conns: %{ + "gun_down_and_up.com:80" => %Conn{ + conn: _, + state: :down, + waiting_pids: _ + } + } + } = Connections.get_state(name) + + conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + + assert is_pid(conn) + assert Process.alive?(conn) + + %Connections{ + conns: %{ + "gun_down_and_up.com:80" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + + test "async processes get same conn for same domain", %{name: name, pid: pid} do + tasks = + for _ <- 1..5 do + Task.async(fn -> + Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + end) + end + + tasks_with_results = Task.yield_many(tasks) + + results = + Enum.map(tasks_with_results, fn {task, res} -> + res || Task.shutdown(task, :brutal_kill) + end) + + conns = for {:ok, value} <- results, do: value + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + + assert Enum.all?(conns, fn res -> res == conn end) + end + + describe "integration test" do + @describetag :integration + + test "opens connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], :gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.get_conn(name, "http://httpbin.org") + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn(name, "http://httpbin.org") + + assert conn == reused_conn + + %Connections{ + conns: %{ + "httpbin.org:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + end +end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 95adae666..56ac8672b 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -332,5 +332,38 @@ defmodule Pleroma.ReverseProxyTest do describe "integration tests" do @describetag :integration + + test "with hackney client", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + end) + + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + end + + test "with tesla client with gun adapter", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + end) + end end end From aee44f3f4bfeb9591e4c469fbfeb1ab0857cf89d Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 13:37:27 +0300 Subject: [PATCH 047/102] added gun connections holder genserver --- lib/pleroma/gun/connections.ex | 25 +++++++++- lib/pleroma/http/http.ex | 23 +++++++++ test/gun/connections_test.exs | 57 ++++++++++++++++++----- test/reverse_proxy/reverse_proxy_test.exs | 7 ++- 4 files changed, 97 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 60ec68d89..d5a9d8607 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -26,8 +26,8 @@ defmodule Pleroma.Gun.Connections do {:ok, %__MODULE__{conns: %{}}} end - @spec get_conn(atom(), String.t(), keyword()) :: pid() - def get_conn(name \\ __MODULE__, url, opts \\ []) do + @spec get_conn(String.t(), keyword(), atom()) :: pid() + def get_conn(url, opts \\ [], name \\ __MODULE__) do opts = Enum.into(opts, %{}) uri = URI.parse(url) @@ -39,6 +39,27 @@ defmodule Pleroma.Gun.Connections do ) end + # TODO: only for testing, add this parameter to the config + @spec try_to_get_gun_conn(String.t(), keyword(), atom()) :: nil | pid() + def try_to_get_gun_conn(url, opts \\ [], name \\ __MODULE__), + do: try_to_get_gun_conn(url, opts, name, 0) + + @spec try_to_get_gun_conn(String.t(), keyword(), atom(), pos_integer()) :: nil | pid() + def try_to_get_gun_conn(_url, _, _, 3), do: nil + + def try_to_get_gun_conn(url, opts, name, acc) do + case Pleroma.Gun.Connections.get_conn(url, opts, name) do + nil -> try_to_get_gun_conn(url, acc + 1) + conn -> conn + end + end + + @spec alive?(atom()) :: boolean() + def alive?(name \\ __MODULE__) do + pid = Process.whereis(name) + if pid, do: Process.alive?(pid), else: false + end + @spec get_state(atom()) :: t() def get_state(name \\ __MODULE__) do GenServer.call(name, {:state}) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 6d7934841..1846749c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -32,6 +32,15 @@ defmodule Pleroma.HTTP do process_request_options(options) |> process_sni_options(url) + adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun + + options = + if adapter_gun? and Pleroma.Gun.Connections.alive?() do + get_conn_for_gun(url, options) + else + options + end + params = Keyword.get(options, :params, []) %{} @@ -52,6 +61,20 @@ defmodule Pleroma.HTTP do end end + defp get_conn_for_gun(url, options) do + case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + nil -> + options + + conn -> + adapter_opts = + Keyword.get(options, :adapter, []) + |> Keyword.put(:conn, conn) + + Keyword.put(options, :adapter, adapter_opts) + end + end + defp process_sni_options(options, nil), do: options defp process_sni_options(options, url) do diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 2ec8f3993..42354d330 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -3,23 +3,56 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Gun.ConnectionsTest do - use ExUnit.Case, async: true + use ExUnit.Case alias Pleroma.Gun.{Connections, Conn, API} setup do name = :test_gun_connections + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) {:ok, pid} = Connections.start_link(name) {:ok, name: name, pid: pid} end + describe "alive?/2" do + test "is alive", %{name: name} do + assert Connections.alive?(name) + end + + test "returns false if not started" do + refute Connections.alive?(:some_random_name) + end + end + + test "try_to_get_gun_conn/1 returns conn", %{name: name, pid: pid} do + conn = Connections.try_to_get_gun_conn("http://some-domain.com", [genserver_pid: pid], name) + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "some-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end + test "opens connection and reuse it on next request", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert conn == reused_conn @@ -35,15 +68,15 @@ defmodule Gun.ConnectionsTest do end test "reuses connection based on protocol", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - https_conn = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + https_conn = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) refute conn == https_conn - reused_https = Connections.get_conn(name, "https://some-domain.com", genserver_pid: pid) + reused_https = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) refute conn == reused_https @@ -66,7 +99,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://gun_down.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down.com", [genserver_pid: pid], name) refute conn @@ -82,7 +115,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message and then gun_up", %{name: name, pid: pid} do - conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) refute conn @@ -96,7 +129,7 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) - conn = Connections.get_conn(name, "http://gun_down_and_up.com", genserver_pid: pid) + conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) @@ -116,7 +149,7 @@ defmodule Gun.ConnectionsTest do tasks = for _ <- 1..5 do Task.async(fn -> - Connections.get_conn(name, "http://some-domain.com", genserver_pid: pid) + Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) end) end @@ -149,12 +182,12 @@ defmodule Gun.ConnectionsTest do api = Pleroma.Config.get([API]) Pleroma.Config.put([API], :gun) on_exit(fn -> Pleroma.Config.put([API], api) end) - conn = Connections.get_conn(name, "http://httpbin.org") + conn = Connections.get_conn("http://httpbin.org", [], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn(name, "http://httpbin.org") + reused_conn = Connections.get_conn("http://httpbin.org", [], name) assert conn == reused_conn diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 56ac8672b..2092b0dc3 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxyTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase import ExUnit.CaptureLog import Mox alias Pleroma.ReverseProxy @@ -354,6 +354,10 @@ defmodule Pleroma.ReverseProxyTest do adapter = Application.get_env(:tesla, :adapter) Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") assert byte_size(conn.resp_body) == 10 @@ -363,6 +367,7 @@ defmodule Pleroma.ReverseProxyTest do on_exit(fn -> Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) end) end end From 9241df642c257bcfc5b697d50f0deed426370694 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:08:10 +0300 Subject: [PATCH 048/102] some debug --- lib/pleroma/gun/connections.ex | 5 +++++ lib/pleroma/http/http.ex | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index d5a9d8607..82579604a 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -69,6 +69,10 @@ defmodule Pleroma.Gun.Connections do def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do key = compose_key(uri) + IO.inspect(state) + + IO.inspect(key) + case state.conns[key] do %{conn: conn, state: conn_state} when conn_state == :up -> {:reply, conn, state} @@ -87,6 +91,7 @@ defmodule Pleroma.Gun.Connections do protocol: String.to_atom(uri.scheme) }) + IO.inspect(state) {:noreply, state} end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 1846749c0..3ad891d11 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + case Pleroma.Gun.Connections.get_conn(url) do nil -> options From 27d5aa7546a3263856581e972e7d79389007840f Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:27:55 +0300 Subject: [PATCH 049/102] ssl fixes --- lib/pleroma/gun/api/mock.ex | 2 +- lib/pleroma/gun/connections.ex | 16 +++++++++------ test/gun/connections_test.exs | 24 +++++++++++++++++++++++ test/reverse_proxy/reverse_proxy_test.exs | 23 ++++++++++++++++++++++ 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index 3348715c4..ff9e13a74 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Gun.API.Mock do def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - send(genserver_pid, {:gun_up, conn_pid, :https}) + send(genserver_pid, {:gun_up, conn_pid, :http2}) {:ok, conn_pid} end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 82579604a..6fcf4332a 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -100,8 +100,8 @@ defmodule Pleroma.Gun.Connections do def handle_call({:state}, _from, state), do: {:reply, state, state} @impl true - def handle_info({:gun_up, conn_pid, protocol}, state) do - {key, conn} = find_conn(state.conns, conn_pid, protocol) + def handle_info({:gun_up, conn_pid, _protocol}, state) do + {key, conn} = find_conn(state.conns, conn_pid) # Send to all waiting processes connection pid Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) @@ -113,8 +113,8 @@ defmodule Pleroma.Gun.Connections do @impl true # Do we need to do something with killed & unprocessed references? - def handle_info({:gun_down, conn_pid, protocol, _reason, _killed, _unprocessed}, state) do - {key, conn} = find_conn(state.conns, conn_pid, protocol) + def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do + {key, conn} = find_conn(state.conns, conn_pid) # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) @@ -125,6 +125,10 @@ defmodule Pleroma.Gun.Connections do defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) - defp find_conn(conns, conn_pid, protocol), - do: Enum.find(conns, fn {_, conn} -> conn.conn == conn_pid and conn.protocol == protocol end) + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {key, conn} -> + protocol = if String.ends_with?(key, ":443"), do: :https, else: :http + conn.conn == conn_pid and conn.protocol == protocol + end) + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 42354d330..aad70a644 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -201,5 +201,29 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) end + + test "opens ssl connection and reuse it on next request", %{name: name} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], :gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + conn = Connections.get_conn("https://httpbin.org", [], name) + + assert is_pid(conn) + assert Process.alive?(conn) + + reused_conn = Connections.get_conn("https://httpbin.org", [], name) + + assert conn == reused_conn + + %Connections{ + conns: %{ + "httpbin.org:443" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + } + } + } = Connections.get_state(name) + end end end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 2092b0dc3..c5563f955 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -370,5 +370,28 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], api) end) end + + test "with tesla client with gun adapter with ssl", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end end end From c0916b66f8f4d9e2917e11d7d24fd5931507a35c Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:34:49 +0300 Subject: [PATCH 050/102] removing debug --- lib/pleroma/gun/connections.ex | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 6fcf4332a..e06cad276 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -69,10 +69,6 @@ defmodule Pleroma.Gun.Connections do def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do key = compose_key(uri) - IO.inspect(state) - - IO.inspect(key) - case state.conns[key] do %{conn: conn, state: conn_state} when conn_state == :up -> {:reply, conn, state} @@ -91,7 +87,6 @@ defmodule Pleroma.Gun.Connections do protocol: String.to_atom(uri.scheme) }) - IO.inspect(state) {:noreply, state} end end From 888ac63df42075f771161a607be1de873ca7d666 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:38:13 +0300 Subject: [PATCH 051/102] some debug --- lib/pleroma/http/http.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3ad891d11..a6c325017 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -41,6 +41,8 @@ defmodule Pleroma.HTTP do options end + IO.inspect(options) + params = Keyword.get(options, :params, []) %{} @@ -62,7 +64,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url) do + case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do nil -> options From db848b2a13b4d47870dcff1c6d8e26f0300ca0a9 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:41:08 +0300 Subject: [PATCH 052/102] tesla adapter options debug --- lib/pleroma/http/http.ex | 2 -- lib/pleroma/reverse_proxy/client/tesla.ex | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a6c325017..1846749c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -41,8 +41,6 @@ defmodule Pleroma.HTTP do options end - IO.inspect(options) - params = Keyword.get(options, :params, []) %{} diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index fad577ec1..282f9dcb7 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -12,6 +12,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do Keyword.get(opts, :adapter, []) |> Keyword.put(:body_as, :chunks) + IO.inspect(adapter_opts) + with {:ok, response} <- Pleroma.HTTP.request( method, From d28051e2847420031873c31278590567d80e4cd1 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 14:45:21 +0300 Subject: [PATCH 053/102] another debug --- lib/pleroma/http/connection.ex | 4 +++- lib/pleroma/reverse_proxy/client/tesla.ex | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 6cb26c0fe..20c83c572 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -23,7 +23,9 @@ defmodule Pleroma.HTTP.Connection do def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) - Tesla.client(middleware, {adapter, options(opts)}) + options = options(opts) + IO.inspect(options) + Tesla.client(middleware, {adapter, options}) end # fetch http options diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 282f9dcb7..fad577ec1 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -12,8 +12,6 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do Keyword.get(opts, :adapter, []) |> Keyword.put(:body_as, :chunks) - IO.inspect(adapter_opts) - with {:ok, response} <- Pleroma.HTTP.request( method, From 43e5307d65874503125584a1110764ce9f2e7e51 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 15:54:25 +0300 Subject: [PATCH 054/102] like this --- lib/pleroma/http/connection.ex | 1 - lib/pleroma/http/http.ex | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 20c83c572..222a8b6aa 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -24,7 +24,6 @@ defmodule Pleroma.HTTP.Connection do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) options = options(opts) - IO.inspect(options) Tesla.client(middleware, {adapter, options}) end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 1846749c0..3ad891d11 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.try_to_get_gun_conn(url) do + case Pleroma.Gun.Connections.get_conn(url) do nil -> options From e0b52ca086b3e3a7c9a4ed5ac3c812fd3a56c45b Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 16:31:13 +0300 Subject: [PATCH 055/102] only http --- lib/pleroma/http/connection.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 222a8b6aa..ef1c1cabd 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,8 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000 + timeout: 20_000, + protocols: [:http] ] @doc """ From 55a5523accf5095e12e2f60b06f19ace49d0a3b8 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 17:48:22 +0300 Subject: [PATCH 056/102] don't close connection in gun adapter on reusing --- lib/pleroma/http/http.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3ad891d11..d493a8b4a 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -70,6 +70,7 @@ defmodule Pleroma.HTTP do adapter_opts = Keyword.get(options, :adapter, []) |> Keyword.put(:conn, conn) + |> Keyword.put(:close_conn, false) Keyword.put(options, :adapter, adapter_opts) end From 59336f14482c369c6c3a7bbc621d0a2828fd577f Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 17:56:16 +0300 Subject: [PATCH 057/102] tesla update --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 9d04b5771..f5200d15d 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "c29a7fd030fa6decbf7091152f563fe322e2b589", + ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index ef5ebda6c..f09b7fece 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "c29a7fd030fa6decbf7091152f563fe322e2b589", [ref: "c29a7fd030fa6decbf7091152f563fe322e2b589"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "99856638d0b8f382eb22e704b76bc30b4c4c379d", [ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 916aeb934f4a961948ba41f9ac9c0f52d986600f Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 14 Aug 2019 20:08:15 +0300 Subject: [PATCH 058/102] adding host header for gun adapter --- lib/pleroma/http/connection.ex | 6 ++---- lib/pleroma/http/http.ex | 2 +- lib/pleroma/http/request_builder.ex | 9 +++++++++ test/http/request_builder_test.exs | 13 ++++++++++++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index ef1c1cabd..6cb26c0fe 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,8 +9,7 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000, - protocols: [:http] + timeout: 20_000 ] @doc """ @@ -24,8 +23,7 @@ defmodule Pleroma.HTTP.Connection do def new(opts \\ []) do middleware = [Tesla.Middleware.FollowRedirects] adapter = Application.get_env(:tesla, :adapter) - options = options(opts) - Tesla.client(middleware, {adapter, options}) + Tesla.client(middleware, {adapter, options(opts)}) end # fetch http options diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index d493a8b4a..21c057f4b 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -45,9 +45,9 @@ defmodule Pleroma.HTTP do %{} |> Builder.method(method) + |> Builder.url(url) |> Builder.headers(headers) |> Builder.opts(options) - |> Builder.url(url) |> Builder.add_param(:body, :body, body) |> Builder.add_param(:query, :query, params) |> Enum.into([]) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 4e77870bd..6edf15bbe 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -53,6 +53,15 @@ defmodule Pleroma.HTTP.RequestBuilder do header_list end + # TODO: maybe we need this header for all adapters, so we won't fail on adapter change + header_list = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + %{host: host} = URI.parse(request.url) + header_list ++ [{"host", host}] + else + header_list + end + Map.put_new(request, :headers, header_list) end diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index ea4dd087e..631730f65 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.RequestBuilderTest do - use ExUnit.Case, async: true + use ExUnit.Case use Pleroma.Tests.Helpers alias Pleroma.HTTP.RequestBuilder @@ -21,6 +21,17 @@ defmodule Pleroma.HTTP.RequestBuilderTest do headers: [{"user-agent", Pleroma.Application.user_agent()}] } end + + test "it adds host header for gun adapter" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) + + assert RequestBuilder.headers(%{url: "https://example.com"}, []) == %{ + headers: [{"host", "example.com"}], + url: "https://example.com" + } + end end describe "add_optional_params/3" do From ff5a0a1d416391756c6c5d6935a1d41bbb162fa7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 15 Aug 2019 15:49:32 +0300 Subject: [PATCH 059/102] receive adapter messages in adapter --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f5200d15d..7c10ab75a 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:poison, "~> 3.0", override: true}, {:tesla, github: "alex-strizhakov/tesla", - ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d", + ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb", override: true}, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index f09b7fece..26be6d663 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "99856638d0b8f382eb22e704b76bc30b4c4c379d", [ref: "99856638d0b8f382eb22e704b76bc30b4c4c379d"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "82dd3ace650b93d6699cf379b1a238d891f45cfb", [ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 66a1573f81021c04063672382ef83716dfc4471d Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 15 Aug 2019 20:43:15 +0300 Subject: [PATCH 060/102] updating tesla --- lib/pleroma/http/http.ex | 3 +++ mix.exs | 10 ++++++---- mix.lock | 2 +- test/reverse_proxy/reverse_proxy_test.exs | 23 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 21c057f4b..6a09b8260 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -67,10 +67,13 @@ defmodule Pleroma.HTTP do options conn -> + %{host: host, port: port} = URI.parse(url) + adapter_opts = Keyword.get(options, :adapter, []) |> Keyword.put(:conn, conn) |> Keyword.put(:close_conn, false) + |> Keyword.put(:original, "#{host}:#{port}") Keyword.put(options, :adapter, adapter_opts) end diff --git a/mix.exs b/mix.exs index 7c10ab75a..d156b9e36 100644 --- a/mix.exs +++ b/mix.exs @@ -111,10 +111,12 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, - github: "alex-strizhakov/tesla", - ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb", - override: true}, + { + :tesla, + github: "alex-strizhakov/tesla", + ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", + override: true + }, {:cowlib, "~> 2.6.0", override: true}, {:gun, "~> 1.3"}, {:jason, "~> 1.0"}, diff --git a/mix.lock b/mix.lock index 26be6d663..606dbfa61 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "82dd3ace650b93d6699cf379b1a238d891f45cfb", [ref: "82dd3ace650b93d6699cf379b1a238d891f45cfb"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", [ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index c5563f955..0a982c56c 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -393,5 +393,28 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], api) end) end + + test "tesla client with gun client follow redirects", %{conn: conn} do + client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) + Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], :gun) + {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/7") + + assert byte_size(conn.resp_body) == 10 + assert conn.state == :chunked + assert conn.status == 200 + + on_exit(fn -> + Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + end end end From 7f431996c5e2ad64b4641385e2358636aa37e92e Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:35:47 +0300 Subject: [PATCH 061/102] updated deps --- lib/pleroma/http/request_builder.ex | 9 --------- mix.exs | 7 ++++--- mix.lock | 6 +++--- test/http/request_builder_test.exs | 11 ----------- test/reverse_proxy/reverse_proxy_test.exs | 3 +-- 5 files changed, 8 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 6edf15bbe..4e77870bd 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -53,15 +53,6 @@ defmodule Pleroma.HTTP.RequestBuilder do header_list end - # TODO: maybe we need this header for all adapters, so we won't fail on adapter change - header_list = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - %{host: host} = URI.parse(request.url) - header_list ++ [{"host", host}] - else - header_list - end - Map.put_new(request, :headers, header_list) end diff --git a/mix.exs b/mix.exs index d156b9e36..29f1c90c5 100644 --- a/mix.exs +++ b/mix.exs @@ -114,11 +114,12 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", + ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", override: true }, - {:cowlib, "~> 2.6.0", override: true}, - {:gun, "~> 1.3"}, + {:cowlib, "~> 2.7.3", override: true}, + {:gun, + github: "ninenines/gun", ref: "491ddf58c0e14824a741852fdc522b390b306ae2", override: true}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 606dbfa61..1d4532fed 100644 --- a/mix.lock +++ b/mix.lock @@ -13,7 +13,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, + "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, @@ -37,7 +37,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, - "gun": {:hex, :gun, "1.3.0", "18e5d269649c987af95aec309f68a27ffc3930531dd227a6eaa0884d6684286e", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm"}, + "gun": {:git, "https://github.com/ninenines/gun.git", "491ddf58c0e14824a741852fdc522b390b306ae2", [ref: "491ddf58c0e14824a741852fdc522b390b306ae2"]}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8", [ref: "33ba3bbd44d6fddd6558aaf4d9768c8de5f279b8"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", [ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 631730f65..77a1e870a 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -21,17 +21,6 @@ defmodule Pleroma.HTTP.RequestBuilderTest do headers: [{"user-agent", Pleroma.Application.user_agent()}] } end - - test "it adds host header for gun adapter" do - adapter = Application.get_env(:tesla, :adapter) - Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) - on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - - assert RequestBuilder.headers(%{url: "https://example.com"}, []) == %{ - headers: [{"host", "example.com"}], - url: "https://example.com" - } - end end describe "add_optional_params/3" do diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 0a982c56c..cd65163d1 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -404,9 +404,8 @@ defmodule Pleroma.ReverseProxyTest do Pleroma.Config.put([Pleroma.Gun.API], :gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) - conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/7") + conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") - assert byte_size(conn.resp_body) == 10 assert conn.state == :chunked assert conn.status == 200 From d052dca152b3f1cbd66873af6f0399a3d97debcc Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:44:00 +0300 Subject: [PATCH 062/102] fix --- lib/pleroma/gun/connections.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index e06cad276..a3f1b0351 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -31,7 +31,10 @@ defmodule Pleroma.Gun.Connections do opts = Enum.into(opts, %{}) uri = URI.parse(url) - opts = if uri.scheme == "https", do: Map.put(opts, :transport, :tls), else: opts + opts = + if uri.scheme == "https" and uri.port != 443, + do: Map.put(opts, :transport, :tls), + else: opts GenServer.call( name, From 7354d8958b66b93fcedb111ee929bc5046cc464f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 09:51:10 +0300 Subject: [PATCH 063/102] debug for media proxy --- lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 8403850ff..7d285a549 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -13,6 +13,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + IO.inspect(sig64), + IO.inspect(url64), + IO.inspect(url), :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else From c97979da776c6da83655b31e8ed20a0ece59de6f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:23:48 +0300 Subject: [PATCH 064/102] passing options for gun --- lib/pleroma/gun/connections.ex | 17 ++++++++++++++++- lib/pleroma/http/http.ex | 13 ++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index a3f1b0351..b8cf2f9b5 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -11,6 +11,20 @@ defmodule Pleroma.Gun.Connections do conns: %{domain() => conn()} } + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :ws_opts + ] + defstruct conns: %{} def start_link(name \\ __MODULE__) do @@ -81,7 +95,8 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + {:ok, conn} = + Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, Map.take(opts, @gun_keys)) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 6a09b8260..a7f42d0c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,7 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url) do + case Pleroma.Gun.Connections.get_conn(url, options) do nil -> options @@ -86,8 +86,15 @@ defmodule Pleroma.HTTP do host = uri.host |> to_charlist() case uri.scheme do - "https" -> options ++ [ssl: [server_name_indication: host]] - _ -> options + "https" -> + tls_opts = + Keyword.get(options, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) + + Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] + + _ -> + options end end From ac0d158c24549bd22b6308c86a20a99ee338c7f7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:33:23 +0300 Subject: [PATCH 065/102] test fixes --- lib/pleroma/gun/api/api.ex | 2 +- lib/pleroma/gun/api/gun.ex | 22 ++++++++++++++++++++++ lib/pleroma/gun/connections.ex | 17 +---------------- mix.exs | 2 +- mix.lock | 2 +- 5 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 lib/pleroma/gun/api/gun.ex diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex index c69ab890e..19adc1bf0 100644 --- a/lib/pleroma/gun/api/api.ex +++ b/lib/pleroma/gun/api/api.ex @@ -10,6 +10,6 @@ defmodule Pleroma.Gun.API do end defp api do - Pleroma.Config.get([Pleroma.Gun.API], :gun) + Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) end end diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex new file mode 100644 index 000000000..14a4b7275 --- /dev/null +++ b/lib/pleroma/gun/api/gun.ex @@ -0,0 +1,22 @@ +defmodule Pleroma.Gun.API.Gun do + @behaviour Pleroma.Gun.API + + @gun_keys [ + :connect_timeout, + :http_opts, + :http2_opts, + :protocols, + :retry, + :retry_timeout, + :trace, + :transport, + :tls_opts, + :tcp_opts, + :ws_opts + ] + + @impl Pleroma.Gun.API + def open(host, port, opts) do + :gun.open(host, port, Map.take(opts, @gun_keys)) + end +end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index b8cf2f9b5..a3f1b0351 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -11,20 +11,6 @@ defmodule Pleroma.Gun.Connections do conns: %{domain() => conn()} } - @gun_keys [ - :connect_timeout, - :http_opts, - :http2_opts, - :protocols, - :retry, - :retry_timeout, - :trace, - :transport, - :tls_opts, - :tcp_opts, - :ws_opts - ] - defstruct conns: %{} def start_link(name \\ __MODULE__) do @@ -95,8 +81,7 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = - Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, Map.take(opts, @gun_keys)) + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/mix.exs b/mix.exs index 29f1c90c5..64c56c4e3 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", + ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 1d4532fed..554b81fa8 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "929d68446a9b9d08149bd92d5b51c7ae9f87cfee", [ref: "929d68446a9b9d08149bd92d5b51c7ae9f87cfee"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", [ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From fb0d06432892bd178baeaadc5b4bdc5e6e21b89e Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 16:45:50 +0300 Subject: [PATCH 066/102] more fixes --- lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 --- test/gun/connections_test.exs | 4 ++-- test/reverse_proxy/reverse_proxy_test.exs | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 7d285a549..8403850ff 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -13,9 +13,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), - IO.inspect(sig64), - IO.inspect(url64), - IO.inspect(url), :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index aad70a644..a63c8eaf9 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -180,7 +180,7 @@ defmodule Gun.ConnectionsTest do test "opens connection and reuse it on next request", %{name: name} do api = Pleroma.Config.get([API]) - Pleroma.Config.put([API], :gun) + Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) conn = Connections.get_conn("http://httpbin.org", [], name) @@ -204,7 +204,7 @@ defmodule Gun.ConnectionsTest do test "opens ssl connection and reuse it on next request", %{name: name} do api = Pleroma.Config.get([API]) - Pleroma.Config.put([API], :gun) + Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) conn = Connections.get_conn("https://httpbin.org", [], name) diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index cd65163d1..476cff643 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -355,7 +355,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") @@ -378,7 +378,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") @@ -401,7 +401,7 @@ defmodule Pleroma.ReverseProxyTest do Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) api = Pleroma.Config.get([Pleroma.Gun.API]) - Pleroma.Config.put([Pleroma.Gun.API], :gun) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") From 5e309605d2087c534e0902cbaff68f0163155929 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 19:21:02 +0300 Subject: [PATCH 067/102] tesla update --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 64c56c4e3..d51b1488d 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", + ref: "81433bd4011217323a1f443661252fdab23f5e59", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 554b81fa8..daec58b4f 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "28b06f772632fe82d9fb4cc39b0477709b1f3d6f", [ref: "28b06f772632fe82d9fb4cc39b0477709b1f3d6f"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "81433bd4011217323a1f443661252fdab23f5e59", [ref: "81433bd4011217323a1f443661252fdab23f5e59"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 6e1408d05ac5d28123b3d62ff555df7fd118f61f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:04:13 +0300 Subject: [PATCH 068/102] tls versions for gun --- lib/pleroma/http/http.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a7f42d0c0..9339532af 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,6 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) + |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', 'tlsv1']) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 4a4a61f4ceac98fde293130f0ac4db164ccd8251 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:07:08 +0300 Subject: [PATCH 069/102] like this --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 9339532af..b6e338aab 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', 'tlsv1']) + |> Keyword.put(:versions, [:tlsv1, :"tlsv1.1", :"tlsv1.2"]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 102ec7bb0ca4b53d59f722f44ad168ca2d79b6da Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:47:24 +0300 Subject: [PATCH 070/102] like thiis --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b6e338aab..40b3e1a29 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, [:tlsv1, :"tlsv1.1", :"tlsv1.2"]) + |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 699fc4f6eb5fa170d0ae56f5949ead7618c264ab Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 16 Aug 2019 20:52:00 +0300 Subject: [PATCH 071/102] order --- lib/pleroma/http/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 40b3e1a29..3e29b7b6c 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,7 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, ['tlsv1.2', 'tlsv1.1', :tlsv1]) + |> Keyword.put(:versions, [:"tlsv1.2", :"tlsv1.1", :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 210e116dc0ee48183bf7b7a363c6d053f88d30fe Mon Sep 17 00:00:00 2001 From: Alex S Date: Sat, 17 Aug 2019 21:50:50 +0300 Subject: [PATCH 072/102] no hardcode of versions --- lib/pleroma/http/http.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 3e29b7b6c..a7f42d0c0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -90,7 +90,6 @@ defmodule Pleroma.HTTP do tls_opts = Keyword.get(options, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - |> Keyword.put(:versions, [:"tlsv1.2", :"tlsv1.1", :tlsv1]) Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] From 814159e668d48ace95d8feac8847e1d68d07810b Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 15:40:06 +0300 Subject: [PATCH 073/102] expanding gun connections closing least frequently used separate pools with settings --- config/config.exs | 18 +++++++ lib/pleroma/application.ex | 14 +++++ lib/pleroma/gun/api/api.ex | 10 ++++ lib/pleroma/gun/api/gun.ex | 6 +++ lib/pleroma/gun/api/mock.ex | 45 ++++++++++++++-- lib/pleroma/gun/conn.ex | 4 +- lib/pleroma/gun/connections.ex | 89 ++++++++++++++++++++++---------- test/gun/connections_test.exs | 94 +++++++++++++++++++++++++++------- 8 files changed, 230 insertions(+), 50 deletions(-) diff --git a/config/config.exs b/config/config.exs index 63162d594..e56b9730d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -550,6 +550,24 @@ config :pleroma, :rate_limit, password_reset: {1_800_000, 5}, account_confirmation_resend: {8_640_000, 5} +config :pleroma, :gun_pools, + federation: [ + max_connections: 50, + timeout: 150_000 + ], + media: [ + max_connections: 50, + timeout: 150_000 + ], + upload: [ + max_connections: 25, + timeout: 300_000 + ], + default: [ + max_connections: 10, + timout: 20_000 + ] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 25e56b9e2..06d1a187e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -39,6 +39,7 @@ defmodule Pleroma.Application do ] ++ cachex_children() ++ hackney_pool_children() ++ + gun_pools() ++ [ Pleroma.Web.Federator.RetryQueue, Pleroma.Stats, @@ -163,6 +164,19 @@ defmodule Pleroma.Application do end end + defp gun_pools do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + for {pool_name, opts} <- Pleroma.Config.get([:gun_pools]) do + %{ + id: :"gun_pool_#{pool_name}", + start: {Pleroma.Gun.Connections, :start_link, [{pool_name, opts}]} + } + end + else + [] + end + end + defp after_supervisor_start do with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], true <- digest_config[:active] do diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex index 19adc1bf0..7e6d2f929 100644 --- a/lib/pleroma/gun/api/api.ex +++ b/lib/pleroma/gun/api/api.ex @@ -4,11 +4,21 @@ defmodule Pleroma.Gun.API do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} + @callback info(pid()) :: map() + @callback close(pid()) :: :ok def open(host, port, opts) do api().open(host, port, opts) end + def info(pid) do + api().info(pid) + end + + def close(pid) do + api().close(pid) + end + defp api do Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) end diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex index 14a4b7275..33e7985a1 100644 --- a/lib/pleroma/gun/api/gun.ex +++ b/lib/pleroma/gun/api/gun.ex @@ -19,4 +19,10 @@ defmodule Pleroma.Gun.API.Gun do def open(host, port, opts) do :gun.open(host, port, Map.take(opts, @gun_keys)) end + + @impl Pleroma.Gun.API + def info(pid), do: :gun.info(pid) + + @impl Pleroma.Gun.API + def close(pid), do: :gun.close(pid) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index ff9e13a74..a80559f0b 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -5,36 +5,75 @@ defmodule Pleroma.Gun.API.Mock do @behaviour Pleroma.Gun.API @impl Pleroma.Gun.API - def open('some-domain.com', 80, %{genserver_pid: genserver_pid}) do + def open(domain, 80, %{genserver_pid: genserver_pid}) + when domain in ['another-domain.com', 'some-domain.com'] do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: domain, + origin_port: 80 + }) + send(genserver_pid, {:gun_up, conn_pid, :http}) {:ok, conn_pid} end def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "https", + origin_host: 'some-domain.com', + origin_port: 443 + }) + send(genserver_pid, {:gun_up, conn_pid, :http2}) {:ok, conn_pid} end @impl Pleroma.Gun.API - def open('gun_down.com', _port, %{genserver_pid: genserver_pid}) do + def open('gun_down.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'gun_down.com', + origin_port: 80 + }) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) {:ok, conn_pid} end @impl Pleroma.Gun.API - def open('gun_down_and_up.com', _port, %{genserver_pid: genserver_pid}) do + def open('gun_down_and_up.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) + + Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + origin_scheme: "http", + origin_host: 'gun_down_and_up.com', + origin_port: 80 + }) + send(genserver_pid, {:gun_down, conn_pid, :http, nil, nil, nil}) {:ok, _} = Task.start_link(fn -> Process.sleep(500) + send(genserver_pid, {:gun_up, conn_pid, :http}) end) {:ok, conn_pid} end + + @impl Pleroma.Gun.API + def info(pid) do + [{_, info}] = Registry.lookup(Pleroma.Gun.API.Mock, pid) + info + end + + @impl Pleroma.Gun.API + def close(_pid), do: :ok end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 62ef146a1..20ddec64c 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Gun.Conn do conn: pid(), state: atom(), waiting_pids: [pid()], - protocol: atom() + used: pos_integer() } - defstruct conn: nil, state: :open, waiting_pids: [], protocol: :http + defstruct conn: nil, state: :open, waiting_pids: [], used: 0 end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index a3f1b0351..cec3de2ca 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -6,29 +6,31 @@ defmodule Pleroma.Gun.Connections do use GenServer @type domain :: String.t() - @type conn :: Gun.Conn.t() + @type conn :: Pleroma.Gun.Conn.t() + @type t :: %__MODULE__{ - conns: %{domain() => conn()} + conns: %{domain() => conn()}, + opts: keyword() } - defstruct conns: %{} + defstruct conns: %{}, opts: [] - def start_link(name \\ __MODULE__) do + @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore + def start_link({name, opts}) do if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - GenServer.start_link(__MODULE__, [], name: name) + GenServer.start_link(__MODULE__, opts, name: name) else :ignore end end @impl true - def init(_) do - {:ok, %__MODULE__{conns: %{}}} - end + def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} @spec get_conn(String.t(), keyword(), atom()) :: pid() - def get_conn(url, opts \\ [], name \\ __MODULE__) do + def get_conn(url, opts \\ [], name \\ :default) do opts = Enum.into(opts, %{}) + uri = URI.parse(url) opts = @@ -58,13 +60,13 @@ defmodule Pleroma.Gun.Connections do end @spec alive?(atom()) :: boolean() - def alive?(name \\ __MODULE__) do + def alive?(name \\ :default) do pid = Process.whereis(name) if pid, do: Process.alive?(pid), else: false end @spec get_state(atom()) :: t() - def get_state(name \\ __MODULE__) do + def get_state(name \\ :default) do GenServer.call(name, {:state}) end @@ -73,7 +75,8 @@ defmodule Pleroma.Gun.Connections do key = compose_key(uri) case state.conns[key] do - %{conn: conn, state: conn_state} when conn_state == :up -> + %{conn: conn, state: conn_state, used: used} when conn_state == :up -> + state = put_in(state.conns[key].used, used + 1) {:reply, conn, state} %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> @@ -81,16 +84,23 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} nil -> - {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + max_connections = state.opts[:max_connections] - state = - put_in(state.conns[key], %Pleroma.Gun.Conn{ - conn: conn, - waiting_pids: [from], - protocol: String.to_atom(uri.scheme) - }) + if Enum.count(state.conns) < max_connections do + open_conn(key, uri, from, state, opts) + else + [{close_key, least_used} | _conns] = Enum.sort_by(state.conns, fn {_k, v} -> v.used end) - {:noreply, state} + :ok = Pleroma.Gun.API.close(least_used.conn) + + state = + put_in( + state.conns, + Map.delete(state.conns, close_key) + ) + + open_conn(key, uri, from, state, opts) + end end end @@ -99,20 +109,29 @@ defmodule Pleroma.Gun.Connections do @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do - {key, conn} = find_conn(state.conns, conn_pid) + conn_key = compose_key_gun_info(conn_pid) + {key, conn} = find_conn(state.conns, conn_pid, conn_key) # Send to all waiting processes connection pid Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) # Update state of the current connection and set waiting_pids to empty list - state = put_in(state.conns[key], %{conn | state: :up, waiting_pids: []}) + state = + put_in(state.conns[key], %{ + conn + | state: :up, + waiting_pids: [], + used: conn.used + length(conn.waiting_pids) + }) + {:noreply, state} end @impl true # Do we need to do something with killed & unprocessed references? def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do - {key, conn} = find_conn(state.conns, conn_pid) + conn_key = compose_key_gun_info(conn_pid) + {key, conn} = find_conn(state.conns, conn_pid, conn_key) # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) @@ -121,12 +140,28 @@ defmodule Pleroma.Gun.Connections do {:noreply, state} end - defp compose_key(uri), do: uri.host <> ":" <> to_string(uri.port) + defp compose_key(uri), do: "#{uri.scheme}:#{uri.host}:#{uri.port}" - defp find_conn(conns, conn_pid) do + defp compose_key_gun_info(pid) do + info = Pleroma.Gun.API.info(pid) + "#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}" + end + + defp find_conn(conns, conn_pid, conn_key) do Enum.find(conns, fn {key, conn} -> - protocol = if String.ends_with?(key, ":443"), do: :https, else: :http - conn.conn == conn_pid and conn.protocol == protocol + key == conn_key and conn.conn == conn_pid end) end + + defp open_conn(key, uri, from, state, opts) do + {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + + state = + put_in(state.conns[key], %Pleroma.Gun.Conn{ + conn: conn, + waiting_pids: [from] + }) + + {:noreply, state} + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index a63c8eaf9..8308b5f9f 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -6,12 +6,17 @@ defmodule Gun.ConnectionsTest do use ExUnit.Case alias Pleroma.Gun.{Connections, Conn, API} + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: API.Mock) + :ok + end + setup do name = :test_gun_connections adapter = Application.get_env(:tesla, :adapter) Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end) - {:ok, pid} = Connections.start_link(name) + {:ok, pid} = Connections.start_link({name, [max_connections: 2, timeout: 10]}) {:ok, name: name, pid: pid} end @@ -37,10 +42,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -58,10 +64,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -84,12 +91,12 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, waiting_pids: [] }, - "some-domain.com:443" => %Conn{ + "https:some-domain.com:443" => %Conn{ conn: ^https_conn, state: :up, waiting_pids: [] @@ -105,7 +112,7 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "gun_down.com:80" => %Conn{ + "http:gun_down.com:80" => %Conn{ conn: _, state: :down, waiting_pids: _ @@ -121,10 +128,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "gun_down_and_up.com:80" => %Conn{ + "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :down, - waiting_pids: _ + waiting_pids: _, + used: 0 } } } = Connections.get_state(name) @@ -136,10 +144,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "gun_down_and_up.com:80" => %Conn{ + "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -164,10 +173,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "some-domain.com:80" => %Conn{ + "http:some-domain.com:80" => %Conn{ conn: conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 5 } } } = Connections.get_state(name) @@ -175,6 +185,52 @@ defmodule Gun.ConnectionsTest do assert Enum.all?(conns, fn res -> res == conn end) end + test "remove frequently used", %{name: name, pid: pid} do + Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) + + for _ <- 1..4 do + Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + end + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + }, + "https:some-domain.com:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + conn = Connections.get_conn("http://another-domain.com", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:another-domain.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + }, + "http:some-domain.com:80" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + describe "integration test" do @describetag :integration @@ -193,10 +249,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "httpbin.org:80" => %Conn{ + "http:httpbin.org:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) @@ -217,10 +274,11 @@ defmodule Gun.ConnectionsTest do %Connections{ conns: %{ - "httpbin.org:443" => %Conn{ + "https:httpbin.org:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [] + waiting_pids: [], + used: 2 } } } = Connections.get_state(name) From 05ca6a7d2c27509b52a985810c0e22f0b1cb5112 Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 16:27:59 +0300 Subject: [PATCH 074/102] test fixes --- lib/pleroma/gun/connections.ex | 2 +- lib/pleroma/http/http.ex | 4 +++- test/reverse_proxy/reverse_proxy_test.exs | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index cec3de2ca..b6845e52d 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -46,7 +46,7 @@ defmodule Pleroma.Gun.Connections do # TODO: only for testing, add this parameter to the config @spec try_to_get_gun_conn(String.t(), keyword(), atom()) :: nil | pid() - def try_to_get_gun_conn(url, opts \\ [], name \\ __MODULE__), + def try_to_get_gun_conn(url, opts \\ [], name \\ :default), do: try_to_get_gun_conn(url, opts, name, 0) @spec try_to_get_gun_conn(String.t(), keyword(), atom(), pos_integer()) :: nil | pid() diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index a7f42d0c0..26ab0bb03 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,7 +62,9 @@ defmodule Pleroma.HTTP do end defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url, options) do + pool = if options[:adapter][:pool], do: options[:adapter][:pool], else: :default + + case Pleroma.Gun.Connections.try_to_get_gun_conn(url, options, pool) do nil -> options diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 476cff643..8235fcc86 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -356,7 +356,9 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + {:ok, _} = + Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") @@ -379,7 +381,9 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + {:ok, _} = + Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") @@ -402,7 +406,9 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = Pleroma.Gun.Connections.start_link(Pleroma.Gun.Connections) + + {:ok, _} = + Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") From 3bc3e61dc5e7b6a10eee4c212dd113e6b2ca788a Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 17:43:06 +0300 Subject: [PATCH 075/102] some clean up and pleroma header --- config/config.exs | 4 ---- lib/pleroma/gun/api/gun.ex | 12 +++++++--- lib/pleroma/gun/api/mock.ex | 24 +++++++++++--------- lib/pleroma/gun/connections.ex | 25 +++++---------------- lib/pleroma/http/connection.ex | 3 ++- lib/pleroma/http/http.ex | 12 +++++----- lib/pleroma/reverse_proxy/client/hackney.ex | 4 ++++ test/gun/connections_test.exs | 21 ----------------- test/reverse_proxy/client/hackney_test.exs | 4 ++++ test/reverse_proxy/client/tesla_test.exs | 4 ++++ test/support/reverse_proxy_client_case.ex | 4 ++++ 11 files changed, 52 insertions(+), 65 deletions(-) diff --git a/config/config.exs b/config/config.exs index e56b9730d..237c61ac9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -562,10 +562,6 @@ config :pleroma, :gun_pools, upload: [ max_connections: 25, timeout: 300_000 - ], - default: [ - max_connections: 10, - timout: 20_000 ] # Import environment specific config. This must remain at the bottom diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex index 33e7985a1..d97f5a7c9 100644 --- a/lib/pleroma/gun/api/gun.ex +++ b/lib/pleroma/gun/api/gun.ex @@ -1,6 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Gun.API.Gun do @behaviour Pleroma.Gun.API + alias Pleroma.Gun.API + @gun_keys [ :connect_timeout, :http_opts, @@ -15,14 +21,14 @@ defmodule Pleroma.Gun.API.Gun do :ws_opts ] - @impl Pleroma.Gun.API + @impl API def open(host, port, opts) do :gun.open(host, port, Map.take(opts, @gun_keys)) end - @impl Pleroma.Gun.API + @impl API def info(pid), do: :gun.info(pid) - @impl Pleroma.Gun.API + @impl API def close(pid), do: :gun.close(pid) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index a80559f0b..b1a30a73c 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -4,12 +4,15 @@ defmodule Pleroma.Gun.API.Mock do @behaviour Pleroma.Gun.API - @impl Pleroma.Gun.API + + alias Pleroma.Gun.API + + @impl API def open(domain, 80, %{genserver_pid: genserver_pid}) when domain in ['another-domain.com', 'some-domain.com'] do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "http", origin_host: domain, origin_port: 80 @@ -19,10 +22,11 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end + @impl API def open('some-domain.com', 443, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "https", origin_host: 'some-domain.com', origin_port: 443 @@ -32,11 +36,11 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end - @impl Pleroma.Gun.API + @impl API def open('gun_down.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "http", origin_host: 'gun_down.com', origin_port: 80 @@ -46,11 +50,11 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end - @impl Pleroma.Gun.API + @impl API def open('gun_down_and_up.com', 80, %{genserver_pid: genserver_pid}) do {:ok, conn_pid} = Task.start_link(fn -> Process.sleep(1_000) end) - Registry.register(Pleroma.Gun.API.Mock, conn_pid, %{ + Registry.register(API.Mock, conn_pid, %{ origin_scheme: "http", origin_host: 'gun_down_and_up.com', origin_port: 80 @@ -68,12 +72,12 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end - @impl Pleroma.Gun.API + @impl API def info(pid) do - [{_, info}] = Registry.lookup(Pleroma.Gun.API.Mock, pid) + [{_, info}] = Registry.lookup(API.Mock, pid) info end - @impl Pleroma.Gun.API + @impl API def close(_pid), do: :ok end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index b6845e52d..361e8aaee 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -15,6 +15,8 @@ defmodule Pleroma.Gun.Connections do defstruct conns: %{}, opts: [] + alias Pleroma.Gun.API + @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore def start_link({name, opts}) do if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do @@ -44,21 +46,6 @@ defmodule Pleroma.Gun.Connections do ) end - # TODO: only for testing, add this parameter to the config - @spec try_to_get_gun_conn(String.t(), keyword(), atom()) :: nil | pid() - def try_to_get_gun_conn(url, opts \\ [], name \\ :default), - do: try_to_get_gun_conn(url, opts, name, 0) - - @spec try_to_get_gun_conn(String.t(), keyword(), atom(), pos_integer()) :: nil | pid() - def try_to_get_gun_conn(_url, _, _, 3), do: nil - - def try_to_get_gun_conn(url, opts, name, acc) do - case Pleroma.Gun.Connections.get_conn(url, opts, name) do - nil -> try_to_get_gun_conn(url, acc + 1) - conn -> conn - end - end - @spec alive?(atom()) :: boolean() def alive?(name \\ :default) do pid = Process.whereis(name) @@ -91,7 +78,7 @@ defmodule Pleroma.Gun.Connections do else [{close_key, least_used} | _conns] = Enum.sort_by(state.conns, fn {_k, v} -> v.used end) - :ok = Pleroma.Gun.API.close(least_used.conn) + :ok = API.close(least_used.conn) state = put_in( @@ -128,12 +115,10 @@ defmodule Pleroma.Gun.Connections do end @impl true - # Do we need to do something with killed & unprocessed references? def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do conn_key = compose_key_gun_info(conn_pid) {key, conn} = find_conn(state.conns, conn_pid, conn_key) - # We don't want to block requests to GenServer if gun send down message, return nil, so we can make some retries, while connection is not up Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) state = put_in(state.conns[key].state, :down) @@ -143,7 +128,7 @@ defmodule Pleroma.Gun.Connections do defp compose_key(uri), do: "#{uri.scheme}:#{uri.host}:#{uri.port}" defp compose_key_gun_info(pid) do - info = Pleroma.Gun.API.info(pid) + info = API.info(pid) "#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}" end @@ -154,7 +139,7 @@ defmodule Pleroma.Gun.Connections do end defp open_conn(key, uri, from, state, opts) do - {:ok, conn} = Pleroma.Gun.API.open(to_charlist(uri.host), uri.port, opts) + {:ok, conn} = API.open(to_charlist(uri.host), uri.port, opts) state = put_in(state.conns[key], %Pleroma.Gun.Conn{ diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 6cb26c0fe..93e661e1b 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -9,7 +9,8 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, - timeout: 20_000 + timeout: 20_000, + pool: :federation ] @doc """ diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 26ab0bb03..ab0fd55de 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -34,9 +34,11 @@ defmodule Pleroma.HTTP do adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun + pool = options[:adapter][:pool] + options = - if adapter_gun? and Pleroma.Gun.Connections.alive?() do - get_conn_for_gun(url, options) + if adapter_gun? and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do + get_conn_for_gun(url, options, pool) else options end @@ -61,10 +63,8 @@ defmodule Pleroma.HTTP do end end - defp get_conn_for_gun(url, options) do - pool = if options[:adapter][:pool], do: options[:adapter][:pool], else: :default - - case Pleroma.Gun.Connections.try_to_get_gun_conn(url, options, pool) do + defp get_conn_for_gun(url, options, pool) do + case Pleroma.Gun.Connections.get_conn(url, options, pool) do nil -> options diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index e6293646a..402c183af 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxy.Client.Hackney do @behaviour Pleroma.ReverseProxy.Client diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 8308b5f9f..4da42d854 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -31,27 +31,6 @@ defmodule Gun.ConnectionsTest do end end - test "try_to_get_gun_conn/1 returns conn", %{name: name, pid: pid} do - conn = Connections.try_to_get_gun_conn("http://some-domain.com", [genserver_pid: pid], name) - assert is_pid(conn) - assert Process.alive?(conn) - - reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) - - assert conn == reused_conn - - %Connections{ - conns: %{ - "http:some-domain.com:80" => %Conn{ - conn: ^conn, - state: :up, - waiting_pids: [], - used: 2 - } - } - } = Connections.get_state(name) - end - test "opens connection and reuse it on next request", %{name: name, pid: pid} do conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs index 577e0b0b2..cf9de912a 100644 --- a/test/reverse_proxy/client/hackney_test.exs +++ b/test/reverse_proxy/client/hackney_test.exs @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxy.Client.HackneyTest do use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Hackney diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index 029a25d0f..a18d609d2 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxy.Client.TeslaTest do use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex index 40cd59ea2..16bc2803b 100644 --- a/test/support/reverse_proxy_client_case.ex +++ b/test/support/reverse_proxy_client_case.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.ReverseProxyClientCase do defmacro __using__(client: client) do quote do From af12d08da7d928f1855df882418ba88870e52047 Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 19 Aug 2019 18:05:57 +0300 Subject: [PATCH 076/102] no need in this --- lib/pleroma/gun/connections.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 361e8aaee..d3c44017d 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -19,11 +19,7 @@ defmodule Pleroma.Gun.Connections do @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore def start_link({name, opts}) do - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do - GenServer.start_link(__MODULE__, opts, name: name) - else - :ignore - end + GenServer.start_link(__MODULE__, opts, name: name) end @impl true From 2a2d11f2b305f415fb7eec7a799fd4c17d6e3446 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 10:51:56 +0300 Subject: [PATCH 077/102] some fixes and tests --- lib/pleroma/application.ex | 2 +- lib/pleroma/http/http.ex | 8 +++- test/gun/connections_test.exs | 54 +++++++++++++++++++++- test/http_test.exs | 24 ++++++++++ test/reverse_proxy/client/hackney_test.exs | 4 ++ test/reverse_proxy/client/tesla_test.exs | 4 ++ test/reverse_proxy/reverse_proxy_test.exs | 9 ---- test/support/reverse_proxy_client_case.ex | 2 +- 8 files changed, 93 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 06d1a187e..347f520bb 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -165,7 +165,7 @@ defmodule Pleroma.Application do end defp gun_pools do - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun || Mix.env() == :test do for {pool_name, opts} <- Pleroma.Config.get([:gun_pools]) do %{ id: :"gun_pool_#{pool_name}", diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index ab0fd55de..b18ce2803 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -89,11 +89,15 @@ defmodule Pleroma.HTTP do case uri.scheme do "https" -> + adapter_opts = Keyword.get(options, :adapter, []) + tls_opts = - Keyword.get(options, :tls_opts, []) + Keyword.get(adapter_opts, :tls_opts, []) |> Keyword.put(:server_name_indication, host) - Keyword.put(options, :tls_opts, tls_opts) ++ [ssl: [server_name_indication: host]] + adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) + + Keyword.put(options, :adapter, adapter_opts) ++ [ssl: [server_name_indication: host]] _ -> options diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 4da42d854..1e41e771b 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -4,7 +4,9 @@ defmodule Gun.ConnectionsTest do use ExUnit.Case - alias Pleroma.Gun.{Connections, Conn, API} + alias Pleroma.Gun.API + alias Pleroma.Gun.Conn + alias Pleroma.Gun.Connections setup_all do {:ok, _} = Registry.start_link(keys: :unique, name: API.Mock) @@ -262,5 +264,55 @@ defmodule Gun.ConnectionsTest do } } = Connections.get_state(name) end + + test "remove frequently used", %{name: name, pid: pid} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + + for _ <- 1..4 do + Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + end + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + }, + "https:www.google.com:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + conn = Connections.get_conn("http://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [], + used: 4 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end end end diff --git a/test/http_test.exs b/test/http_test.exs index 5f9522cf0..5ffd351e4 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -56,4 +56,28 @@ defmodule Pleroma.HTTPTest do } end end + + @tag :integration + test "get_conn_for_gun/3" do + adapter = Application.get_env(:tesla, :adapter) + Application.put_env(:tesla, :adapter, Tesla.Adapter.Gun) + api = Pleroma.Config.get([Pleroma.Gun.API]) + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + + on_exit(fn -> + Application.put_env(:tesla, :adapter, adapter) + Pleroma.Config.put([Pleroma.Gun.API], api) + end) + + options = [adapter: [pool: :federation]] + + assert {:ok, resp} = + Pleroma.HTTP.request(:get, "https://httpbin.org/user-agent", "", [], options) + + adapter_opts = resp.opts[:adapter] + + assert adapter_opts[:original] == "httpbin.org:443" + refute adapter_opts[:close_conn] + assert adapter_opts[:pool] == :federation + end end diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs index cf9de912a..3c552dc83 100644 --- a/test/reverse_proxy/client/hackney_test.exs +++ b/test/reverse_proxy/client/hackney_test.exs @@ -8,4 +8,8 @@ defmodule Pleroma.ReverseProxy.Client.HackneyTest do defp check_ref(ref) do assert is_reference(ref) end + + defp close(ref) do + Pleroma.ReverseProxy.Client.Hackney.close(ref) + end end diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index a18d609d2..8adefbe59 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -10,4 +10,8 @@ defmodule Pleroma.ReverseProxy.Client.TeslaTest do assert is_reference(stream) assert ref[:fin] end + + defp close(%{pid: pid}) do + Pleroma.ReverseProxy.Client.Tesla.close(pid) + end end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 8235fcc86..1d3e554b3 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -357,9 +357,6 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = - Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) - conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") assert byte_size(conn.resp_body) == 10 @@ -382,9 +379,6 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = - Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) - conn = ReverseProxy.call(conn, "https://httpbin.org/stream-bytes/10") assert byte_size(conn.resp_body) == 10 @@ -407,9 +401,6 @@ defmodule Pleroma.ReverseProxyTest do api = Pleroma.Config.get([Pleroma.Gun.API]) Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - {:ok, _} = - Pleroma.Gun.Connections.start_link({:media, [max_connections: 5, timeout: 5_000]}) - conn = ReverseProxy.call(conn, "https://httpbin.org/redirect/5") assert conn.state == :chunked diff --git a/test/support/reverse_proxy_client_case.ex b/test/support/reverse_proxy_client_case.ex index 16bc2803b..36df1ed95 100644 --- a/test/support/reverse_proxy_client_case.ex +++ b/test/support/reverse_proxy_client_case.ex @@ -55,7 +55,7 @@ defmodule Pleroma.ReverseProxyClientCase do assert headers != [] check_ref(ref) - assert :ok = @client.close(ref) + assert :ok == close(ref) {:ok, status, headers} -> assert headers != [] From 5209abf09b3b0ec4c2cf40ac2d7c5eb38472ee0f Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 12:11:54 +0300 Subject: [PATCH 078/102] testing without sni --- lib/pleroma/http/http.ex | 5 ++--- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b18ce2803..54ea04764 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -91,9 +91,8 @@ defmodule Pleroma.HTTP do "https" -> adapter_opts = Keyword.get(options, :adapter, []) - tls_opts = - Keyword.get(adapter_opts, :tls_opts, []) - |> Keyword.put(:server_name_indication, host) + tls_opts = Keyword.get(adapter_opts, :tls_opts, []) + # |> Keyword.put(:server_name_indication, host) adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) diff --git a/mix.exs b/mix.exs index d51b1488d..8d9a63362 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "81433bd4011217323a1f443661252fdab23f5e59", + ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index daec58b4f..f0633f8a9 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "81433bd4011217323a1f443661252fdab23f5e59", [ref: "81433bd4011217323a1f443661252fdab23f5e59"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", [ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 6c3990b1eaa57f4a633b3e136089ebaa9edc2d38 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 12:25:58 +0300 Subject: [PATCH 079/102] with sni --- lib/pleroma/http/http.ex | 2 +- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 54ea04764..aca378c9d 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -92,7 +92,7 @@ defmodule Pleroma.HTTP do adapter_opts = Keyword.get(options, :adapter, []) tls_opts = Keyword.get(adapter_opts, :tls_opts, []) - # |> Keyword.put(:server_name_indication, host) + |> Keyword.put(:server_name_indication, host) adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) diff --git a/mix.exs b/mix.exs index 8d9a63362..8b1a898bf 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", + ref: "245f75502052e53d45cdb44f6afb481f4bd0f576", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index f0633f8a9..484a3a1a9 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d", [ref: "a64f34a83b579d46cc4ec7bac2ef561cc8b03f1d"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "245f75502052e53d45cdb44f6afb481f4bd0f576", [ref: "245f75502052e53d45cdb44f6afb481f4bd0f576"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From b8ee02579275b2c2fd7b511b357ae039b9c82fb7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 12:47:16 +0300 Subject: [PATCH 080/102] rebase fixes --- test/reverse_proxy/reverse_proxy_test.exs | 32 ----------------------- 1 file changed, 32 deletions(-) diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index 1d3e554b3..a1adaedf6 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -108,38 +108,6 @@ defmodule Pleroma.ReverseProxyTest do "[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large" end - defp stream_mock(invokes, with_close? \\ false) do - ClientMock - |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> - Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) - - {:ok, 200, [{"content-type", "application/octet-stream"}], - %{url: "/stream-bytes/" <> length}} - end) - |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> - max = String.to_integer(length) - - case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do - [{_, current}] when current < max -> - Registry.update_value( - Pleroma.ReverseProxy.ClientMock, - "/stream-bytes/" <> length, - &(&1 + 10) - ) - - {:ok, "0123456789"} - - [{_, ^max}] -> - Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) - :done - end - end) - - if with_close? do - expect(ClientMock, :close, fn _ -> :ok end) - end - end - test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) From 70c4f003497ca6a324014c112a4276666e2e8ed0 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 13:10:28 +0300 Subject: [PATCH 081/102] formatting and pool fix --- lib/pleroma/gun/connections.ex | 10 ++++++++-- lib/pleroma/http/http.ex | 5 +++-- test/web/activity_pub/activity_pub_test.exs | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index d3c44017d..5b0605026 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -112,8 +112,8 @@ defmodule Pleroma.Gun.Connections do @impl true def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed, _unprocessed}, state) do - conn_key = compose_key_gun_info(conn_pid) - {key, conn} = find_conn(state.conns, conn_pid, conn_key) + # we can't get info on this pid, because pid is dead + {key, conn} = find_conn(state.conns, conn_pid) Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) @@ -128,6 +128,12 @@ defmodule Pleroma.Gun.Connections do "#{info.origin_scheme}:#{info.origin_host}:#{info.origin_port}" end + defp find_conn(conns, conn_pid) do + Enum.find(conns, fn {_key, conn} -> + conn.conn == conn_pid + end) + end + defp find_conn(conns, conn_pid, conn_key) do Enum.find(conns, fn {key, conn} -> key == conn_key and conn.conn == conn_pid diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index aca378c9d..b18ce2803 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -91,8 +91,9 @@ defmodule Pleroma.HTTP do "https" -> adapter_opts = Keyword.get(options, :adapter, []) - tls_opts = Keyword.get(adapter_opts, :tls_opts, []) - |> Keyword.put(:server_name_indication, host) + tls_opts = + Keyword.get(adapter_opts, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index f20cd2840..1515f4eb6 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -555,7 +555,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do note_two = insert(:note, data: %{"context" => "suya.."}) activity_two = insert(:note_activity, note: note_two) - {:ok, activity_two} = CommonAPI.add_mute(user, activity_two) + {:ok, _activity_two} = CommonAPI.add_mute(user, activity_two) assert [_activity_two, _activity_one] = ActivityPub.fetch_activities([], %{"muting_user" => user, "with_muted" => true}) From 50bc6c5ea98494ceeda7cdb18d75defe485d9a64 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 16:09:50 +0300 Subject: [PATCH 082/102] tesla gun adapter supports sni --- lib/pleroma/http/connection.ex | 3 ++- lib/pleroma/http/http.ex | 10 +--------- mix.exs | 2 +- mix.lock | 2 +- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 93e661e1b..69589bc5e 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -10,7 +10,8 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, timeout: 20_000, - pool: :federation + pool: :federation, + version: :master ] @doc """ diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b18ce2803..89afba720 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -89,15 +89,7 @@ defmodule Pleroma.HTTP do case uri.scheme do "https" -> - adapter_opts = Keyword.get(options, :adapter, []) - - tls_opts = - Keyword.get(adapter_opts, :tls_opts, []) - |> Keyword.put(:server_name_indication, host) - - adapter_opts = Keyword.put(adapter_opts, :tls_opts, tls_opts) - - Keyword.put(options, :adapter, adapter_opts) ++ [ssl: [server_name_indication: host]] + options ++ [ssl: [server_name_indication: host]] _ -> options diff --git a/mix.exs b/mix.exs index 8b1a898bf..9926a0836 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "245f75502052e53d45cdb44f6afb481f4bd0f576", + ref: "9f56f950a88a9ab0c21ad351e2cfd47b98594310", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 484a3a1a9..a10677c49 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "245f75502052e53d45cdb44f6afb481f4bd0f576", [ref: "245f75502052e53d45cdb44f6afb481f4bd0f576"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9f56f950a88a9ab0c21ad351e2cfd47b98594310", [ref: "9f56f950a88a9ab0c21ad351e2cfd47b98594310"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 6f08e772693d3b45da3b548404a56be1c88ec2da Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 18:06:08 +0300 Subject: [PATCH 083/102] sni option if we open connection in gun pool --- lib/pleroma/gun/connections.ex | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 5b0605026..3716d9f74 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -36,6 +36,19 @@ defmodule Pleroma.Gun.Connections do do: Map.put(opts, :transport, :tls), else: opts + opts = + if uri.scheme == "https" do + host = uri.host |> to_charlist() + + tls_opts = + Map.get(opts, :tls_opts, []) + |> Keyword.put(:server_name_indication, host) + + Map.put(opts, :tls_opts, tls_opts) + else + opts + end + GenServer.call( name, {:conn, %{opts: opts, uri: uri}} From cee945bead8d6e95e05ccb661330cd5561caf7a3 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 18:37:22 +0300 Subject: [PATCH 084/102] set adapter opts in connection --- lib/pleroma/http/connection.ex | 32 ++++++++++++++++++++++++++++---- lib/pleroma/http/http.ex | 28 ++++++---------------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 69589bc5e..ef2ee918d 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -35,9 +35,33 @@ defmodule Pleroma.HTTP.Connection do adapter_options = Pleroma.Config.get([:http, :adapter], []) proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) - @options - |> Keyword.merge(adapter_options) - |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + options = + @options + |> Keyword.merge(adapter_options) + |> Keyword.merge(options) + |> Keyword.merge(proxy: proxy_url) + + pool = options[:pool] + url = options[:url] + + if not is_nil(url) and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do + get_conn_for_gun(url, options, pool) + else + options + end + end + + defp get_conn_for_gun(url, options, pool) do + case Pleroma.Gun.Connections.get_conn(url, options, pool) do + nil -> + options + + conn -> + %{host: host, port: port} = URI.parse(url) + + Keyword.put(options, :conn, conn) + |> Keyword.put(:close_conn, false) + |> Keyword.put(:original, "#{host}:#{port}") + end end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 89afba720..da86ebcc7 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -34,11 +34,13 @@ defmodule Pleroma.HTTP do adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun - pool = options[:adapter][:pool] - options = - if adapter_gun? and not is_nil(pool) and Pleroma.Gun.Connections.alive?(pool) do - get_conn_for_gun(url, options, pool) + if adapter_gun? do + adapter_opts = + Keyword.get(options, :adapter, []) + |> Keyword.put(:url, url) + + Keyword.put(options, :adapter, adapter_opts) else options end @@ -63,24 +65,6 @@ defmodule Pleroma.HTTP do end end - defp get_conn_for_gun(url, options, pool) do - case Pleroma.Gun.Connections.get_conn(url, options, pool) do - nil -> - options - - conn -> - %{host: host, port: port} = URI.parse(url) - - adapter_opts = - Keyword.get(options, :adapter, []) - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - |> Keyword.put(:original, "#{host}:#{port}") - - Keyword.put(options, :adapter, adapter_opts) - end - end - defp process_sni_options(options, nil), do: options defp process_sni_options(options, url) do From 4930bc3ca100f4ee299a30377545dce66c7c88c8 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 20 Aug 2019 18:56:15 +0300 Subject: [PATCH 085/102] test fix --- test/http_test.exs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/http_test.exs b/test/http_test.exs index 5ffd351e4..460f8f7c2 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -76,8 +76,6 @@ defmodule Pleroma.HTTPTest do adapter_opts = resp.opts[:adapter] - assert adapter_opts[:original] == "httpbin.org:443" - refute adapter_opts[:close_conn] - assert adapter_opts[:pool] == :federation + assert adapter_opts[:url] == "https://httpbin.org/user-agent" end end From b7f677dc281f61d449abcf85494a41b179cd8dcc Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 20 Aug 2019 20:11:17 +0000 Subject: [PATCH 086/102] http: remove unused get_conn_for_gun() --- lib/pleroma/http/http.ex | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 00558d70a..da86ebcc7 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,24 +65,6 @@ defmodule Pleroma.HTTP do end end - defp get_conn_for_gun(url, options) do - case Pleroma.Gun.Connections.get_conn(url, options) do - nil -> - options - - conn -> - %{host: host, port: port} = URI.parse(url) - - adapter_opts = - Keyword.get(options, :adapter, []) - |> Keyword.put(:conn, conn) - |> Keyword.put(:close_conn, false) - |> Keyword.put(:original, "#{host}:#{port}") - - Keyword.put(options, :adapter, adapter_opts) - end - end - defp process_sni_options(options, nil), do: options defp process_sni_options(options, url) do From 27f7625dd20c1580e9f29f2abe421955e1ee97b1 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 20 Aug 2019 20:13:48 +0000 Subject: [PATCH 087/102] application: don't try to start Pleroma.Gun.Connections automatically --- lib/pleroma/application.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index cf0b99fe6..347f520bb 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,8 +35,7 @@ defmodule Pleroma.Application do Pleroma.Emoji, Pleroma.Captcha, Pleroma.FlakeId, - Pleroma.ScheduledActivityWorker, - Pleroma.Gun.Connections + Pleroma.ScheduledActivityWorker ] ++ cachex_children() ++ hackney_pool_children() ++ From 8f17204221e863a14b492d38c6e8853db9ad5122 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 20 Aug 2019 20:19:23 +0000 Subject: [PATCH 088/102] test: rich media: aws signed url: increase TTL delta check to 2 seconds from 1 --- test/web/rich_media/aws_signed_url_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs index 122787bc2..a3a50cbb1 100644 --- a/test/web/rich_media/aws_signed_url_test.exs +++ b/test/web/rich_media/aws_signed_url_test.exs @@ -60,7 +60,8 @@ defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url) # as there is delay in setting and pulling the data from cache we ignore 1 second - assert_in_delta(valid_till * 1000, cache_ttl, 1000) + # make it 2 seconds for flakyness + assert_in_delta(valid_till * 1000, cache_ttl, 2000) end defp construct_s3_url(timestamp, valid_till) do From dd9446474f56546526fb30995c4276077037edc7 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 20 Aug 2019 21:00:08 +0000 Subject: [PATCH 089/102] purge all remaining hackney usage and references from the tree --- config/config.exs | 24 ++++--------------- lib/pleroma/application.ex | 22 ----------------- lib/pleroma/reverse_proxy/client/hackney.ex | 21 ---------------- lib/pleroma/reverse_proxy/reverse_proxy.ex | 6 ++--- lib/pleroma/web/admin_api/config.ex | 6 ----- test/reverse_proxy/client/hackney_test.exs | 15 ------------ test/reverse_proxy/reverse_proxy_test.exs | 15 ------------ .../admin_api/admin_api_controller_test.exs | 6 ++--- test/web/admin_api/config_test.exs | 8 ------- 9 files changed, 10 insertions(+), 113 deletions(-) delete mode 100644 lib/pleroma/reverse_proxy/client/hackney.ex delete mode 100644 test/reverse_proxy/client/hackney_test.exs diff --git a/config/config.exs b/config/config.exs index 237c61ac9..30c9f248b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -56,20 +56,6 @@ config :pleroma, Pleroma.Captcha, seconds_valid: 60, method: Pleroma.Captcha.Kocaptcha -config :pleroma, :hackney_pools, - federation: [ - max_connections: 50, - timeout: 150_000 - ], - media: [ - max_connections: 50, - timeout: 150_000 - ], - upload: [ - max_connections: 25, - timeout: 300_000 - ] - config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" # Upload configuration @@ -193,11 +179,11 @@ config :pleroma, :http, proxy_url: nil, send_user_agent: true, adapter: [ - ssl_options: [ - # Workaround for remote server certificate chain issues - partial_chain: &:hackney_connect.partial_chain/1, - # We don't support TLS v1.3 yet - versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"] + tls_opts: [ + verify: :verify_peer, + cacerts: :certifi.cacerts(), + depth: 20, + reuse_sessions: false ] ] diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 347f520bb..5ff2e06b0 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -38,7 +38,6 @@ defmodule Pleroma.Application do Pleroma.ScheduledActivityWorker ] ++ cachex_children() ++ - hackney_pool_children() ++ gun_pools() ++ [ Pleroma.Web.Federator.RetryQueue, @@ -95,20 +94,6 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint.Instrumenter.setup() end - def enabled_hackney_pools do - [:media] ++ - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - [:federation] - else - [] - end ++ - if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do - [:upload] - else - [] - end - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), @@ -157,13 +142,6 @@ defmodule Pleroma.Application do defp chat_child(_, _), do: [] - defp hackney_pool_children do - for pool <- enabled_hackney_pools() do - options = Pleroma.Config.get([:hackney_pools, pool]) - :hackney_pool.child_spec(pool, options) - end - end - defp gun_pools do if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun || Mix.env() == :test do for {pool_name, opts} <- Pleroma.Config.get([:gun_pools]) do diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex deleted file mode 100644 index 402c183af..000000000 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ /dev/null @@ -1,21 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ReverseProxy.Client.Hackney do - @behaviour Pleroma.ReverseProxy.Client - - def request(method, url, headers, body, opts \\ []) do - :hackney.request(method, url, headers, body, opts) - end - - def stream_body(ref) do - case :hackney.stream_body(ref) do - :done -> :done - {:ok, data} -> {:ok, data, ref} - {:error, error} -> {:error, error} - end - end - - def close(ref), do: :hackney.close(ref) -end diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index a2cdcf393..df4eca207 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -58,7 +58,7 @@ defmodule Pleroma.ReverseProxy do * `req_headers`, `resp_headers` additional headers. - * `http`: options for [hackney](https://github.com/benoitc/hackney). + * `http`: options for [gun](https://github.com/ninenines/gun). """ @default_options [pool: :media] @@ -147,11 +147,11 @@ defmodule Pleroma.ReverseProxy do |> halt() end - defp request(method, url, headers, hackney_opts) do + defp request(method, url, headers, opts) do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case client().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} diff --git a/lib/pleroma/web/admin_api/config.ex b/lib/pleroma/web/admin_api/config.ex index a10cc779b..25e91eee6 100644 --- a/lib/pleroma/web/admin_api/config.ex +++ b/lib/pleroma/web/admin_api/config.ex @@ -95,7 +95,6 @@ defmodule Pleroma.Web.AdminAPI.Config do end defp do_convert({:dispatch, [entity]}), do: %{"tuple" => [":dispatch", [inspect(entity)]]} - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} defp do_convert(entity) when is_tuple(entity), do: %{"tuple" => do_convert(Tuple.to_list(entity))} @@ -129,11 +128,6 @@ defmodule Pleroma.Web.AdminAPI.Config do {:dispatch, [dispatch_settings]} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do - {partial_chain, []} = do_eval(entity) - {:partial_chain, partial_chain} - end - defp do_transform(%{"tuple" => entity}) do Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) end diff --git a/test/reverse_proxy/client/hackney_test.exs b/test/reverse_proxy/client/hackney_test.exs deleted file mode 100644 index 3c552dc83..000000000 --- a/test/reverse_proxy/client/hackney_test.exs +++ /dev/null @@ -1,15 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ReverseProxy.Client.HackneyTest do - use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Hackney - - defp check_ref(ref) do - assert is_reference(ref) - end - - defp close(ref) do - Pleroma.ReverseProxy.Client.Hackney.close(ref) - end -end diff --git a/test/reverse_proxy/reverse_proxy_test.exs b/test/reverse_proxy/reverse_proxy_test.exs index a1adaedf6..b2f7932bf 100644 --- a/test/reverse_proxy/reverse_proxy_test.exs +++ b/test/reverse_proxy/reverse_proxy_test.exs @@ -301,21 +301,6 @@ defmodule Pleroma.ReverseProxyTest do describe "integration tests" do @describetag :integration - test "with hackney client", %{conn: conn} do - client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) - Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Hackney) - - on_exit(fn -> - Pleroma.Config.put([Pleroma.ReverseProxy.Client], client) - end) - - conn = ReverseProxy.call(conn, "http://httpbin.org/stream-bytes/10") - - assert byte_size(conn.resp_body) == 10 - assert conn.state == :chunked - assert conn.status == 200 - end - test "with tesla client with gun adapter", %{conn: conn} do client = Pleroma.Config.get([Pleroma.ReverseProxy.Client]) Pleroma.Config.put([Pleroma.ReverseProxy.Client], Pleroma.ReverseProxy.Client.Tesla) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 844cd0732..0472a3f6d 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1551,8 +1551,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do %{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]}, %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, - %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} + %{"tuple" => [":key1", nil]} ] } ] @@ -1568,8 +1567,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do %{"tuple" => [":method", "Pleroma.Captcha.Kocaptcha"]}, %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, - %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} + %{"tuple" => [":key1", nil]} ] } ] diff --git a/test/web/admin_api/config_test.exs b/test/web/admin_api/config_test.exs index 3190dc1c8..d41666ef3 100644 --- a/test/web/admin_api/config_test.exs +++ b/test/web/admin_api/config_test.exs @@ -238,14 +238,6 @@ defmodule Pleroma.Web.AdminAPI.ConfigTest do assert Config.from_binary(binary) == [key: "value"] end - test "keyword with partial_chain key" do - binary = - Config.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}]) - - assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1) - assert Config.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1] - end - test "keyword" do binary = Config.transform([ From 79a0c3204a9fd271dca9b03a46c2702de7e37c07 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 20 Aug 2019 21:08:29 +0000 Subject: [PATCH 090/102] config: tls_opts needs to be done at runtime --- config/config.exs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config/config.exs b/config/config.exs index 30c9f248b..4af1b94d3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -179,12 +179,6 @@ config :pleroma, :http, proxy_url: nil, send_user_agent: true, adapter: [ - tls_opts: [ - verify: :verify_peer, - cacerts: :certifi.cacerts(), - depth: 20, - reuse_sessions: false - ] ] config :pleroma, :instance, From 125ba1dc3d9650bb34a4b405fb0d7ddf959e09e8 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Tue, 20 Aug 2019 21:32:21 +0000 Subject: [PATCH 091/102] tests: fix up tesla test --- test/reverse_proxy/client/tesla_test.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index 8adefbe59..db58c805b 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -5,6 +5,13 @@ defmodule Pleroma.ReverseProxy.Client.TeslaTest do use Pleroma.ReverseProxyClientCase, client: Pleroma.ReverseProxy.Client.Tesla + setup_all do + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + on_exit(fn -> + Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Mock) + end) + end + defp check_ref(%{pid: pid, stream: stream} = ref) do assert is_pid(pid) assert is_reference(stream) From 540338aaa0cc55f969aedbe221b6a882b3c3d5f9 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 22 Aug 2019 12:36:57 +0300 Subject: [PATCH 092/102] tesla version bump --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 9926a0836..2e7d4fc6c 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "9f56f950a88a9ab0c21ad351e2cfd47b98594310", + ref: "893a5d0d38c1ad0c08339586a3235bf6149ff574", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index a10677c49..1d83e6832 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "9f56f950a88a9ab0c21ad351e2cfd47b98594310", [ref: "9f56f950a88a9ab0c21ad351e2cfd47b98594310"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "893a5d0d38c1ad0c08339586a3235bf6149ff574", [ref: "893a5d0d38c1ad0c08339586a3235bf6149ff574"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 6e66bb35d1c2ab87e65e3249fbb427b80ca5b015 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 22 Aug 2019 12:37:33 +0300 Subject: [PATCH 093/102] was used with hackney --- lib/pleroma/http/http.ex | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index da86ebcc7..5c0d66955 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -28,9 +28,7 @@ defmodule Pleroma.HTTP do """ def request(method, url, body \\ "", headers \\ [], options \\ []) do try do - options = - process_request_options(options) - |> process_sni_options(url) + options = process_request_options(options) adapter_gun? = Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun @@ -65,21 +63,6 @@ defmodule Pleroma.HTTP do end end - defp process_sni_options(options, nil), do: options - - defp process_sni_options(options, url) do - uri = URI.parse(url) - host = uri.host |> to_charlist() - - case uri.scheme do - "https" -> - options ++ [ssl: [server_name_indication: host]] - - _ -> - options - end - end - def process_request_options(options) do Keyword.merge(Pleroma.HTTP.Connection.options([]), options) end From d812c8bbc7ed5605650e7baca3397617a86fc154 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 22 Aug 2019 12:38:39 +0300 Subject: [PATCH 094/102] opts to verify certificates by gun --- lib/pleroma/http/connection.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index ef2ee918d..fbf135bf9 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -59,9 +59,20 @@ defmodule Pleroma.HTTP.Connection do conn -> %{host: host, port: port} = URI.parse(url) + # verify sertificates opts for gun + tls_opts = [ + verify: :verify_peer, + cacerts: :certifi.cacerts(), + depth: 20, + server_name_indication: to_charlist(host), + reuse_sessions: false, + verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: to_charlist(host)]} + ] + Keyword.put(options, :conn, conn) |> Keyword.put(:close_conn, false) |> Keyword.put(:original, "#{host}:#{port}") + |> Keyword.put(:tls_opts, tls_opts) end end end From e8ee0c19e871295965fb65512427b88d52f3b686 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 22 Aug 2019 12:57:42 +0300 Subject: [PATCH 095/102] formatting --- config/config.exs | 3 +-- test/reverse_proxy/client/tesla_test.exs | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4af1b94d3..ca1de3fa2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -178,8 +178,7 @@ config :tesla, adapter: Tesla.Adapter.Gun config :pleroma, :http, proxy_url: nil, send_user_agent: true, - adapter: [ - ] + adapter: [] config :pleroma, :instance, name: "Pleroma", diff --git a/test/reverse_proxy/client/tesla_test.exs b/test/reverse_proxy/client/tesla_test.exs index db58c805b..c9ea3f6c0 100644 --- a/test/reverse_proxy/client/tesla_test.exs +++ b/test/reverse_proxy/client/tesla_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.ReverseProxy.Client.TeslaTest do setup_all do Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Gun) + on_exit(fn -> Pleroma.Config.put([Pleroma.Gun.API], Pleroma.Gun.API.Mock) end) From e34ca5174c6860cc97bed18d9baaece77687e23b Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 23 Aug 2019 12:57:52 +0300 Subject: [PATCH 096/102] basic support for proxies --- lib/pleroma/gun/api/api.ex | 25 +++---- lib/pleroma/gun/api/gun.ex | 9 +++ lib/pleroma/gun/api/mock.ex | 35 ++++++++++ lib/pleroma/gun/connections.ex | 70 +++++++++++++++++-- lib/pleroma/http/connection.ex | 56 ++++++++++++++- test/gun/connections_test.exs | 122 +++++++++++++++++++++++++++++++++ test/http/connection_test.exs | 65 ++++++++++++++++++ 7 files changed, 362 insertions(+), 20 deletions(-) create mode 100644 test/http/connection_test.exs diff --git a/lib/pleroma/gun/api/api.ex b/lib/pleroma/gun/api/api.ex index 7e6d2f929..43ee7f354 100644 --- a/lib/pleroma/gun/api/api.ex +++ b/lib/pleroma/gun/api/api.ex @@ -6,20 +6,21 @@ defmodule Pleroma.Gun.API do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} @callback info(pid()) :: map() @callback close(pid()) :: :ok + @callback await_up(pid) :: {:ok, atom()} | {:error, atom()} + @callback connect(pid(), map()) :: reference() + @callback await(pid(), reference()) :: {:response, :fin, 200, []} - def open(host, port, opts) do - api().open(host, port, opts) - end + def open(host, port, opts), do: api().open(host, port, opts) - def info(pid) do - api().info(pid) - end + def info(pid), do: api().info(pid) - def close(pid) do - api().close(pid) - end + def close(pid), do: api().close(pid) - defp api do - Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) - end + def await_up(pid), do: api().await_up(pid) + + def connect(pid, opts), do: api().connect(pid, opts) + + def await(pid, ref), do: api().await(pid, ref) + + defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun.API.Gun) end diff --git a/lib/pleroma/gun/api/gun.ex b/lib/pleroma/gun/api/gun.ex index d97f5a7c9..603dd700e 100644 --- a/lib/pleroma/gun/api/gun.ex +++ b/lib/pleroma/gun/api/gun.ex @@ -31,4 +31,13 @@ defmodule Pleroma.Gun.API.Gun do @impl API def close(pid), do: :gun.close(pid) + + @impl API + def await_up(pid), do: :gun.await_up(pid) + + @impl API + def connect(pid, opts), do: :gun.connect(pid, opts) + + @impl API + def await(pid, ref), do: :gun.await(pid, ref) end diff --git a/lib/pleroma/gun/api/mock.ex b/lib/pleroma/gun/api/mock.ex index b1a30a73c..5e1bb8abc 100644 --- a/lib/pleroma/gun/api/mock.ex +++ b/lib/pleroma/gun/api/mock.ex @@ -72,6 +72,41 @@ defmodule Pleroma.Gun.API.Mock do {:ok, conn_pid} end + @impl API + def open({127, 0, 0, 1}, 8123, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def open('localhost', 9050, _) do + Task.start_link(fn -> Process.sleep(1_000) end) + end + + @impl API + def await_up(_pid) do + {:ok, :http} + end + + @impl API + def connect(pid, %{host: _, port: 80}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def connect(pid, %{host: _, port: 443, protocols: [:http2], transport: :tls}) do + ref = make_ref() + Registry.register(API.Mock, ref, pid) + ref + end + + @impl API + def await(pid, ref) do + [{_, ^pid}] = Registry.lookup(API.Mock, ref) + {:response, :fin, 200, []} + end + @impl API def info(pid) do [{_, info}] = Registry.lookup(API.Mock, pid) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 3716d9f74..6cec4277a 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Gun.Connections do use GenServer + require Logger @type domain :: String.t() @type conn :: Pleroma.Gun.Conn.t() @@ -154,14 +155,69 @@ defmodule Pleroma.Gun.Connections do end defp open_conn(key, uri, from, state, opts) do - {:ok, conn} = API.open(to_charlist(uri.host), uri.port, opts) + host = to_charlist(uri.host) + port = uri.port - state = - put_in(state.conns[key], %Pleroma.Gun.Conn{ - conn: conn, - waiting_pids: [from] - }) + result = + if opts[:proxy] do + with {proxy_host, proxy_port} <- opts[:proxy], + {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + {:ok, _} <- API.await_up(conn) do + connect_opts = %{host: host, port: port} - {:noreply, state} + connect_opts = + if uri.scheme == "https" do + Map.put(connect_opts, :protocols, [:http2]) + |> Map.put(:transport, :tls) + else + connect_opts + end + + with stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream) do + {:ok, conn, true} + end + else + {:error, error} -> + {:error, error} + + error -> + Logger.warn(inspect(error)) + {:error, :error_connection_to_proxy} + end + else + with {:ok, conn} <- API.open(host, port, opts) do + {:ok, conn, false} + else + {:error, error} -> + {:error, error} + + error -> + Logger.warn(inspect(error)) + {:error, :error_connection} + end + end + + case result do + {:ok, conn, is_up} -> + {from_list, used, conn_state} = if is_up, do: {[], 1, :up}, else: {[from], 0, :open} + + state = + put_in(state.conns[key], %Pleroma.Gun.Conn{ + conn: conn, + waiting_pids: from_list, + used: used, + state: conn_state + }) + + if is_up do + {:reply, conn, state} + else + {:noreply, state} + end + + {:error, _error} -> + {:reply, nil, state} + end end end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index fbf135bf9..39c0fff43 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -14,6 +14,8 @@ defmodule Pleroma.HTTP.Connection do version: :master ] + require Logger + @doc """ Configure a client connection @@ -33,13 +35,20 @@ defmodule Pleroma.HTTP.Connection do def options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) + proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) + proxy = + case parse_proxy(proxy_url) do + {:ok, proxy_host, proxy_port} -> {proxy_host, proxy_port} + _ -> nil + end + options = @options |> Keyword.merge(adapter_options) |> Keyword.merge(options) - |> Keyword.merge(proxy: proxy_url) + |> Keyword.merge(proxy: proxy) pool = options[:pool] url = options[:url] @@ -75,4 +84,49 @@ defmodule Pleroma.HTTP.Connection do |> Keyword.put(:tls_opts, tls_opts) end end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {tuple, pos_integer()} | {:error, atom()} | nil + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + :error -> + Logger.warn("parsing port in proxy fail #{inspect(proxy)}") + {:error, :error_parsing_port_in_proxy} + + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {_type, host, port} <- proxy do + {:ok, parse_host(host), port} + else + _ -> + Logger.warn("parsing proxy fail #{inspect(proxy)}") + {:error, :error_parsing_proxy} + end + end + + @spec parse_host(String.t() | tuple()) :: charlist() | atom() + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 1e41e771b..4d84821a0 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -315,4 +315,126 @@ defmodule Gun.ConnectionsTest do } = Connections.get_state(name) end end + + describe "with proxy usage" do + test "proxy as ip", %{name: name, pid: pid} do + conn = + Connections.get_conn( + "http://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + %Connections{ + conns: %{ + "http:proxy_string.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.get_conn( + "http://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + assert reused_conn == conn + end + + test "proxy as host", %{name: name, pid: pid} do + conn = + Connections.get_conn( + "http://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + %Connections{ + conns: %{ + "http:proxy_tuple_atom.com:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.get_conn( + "http://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + assert reused_conn == conn + end + + test "proxy as ip and ssl", %{name: name, pid: pid} do + conn = + Connections.get_conn( + "https://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + %Connections{ + conns: %{ + "https:proxy_string.com:443" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.get_conn( + "https://proxy_string.com", + [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], + name + ) + + assert reused_conn == conn + end + + test "proxy as host and ssl", %{name: name, pid: pid} do + conn = + Connections.get_conn( + "https://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + %Connections{ + conns: %{ + "https:proxy_tuple_atom.com:443" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [], + used: 1 + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + reused_conn = + Connections.get_conn( + "https://proxy_tuple_atom.com", + [genserver_pid: pid, proxy: {'localhost', 9050}], + name + ) + + assert reused_conn == conn + end + end end diff --git a/test/http/connection_test.exs b/test/http/connection_test.exs new file mode 100644 index 000000000..99eab4026 --- /dev/null +++ b/test/http/connection_test.exs @@ -0,0 +1,65 @@ +defmodule Pleroma.HTTP.ConnectionTest do + use ExUnit.Case, async: true + import ExUnit.CaptureLog + alias Pleroma.HTTP.Connection + + describe "parse_host/1" do + test "as atom" do + assert Connection.parse_host(:localhost) == 'localhost' + end + + test "as string" do + assert Connection.parse_host("localhost.com") == 'localhost.com' + end + + test "as string ip" do + assert Connection.parse_host("127.0.0.1") == {127, 0, 0, 1} + end + end + + describe "parse_proxy/1" do + test "ip with port" do + assert Connection.parse_proxy("127.0.0.1:8123") == {:ok, {127, 0, 0, 1}, 8123} + end + + test "host with port" do + assert Connection.parse_proxy("localhost:8123") == {:ok, 'localhost', 8123} + end + + test "as tuple" do + assert Connection.parse_proxy({:socks5, :localhost, 9050}) == {:ok, 'localhost', 9050} + end + + test "as tuple with string host" do + assert Connection.parse_proxy({:socks5, "localhost", 9050}) == {:ok, 'localhost', 9050} + end + + test "ip without port" do + capture_log(fn -> + assert Connection.parse_proxy("127.0.0.1") == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail \"127.0.0.1\"" + end + + test "host without port" do + capture_log(fn -> + assert Connection.parse_proxy("localhost") == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail \"localhost\"" + end + + test "host with bad port" do + capture_log(fn -> + assert Connection.parse_proxy("localhost:port") == {:error, :error_parsing_port_in_proxy} + end) =~ "parsing port in proxy fail \"localhost:port\"" + end + + test "as tuple without port" do + capture_log(fn -> + assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :error_parsing_proxy} + end) =~ "parsing proxy fail {:socks5, :localhost}" + end + + test "with nil" do + assert Connection.parse_proxy(nil) == nil + end + end +end From 65fd5bdf4cd1f70edd8669f0c90fcc6d9b4385ff Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 23 Aug 2019 17:12:43 +0300 Subject: [PATCH 097/102] added tls_opts to connect method --- lib/pleroma/gun/connections.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 6cec4277a..a0fa8e6eb 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -161,7 +161,9 @@ defmodule Pleroma.Gun.Connections do result = if opts[:proxy] do with {proxy_host, proxy_port} <- opts[:proxy], - {:ok, conn} <- API.open(proxy_host, proxy_port, opts), + tls_opts <- Map.get(opts, :tls_opts, []), + open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), {:ok, _} <- API.await_up(conn) do connect_opts = %{host: host, port: port} @@ -169,6 +171,7 @@ defmodule Pleroma.Gun.Connections do if uri.scheme == "https" do Map.put(connect_opts, :protocols, [:http2]) |> Map.put(:transport, :tls) + |> Map.put(:tls_opts, tls_opts) else connect_opts end From 431cf60510e05a8483847fe2c2025b79a3602a08 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 23 Aug 2019 18:17:13 +0300 Subject: [PATCH 098/102] little refactoring --- lib/pleroma/gun/connections.ex | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index a0fa8e6eb..95398dd6c 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -160,26 +160,25 @@ defmodule Pleroma.Gun.Connections do result = if opts[:proxy] do + tls_opts = Map.get(opts, :tls_opts, []) + connect_opts = %{host: host, port: port} + + connect_opts = + if uri.scheme == "https" do + Map.put(connect_opts, :protocols, [:http2]) + |> Map.put(:transport, :tls) + |> Map.put(:tls_opts, tls_opts) + else + connect_opts + end + with {proxy_host, proxy_port} <- opts[:proxy], - tls_opts <- Map.get(opts, :tls_opts, []), open_opts <- Map.delete(opts, :tls_opts), {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn) do - connect_opts = %{host: host, port: port} - - connect_opts = - if uri.scheme == "https" do - Map.put(connect_opts, :protocols, [:http2]) - |> Map.put(:transport, :tls) - |> Map.put(:tls_opts, tls_opts) - else - connect_opts - end - - with stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream) do - {:ok, conn, true} - end + {:ok, _} <- API.await_up(conn), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream) do + {:ok, conn, true} else {:error, error} -> {:error, error} From 487e05a9c9074a2ebf842a42426c82cadc8d77f2 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 23 Aug 2019 19:32:14 +0300 Subject: [PATCH 099/102] bump tesla version --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 2e7d4fc6c..a7ed3291e 100644 --- a/mix.exs +++ b/mix.exs @@ -114,7 +114,7 @@ defmodule Pleroma.Mixfile do { :tesla, github: "alex-strizhakov/tesla", - ref: "893a5d0d38c1ad0c08339586a3235bf6149ff574", + ref: "199e77f6e4390495eef7c31f2d830da855571b64", override: true }, {:cowlib, "~> 2.7.3", override: true}, diff --git a/mix.lock b/mix.lock index 1d83e6832..3a6ef325c 100644 --- a/mix.lock +++ b/mix.lock @@ -85,7 +85,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "893a5d0d38c1ad0c08339586a3235bf6149ff574", [ref: "893a5d0d38c1ad0c08339586a3235bf6149ff574"]}, + "tesla": {:git, "https://github.com/alex-strizhakov/tesla.git", "199e77f6e4390495eef7c31f2d830da855571b64", [ref: "199e77f6e4390495eef7c31f2d830da855571b64"]}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From 26a98737a839845d4d1f3c257eddd971e37134b8 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 23 Aug 2019 20:11:02 +0300 Subject: [PATCH 100/102] some refactoring --- lib/pleroma/gun/connections.ex | 108 +++++++++++++++------------------ 1 file changed, 48 insertions(+), 60 deletions(-) diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 95398dd6c..5b1c75d78 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Gun.Connections do defstruct conns: %{}, opts: [] alias Pleroma.Gun.API + alias Pleroma.Gun.Conn @spec start_link({atom(), keyword()}) :: {:ok, pid()} | :ignore def start_link({name, opts}) do @@ -154,71 +155,58 @@ defmodule Pleroma.Gun.Connections do end) end + defp open_conn(key, uri, _from, state, %{proxy: {proxy_host, proxy_port}} = opts) do + host = to_charlist(uri.host) + port = uri.port + + tls_opts = Map.get(opts, :tls_opts, []) + connect_opts = %{host: host, port: port} + + connect_opts = + if uri.scheme == "https" do + Map.put(connect_opts, :protocols, [:http2]) + |> Map.put(:transport, :tls) + |> Map.put(:tls_opts, tls_opts) + else + connect_opts + end + + with open_opts <- Map.delete(opts, :tls_opts), + {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), + {:ok, _} <- API.await_up(conn), + stream <- API.connect(conn, connect_opts), + {:response, :fin, 200, _} <- API.await(conn, stream) do + state = + put_in(state.conns[key], %Conn{ + conn: conn, + waiting_pids: [], + used: 1, + state: :up + }) + + {:reply, conn, state} + else + error -> + Logger.warn(inspect(error)) + {:reply, nil, state} + end + end + defp open_conn(key, uri, from, state, opts) do host = to_charlist(uri.host) port = uri.port - result = - if opts[:proxy] do - tls_opts = Map.get(opts, :tls_opts, []) - connect_opts = %{host: host, port: port} + with {:ok, conn} <- API.open(host, port, opts) do + state = + put_in(state.conns[key], %Conn{ + conn: conn, + waiting_pids: [from] + }) - connect_opts = - if uri.scheme == "https" do - Map.put(connect_opts, :protocols, [:http2]) - |> Map.put(:transport, :tls) - |> Map.put(:tls_opts, tls_opts) - else - connect_opts - end - - with {proxy_host, proxy_port} <- opts[:proxy], - open_opts <- Map.delete(opts, :tls_opts), - {:ok, conn} <- API.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- API.await_up(conn), - stream <- API.connect(conn, connect_opts), - {:response, :fin, 200, _} <- API.await(conn, stream) do - {:ok, conn, true} - else - {:error, error} -> - {:error, error} - - error -> - Logger.warn(inspect(error)) - {:error, :error_connection_to_proxy} - end - else - with {:ok, conn} <- API.open(host, port, opts) do - {:ok, conn, false} - else - {:error, error} -> - {:error, error} - - error -> - Logger.warn(inspect(error)) - {:error, :error_connection} - end - end - - case result do - {:ok, conn, is_up} -> - {from_list, used, conn_state} = if is_up, do: {[], 1, :up}, else: {[from], 0, :open} - - state = - put_in(state.conns[key], %Pleroma.Gun.Conn{ - conn: conn, - waiting_pids: from_list, - used: used, - state: conn_state - }) - - if is_up do - {:reply, conn, state} - else - {:noreply, state} - end - - {:error, _error} -> + {:noreply, state} + else + error -> + Logger.warn(inspect(error)) {:reply, nil, state} end end From a7aa39cfe4436dbe211d340891f9aaac6201edaf Mon Sep 17 00:00:00 2001 From: Alex S Date: Sat, 24 Aug 2019 17:44:34 +0300 Subject: [PATCH 101/102] calculated crf for closing connections don't close conn where are waiting pids --- lib/pleroma/gun/conn.ex | 9 +- lib/pleroma/gun/connections.ex | 39 +++++++-- test/gun/connections_test.exs | 149 +++++++++++++++++++++++++-------- 3 files changed, 154 insertions(+), 43 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 20ddec64c..9e5c2b184 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -10,8 +10,13 @@ defmodule Pleroma.Gun.Conn do conn: pid(), state: atom(), waiting_pids: [pid()], - used: pos_integer() + last_reference: pos_integer(), + crf: float() } - defstruct conn: nil, state: :open, waiting_pids: [], used: 0 + defstruct conn: nil, + state: :open, + waiting_pids: [], + last_reference: :os.system_time(:second), + crf: 1 end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 5b1c75d78..73d54e94d 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -73,8 +73,20 @@ defmodule Pleroma.Gun.Connections do key = compose_key(uri) case state.conns[key] do - %{conn: conn, state: conn_state, used: used} when conn_state == :up -> - state = put_in(state.conns[key].used, used + 1) + %{conn: conn, state: conn_state, last_reference: reference, crf: last_crf} = current_conn + when conn_state == :up -> + time = current_time() + last_reference = time - reference + + current_crf = crf(last_reference, 100, last_crf) + + state = + put_in(state.conns[key], %{ + current_conn + | last_reference: time, + crf: current_crf + }) + {:reply, conn, state} %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> @@ -87,7 +99,12 @@ defmodule Pleroma.Gun.Connections do if Enum.count(state.conns) < max_connections do open_conn(key, uri, from, state, opts) else - [{close_key, least_used} | _conns] = Enum.sort_by(state.conns, fn {_k, v} -> v.used end) + [{close_key, least_used} | _conns] = + state.conns + |> Enum.filter(fn {_k, v} -> v.waiting_pids == [] end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf < y.crf and x.last_reference < y.last_reference + end) :ok = API.close(least_used.conn) @@ -114,12 +131,17 @@ defmodule Pleroma.Gun.Connections do Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) # Update state of the current connection and set waiting_pids to empty list + time = current_time() + last_reference = time - conn.last_reference + current_crf = crf(last_reference, 100, conn.crf) + state = put_in(state.conns[key], %{ conn | state: :up, waiting_pids: [], - used: conn.used + length(conn.waiting_pids) + last_reference: time, + crf: current_crf }) {:noreply, state} @@ -180,7 +202,6 @@ defmodule Pleroma.Gun.Connections do put_in(state.conns[key], %Conn{ conn: conn, waiting_pids: [], - used: 1, state: :up }) @@ -210,4 +231,12 @@ defmodule Pleroma.Gun.Connections do {:reply, nil, state} end end + + defp current_time do + :os.system_time(:second) + end + + def crf(current, steps, crf) do + 1 + :math.pow(0.5, current / steps) * crf + end end diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index 4d84821a0..cc3675ad9 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -48,8 +48,7 @@ defmodule Gun.ConnectionsTest do "http:some-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -112,8 +111,7 @@ defmodule Gun.ConnectionsTest do "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :down, - waiting_pids: _, - used: 0 + waiting_pids: _ } } } = Connections.get_state(name) @@ -128,8 +126,7 @@ defmodule Gun.ConnectionsTest do "http:gun_down_and_up.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -157,8 +154,7 @@ defmodule Gun.ConnectionsTest do "http:some-domain.com:80" => %Conn{ conn: conn, state: :up, - waiting_pids: [], - used: 5 + waiting_pids: [] } } } = Connections.get_state(name) @@ -178,14 +174,12 @@ defmodule Gun.ConnectionsTest do "http:some-domain.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] }, "https:some-domain.com:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -198,14 +192,12 @@ defmodule Gun.ConnectionsTest do "http:another-domain.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] }, "http:some-domain.com:80" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -233,8 +225,7 @@ defmodule Gun.ConnectionsTest do "http:httpbin.org:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -258,8 +249,7 @@ defmodule Gun.ConnectionsTest do "https:httpbin.org:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 2 + waiting_pids: [] } } } = Connections.get_state(name) @@ -281,14 +271,12 @@ defmodule Gun.ConnectionsTest do "https:httpbin.org:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] }, "https:www.google.com:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -301,14 +289,60 @@ defmodule Gun.ConnectionsTest do "http:httpbin.org:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] }, "https:httpbin.org:443" => %Conn{ conn: _, state: :up, - waiting_pids: [], - used: 4 + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end + + test "remove earlier used", %{name: name, pid: pid} do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + + Process.sleep(1_000) + Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] + }, + "https:www.google.com:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + Process.sleep(1_000) + conn = Connections.get_conn("http://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + state: :up, + waiting_pids: [] + }, + "https:httpbin.org:443" => %Conn{ + conn: _, + state: :up, + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -330,8 +364,7 @@ defmodule Gun.ConnectionsTest do "http:proxy_string.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -360,8 +393,7 @@ defmodule Gun.ConnectionsTest do "http:proxy_tuple_atom.com:80" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -390,8 +422,7 @@ defmodule Gun.ConnectionsTest do "https:proxy_string.com:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -420,8 +451,7 @@ defmodule Gun.ConnectionsTest do "https:proxy_tuple_atom.com:443" => %Conn{ conn: ^conn, state: :up, - waiting_pids: [], - used: 1 + waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] @@ -437,4 +467,51 @@ defmodule Gun.ConnectionsTest do assert reused_conn == conn end end + + describe "crf/3" do + setup do + crf = Connections.crf(1, 10, 1) + {:ok, crf: crf} + end + + test "more used will have crf higher", %{crf: crf} do + # used 3 times + crf1 = Connections.crf(1, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 > crf2 + end + + test "recently used will have crf higher on equal references", %{crf: crf} do + # used 4 sec ago + crf1 = Connections.crf(3, 10, crf) + + # used 3 sec ago + crf2 = Connections.crf(4, 10, crf) + + assert crf1 > crf2 + end + + test "equal crf on equal reference and time", %{crf: crf} do + # used 2 times + crf1 = Connections.crf(1, 10, crf) + + # used 2 times + crf2 = Connections.crf(1, 10, crf) + + assert crf1 == crf2 + end + + test "recently used will have higher crf", %{crf: crf} do + crf1 = Connections.crf(2, 10, crf) + crf1 = Connections.crf(1, 10, crf1) + + crf2 = Connections.crf(3, 10, crf) + crf2 = Connections.crf(4, 10, crf2) + assert crf1 > crf2 + end + end end From c19d4eeaeebec3d938324fe24b67182fc5d2be4e Mon Sep 17 00:00:00 2001 From: Alex S Date: Sun, 25 Aug 2019 15:05:34 +0300 Subject: [PATCH 102/102] no drop active connections --- lib/pleroma/gun/conn.ex | 11 +- lib/pleroma/gun/connections.ex | 132 ++++++++++---- lib/pleroma/http/connection.ex | 5 +- lib/pleroma/http/http.ex | 30 +++- test/gun/connections_test.exs | 304 +++++++++++++++++++++++++-------- test/http_test.exs | 11 +- 6 files changed, 374 insertions(+), 119 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 9e5c2b184..906607b28 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -6,17 +6,24 @@ defmodule Pleroma.Gun.Conn do @moduledoc """ Struct for gun connection data """ + @type gun_state :: :open | :up | :down + @type conn_state :: :init | :active | :idle + @type t :: %__MODULE__{ conn: pid(), - state: atom(), + gun_state: gun_state(), waiting_pids: [pid()], + conn_state: conn_state(), + used_by: [pid()], last_reference: pos_integer(), crf: float() } defstruct conn: nil, - state: :open, + gun_state: :open, waiting_pids: [], + conn_state: :init, + used_by: [], last_reference: :os.system_time(:second), crf: 1 end diff --git a/lib/pleroma/gun/connections.ex b/lib/pleroma/gun/connections.ex index 73d54e94d..e3d392de7 100644 --- a/lib/pleroma/gun/connections.ex +++ b/lib/pleroma/gun/connections.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Gun.Connections do opts: keyword() } - defstruct conns: %{}, opts: [] + defstruct conns: %{}, opts: [], queue: [] alias Pleroma.Gun.API alias Pleroma.Gun.Conn @@ -27,8 +27,8 @@ defmodule Pleroma.Gun.Connections do @impl true def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} - @spec get_conn(String.t(), keyword(), atom()) :: pid() - def get_conn(url, opts \\ [], name \\ :default) do + @spec checkin(String.t(), keyword(), atom()) :: pid() + def checkin(url, opts \\ [], name \\ :default) do opts = Enum.into(opts, %{}) uri = URI.parse(url) @@ -53,7 +53,7 @@ defmodule Pleroma.Gun.Connections do GenServer.call( name, - {:conn, %{opts: opts, uri: uri}} + {:checkin, %{opts: opts, uri: uri}} ) end @@ -68,28 +68,57 @@ defmodule Pleroma.Gun.Connections do GenServer.call(name, {:state}) end + def checkout(conn, pid, name \\ :default) do + GenServer.cast(name, {:checkout, conn, pid}) + end + + def process_queue(name \\ :default) do + GenServer.cast(name, {:process_queue}) + end + @impl true - def handle_call({:conn, %{opts: opts, uri: uri}}, from, state) do + def handle_cast({:checkout, conn_pid, pid}, state) do + {key, conn} = find_conn(state.conns, conn_pid) + used_by = List.keydelete(conn.used_by, pid, 0) + conn_state = if used_by == [], do: :idle, else: conn.conn_state + state = put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) + {:noreply, state} + end + + @impl true + def handle_cast({:process_queue}, state) do + case state.queue do + [{from, key, uri, opts} | _queue] -> + try_to_checkin(key, uri, from, state, Map.put(opts, :from_cast, true)) + + [] -> + {:noreply, state} + end + end + + @impl true + def handle_call({:checkin, %{opts: opts, uri: uri}}, from, state) do key = compose_key(uri) case state.conns[key] do - %{conn: conn, state: conn_state, last_reference: reference, crf: last_crf} = current_conn - when conn_state == :up -> + %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> time = current_time() - last_reference = time - reference + last_reference = time - current_conn.last_reference - current_crf = crf(last_reference, 100, last_crf) + current_crf = crf(last_reference, 100, current_conn.crf) state = put_in(state.conns[key], %{ current_conn | last_reference: time, - crf: current_crf + crf: current_crf, + conn_state: :active, + used_by: [from | current_conn.used_by] }) {:reply, conn, state} - %{state: conn_state, waiting_pids: pids} when conn_state in [:open, :down] -> + %{gun_state: gun_state, waiting_pids: pids} when gun_state in [:open, :down] -> state = put_in(state.conns[key].waiting_pids, [from | pids]) {:noreply, state} @@ -99,22 +128,7 @@ defmodule Pleroma.Gun.Connections do if Enum.count(state.conns) < max_connections do open_conn(key, uri, from, state, opts) else - [{close_key, least_used} | _conns] = - state.conns - |> Enum.filter(fn {_k, v} -> v.waiting_pids == [] end) - |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> - x.crf < y.crf and x.last_reference < y.last_reference - end) - - :ok = API.close(least_used.conn) - - state = - put_in( - state.conns, - Map.delete(state.conns, close_key) - ) - - open_conn(key, uri, from, state, opts) + try_to_checkin(key, uri, from, state, opts) end end end @@ -122,14 +136,44 @@ defmodule Pleroma.Gun.Connections do @impl true def handle_call({:state}, _from, state), do: {:reply, state, state} + defp try_to_checkin(key, uri, from, state, opts) do + unused_conns = + state.conns + |> Enum.filter(fn {_k, v} -> + v.conn_state == :idle and v.waiting_pids == [] and v.used_by == [] + end) + |> Enum.sort(fn {_x_k, x}, {_y_k, y} -> + x.crf < y.crf and x.last_reference < y.last_reference + end) + + case unused_conns do + [{close_key, least_used} | _conns] -> + :ok = API.close(least_used.conn) + + state = + put_in( + state.conns, + Map.delete(state.conns, close_key) + ) + + open_conn(key, uri, from, state, opts) + + [] -> + queue = + if List.keymember?(state.queue, from, 0), + do: state.queue, + else: state.queue ++ [{from, key, uri, opts}] + + state = put_in(state.queue, queue) + {:noreply, state} + end + end + @impl true def handle_info({:gun_up, conn_pid, _protocol}, state) do conn_key = compose_key_gun_info(conn_pid) {key, conn} = find_conn(state.conns, conn_pid, conn_key) - # Send to all waiting processes connection pid - Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) - # Update state of the current connection and set waiting_pids to empty list time = current_time() last_reference = time - conn.last_reference @@ -138,12 +182,17 @@ defmodule Pleroma.Gun.Connections do state = put_in(state.conns[key], %{ conn - | state: :up, + | gun_state: :up, waiting_pids: [], last_reference: time, - crf: current_crf + crf: current_crf, + conn_state: :active, + used_by: conn.waiting_pids ++ conn.used_by }) + # Send to all waiting processes connection pid + Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, conn_pid) end) + {:noreply, state} end @@ -154,7 +203,7 @@ defmodule Pleroma.Gun.Connections do Enum.each(conn.waiting_pids, fn waiting_pid -> GenServer.reply(waiting_pid, nil) end) - state = put_in(state.conns[key].state, :down) + state = put_in(state.conns[key].gun_state, :down) {:noreply, state} end @@ -177,7 +226,7 @@ defmodule Pleroma.Gun.Connections do end) end - defp open_conn(key, uri, _from, state, %{proxy: {proxy_host, proxy_port}} = opts) do + defp open_conn(key, uri, from, state, %{proxy: {proxy_host, proxy_port}} = opts) do host = to_charlist(uri.host) port = uri.port @@ -202,9 +251,15 @@ defmodule Pleroma.Gun.Connections do put_in(state.conns[key], %Conn{ conn: conn, waiting_pids: [], - state: :up + gun_state: :up, + conn_state: :active, + used_by: [from] }) + if opts[:from_cast] do + GenServer.reply(from, conn) + end + {:reply, conn, state} else error -> @@ -218,6 +273,13 @@ defmodule Pleroma.Gun.Connections do port = uri.port with {:ok, conn} <- API.open(host, port, opts) do + state = + if opts[:from_cast] do + put_in(state.queue, List.keydelete(state.queue, from, 0)) + else + state + end + state = put_in(state.conns[key], %Conn{ conn: conn, diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 39c0fff43..d4e6d0f99 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -10,8 +10,7 @@ defmodule Pleroma.HTTP.Connection do @options [ connect_timeout: 10_000, timeout: 20_000, - pool: :federation, - version: :master + pool: :federation ] require Logger @@ -61,7 +60,7 @@ defmodule Pleroma.HTTP.Connection do end defp get_conn_for_gun(url, options, pool) do - case Pleroma.Gun.Connections.get_conn(url, options, pool) do + case Pleroma.Gun.Connections.checkin(url, options, pool) do nil -> options diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 5c0d66955..0a7db737f 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -45,15 +45,27 @@ defmodule Pleroma.HTTP do params = Keyword.get(options, :params, []) - %{} - |> Builder.method(method) - |> Builder.url(url) - |> Builder.headers(headers) - |> Builder.opts(options) - |> Builder.add_param(:body, :body, body) - |> Builder.add_param(:query, :query, params) - |> Enum.into([]) - |> (&Tesla.request(Connection.new(options), &1)).() + request = + %{} + |> Builder.method(method) + |> Builder.url(url) + |> Builder.headers(headers) + |> Builder.opts(options) + |> Builder.add_param(:body, :body, body) + |> Builder.add_param(:query, :query, params) + |> Enum.into([]) + + client = Connection.new(options) + response = Tesla.request(client, request) + + if adapter_gun? do + %{adapter: {_, _, [adapter_options]}} = client + pool = adapter_options[:pool] + Pleroma.Gun.Connections.checkout(adapter_options[:conn], self(), pool) + Pleroma.Gun.Connections.process_queue(pool) + end + + response rescue e -> {:error, e} diff --git a/test/gun/connections_test.exs b/test/gun/connections_test.exs index cc3675ad9..39f77070a 100644 --- a/test/gun/connections_test.exs +++ b/test/gun/connections_test.exs @@ -34,12 +34,26 @@ defmodule Gun.ConnectionsTest do end test "opens connection and reuse it on next request", %{name: name, pid: pid} do - conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + self = self() + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + reused_conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) assert conn == reused_conn @@ -47,23 +61,53 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:some-domain.com:80" => %Conn{ conn: ^conn, - state: :up, - waiting_pids: [] + gun_state: :up, + waiting_pids: [], + used_by: [{^self, _}, {^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + used_by: [{^self, _}], + conn_state: :active + } + } + } = Connections.get_state(name) + + :ok = Connections.checkout(conn, self, name) + + %Connections{ + conns: %{ + "http:some-domain.com:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + used_by: [], + conn_state: :idle } } } = Connections.get_state(name) end test "reuses connection based on protocol", %{name: name, pid: pid} do - conn = Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + conn = Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) - https_conn = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) + https_conn = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name) refute conn == https_conn - reused_https = Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) + reused_https = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name) refute conn == reused_https @@ -73,12 +117,12 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:some-domain.com:80" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] }, "https:some-domain.com:443" => %Conn{ conn: ^https_conn, - state: :up, + gun_state: :up, waiting_pids: [] } } @@ -86,7 +130,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message", %{name: name, pid: pid} do - conn = Connections.get_conn("http://gun_down.com", [genserver_pid: pid], name) + conn = Connections.checkin("http://gun_down.com", [genserver_pid: pid], name) refute conn @@ -94,7 +138,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:gun_down.com:80" => %Conn{ conn: _, - state: :down, + gun_state: :down, waiting_pids: _ } } @@ -102,7 +146,7 @@ defmodule Gun.ConnectionsTest do end test "process gun_down message and then gun_up", %{name: name, pid: pid} do - conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) + conn = Connections.checkin("http://gun_down_and_up.com", [genserver_pid: pid], name) refute conn @@ -110,13 +154,13 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:gun_down_and_up.com:80" => %Conn{ conn: _, - state: :down, + gun_state: :down, waiting_pids: _ } } } = Connections.get_state(name) - conn = Connections.get_conn("http://gun_down_and_up.com", [genserver_pid: pid], name) + conn = Connections.checkin("http://gun_down_and_up.com", [genserver_pid: pid], name) assert is_pid(conn) assert Process.alive?(conn) @@ -125,7 +169,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:gun_down_and_up.com:80" => %Conn{ conn: _, - state: :up, + gun_state: :up, waiting_pids: [] } } @@ -136,7 +180,7 @@ defmodule Gun.ConnectionsTest do tasks = for _ <- 1..5 do Task.async(fn -> - Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) + Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) end) end @@ -153,7 +197,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:some-domain.com:80" => %Conn{ conn: conn, - state: :up, + gun_state: :up, waiting_pids: [] } } @@ -162,41 +206,49 @@ defmodule Gun.ConnectionsTest do assert Enum.all?(conns, fn res -> res == conn end) end - test "remove frequently used", %{name: name, pid: pid} do - Connections.get_conn("https://some-domain.com", [genserver_pid: pid], name) + test "remove frequently used and idle", %{name: name, pid: pid} do + self = self() + conn1 = Connections.checkin("https://some-domain.com", [genserver_pid: pid], name) - for _ <- 1..4 do - Connections.get_conn("http://some-domain.com", [genserver_pid: pid], name) - end + [conn2 | _conns] = + for _ <- 1..4 do + Connections.checkin("http://some-domain.com", [genserver_pid: pid], name) + end %Connections{ conns: %{ "http:some-domain.com:80" => %Conn{ - conn: _, - state: :up, - waiting_pids: [] + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}, {^self, _}, {^self, _}, {^self, _}] }, "https:some-domain.com:443" => %Conn{ - conn: _, - state: :up, - waiting_pids: [] + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] } }, opts: [max_connections: 2, timeout: 10] } = Connections.get_state(name) - conn = Connections.get_conn("http://another-domain.com", [genserver_pid: pid], name) + :ok = Connections.checkout(conn1, self, name) + + conn = Connections.checkin("http://another-domain.com", [genserver_pid: pid], name) %Connections{ conns: %{ "http:another-domain.com:80" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] }, "http:some-domain.com:80" => %Conn{ conn: _, - state: :up, + gun_state: :up, waiting_pids: [] } }, @@ -211,12 +263,12 @@ defmodule Gun.ConnectionsTest do api = Pleroma.Config.get([API]) Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) - conn = Connections.get_conn("http://httpbin.org", [], name) + conn = Connections.checkin("http://httpbin.org", [], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn("http://httpbin.org", [], name) + reused_conn = Connections.checkin("http://httpbin.org", [], name) assert conn == reused_conn @@ -224,7 +276,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:httpbin.org:80" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] } } @@ -235,12 +287,12 @@ defmodule Gun.ConnectionsTest do api = Pleroma.Config.get([API]) Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) - conn = Connections.get_conn("https://httpbin.org", [], name) + conn = Connections.checkin("https://httpbin.org", [], name) assert is_pid(conn) assert Process.alive?(conn) - reused_conn = Connections.get_conn("https://httpbin.org", [], name) + reused_conn = Connections.checkin("https://httpbin.org", [], name) assert conn == reused_conn @@ -248,52 +300,54 @@ defmodule Gun.ConnectionsTest do conns: %{ "https:httpbin.org:443" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] } } } = Connections.get_state(name) end - test "remove frequently used", %{name: name, pid: pid} do + test "remove frequently used and idle", %{name: name, pid: pid} do + self = self() api = Pleroma.Config.get([API]) Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) - Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + conn = Connections.checkin("https://www.google.com", [genserver_pid: pid], name) for _ <- 1..4 do - Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) end %Connections{ conns: %{ "https:httpbin.org:443" => %Conn{ conn: _, - state: :up, + gun_state: :up, waiting_pids: [] }, "https:www.google.com:443" => %Conn{ conn: _, - state: :up, + gun_state: :up, waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] } = Connections.get_state(name) - conn = Connections.get_conn("http://httpbin.org", [genserver_pid: pid], name) + :ok = Connections.checkout(conn, self, name) + conn = Connections.checkin("http://httpbin.org", [genserver_pid: pid], name) %Connections{ conns: %{ "http:httpbin.org:80" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] }, "https:httpbin.org:443" => %Conn{ conn: _, - state: :up, + gun_state: :up, waiting_pids: [] } }, @@ -301,59 +355,173 @@ defmodule Gun.ConnectionsTest do } = Connections.get_state(name) end - test "remove earlier used", %{name: name, pid: pid} do + test "remove earlier used and idle", %{name: name, pid: pid} do + self = self() api = Pleroma.Config.get([API]) Pleroma.Config.put([API], API.Gun) on_exit(fn -> Pleroma.Config.put([API], api) end) - Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) - Connections.get_conn("https://www.google.com", [genserver_pid: pid], name) + Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + conn = Connections.checkin("https://www.google.com", [genserver_pid: pid], name) Process.sleep(1_000) - Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) - Connections.get_conn("https://httpbin.org", [genserver_pid: pid], name) + Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) + Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) %Connections{ conns: %{ "https:httpbin.org:443" => %Conn{ conn: _, - state: :up, + gun_state: :up, waiting_pids: [] }, "https:www.google.com:443" => %Conn{ - conn: _, - state: :up, + conn: ^conn, + gun_state: :up, waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] } = Connections.get_state(name) + :ok = Connections.checkout(conn, self, name) + :ok = Connections.checkout(conn, self, name) Process.sleep(1_000) - conn = Connections.get_conn("http://httpbin.org", [genserver_pid: pid], name) + conn = Connections.checkin("http://httpbin.org", [genserver_pid: pid], name) %Connections{ conns: %{ "http:httpbin.org:80" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] }, "https:httpbin.org:443" => %Conn{ conn: _, - state: :up, + gun_state: :up, waiting_pids: [] } }, opts: [max_connections: 2, timeout: 10] } = Connections.get_state(name) end + + test "doesn't drop active connections on pool overflow addinng new requests to the queue", %{ + name: name, + pid: pid + } do + api = Pleroma.Config.get([API]) + Pleroma.Config.put([API], API.Gun) + on_exit(fn -> Pleroma.Config.put([API], api) end) + + self = self() + Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + conn1 = Connections.checkin("https://www.google.com", [genserver_pid: pid], name) + conn2 = Connections.checkin("https://httpbin.org", [genserver_pid: pid], name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}, {^self, _}] + } + }, + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + task = + Task.async(fn -> Connections.checkin("http://httpbin.org", [genserver_pid: pid], name) end) + + task_pid = task.pid + + :ok = Connections.checkout(conn1, self, name) + + Process.sleep(1_000) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + } + }, + queue: [{{^task_pid, _}, "http:httpbin.org:80", _, _}], + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + :ok = Connections.checkout(conn1, self, name) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "https:www.google.com:443" => %Conn{ + conn: ^conn1, + gun_state: :up, + waiting_pids: [], + conn_state: :idle, + used_by: [] + } + }, + queue: [{{^task_pid, _}, "http:httpbin.org:80", _, _}], + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + + :ok = Connections.process_queue(name) + conn = Task.await(task) + + %Connections{ + conns: %{ + "https:httpbin.org:443" => %Conn{ + conn: ^conn2, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^self, _}] + }, + "http:httpbin.org:80" => %Conn{ + conn: ^conn, + gun_state: :up, + waiting_pids: [], + conn_state: :active, + used_by: [{^task_pid, _}] + } + }, + queue: [], + opts: [max_connections: 2, timeout: 10] + } = Connections.get_state(name) + end end describe "with proxy usage" do test "proxy as ip", %{name: name, pid: pid} do conn = - Connections.get_conn( + Connections.checkin( "http://proxy_string.com", [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], name @@ -363,7 +531,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:proxy_string.com:80" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] } }, @@ -371,7 +539,7 @@ defmodule Gun.ConnectionsTest do } = Connections.get_state(name) reused_conn = - Connections.get_conn( + Connections.checkin( "http://proxy_string.com", [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], name @@ -382,7 +550,7 @@ defmodule Gun.ConnectionsTest do test "proxy as host", %{name: name, pid: pid} do conn = - Connections.get_conn( + Connections.checkin( "http://proxy_tuple_atom.com", [genserver_pid: pid, proxy: {'localhost', 9050}], name @@ -392,7 +560,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "http:proxy_tuple_atom.com:80" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] } }, @@ -400,7 +568,7 @@ defmodule Gun.ConnectionsTest do } = Connections.get_state(name) reused_conn = - Connections.get_conn( + Connections.checkin( "http://proxy_tuple_atom.com", [genserver_pid: pid, proxy: {'localhost', 9050}], name @@ -411,7 +579,7 @@ defmodule Gun.ConnectionsTest do test "proxy as ip and ssl", %{name: name, pid: pid} do conn = - Connections.get_conn( + Connections.checkin( "https://proxy_string.com", [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], name @@ -421,7 +589,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "https:proxy_string.com:443" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] } }, @@ -429,7 +597,7 @@ defmodule Gun.ConnectionsTest do } = Connections.get_state(name) reused_conn = - Connections.get_conn( + Connections.checkin( "https://proxy_string.com", [genserver_pid: pid, proxy: {{127, 0, 0, 1}, 8123}], name @@ -440,7 +608,7 @@ defmodule Gun.ConnectionsTest do test "proxy as host and ssl", %{name: name, pid: pid} do conn = - Connections.get_conn( + Connections.checkin( "https://proxy_tuple_atom.com", [genserver_pid: pid, proxy: {'localhost', 9050}], name @@ -450,7 +618,7 @@ defmodule Gun.ConnectionsTest do conns: %{ "https:proxy_tuple_atom.com:443" => %Conn{ conn: ^conn, - state: :up, + gun_state: :up, waiting_pids: [] } }, @@ -458,7 +626,7 @@ defmodule Gun.ConnectionsTest do } = Connections.get_state(name) reused_conn = - Connections.get_conn( + Connections.checkin( "https://proxy_tuple_atom.com", [genserver_pid: pid, proxy: {'localhost', 9050}], name diff --git a/test/http_test.exs b/test/http_test.exs index 460f8f7c2..b88e3b605 100644 --- a/test/http_test.exs +++ b/test/http_test.exs @@ -71,11 +71,18 @@ defmodule Pleroma.HTTPTest do options = [adapter: [pool: :federation]] - assert {:ok, resp} = - Pleroma.HTTP.request(:get, "https://httpbin.org/user-agent", "", [], options) + assert {:ok, resp} = Pleroma.HTTP.get("https://httpbin.org/user-agent", [], options) adapter_opts = resp.opts[:adapter] + assert resp.status == 200 + assert adapter_opts[:url] == "https://httpbin.org/user-agent" + state = Pleroma.Gun.Connections.get_state(:federation) + conn = state.conns["https:httpbin.org:443"] + + assert conn.conn_state == :idle + assert conn.used_by == [] + assert state.queue == [] end end