From ac55764599c054a03299470d92fd0d274509a024 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 22 Feb 2024 14:07:46 -0500 Subject: [PATCH 1/5] Gun Connection Pool was not attempting to free a connection and retry once if the pool was full. --- lib/pleroma/gun/connection_pool/worker_supervisor.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index 24ad61117..edaffcc25 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do def start_worker(opts, retry \\ false) do case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do {:error, :max_children} -> - funs = [fn -> !retry end, fn -> match?(:error, free_pool()) end] + funs = [fn -> retry end, fn -> match?(:error, free_pool()) end] if Enum.any?(funs, fn fun -> fun.() end) do :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) From 72fc41d89196dc76e6c2c1c6213e10cbaadd7d3c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 22 Feb 2024 14:11:02 -0500 Subject: [PATCH 2/5] Prevent publisher jobs from erroring if the connection pool is full A full pool is a soft-error. Snooze the job for 30 seconds and try again. --- changelog.d/gun_pool2.fix | 1 + lib/pleroma/web/activity_pub/publisher.ex | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelog.d/gun_pool2.fix diff --git a/changelog.d/gun_pool2.fix b/changelog.d/gun_pool2.fix new file mode 100644 index 000000000..a1f98b49c --- /dev/null +++ b/changelog.d/gun_pool2.fix @@ -0,0 +1 @@ +Connection pool errors when publishing an activity is a soft-error that will be retried shortly. diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index c27612697..9e7d00519 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -129,6 +129,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do _ -> {:error, e} end + {:error, :pool_full} -> + Logger.debug("Publisher snoozing worker job due to full connection pool") + {:snooze, 30} + e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) Logger.metadata(activity: id, inbox: inbox) From 6af6a9704d2d0a04a43967805368eed419b9dc67 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 23 Feb 2024 01:57:43 +0100 Subject: [PATCH 3/5] RemoteFetcherWorker: Make sure {:error, _} is returned on failure Otherwise jobs are considered a success. --- changelog.d/remote-fetcher-error.skip | 0 lib/pleroma/workers/remote_fetcher_worker.ex | 7 +++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/remote-fetcher-error.skip diff --git a/changelog.d/remote-fetcher-error.skip b/changelog.d/remote-fetcher-error.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index d526a99cb..c26418483 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -22,8 +22,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:error, :allowed_depth} -> {:discard, :allowed_depth} - _ -> - :error + {:error, _} = e -> + e + + e -> + {:error, e} end end From f4e48bc53ee5ffeb9e7387219c0d46dc33d04521 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 23 Feb 2024 11:12:10 -0500 Subject: [PATCH 4/5] Rename variable to make the worker retry logic easier to read The boolean value matches the intent of the "last_attempt" variable name now --- changelog.d/gun_pool3.skip | 0 lib/pleroma/gun/connection_pool/worker_supervisor.ex | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/gun_pool3.skip diff --git a/changelog.d/gun_pool3.skip b/changelog.d/gun_pool3.skip new file mode 100644 index 000000000..e69de29bb diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index edaffcc25..eb83962d8 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -18,10 +18,10 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do ) end - def start_worker(opts, retry \\ false) do + def start_worker(opts, last_attempt \\ false) do case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do {:error, :max_children} -> - funs = [fn -> retry end, fn -> match?(:error, free_pool()) end] + funs = [fn -> last_attempt end, fn -> match?(:error, free_pool()) end] if Enum.any?(funs, fn fun -> fun.() end) do :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) From 267e20dbcdbe9fcf0207f1873a624492a7206c21 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 23 Feb 2024 15:36:07 -0500 Subject: [PATCH 5/5] Exile: change to upstream pre-release commit that fixes build on FreeBSD --- changelog.d/exile-freebsd.skip | 0 mix.exs | 4 ++-- mix.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 changelog.d/exile-freebsd.skip diff --git a/changelog.d/exile-freebsd.skip b/changelog.d/exile-freebsd.skip new file mode 100644 index 000000000..e69de29bb diff --git a/mix.exs b/mix.exs index b707a20a8..3edae5046 100644 --- a/mix.exs +++ b/mix.exs @@ -186,8 +186,8 @@ defmodule Pleroma.Mixfile do {:elixir_make, "~> 0.7.7", override: true}, {:blurhash, "~> 0.1.0", hex: :rinpatch_blurhash}, {:exile, - git: "https://git.pleroma.social/pleroma/elixir-libraries/exile.git", - ref: "0d6337cf68e7fbc8a093cae000955aa93b067f91"}, + git: "https://github.com/akash-akya/exile.git", + ref: "be87c33b02a7c3c5d22d2ece01fbd462355b28ef"}, {:bandit, "~> 1.2"}, ## dev & test diff --git a/mix.lock b/mix.lock index a5fccbec4..86545adcf 100644 --- a/mix.lock +++ b/mix.lock @@ -47,7 +47,7 @@ "ex_doc": {:hex, :ex_doc, "0.31.1", "8a2355ac42b1cc7b2379da9e40243f2670143721dd50748bf6c3b1184dae2089", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "3178c3a407c557d8343479e1ff117a96fd31bafe52a039079593fb0524ef61b0"}, "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, - "exile": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/exile.git", "0d6337cf68e7fbc8a093cae000955aa93b067f91", [ref: "0d6337cf68e7fbc8a093cae000955aa93b067f91"]}, + "exile": {:git, "https://github.com/akash-akya/exile.git", "be87c33b02a7c3c5d22d2ece01fbd462355b28ef", [ref: "be87c33b02a7c3c5d22d2ece01fbd462355b28ef"]}, "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"}, "fast_html": {:hex, :fast_html, "2.2.0", "6c5ef1be087a4ed613b0379c13f815c4d11742b36b67bb52cee7859847c84520", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "064c4f23b4a6168f9187dac8984b056f2c531bb0787f559fd6a8b34b38aefbae"}, "fast_sanitize": {:hex, :fast_sanitize, "0.2.3", "67b93dfb34e302bef49fec3aaab74951e0f0602fd9fa99085987af05bd91c7a5", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e8ad286d10d0386e15d67d0ee125245ebcfbc7d7290b08712ba9013c8c5e56e2"},