From 0e9be6bafa81b4a55546b41e8b217fbe559cd16d Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 25 Nov 2019 14:32:20 +0300 Subject: [PATCH 01/17] moved OAuth.Token.CleanWorker to Oban Periodic jobs --- config/config.exs | 6 ++- config/description.exs | 7 ---- docs/configuration/cheatsheet.md | 13 ++++++- lib/pleroma/application.ex | 9 ----- lib/pleroma/web/oauth/token/clean_worker.ex | 38 ------------------- lib/pleroma/workers/background_worker.ex | 5 --- .../workers/cron/clear_oauth_token_worker.ex | 21 ++++++++++ mix.exs | 2 +- mix.lock | 8 ++-- 9 files changed, 41 insertions(+), 68 deletions(-) delete mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex create mode 100644 lib/pleroma/workers/cron/clear_oauth_token_worker.ex diff --git a/config/config.exs b/config/config.exs index 1e36d3314..79d4ee55f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -505,6 +505,9 @@ config :pleroma, Oban, transmogrifier: 20, scheduled_activities: 10, background: 5 + ], + crontab: [ + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} ] config :pleroma, :workers, @@ -594,8 +597,7 @@ config :pleroma, :email_notifications, config :pleroma, :oauth2, token_expires_in: 600, issue_new_refresh_token: true, - clean_expired_tokens: false, - clean_expired_tokens_interval: 86_400_000 + clean_expired_tokens: false config :pleroma, :database, rum_enabled: false diff --git a/config/description.exs b/config/description.exs index 70e963399..6b89d7c67 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2251,13 +2251,6 @@ config :pleroma, :config_description, [ key: :clean_expired_tokens, type: :boolean, description: "Enable a background job to clean expired oauth tokens. Defaults to false" - }, - %{ - key: :clean_expired_tokens_interval, - type: :integer, - description: - "Interval to run the job to clean expired tokens. Defaults to 86_400_000 (24 hours).", - suggestions: [86_400_000] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 07d9a1d45..e414d0b11 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -483,6 +483,7 @@ Configuration options described in [Oban readme](https://github.com/sorentwo/oba * `verbose` - logs verbosity * `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`) * `queues` - job queues (see below) +* `crontab` - periodic jobs, see [`Oban.Cron`](#obancron) Pleroma has the following queues: @@ -494,6 +495,12 @@ Pleroma has the following queues: * `web_push` - Web push notifications * `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity) +#### Oban.Cron + +Pleroma has the periodic jobs: + +`Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. + Example: ```elixir @@ -504,6 +511,9 @@ config :pleroma, Oban, queues: [ federator_incoming: 50, federator_outgoing: 50 + ], + crontab: [ + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} ] ``` @@ -786,8 +796,7 @@ Configure OAuth 2 provider capabilities: * `token_expires_in` - The lifetime in seconds of the access token. * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. -* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. -* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. Interval settings sets in configuration periodic jobs [`Oban.Cron`](#obancron) ## Link parsing diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2b6a55f98..69a1e4a50 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -47,7 +47,6 @@ defmodule Pleroma.Application do {Oban, Pleroma.Config.get(Oban)} ] ++ task_children(@env) ++ - oauth_cleanup_child(oauth_cleanup_enabled?()) ++ streamer_child(@env) ++ chat_child(@env, chat_enabled?()) ++ [ @@ -127,20 +126,12 @@ defmodule Pleroma.Application do defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) - defp oauth_cleanup_enabled?, - do: Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) - defp streamer_child(:test), do: [] defp streamer_child(_) do [Pleroma.Web.Streamer.supervisor()] end - defp oauth_cleanup_child(true), - do: [Pleroma.Web.OAuth.Token.CleanWorker] - - defp oauth_cleanup_child(_), do: [] - defp chat_child(:test, _), do: [] defp chat_child(_env, true) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex deleted file mode 100644 index f639f9c6f..000000000 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ /dev/null @@ -1,38 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OAuth.Token.CleanWorker do - @moduledoc """ - The module represents functions to clean an expired oauth tokens. - """ - use GenServer - - @ten_seconds 10_000 - @one_day 86_400_000 - - @interval Pleroma.Config.get( - [:oauth2, :clean_expired_tokens_interval], - @one_day - ) - - alias Pleroma.Web.OAuth.Token - alias Pleroma.Workers.BackgroundWorker - - def start_link(_), do: GenServer.start_link(__MODULE__, %{}) - - def init(_) do - Process.send_after(self(), :perform, @ten_seconds) - {:ok, nil} - end - - @doc false - def handle_info(:perform, state) do - BackgroundWorker.enqueue("clean_expired_tokens", %{}) - - Process.send_after(self(), :perform, @interval) - {:noreply, state} - end - - def perform(:clean), do: Token.delete_expired_tokens() -end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 7ffc8eabe..bb2b37b18 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.Activity alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy - alias Pleroma.Web.OAuth.Token.CleanWorker use Pleroma.Workers.WorkerHelper, queue: "background" @@ -55,10 +54,6 @@ defmodule Pleroma.Workers.BackgroundWorker do User.perform(:follow_import, follower, followed_identifiers) end - def perform(%{"op" => "clean_expired_tokens"}, _job) do - CleanWorker.perform(:clean) - end - def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do MediaProxyWarmingPolicy.perform(:preload, message) end diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex new file mode 100644 index 000000000..1a4cdc198 --- /dev/null +++ b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do + @moduledoc """ + The worker to clean an expired oauth tokens. + """ + + use Oban.Worker, queue: "background" + + alias Pleroma.Config + alias Pleroma.Web.OAuth.Token + + @impl Oban.Worker + def perform(_opts, _job) do + if Config.get([:oauth2, :clean_expired_tokens], false) do + Token.delete_expired_tokens() + end + end +end diff --git a/mix.exs b/mix.exs index 60c7fe3f6..31deae96e 100644 --- a/mix.exs +++ b/mix.exs @@ -102,7 +102,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.8.1"}, + {:oban, "~> 0.11.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 3a664287c..c212b39e0 100644 --- a/mix.lock +++ b/mix.lock @@ -23,8 +23,8 @@ "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.2.3", "51274df79862845b388733fddcf6f107d0c8c86e27abe7131fa98f8d30761bda", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.5", "76c864b77948a479e18e69cc1d0f0f4ee7cced1148ffe6a093ff91eba644f0b5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.2.1", "4eed4100cbb2abcff10c46660d6613693807bf64f1b865f414daccf762d3758d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -67,7 +67,7 @@ "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.11.1", "e34964fad7f188c2c3d006485601a8897e537f7b88a31928be2833ae1cab59af", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -97,7 +97,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, From ac3abb5414bd7a5bbf53678cdf02b6f59063124c Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 26 Nov 2019 10:53:07 +0300 Subject: [PATCH 02/17] moved Pleroma.Stats to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/stats.ex | 39 ++++++++++++++++++------ lib/pleroma/workers/cron/stats_worker.ex | 16 ++++++++++ test/web/node_info_test.exs | 1 + 4 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/workers/cron/stats_worker.ex diff --git a/config/config.exs b/config/config.exs index 79d4ee55f..5fc92ca1b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -507,7 +507,8 @@ config :pleroma, Oban, background: 5 ], crontab: [ - {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, + {"0 * * * *", Pleroma.Workers.Cron.StatsWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 8154a09b7..cf590fb01 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -9,22 +9,43 @@ defmodule Pleroma.Stats do use GenServer - @interval 1000 * 60 * 60 + @init_state %{ + peers: [], + stats: %{ + domain_count: 0, + status_count: 0, + user_count: 0 + } + } def start_link(_) do - GenServer.start_link(__MODULE__, initial_data(), name: __MODULE__) + GenServer.start_link( + __MODULE__, + @init_state, + name: __MODULE__ + ) end + @doc "Performs update stats" def force_update do GenServer.call(__MODULE__, :force_update) end + @doc "Performs collect stats" + def do_collect do + GenServer.cast(__MODULE__, :run_update) + end + + @doc "Returns stats data" + @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()} def get_stats do %{stats: stats} = GenServer.call(__MODULE__, :get_state) stats end + @doc "Returns list peers" + @spec get_peers() :: list(String.t()) def get_peers do %{peers: peers} = GenServer.call(__MODULE__, :get_state) @@ -32,7 +53,6 @@ defmodule Pleroma.Stats do end def init(args) do - Process.send(self(), :run_update, []) {:ok, args} end @@ -45,17 +65,12 @@ defmodule Pleroma.Stats do {:reply, state, state} end - def handle_info(:run_update, _state) do + def handle_cast(:run_update, _state) do new_stats = get_stat_data() - Process.send_after(self(), :run_update, @interval) {:noreply, new_stats} end - defp initial_data do - %{peers: [], stats: %{}} - end - defp get_stat_data do peers = from( @@ -74,7 +89,11 @@ defmodule Pleroma.Stats do %{ peers: peers, - stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} + stats: %{ + domain_count: domain_count, + status_count: status_count, + user_count: user_count + } } end end diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex new file mode 100644 index 000000000..425ad41ca --- /dev/null +++ b/lib/pleroma/workers/cron/stats_worker.ex @@ -0,0 +1,16 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.StatsWorker do + @moduledoc """ + The worker to update peers statistics. + """ + + use Oban.Worker, queue: "background" + + @impl Oban.Worker + def perform(_opts, _job) do + Pleroma.Stats.do_collect() + end +end diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index 9a574a38d..39dd72cec 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.NodeInfoTest do use Pleroma.Web.ConnCase import Pleroma.Factory + clear_config([:mrf_simple]) test "GET /.well-known/nodeinfo", %{conn: conn} do links = From 6f202a401babbc73e93fb09e697c4c844688f557 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 09:26:37 +0300 Subject: [PATCH 03/17] moved ScheduledActivity to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/application.ex | 1 - .../cron/scheduled_activity_worker.ex} | 41 +++++-------------- .../workers/scheduled_activity_worker.ex | 12 ------ .../scheduled_activity_daemon_test.exs | 19 --------- test/scheduled_activity_test.exs | 28 +++++++++++++ 6 files changed, 40 insertions(+), 64 deletions(-) rename lib/pleroma/{daemons/scheduled_activity_daemon.ex => workers/cron/scheduled_activity_worker.ex} (51%) delete mode 100644 lib/pleroma/workers/scheduled_activity_worker.ex delete mode 100644 test/daemons/scheduled_activity_daemon_test.exs diff --git a/config/config.exs b/config/config.exs index 5fc92ca1b..f4f7fcce7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -508,7 +508,8 @@ config :pleroma, Oban, ], crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, - {"0 * * * *", Pleroma.Workers.Cron.StatsWorker} + {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, + {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 69a1e4a50..71670d9fc 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ defmodule Pleroma.Application do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.Daemons.ScheduledActivityDaemon, Pleroma.Daemons.ActivityExpirationDaemon, Pleroma.Plugs.RateLimiter.Supervisor ] ++ diff --git a/lib/pleroma/daemons/scheduled_activity_daemon.ex b/lib/pleroma/workers/cron/scheduled_activity_worker.ex similarity index 51% rename from lib/pleroma/daemons/scheduled_activity_daemon.ex rename to lib/pleroma/workers/cron/scheduled_activity_worker.ex index aee5f723a..407ab687a 100644 --- a/lib/pleroma/daemons/scheduled_activity_daemon.ex +++ b/lib/pleroma/workers/cron/scheduled_activity_worker.ex @@ -2,37 +2,33 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Daemons.ScheduledActivityDaemon do +defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do @moduledoc """ - Sends scheduled activities to the job queue. + The worker to post scheduled actvities. """ + use Oban.Worker, queue: "scheduled_activities" alias Pleroma.Config alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI - use GenServer require Logger @schedule_interval :timer.minutes(1) - def start_link(_) do - GenServer.start_link(__MODULE__, nil) - end - - def init(_) do + @impl Oban.Worker + def perform(_opts, _job) do if Config.get([ScheduledActivity, :enabled]) do - schedule_next() - {:ok, nil} - else - :ignore + @schedule_interval + |> ScheduledActivity.due_activities() + |> Enum.each(&post_activity/1) end end - def perform(:execute, scheduled_activity_id) do + def post_activity(scheduled_activity) do try do - {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity_id) + {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) {:ok, _result} = CommonAPI.post(user, scheduled_activity.params) rescue @@ -42,21 +38,4 @@ defmodule Pleroma.Daemons.ScheduledActivityDaemon do ) end end - - def handle_info(:perform, state) do - ScheduledActivity.due_activities(@schedule_interval) - |> Enum.each(fn scheduled_activity -> - Pleroma.Workers.ScheduledActivityWorker.enqueue( - "execute", - %{"activity_id" => scheduled_activity.id} - ) - end) - - schedule_next() - {:noreply, state} - end - - defp schedule_next do - Process.send_after(self(), :perform, @schedule_interval) - end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex deleted file mode 100644 index ca7d53af1..000000000 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ /dev/null @@ -1,12 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.ScheduledActivityWorker do - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" - - @impl Oban.Worker - def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do - Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, activity_id) - end -end diff --git a/test/daemons/scheduled_activity_daemon_test.exs b/test/daemons/scheduled_activity_daemon_test.exs deleted file mode 100644 index c8e464491..000000000 --- a/test/daemons/scheduled_activity_daemon_test.exs +++ /dev/null @@ -1,19 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ScheduledActivityDaemonTest do - use Pleroma.DataCase - alias Pleroma.ScheduledActivity - import Pleroma.Factory - - test "creates a status from the scheduled activity" do - user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user, params: %{status: "hi"}) - Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, scheduled_activity.id) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end -end diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index dcf12fb49..d3d05745f 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -8,6 +8,8 @@ defmodule Pleroma.ScheduledActivityTest do alias Pleroma.ScheduledActivity import Pleroma.Factory + clear_config([ScheduledActivity, :enabled]) + setup context do DataCase.ensure_local_uploader(context) end @@ -61,4 +63,30 @@ defmodule Pleroma.ScheduledActivityTest do assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] end end + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end end From c5766a8100de465669178a98f8267425ecfe26e3 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 13:35:02 +0300 Subject: [PATCH 04/17] moved ActivityExpiration to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/application.ex | 1 - .../daemons/activity_expiration_daemon.ex | 66 ------------------- .../workers/activity_expiration_worker.ex | 18 ----- .../cron/purge_expired_activities_worker.ex | 39 +++++++++++ test/activity_expiration_test.exs | 25 +++++++ .../activity_expiration_daemon_test.exs | 17 ----- 7 files changed, 66 insertions(+), 103 deletions(-) delete mode 100644 lib/pleroma/daemons/activity_expiration_daemon.ex delete mode 100644 lib/pleroma/workers/activity_expiration_worker.ex create mode 100644 lib/pleroma/workers/cron/purge_expired_activities_worker.ex delete mode 100644 test/daemons/activity_expiration_daemon_test.exs diff --git a/config/config.exs b/config/config.exs index f4f7fcce7..be8e3ee15 100644 --- a/config/config.exs +++ b/config/config.exs @@ -509,7 +509,8 @@ config :pleroma, Oban, crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, - {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker} + {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, + {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 71670d9fc..15face96e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ defmodule Pleroma.Application do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.Daemons.ActivityExpirationDaemon, Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ diff --git a/lib/pleroma/daemons/activity_expiration_daemon.ex b/lib/pleroma/daemons/activity_expiration_daemon.ex deleted file mode 100644 index cab7628c4..000000000 --- a/lib/pleroma/daemons/activity_expiration_daemon.ex +++ /dev/null @@ -1,66 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Daemons.ActivityExpirationDaemon do - alias Pleroma.Activity - alias Pleroma.ActivityExpiration - alias Pleroma.Config - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.CommonAPI - - require Logger - use GenServer - import Ecto.Query - - @schedule_interval :timer.minutes(1) - - def start_link(_) do - GenServer.start_link(__MODULE__, nil) - end - - @impl true - def init(_) do - if Config.get([ActivityExpiration, :enabled]) do - schedule_next() - {:ok, nil} - else - :ignore - end - end - - def perform(:execute, expiration_id) do - try do - expiration = - ActivityExpiration - |> where([e], e.id == ^expiration_id) - |> Repo.one!() - - activity = Activity.get_by_id_with_object(expiration.activity_id) - user = User.get_by_ap_id(activity.object.data["actor"]) - CommonAPI.delete(activity.id, user) - rescue - error -> - Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") - end - end - - @impl true - def handle_info(:perform, state) do - ActivityExpiration.due_expirations(@schedule_interval) - |> Enum.each(fn expiration -> - Pleroma.Workers.ActivityExpirationWorker.enqueue( - "activity_expiration", - %{"activity_expiration_id" => expiration.id} - ) - end) - - schedule_next() - {:noreply, state} - end - - defp schedule_next do - Process.send_after(self(), :perform, @schedule_interval) - end -end diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex deleted file mode 100644 index 4e3e4195f..000000000 --- a/lib/pleroma/workers/activity_expiration_worker.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.ActivityExpirationWorker do - use Pleroma.Workers.WorkerHelper, queue: "activity_expiration" - - @impl Oban.Worker - def perform( - %{ - "op" => "activity_expiration", - "activity_expiration_id" => activity_expiration_id - }, - _job - ) do - Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, activity_expiration_id) - end -end diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex new file mode 100644 index 000000000..2a7274207 --- /dev/null +++ b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do + @moduledoc """ + The worker to purge expired activities. + """ + + use Oban.Worker, queue: "activity_expiration" + + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + require Logger + + @interval :timer.minutes(1) + + @impl Oban.Worker + def perform(_opts, _job) do + if Config.get([ActivityExpiration, :enabled]) do + Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1) + end + end + + def delete_activity(expiration) do + try do + activity = Activity.get_by_id_with_object(expiration.activity_id) + user = User.get_by_ap_id(activity.object.data["actor"]) + CommonAPI.delete(activity.id, user) + rescue + error -> + Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + end + end +end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs index 4948fae16..2fc593b8c 100644 --- a/test/activity_expiration_test.exs +++ b/test/activity_expiration_test.exs @@ -7,6 +7,8 @@ defmodule Pleroma.ActivityExpirationTest do alias Pleroma.ActivityExpiration import Pleroma.Factory + clear_config([ActivityExpiration, :enabled]) + test "finds activities due to be deleted only" do activity = insert(:note_activity) expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id}) @@ -24,4 +26,27 @@ defmodule Pleroma.ActivityExpirationTest do now = NaiveDateTime.utc_now() assert {:error, _} = ActivityExpiration.create(activity, now) end + + test "deletes an expiration activity" do + Pleroma.Config.put([ActivityExpiration, :enabled], true) + activity = insert(:note_activity) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + expiration = + insert( + :expiration_in_the_past, + %{activity_id: activity.id, scheduled_at: naive_datetime} + ) + + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(:ops, :pid) + + refute Pleroma.Repo.get(Pleroma.Activity, activity.id) + refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) + end end diff --git a/test/daemons/activity_expiration_daemon_test.exs b/test/daemons/activity_expiration_daemon_test.exs deleted file mode 100644 index b51132fb0..000000000 --- a/test/daemons/activity_expiration_daemon_test.exs +++ /dev/null @@ -1,17 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ActivityExpirationWorkerTest do - use Pleroma.DataCase - alias Pleroma.Activity - import Pleroma.Factory - - test "deletes an activity" do - activity = insert(:note_activity) - expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) - Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, expiration.id) - - refute Repo.get(Activity, activity.id) - end -end From a4f834a687d82e7883c7dabf55b86a7e2c1dad33 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 15:59:13 +0300 Subject: [PATCH 05/17] moved DigestEmail to Oban Periodic jobs --- config/config.exs | 18 +----- config/test.exs | 2 - lib/pleroma/application.ex | 1 - lib/pleroma/daemons/digest_email_daemon.ex | 42 -------------- lib/pleroma/scheduler.ex | 7 --- .../workers/cron/digest_emails_worker.ex | 58 +++++++++++++++++++ lib/pleroma/workers/digest_emails_worker.ex | 16 ----- mix.exs | 1 - .../cron/digest_emails_worker_test.exs} | 15 +++-- 9 files changed, 71 insertions(+), 89 deletions(-) delete mode 100644 lib/pleroma/daemons/digest_email_daemon.ex delete mode 100644 lib/pleroma/scheduler.ex create mode 100644 lib/pleroma/workers/cron/digest_emails_worker.ex delete mode 100644 lib/pleroma/workers/digest_emails_worker.ex rename test/{daemons/digest_email_daemon_test.exs => workers/cron/digest_emails_worker_test.exs} (74%) diff --git a/config/config.exs b/config/config.exs index be8e3ee15..ef9b138d1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -51,20 +51,6 @@ config :pleroma, Pleroma.Repo, telemetry_event: [Pleroma.Repo.Instrumenter], migration_lock: nil -scheduled_jobs = - with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], - true <- digest_config[:active] do - [{digest_config[:schedule], {Pleroma.Daemons.DigestEmailDaemon, :perform, []}}] - else - _ -> [] - end - -config :pleroma, Pleroma.Scheduler, - global: true, - overlap: true, - timezone: :utc, - jobs: scheduled_jobs - config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -510,7 +496,8 @@ config :pleroma, Oban, {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, - {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker} + {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, + {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker} ] config :pleroma, :workers, @@ -592,7 +579,6 @@ config :pleroma, Pleroma.ScheduledActivity, config :pleroma, :email_notifications, digest: %{ active: false, - schedule: "0 0 * * 0", interval: 7, inactivity_threshold: 7 } diff --git a/config/test.exs b/config/test.exs index 9b737d4d7..fff709d65 100644 --- a/config/test.exs +++ b/config/test.exs @@ -68,8 +68,6 @@ config :pleroma, Oban, queues: false, prune: :disabled -config :pleroma, Pleroma.Scheduler, jobs: [] - config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 15face96e..695cc28d6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -31,7 +31,6 @@ defmodule Pleroma.Application do children = [ Pleroma.Repo, - Pleroma.Scheduler, Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, diff --git a/lib/pleroma/daemons/digest_email_daemon.ex b/lib/pleroma/daemons/digest_email_daemon.ex deleted file mode 100644 index b4c8eaad9..000000000 --- a/lib/pleroma/daemons/digest_email_daemon.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Daemons.DigestEmailDaemon do - alias Pleroma.Repo - alias Pleroma.Workers.DigestEmailsWorker - - import Ecto.Query - - def perform do - config = Pleroma.Config.get([:email_notifications, :digest]) - negative_interval = -Map.fetch!(config, :interval) - inactivity_threshold = Map.fetch!(config, :inactivity_threshold) - inactive_users_query = Pleroma.User.list_inactive_users_query(inactivity_threshold) - - now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) - - from(u in inactive_users_query, - where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications), - where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), - select: u - ) - |> Repo.all() - |> Enum.each(fn user -> - DigestEmailsWorker.enqueue("digest_email", %{"user_id" => user.id}) - end) - end - - @doc """ - Send digest email to the given user. - Updates `last_digest_emailed_at` field for the user and returns the updated user. - """ - @spec perform(Pleroma.User.t()) :: Pleroma.User.t() - def perform(user) do - with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(user) do - Pleroma.Emails.Mailer.deliver_async(email) - end - - Pleroma.User.touch_last_digest_emailed_at(user) - end -end diff --git a/lib/pleroma/scheduler.ex b/lib/pleroma/scheduler.ex deleted file mode 100644 index d84cd99ad..000000000 --- a/lib/pleroma/scheduler.ex +++ /dev/null @@ -1,7 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Scheduler do - use Quantum.Scheduler, otp_app: :pleroma -end diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex new file mode 100644 index 000000000..0a00129df --- /dev/null +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.DigestEmailsWorker do + @moduledoc """ + The worker to send digest emails. + """ + + use Oban.Worker, queue: "digest_emails" + + alias Pleroma.Config + alias Pleroma.Emails + alias Pleroma.Repo + alias Pleroma.User + + import Ecto.Query + + require Logger + + @impl Oban.Worker + def perform(_opts, _job) do + config = Config.get([:email_notifications, :digest]) + + if config[:active] do + negative_interval = -Map.fetch!(config, :interval) + inactivity_threshold = Map.fetch!(config, :inactivity_threshold) + inactive_users_query = User.list_inactive_users_query(inactivity_threshold) + + now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + + from(u in inactive_users_query, + where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications), + where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), + select: u + ) + |> Repo.all() + |> send_emails + end + end + + def send_emails(users) do + Enum.each(users, &send_email/1) + end + + @doc """ + Send digest email to the given user. + Updates `last_digest_emailed_at` field for the user and returns the updated user. + """ + @spec send_email(User.t()) :: User.t() + def send_email(user) do + with %Swoosh.Email{} = email <- Emails.UserEmail.digest_email(user) do + Emails.Mailer.deliver_async(email) + end + + User.touch_last_digest_emailed_at(user) + end +end diff --git a/lib/pleroma/workers/digest_emails_worker.ex b/lib/pleroma/workers/digest_emails_worker.ex deleted file mode 100644 index 3e5a836d0..000000000 --- a/lib/pleroma/workers/digest_emails_worker.ex +++ /dev/null @@ -1,16 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.DigestEmailsWorker do - alias Pleroma.User - - use Pleroma.Workers.WorkerHelper, queue: "digest_emails" - - @impl Oban.Worker - def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do - user_id - |> User.get_cached_by_id() - |> Pleroma.Daemons.DigestEmailDaemon.perform() - end -end diff --git a/mix.exs b/mix.exs index 31deae96e..23d981dac 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,6 @@ defmodule Pleroma.Mixfile do {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, - {:postgrex, ">= 0.13.5"}, {:oban, "~> 0.11.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, diff --git a/test/daemons/digest_email_daemon_test.exs b/test/workers/cron/digest_emails_worker_test.exs similarity index 74% rename from test/daemons/digest_email_daemon_test.exs rename to test/workers/cron/digest_emails_worker_test.exs index faf592d5f..073615d9e 100644 --- a/test/daemons/digest_email_daemon_test.exs +++ b/test/workers/cron/digest_emails_worker_test.exs @@ -2,16 +2,24 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.DigestEmailDaemonTest do +defmodule Pleroma.Workers.Cron.DigestEmailsWorkerTest do use Pleroma.DataCase + import Pleroma.Factory - alias Pleroma.Daemons.DigestEmailDaemon alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.CommonAPI + clear_config([:email_notifications, :digest]) + test "it sends digest emails" do + Pleroma.Config.put([:email_notifications, :digest], %{ + active: true, + inactivity_threshold: 7, + interval: 7 + }) + user = insert(:user) date = @@ -23,8 +31,7 @@ defmodule Pleroma.DigestEmailDaemonTest do {:ok, _} = User.switch_email_notifications(user2, "digest", true) CommonAPI.post(user, %{"status" => "hey @#{user2.nickname}!"}) - DigestEmailDaemon.perform() - ObanHelpers.perform_all() + Pleroma.Workers.Cron.DigestEmailsWorker.perform(:opts, :pid) # Performing job(s) enqueued at previous step ObanHelpers.perform_all() From d55859f5a4cabbe7849dc3f39079b17bd450a303 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 16:35:12 +0300 Subject: [PATCH 06/17] remove `quantum` package --- config/config.exs | 1 - mix.exs | 3 +-- mix.lock | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index ef9b138d1..54d167a82 100644 --- a/config/config.exs +++ b/config/config.exs @@ -607,7 +607,6 @@ config :pleroma, :web_cache_ttl, activity_pub: nil, activity_pub_question: 30_000 -config :swarm, node_blacklist: [~r/myhtml_.*$/] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/mix.exs b/mix.exs index 23d981dac..72e37cfe6 100644 --- a/mix.exs +++ b/mix.exs @@ -63,7 +63,7 @@ defmodule Pleroma.Mixfile do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize, :swarm], + extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize], included_applications: [:ex_syslogger] ] end @@ -102,7 +102,6 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, {:oban, "~> 0.11.1"}, - {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index c212b39e0..ff64b1d75 100644 --- a/mix.lock +++ b/mix.lock @@ -88,12 +88,10 @@ "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, - "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, - "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 1403a1441de36e8a58da5e996c208a9c10f65440 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 28 Nov 2019 09:02:23 +0300 Subject: [PATCH 07/17] added tests --- .../cron/clear_oauth_token_worker_test.exs | 22 +++++++++++ .../purge_expired_activities_worker_test.exs | 34 +++++++++++++++++ .../cron/scheduled_activity_worker_test.exs | 37 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 test/workers/cron/clear_oauth_token_worker_test.exs create mode 100644 test/workers/cron/purge_expired_activities_worker_test.exs create mode 100644 test/workers/cron/scheduled_activity_worker_test.exs diff --git a/test/workers/cron/clear_oauth_token_worker_test.exs b/test/workers/cron/clear_oauth_token_worker_test.exs new file mode 100644 index 000000000..adea47326 --- /dev/null +++ b/test/workers/cron/clear_oauth_token_worker_test.exs @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ClearOauthTokenWorkerTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Workers.Cron.ClearOauthTokenWorker + + clear_config([:oauth2, :clean_expired_tokens]) + + test "deletes expired tokens" do + insert(:oauth_token, + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -60 * 10) + ) + + Pleroma.Config.put([:oauth2, :clean_expired_tokens], true) + ClearOauthTokenWorker.perform(:opts, :job) + assert Pleroma.Repo.all(Pleroma.Web.OAuth.Token) == [] + end +end diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs new file mode 100644 index 000000000..07980bcd0 --- /dev/null +++ b/test/workers/cron/purge_expired_activities_worker_test.exs @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do + use Pleroma.DataCase + alias Pleroma.ActivityExpiration + import Pleroma.Factory + + clear_config([ActivityExpiration, :enabled]) + + test "deletes an expiration activity" do + Pleroma.Config.put([ActivityExpiration, :enabled], true) + activity = insert(:note_activity) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + expiration = + insert( + :expiration_in_the_past, + %{activity_id: activity.id, scheduled_at: naive_datetime} + ) + + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(:ops, :pid) + + refute Pleroma.Repo.get(Pleroma.Activity, activity.id) + refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) + end +end diff --git a/test/workers/cron/scheduled_activity_worker_test.exs b/test/workers/cron/scheduled_activity_worker_test.exs new file mode 100644 index 000000000..6f17d6f6c --- /dev/null +++ b/test/workers/cron/scheduled_activity_worker_test.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ScheduledActivityWorkerTest do + use Pleroma.DataCase + alias Pleroma.ScheduledActivity + import Pleroma.Factory + + clear_config([ScheduledActivity, :enabled]) + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end +end From 652cc6ba4b7a0494cc96ef4a9bfcaa3b5e5be60e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 3 Dec 2019 21:30:10 +0300 Subject: [PATCH 08/17] updated ScheduledActivity --- config/config.exs | 1 - lib/pleroma/scheduled_activity.ex | 66 +++++++++++++++---- .../scheduled_activity_controller.ex | 3 +- .../controllers/status_controller.ex | 19 +++--- .../{cron => }/scheduled_activity_worker.ex | 23 ++++--- test/scheduled_activity_test.exs | 6 +- test/support/helpers.ex | 6 ++ .../scheduled_activity_controller_test.exs | 53 ++++++++++++++- .../cron/scheduled_activity_worker_test.exs | 37 ----------- 9 files changed, 142 insertions(+), 72 deletions(-) rename lib/pleroma/workers/{cron => }/scheduled_activity_worker.ex (59%) delete mode 100644 test/workers/cron/scheduled_activity_worker_test.exs diff --git a/config/config.exs b/config/config.exs index e7e17669e..259529f97 100644 --- a/config/config.exs +++ b/config/config.exs @@ -496,7 +496,6 @@ config :pleroma, Oban, crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, - {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker} ] diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index fea2cf3ff..96fa6a987 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -5,11 +5,13 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema + alias Ecto.Multi alias Pleroma.Config alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Workers.ScheduledActivityWorker import Ecto.Query import Ecto.Changeset @@ -105,14 +107,29 @@ defmodule Pleroma.ScheduledActivity do end def new(%User{} = user, attrs) do - %ScheduledActivity{user_id: user.id} - |> changeset(attrs) + changeset(%ScheduledActivity{user_id: user.id}, attrs) end + @doc """ + Creates ScheduledActivity and add to queue to perform at scheduled_at date + """ + @spec create(User.t(), map()) :: {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} def create(%User{} = user, attrs) do - user - |> new(attrs) - |> Repo.insert() + Multi.new() + |> Multi.insert(:scheduled_activity, new(user, attrs)) + |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> + %{activity_id: activity.id} + |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) + |> Oban.insert() + end) + |> Repo.transaction() + |> case do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} + end end def get(%User{} = user, scheduled_activity_id) do @@ -122,15 +139,35 @@ defmodule Pleroma.ScheduledActivity do |> Repo.one() end - def update(%ScheduledActivity{} = scheduled_activity, attrs) do - scheduled_activity - |> update_changeset(attrs) - |> Repo.update() + @spec update(ScheduledActivity.t(), map()) :: + {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} + def update(%ScheduledActivity{id: id} = scheduled_activity, attrs) do + with {:error, %Ecto.Changeset{valid?: true} = changeset} <- + {:error, update_changeset(scheduled_activity, attrs)} do + Multi.new() + |> Multi.update(:scheduled_activity, changeset) + |> Multi.update_all(:scheduled_job, job_query(id), + set: [scheduled_at: changeset.changes[:scheduled_at]] + ) + |> Repo.transaction() + |> case do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} + end + end + end + + def delete_job(%ScheduledActivity{id: id} = _scheduled_activity) do + id + |> job_query + |> Repo.delete_all() end def delete(%ScheduledActivity{} = scheduled_activity) do - scheduled_activity - |> Repo.delete() + Repo.delete(scheduled_activity) end def delete(id) when is_binary(id) or is_integer(id) do @@ -158,4 +195,11 @@ defmodule Pleroma.ScheduledActivity do |> where([sa], sa.scheduled_at < ^naive_datetime) |> Repo.all() end + + def job_query(scheduled_activity_id) do + from(j in Oban.Job, + where: j.queue == "scheduled_activities", + where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) + ) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex index ff9276541..4f9a8bdbe 100644 --- a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex @@ -45,7 +45,8 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityController do @doc "DELETE /api/v1/scheduled_statuses/:id" def delete(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, _params) do - with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity) do + with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity), + _ <- ScheduledActivity.delete_job(scheduled_activity) do render(conn, "show.json", scheduled_activity: scheduled_activity) end end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 74b223cf4..d70749dfa 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -124,15 +124,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do ) do params = Map.put(params, "in_reply_to_status_id", params["in_reply_to_id"]) - if ScheduledActivity.far_enough?(scheduled_at) do - with {:ok, scheduled_activity} <- - ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) do - conn - |> put_view(ScheduledActivityView) - |> render("show.json", scheduled_activity: scheduled_activity) - end + with {:far_enough, true} <- {:far_enough, ScheduledActivity.far_enough?(scheduled_at)}, + attrs <- %{"params" => params, "scheduled_at" => scheduled_at}, + {:ok, scheduled_activity} <- ScheduledActivity.create(user, attrs) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", scheduled_activity: scheduled_activity) else - create(conn, Map.drop(params, ["scheduled_at"])) + {:far_enough, _} -> + create(conn, Map.drop(params, ["scheduled_at"])) + + error -> + error end end diff --git a/lib/pleroma/workers/cron/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex similarity index 59% rename from lib/pleroma/workers/cron/scheduled_activity_worker.ex rename to lib/pleroma/workers/scheduled_activity_worker.ex index 407ab687a..5109d7f75 100644 --- a/lib/pleroma/workers/cron/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -2,12 +2,13 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do +defmodule Pleroma.Workers.ScheduledActivityWorker do @moduledoc """ - The worker to post scheduled actvities. + The worker to post scheduled activity. """ - use Oban.Worker, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + alias Pleroma.Config alias Pleroma.ScheduledActivity alias Pleroma.User @@ -15,18 +16,20 @@ defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do require Logger - @schedule_interval :timer.minutes(1) - @impl Oban.Worker - def perform(_opts, _job) do + def perform(%{"activity_id" => activity_id}, _job) do if Config.get([ScheduledActivity, :enabled]) do - @schedule_interval - |> ScheduledActivity.due_activities() - |> Enum.each(&post_activity/1) + case Pleroma.Repo.get(ScheduledActivity, activity_id) do + %ScheduledActivity{} = scheduled_activity -> + post_activity(scheduled_activity) + + _ -> + Logger.error("#{__MODULE__} Couldn't find scheduled activity: #{activity_id}") + end end end - def post_activity(scheduled_activity) do + defp post_activity(%ScheduledActivity{} = scheduled_activity) do try do {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index d3d05745f..663cfdd34 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -26,6 +26,7 @@ defmodule Pleroma.ScheduledActivityTest do attrs = %{params: %{}, scheduled_at: today} {:ok, _} = ScheduledActivity.create(user, attrs) {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end @@ -83,7 +84,10 @@ defmodule Pleroma.ScheduledActivityTest do params: %{status: "hi"} ) - Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + Pleroma.Workers.ScheduledActivityWorker.perform( + %{"activity_id" => scheduled_activity.id}, + :pid + ) refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) diff --git a/test/support/helpers.ex b/test/support/helpers.ex index ce39dd9d8..ec556a916 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -53,6 +53,12 @@ defmodule Pleroma.Tests.Helpers do clear_config_all: 2 ] + def to_datetime(naive_datetime) do + naive_datetime + |> DateTime.from_naive!("Etc/UTC") + |> DateTime.truncate(:second) + end + def collect_ids(collection) do collection |> Enum.map(& &1.id) diff --git a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index ae5fee2bc..5f3a376be 100644 --- a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do alias Pleroma.ScheduledActivity import Pleroma.Factory + import Ecto.Query test "shows scheduled activities", %{conn: conn} do user = insert(:user) @@ -68,7 +69,30 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do test "updates a scheduled activity", %{conn: conn} do user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user) + + scheduled_at = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + :timer.minutes(60), + :millisecond + ) + + {:ok, scheduled_activity} = + ScheduledActivity.create( + user, + %{ + scheduled_at: scheduled_at, + params: build(:note).data + } + ) + + scheduled_activity_job = + Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + + assert scheduled_activity_job.args == %{"activity_id" => scheduled_activity.id} + + assert DateTime.truncate(scheduled_activity_job.scheduled_at, :second) == + to_datetime(scheduled_at) new_scheduled_at = NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) @@ -82,6 +106,10 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do assert %{"scheduled_at" => expected_scheduled_at} = json_response(res_conn, 200) assert expected_scheduled_at == Pleroma.Web.CommonAPI.Utils.to_masto_date(new_scheduled_at) + scheduled_activity_job = refresh_record(scheduled_activity_job) + + assert DateTime.truncate(scheduled_activity_job.scheduled_at, :second) == + to_datetime(new_scheduled_at) res_conn = conn @@ -93,7 +121,25 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do test "deletes a scheduled activity", %{conn: conn} do user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user) + + {:ok, scheduled_activity} = + ScheduledActivity.create( + user, + %{ + scheduled_at: + NaiveDateTime.add( + NaiveDateTime.utc_now(), + :timer.minutes(60), + :millisecond + ), + params: build(:note).data + } + ) + + scheduled_activity_job = + Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + + assert scheduled_activity_job.args == %{"activity_id" => scheduled_activity.id} res_conn = conn @@ -101,7 +147,8 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do |> delete("/api/v1/scheduled_statuses/#{scheduled_activity.id}") assert %{} = json_response(res_conn, 200) - assert nil == Repo.get(ScheduledActivity, scheduled_activity.id) + refute Repo.get(ScheduledActivity, scheduled_activity.id) + refute Repo.get(Oban.Job, scheduled_activity_job.id) res_conn = conn diff --git a/test/workers/cron/scheduled_activity_worker_test.exs b/test/workers/cron/scheduled_activity_worker_test.exs deleted file mode 100644 index 6f17d6f6c..000000000 --- a/test/workers/cron/scheduled_activity_worker_test.exs +++ /dev/null @@ -1,37 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ScheduledActivityWorkerTest do - use Pleroma.DataCase - alias Pleroma.ScheduledActivity - import Pleroma.Factory - - clear_config([ScheduledActivity, :enabled]) - - test "creates a status from the scheduled activity" do - Pleroma.Config.put([ScheduledActivity, :enabled], true) - user = insert(:user) - - naive_datetime = - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -:timer.minutes(2), - :millisecond - ) - - scheduled_activity = - insert( - :scheduled_activity, - scheduled_at: naive_datetime, - user: user, - params: %{status: "hi"} - ) - - Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end -end From 469c0956cd937b28a38f8c982b670207ec14b843 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 09:43:01 +0300 Subject: [PATCH 09/17] update excoveralls --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 618864eff..fc84ae962 100644 --- a/mix.exs +++ b/mix.exs @@ -156,7 +156,7 @@ defmodule Pleroma.Mixfile do {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, - {:excoveralls, "~> 0.11.1", only: :test}, + {:excoveralls, "~> 0.12.1", only: :test}, {:flake_id, "~> 0.1.0"}, {:remote_ip, git: "https://git.pleroma.social/pleroma/remote_ip.git", diff --git a/mix.lock b/mix.lock index 2e12d9b73..c76874a4d 100644 --- a/mix.lock +++ b/mix.lock @@ -34,7 +34,7 @@ "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, - "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "excoveralls": {:hex, :excoveralls, "0.12.1", "a553c59f6850d0aff3770e4729515762ba7c8e41eedde03208182a8dc9d0ce07", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "fast_html": {:hex, :fast_html, "0.99.4", "d80812664f0429607e1d880fba0ef04da87a2e4fa596701bcaae17953535695c", [:make, :mix], [], "hexpm"}, "fast_sanitize": {:hex, :fast_sanitize, "0.1.4", "6c2e7203ca2f8275527a3021ba6e9d5d4ee213a47dc214a97c128737c9e56df1", [:mix], [{:fast_html, "~> 0.99", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, From 3a0a400fe154f44ab37b79eeb3a07f919517743e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 09:53:01 +0300 Subject: [PATCH 10/17] add @type to ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 96fa6a987..ea78dce6a 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -16,6 +16,8 @@ defmodule Pleroma.ScheduledActivity do import Ecto.Query import Ecto.Changeset + @type t :: %__MODULE__{} + @min_offset :timer.minutes(5) schema "scheduled_activities" do From c0d572d0bf842fae08c609aa58c82554bee3a263 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 17:28:57 +0300 Subject: [PATCH 11/17] added test --- test/scheduled_activity_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index 663cfdd34..b367ae5fb 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -24,9 +24,13 @@ defmodule Pleroma.ScheduledActivityTest do |> NaiveDateTime.to_iso8601() attrs = %{params: %{}, scheduled_at: today} - {:ok, _} = ScheduledActivity.create(user, attrs) - {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, sa1} = ScheduledActivity.create(user, attrs) + {:ok, sa2} = ScheduledActivity.create(user, attrs) + jobs = + Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + + assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end From 3c3bba0b7c65187b3270ef3402442cf870a55198 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 21:18:05 +0300 Subject: [PATCH 12/17] fix ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 17 ++++++--- test/scheduled_activity_test.exs | 36 ++++++++++++++++++- .../scheduled_activity_controller_test.exs | 4 +++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index ea78dce6a..d01100702 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -119,11 +119,7 @@ defmodule Pleroma.ScheduledActivity do def create(%User{} = user, attrs) do Multi.new() |> Multi.insert(:scheduled_activity, new(user, attrs)) - |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> - %{activity_id: activity.id} - |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) - |> Oban.insert() - end) + |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) |> Repo.transaction() |> case do {:ok, %{scheduled_activity: scheduled_activity}} -> @@ -134,6 +130,17 @@ defmodule Pleroma.ScheduledActivity do end end + defp maybe_add_jobs(multi, true) do + multi + |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> + %{activity_id: activity.id} + |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) + |> Oban.insert() + end) + end + + defp maybe_add_jobs(multi, _), do: multi + def get(%User{} = user, scheduled_activity_id) do ScheduledActivity |> where(user_id: ^user.id) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index b367ae5fb..d2c5f5aa2 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -15,7 +15,8 @@ defmodule Pleroma.ScheduledActivityTest do end describe "creation" do - test "when daily user limit is exceeded" do + test "scheduled activities with jobs when ScheduledActivity enabled" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) today = @@ -31,6 +32,39 @@ defmodule Pleroma.ScheduledActivityTest do Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] + end + + test "scheduled activities without jobs when ScheduledActivity disabled" do + Pleroma.Config.put([ScheduledActivity, :enabled], false) + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _sa1} = ScheduledActivity.create(user, attrs) + {:ok, _sa2} = ScheduledActivity.create(user, attrs) + + jobs = + Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + + assert jobs == [] + end + + test "when daily user limit is exceeded" do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end diff --git a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index 5f3a376be..478631a12 100644 --- a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -11,6 +11,8 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do import Pleroma.Factory import Ecto.Query + clear_config([ScheduledActivity, :enabled]) + test "shows scheduled activities", %{conn: conn} do user = insert(:user) scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() @@ -68,6 +70,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do end test "updates a scheduled activity", %{conn: conn} do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) scheduled_at = @@ -120,6 +123,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do end test "deletes a scheduled activity", %{conn: conn} do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) {:ok, scheduled_activity} = From d43c9ab286c410abfe903b56911888d75f492155 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 Jan 2020 16:58:43 +0000 Subject: [PATCH 13/17] Apply suggestion to docs/configuration/cheatsheet.md --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index f690259bf..22f099581 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -515,7 +515,7 @@ Pleroma has the following queues: #### Oban.Cron -Pleroma has the periodic jobs: +Pleroma has these periodic job workers: `Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. From e761d76e4fe95a8fadd8eb05d5c55ef5fc4bc8ec Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 Jan 2020 16:58:50 +0000 Subject: [PATCH 14/17] Apply suggestion to docs/configuration/cheatsheet.md --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 22f099581..48d980aea 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -517,7 +517,7 @@ Pleroma has the following queues: Pleroma has these periodic job workers: -`Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. +`Pleroma.Workers.Cron.ClearOauthTokenWorker` - a job worker to cleanup expired oauth tokens. Example: From 6f77b7742837f476fb2bf732e5c4ba6bc4fb3799 Mon Sep 17 00:00:00 2001 From: Maksim Date: Thu, 23 Jan 2020 05:49:00 +0000 Subject: [PATCH 15/17] Apply suggestion to lib/pleroma/workers/cron/clear_oauth_token_worker.ex --- lib/pleroma/workers/cron/clear_oauth_token_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex index 1a4cdc198..a24407874 100644 --- a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex +++ b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do @moduledoc """ - The worker to clean an expired oauth tokens. + The worker to cleanup expired oAuth tokens. """ use Oban.Worker, queue: "background" From ce7c887a27ad5af59f540650637a50da7f91fa52 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 23 Jan 2020 11:05:08 +0300 Subject: [PATCH 16/17] removed try/rescue --- lib/pleroma/scheduled_activity.ex | 2 +- .../cron/purge_expired_activities_worker.ex | 21 +++++--- .../workers/scheduled_activity_worker.ex | 12 ++--- test/scheduled_activity_test.exs | 29 ----------- .../purge_expired_activities_worker_test.exs | 22 ++++++++ .../scheduled_activity_worker_test.exs | 52 +++++++++++++++++++ 6 files changed, 95 insertions(+), 43 deletions(-) create mode 100644 test/workers/scheduled_activity_worker_test.exs diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index d01100702..68da0550c 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -156,7 +156,7 @@ defmodule Pleroma.ScheduledActivity do Multi.new() |> Multi.update(:scheduled_activity, changeset) |> Multi.update_all(:scheduled_job, job_query(id), - set: [scheduled_at: changeset.changes[:scheduled_at]] + set: [scheduled_at: get_field(changeset, :scheduled_at)] ) |> Repo.transaction() |> case do diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex index 2a7274207..7a52860a9 100644 --- a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex +++ b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex @@ -26,14 +26,21 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do end end - def delete_activity(expiration) do - try do - activity = Activity.get_by_id_with_object(expiration.activity_id) - user = User.get_by_ap_id(activity.object.data["actor"]) + def delete_activity(%ActivityExpiration{activity_id: activity_id}) do + with {:activity, %Activity{} = activity} <- + {:activity, Activity.get_by_id_with_object(activity_id)}, + {:user, %User{} = user} <- {:user, User.get_by_ap_id(activity.object.data["actor"])} do CommonAPI.delete(activity.id, user) - rescue - error -> - Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + else + {:activity, _} -> + Logger.error( + "#{__MODULE__} Couldn't delete expired activity: not found activity ##{activity_id}" + ) + + {:user, _} -> + Logger.error( + "#{__MODULE__} Couldn't delete expired activity: not found actorof ##{activity_id}" + ) end end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 5109d7f75..bd41ab4ce 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -29,12 +29,12 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do end end - defp post_activity(%ScheduledActivity{} = scheduled_activity) do - try do - {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) - %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) - {:ok, _result} = CommonAPI.post(user, scheduled_activity.params) - rescue + defp post_activity(%ScheduledActivity{user_id: user_id, params: params} = scheduled_activity) do + with {:delete, {:ok, _}} <- {:delete, ScheduledActivity.delete(scheduled_activity)}, + {:user, %User{} = user} <- {:user, User.get_cached_by_id(user_id)}, + {:post, {:ok, _}} <- {:post, CommonAPI.post(user, params)} do + :ok + else error -> Logger.error( "#{__MODULE__} Couldn't create a status from the scheduled activity: #{inspect(error)}" diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index d2c5f5aa2..6c13d300a 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -102,33 +102,4 @@ defmodule Pleroma.ScheduledActivityTest do assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] end end - - test "creates a status from the scheduled activity" do - Pleroma.Config.put([ScheduledActivity, :enabled], true) - user = insert(:user) - - naive_datetime = - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -:timer.minutes(2), - :millisecond - ) - - scheduled_activity = - insert( - :scheduled_activity, - scheduled_at: naive_datetime, - user: user, - params: %{status: "hi"} - ) - - Pleroma.Workers.ScheduledActivityWorker.perform( - %{"activity_id" => scheduled_activity.id}, - :pid - ) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end end diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs index 07980bcd0..c2561683e 100644 --- a/test/workers/cron/purge_expired_activities_worker_test.exs +++ b/test/workers/cron/purge_expired_activities_worker_test.exs @@ -4,8 +4,12 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do use Pleroma.DataCase + alias Pleroma.ActivityExpiration + alias Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker + import Pleroma.Factory + import ExUnit.CaptureLog clear_config([ActivityExpiration, :enabled]) @@ -31,4 +35,22 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do refute Pleroma.Repo.get(Pleroma.Activity, activity.id) refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) end + + describe "delete_activity/1" do + test "adds log message if activity isn't find" do + assert capture_log([level: :error], fn -> + PurgeExpiredActivitiesWorker.delete_activity(%ActivityExpiration{ + activity_id: "test-activity" + }) + end) =~ "Couldn't delete expired activity: not found activity" + end + + test "adds log message if actor isn't find" do + assert capture_log([level: :error], fn -> + PurgeExpiredActivitiesWorker.delete_activity(%ActivityExpiration{ + activity_id: "test-activity" + }) + end) =~ "Couldn't delete expired activity: not found activity" + end + end end diff --git a/test/workers/scheduled_activity_worker_test.exs b/test/workers/scheduled_activity_worker_test.exs new file mode 100644 index 000000000..1405d7142 --- /dev/null +++ b/test/workers/scheduled_activity_worker_test.exs @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ScheduledActivityWorkerTest do + use Pleroma.DataCase + + alias Pleroma.ScheduledActivity + alias Pleroma.Workers.ScheduledActivityWorker + + import Pleroma.Factory + import ExUnit.CaptureLog + + clear_config([ScheduledActivity, :enabled]) + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + ScheduledActivityWorker.perform( + %{"activity_id" => scheduled_activity.id}, + :pid + ) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end + + test "adds log message if ScheduledActivity isn't find" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + + assert capture_log([level: :error], fn -> + ScheduledActivityWorker.perform(%{"activity_id" => 42}, :pid) + end) =~ "Couldn't find scheduled activity" + end +end From 8589632d0905c5d028ed7f01f63a1103788d3571 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 23 Jan 2020 17:18:23 +0300 Subject: [PATCH 17/17] fixed delete ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 50 ++++++++----------- .../scheduled_activity_controller.ex | 3 +- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 68da0550c..e81bfcd7d 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -121,13 +121,7 @@ defmodule Pleroma.ScheduledActivity do |> Multi.insert(:scheduled_activity, new(user, attrs)) |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) |> Repo.transaction() - |> case do - {:ok, %{scheduled_activity: scheduled_activity}} -> - {:ok, scheduled_activity} - - {:error, _, changeset, _} -> - {:error, changeset} - end + |> transaction_response end defp maybe_add_jobs(multi, true) do @@ -159,34 +153,32 @@ defmodule Pleroma.ScheduledActivity do set: [scheduled_at: get_field(changeset, :scheduled_at)] ) |> Repo.transaction() - |> case do - {:ok, %{scheduled_activity: scheduled_activity}} -> - {:ok, scheduled_activity} - - {:error, _, changeset, _} -> - {:error, changeset} - end + |> transaction_response end end - def delete_job(%ScheduledActivity{id: id} = _scheduled_activity) do - id - |> job_query - |> Repo.delete_all() - end - - def delete(%ScheduledActivity{} = scheduled_activity) do - Repo.delete(scheduled_activity) + @doc "Deletes a ScheduledActivity and linked jobs." + @spec delete(ScheduledActivity.t() | binary() | integer) :: + {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} + def delete(%ScheduledActivity{id: id} = scheduled_activity) do + Multi.new() + |> Multi.delete(:scheduled_activity, scheduled_activity, stale_error_field: :id) + |> Multi.delete_all(:jobs, job_query(id)) + |> Repo.transaction() + |> transaction_response end def delete(id) when is_binary(id) or is_integer(id) do - ScheduledActivity - |> where(id: ^id) - |> select([sa], sa) - |> Repo.delete_all() - |> case do - {1, [scheduled_activity]} -> {:ok, scheduled_activity} - _ -> :error + delete(%__MODULE__{id: id}) + end + + defp transaction_response(result) do + case result do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} end end diff --git a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex index 4f9a8bdbe..ff9276541 100644 --- a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex @@ -45,8 +45,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityController do @doc "DELETE /api/v1/scheduled_statuses/:id" def delete(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, _params) do - with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity), - _ <- ScheduledActivity.delete_job(scheduled_activity) do + with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity) do render(conn, "show.json", scheduled_activity: scheduled_activity) end end