From db690bede92e9ba65b1afaa8605aa5ddb3c992a4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 2 Aug 2019 21:33:12 +0300 Subject: [PATCH 01/84] temp commit --- config/benchmark.exs | 88 +++++++++++++++++++ lib/load_testing/fetcher.ex | 116 ++++++++++++++++++++++++++ lib/load_testing/generator.ex | 83 ++++++++++++++++++ lib/load_testing/helper.ex | 16 ++++ lib/mix/tasks/pleroma/load_testing.ex | 100 ++++++++++++++++++++++ 5 files changed, 403 insertions(+) create mode 100644 config/benchmark.exs create mode 100644 lib/load_testing/fetcher.ex create mode 100644 lib/load_testing/generator.ex create mode 100644 lib/load_testing/helper.ex create mode 100644 lib/mix/tasks/pleroma/load_testing.ex diff --git a/config/benchmark.exs b/config/benchmark.exs new file mode 100644 index 000000000..b4f5dbdbd --- /dev/null +++ b/config/benchmark.exs @@ -0,0 +1,88 @@ +use Mix.Config + +# We don't run a server during test. If one is required, +# you can enable the server option below. +config :pleroma, Pleroma.Web.Endpoint, + http: [port: 4001], + url: [port: 4001], + server: true + +# Disable captha for tests +config :pleroma, Pleroma.Captcha, + # It should not be enabled for automatic tests + enabled: false, + # A fake captcha service for tests + method: Pleroma.Captcha.Mock + +# Print only warnings and errors during test +config :logger, level: :warn + +config :pleroma, :auth, oauth_consumer_strategies: [] + +config :pleroma, Pleroma.Upload, filters: [], link_name: false + +config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads" + +config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled: true + +config :pleroma, :instance, + email: "admin@example.com", + notify_email: "noreply@example.com", + skip_thread_containment: false, + federating: false + +config :pleroma, :activitypub, sign_object_fetches: false + +# Configure your database +config :pleroma, Pleroma.Repo, + adapter: Ecto.Adapters.Postgres, + username: System.get_env("DB_USER") || "postgres", + database: System.get_env("DB_DATABASE") || "pleroma_test", + hostname: System.get_env("DB_HOST") || "localhost", + # username: "pleroma", + password: + System.get_env("DB_PASS") || + "cAUrGezwXjRwd/lIPzZAcwjb/hiZiGi3FIaSGy9l/XsTcGA61FMy7eCBiRcg1DyQ", + # password: "", + pool_size: 10, + timeout: 180_000 + +# Reduce hash rounds for testing +config :pbkdf2_elixir, rounds: 1 + +config :tesla, adapter: Tesla.Mock + +config :pleroma, :rich_media, + enabled: false, + ignore_hosts: [], + ignore_tld: ["local", "localdomain", "lan"] + +config :web_push_encryption, :vapid_details, + subject: "mailto:administrator@example.com", + public_key: + "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4", + private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA" + +config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock + +config :pleroma_job_queue, disabled: true + +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 2, + total_user_limit: 3, + enabled: false + +config :pleroma, :rate_limit, + search: [{1000, 30}, {1000, 30}], + app_account_creation: {10_000, 5}, + password_reset: {1000, 30} + +config :pleroma, :http_security, report_uri: "https://endpoint.com" + +config :pleroma, :http, send_user_agent: false + +rum_enabled = System.get_env("RUM_ENABLED") == "true" +config :pleroma, :database, rum_enabled: rum_enabled +IO.puts("RUM enabled: #{rum_enabled}") + +config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex new file mode 100644 index 000000000..70c0fcd0c --- /dev/null +++ b/lib/load_testing/fetcher.ex @@ -0,0 +1,116 @@ +defmodule Pleroma.LoadTesting.Fetcher do + use Pleroma.LoadTesting.Helper + + def fetch_user(user) do + IO.puts("=================================") + + {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end) + + IO.puts("Query user by id: #{to_sec(time)} sec.") + + {time, _value} = + :timer.tc(fn -> + Repo.get_by(User, ap_id: user.ap_id) + end) + + IO.puts("Query user by ap_id: #{to_sec(time)} sec.") + + {time, _value} = + :timer.tc(fn -> + Repo.get_by(User, email: user.email) + end) + + IO.puts("Query user by email: #{to_sec(time)} sec.") + + {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end) + + IO.puts("Query user by nickname: #{to_sec(time)} sec.") + end + + def query_timelines(user) do + IO.puts("\n=================================") + + params = %{ + "count" => 20, + "with_muted" => true, + "type" => ["Create", "Announce"], + "blocking_user" => user, + "muting_user" => user, + "user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params) + end) + + IO.puts("Query user home timeline: #{to_sec(time)} sec.") + + params = %{ + "count" => 20, + "local_only" => true, + "only_media" => "false", + "type" => ["Create", "Announce"], + "with_muted" => "true", + "blocking_user" => user, + "muting_user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_public_activities(params) + end) + + IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.") + + params = %{ + "count" => 20, + "only_media" => "false", + "type" => ["Create", "Announce"], + "with_muted" => "true", + "blocking_user" => user, + "muting_user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_public_activities(params) + end) + + IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.") + end + + def query_notifications(user) do + IO.puts("\n=================================") + params = %{"count" => "20", "with_muted" => "false"} + + {time, _} = + :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + + IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.") + + params = %{"count" => "20", "with_muted" => "true"} + + {time, _} = + :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + + IO.puts("Query user notifications with muted: #{to_sec(time)} sec.") + end + + def query_long_thread(user, activity) do + IO.puts("\n=================================") + + {time, replies} = + :timer.tc(fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( + activity.data["context"], + %{ + "blocking_user" => user, + "user" => user + } + ) + end) + + IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.") + end +end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex new file mode 100644 index 000000000..a9016b9e8 --- /dev/null +++ b/lib/load_testing/generator.ex @@ -0,0 +1,83 @@ +defmodule Pleroma.LoadTesting.Generator do + use Pleroma.LoadTesting.Helper + + def generate_users(opts) do + IO.puts("Starting generating #{opts[:users_max]} users...") + {time, _} = :timer.tc(fn -> do_generate_users(opts) end) + IO.puts("Inserting users take #{to_sec(time)} sec.\n") + end + + defp do_generate_users(opts) do + min = Keyword.get(opts, :users_min, 1) + max = Keyword.get(opts, :users_max) + + query = + "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\", + \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n" + + users = + Task.async_stream( + min..max, + &generate_user_data(&1), + max_concurrency: 10, + timeout: 30_000 + ) + |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end) + + query = query <> String.replace_trailing(users, ", \n", ";") + + Ecto.Adapters.SQL.query!(Repo, query) + end + + defp generate_user_data(i) do + user = %User{ + name: "Test テスト User #{i}", + email: "user#{i}@example.com", + nickname: "nick#{i}", + password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + bio: "Tester Number #{i}", + info: %{} + } + + user = %{ + user + | ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user), + following: [User.ap_id(user)] + } + + "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{ + user.following + }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{ + user.name + }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())" + end + + def generate_activities(users, opts) do + IO.puts("Starting generating #{opts[:activities_max]} activities...") + {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end) + IO.puts("Inserting activities take #{to_sec(time)} sec.\n") + end + + defp do_generate_activities(users, opts) do + Task.async_stream( + 1..opts[:activities_max], + fn _ -> + do_generate_activity(users, opts) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end + + defp do_generate_activity(users, opts) do + status = + if opts[:mention], + do: "some status with @#{opts[:mention].nickname}", + else: "some status" + + Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) + end +end diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex new file mode 100644 index 000000000..338dba323 --- /dev/null +++ b/lib/load_testing/helper.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.LoadTesting.Helper do + defmacro __using__(_) do + quote do + import Ecto.Query + alias Pleroma.Activity + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.ActivityPub + alias Pleroma.Web.CommonAPI + + defp to_sec(microseconds), do: microseconds / 1_000_000 + end + end +end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex new file mode 100644 index 000000000..9ed30db7e --- /dev/null +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -0,0 +1,100 @@ +defmodule Mix.Tasks.Pleroma.LoadTesting do + use Mix.Task + use Pleroma.LoadTesting.Helper + import Mix.Pleroma + import Pleroma.LoadTesting.Generator + import Pleroma.LoadTesting.Fetcher + + # tODO: remove autovacuum worker until generation is not ended + @shortdoc "Factory for generation data" + @moduledoc """ + Generates data like: + - users + - activities with notifications + + ## Generate data + MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000 + MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000 + + Options: + - `--users NUMBER` - number of users to generate (default: 10000) + - `--activities NUMBER` - number of activities to generate (default: 20000) + """ + + @aliases [u: :users, a: :activities, d: :delete] + @switches [users: :integer, activities: :integer, delete: :boolean] + @users_default 20_000 + @activities_default 50_000 + + def run(args) do + {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) + start_pleroma() + + current_max = Keyword.get(opts, :users, @users_default) + activities_max = Keyword.get(opts, :activities, @activities_default) + + {users_min, users_max} = + if opts[:delete] do + clean_tables() + {1, current_max} + else + current_count = Repo.aggregate(from(u in User), :count, :id) + 1 + {current_count, current_max + current_count} + end + + opts = + Keyword.put(opts, :users_min, users_min) + |> Keyword.put(:users_max, users_max) + |> Keyword.put(:activities_max, activities_max) + + generate_users(opts) + + # main user for queries + IO.puts("Fetching main user...") + + {time, user} = + :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end) + + IO.puts("Fetching main user take #{to_sec(time)} sec.\n") + + IO.puts("Fetching users...") + + {time, users} = + :timer.tc(fn -> + Repo.all( + from(u in User, + where: u.id != ^user.id, + order_by: fragment("RANDOM()"), + limit: 10 + ) + ) + end) + + IO.puts("Fetching users take #{to_sec(time)} sec.\n") + + generate_activities(users, opts) + + generate_activities(users, Keyword.put(opts, :mention, user)) + + # generate_replies(user, users, activities) + + # activity = Enum.random(activities) + # generate_long_thread(user, users, activity) + + IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}") + IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}") + IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") + IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + + query_timelines(user) + query_notifications(user) + # query_long_thread(user, activity) + end + + defp clean_tables do + IO.puts("\n\nDeleting old data...\n") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") + end +end From 79dde5804427ab24c5bc9aa1b5a44d2259cecc6d Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 4 Sep 2019 20:18:11 +0300 Subject: [PATCH 02/84] one more temp commit --- lib/load_testing/fetcher.ex | 120 +++++++++--------- lib/load_testing/generator.ex | 127 +++++++++++++++---- lib/mix/tasks/pleroma/load_testing.ex | 46 ++++--- lib/pleroma/web/activity_pub/activity_pub.ex | 5 + 4 files changed, 202 insertions(+), 96 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index 70c0fcd0c..ed744db9b 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -4,33 +4,18 @@ defmodule Pleroma.LoadTesting.Fetcher do def fetch_user(user) do IO.puts("=================================") - {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end) - - IO.puts("Query user by id: #{to_sec(time)} sec.") - - {time, _value} = - :timer.tc(fn -> - Repo.get_by(User, ap_id: user.ap_id) - end) - - IO.puts("Query user by ap_id: #{to_sec(time)} sec.") - - {time, _value} = - :timer.tc(fn -> - Repo.get_by(User, email: user.email) - end) - - IO.puts("Query user by email: #{to_sec(time)} sec.") - - {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end) - - IO.puts("Query user by nickname: #{to_sec(time)} sec.") + Benchee.run(%{ + "By id" => fn -> Repo.get_by(User, id: user.id) end, + "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, + "By email" => fn -> Repo.get_by(User, email: user.email) end, + "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end + }) end def query_timelines(user) do IO.puts("\n=================================") - params = %{ + home_timeline_params = %{ "count" => 20, "with_muted" => true, "type" => ["Create", "Announce"], @@ -39,14 +24,7 @@ defmodule Pleroma.LoadTesting.Fetcher do "user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params) - end) - - IO.puts("Query user home timeline: #{to_sec(time)} sec.") - - params = %{ + mastodon_public_timeline_params = %{ "count" => 20, "local_only" => true, "only_media" => "false", @@ -56,14 +34,7 @@ defmodule Pleroma.LoadTesting.Fetcher do "muting_user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_public_activities(params) - end) - - IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.") - - params = %{ + mastodon_federated_timeline_params = %{ "count" => 20, "only_media" => "false", "type" => ["Create", "Announce"], @@ -72,45 +43,76 @@ defmodule Pleroma.LoadTesting.Fetcher do "muting_user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_public_activities(params) - end) - - IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.") + Benchee.run(%{ + "User home timeline" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( + [user.ap_id | user.following], + home_timeline_params + ) + end, + "User mastodon public timeline" => fn -> + ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + end, + "User mastodon federated public timeline" => fn -> + ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params) + end + }) end def query_notifications(user) do IO.puts("\n=================================") - params = %{"count" => "20", "with_muted" => "false"} + without_muted_params = %{"count" => "20", "with_muted" => "false"} + with_muted_params = %{"count" => "20", "with_muted" => "true"} - {time, _} = - :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + Benchee.run(%{ + "Notifications without muted" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) + end, + "Notifications with muted" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) + end + }) + end - IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.") + def query_dms(user) do + IO.puts("\n=================================") - params = %{"count" => "20", "with_muted" => "true"} + params = %{ + "count" => "20", + "with_muted" => "true", + "type" => "Create", + "blocking_user" => user, + "user" => user, + visibility: "direct" + } - {time, _} = - :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) - - IO.puts("Query user notifications with muted: #{to_sec(time)} sec.") + Benchee.run(%{ + "Direct messages with muted" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(params) + end, + "Direct messages without muted" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) + end + }) end def query_long_thread(user, activity) do IO.puts("\n=================================") - {time, replies} = - :timer.tc(fn -> + Benchee.run(%{ + "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end, + "Fetch context of main post" => fn -> Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( activity.data["context"], %{ "blocking_user" => user, - "user" => user + "user" => user, + "exclude_id" => activity.id } ) - end) - - IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.") + end + }) end end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index a9016b9e8..7f50ee68e 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -8,25 +8,15 @@ defmodule Pleroma.LoadTesting.Generator do end defp do_generate_users(opts) do - min = Keyword.get(opts, :users_min, 1) max = Keyword.get(opts, :users_max) - query = - "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\", - \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n" - - users = - Task.async_stream( - min..max, - &generate_user_data(&1), - max_concurrency: 10, - timeout: 30_000 - ) - |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end) - - query = query <> String.replace_trailing(users, ", \n", ";") - - Ecto.Adapters.SQL.query!(Repo, query) + Task.async_stream( + 1..max, + &generate_user_data(&1), + max_concurrency: 10, + timeout: 30_000 + ) + |> Enum.to_list() end defp generate_user_data(i) do @@ -47,11 +37,7 @@ defmodule Pleroma.LoadTesting.Generator do following: [User.ap_id(user)] } - "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{ - user.following - }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{ - user.name - }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())" + Pleroma.Repo.insert!(user) end def generate_activities(users, opts) do @@ -80,4 +66,101 @@ defmodule Pleroma.LoadTesting.Generator do Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) end + + def generate_dms(user, users, opts) do + IO.puts("Starting generating #{opts[:dms_max]} DMs") + {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end) + IO.puts("Inserting dms take #{to_sec(time)} sec.\n") + end + + defp do_generate_dms(user, users, opts) do + Task.async_stream( + 1..opts[:dms_max], + fn _ -> + do_generate_dm(user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end + + defp do_generate_dm(user, users) do + post = %{ + "status" => "@#{user.nickname} some direct message", + "visibility" => "direct" + } + + Pleroma.Web.CommonAPI.post(Enum.random(users), post) + end + + def generate_long_thread(user, users, opts) do + IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies") + {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end) + IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + {:ok, activity} + end + + defp do_generate_long_thread(user, users, opts) do + {:ok, %{id: id} = activity} = + Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"}) + + Task.async_stream( + 1..opts[:long_thread_length], + fn _ -> do_generate_thread(users, id) end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + + activity + end + + defp do_generate_thread(users, activity_id) do + Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + "status" => "reply to main post", + "in_reply_to_status_id" => activity_id + }) + end + + def generate_private_thread(users, opts) do + IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies") + {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end) + IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + end + + defp do_generate_non_visible_posts(users, opts) do + [user1, user2] = Enum.take(users, 2) + {:ok, user1} = Pleroma.User.follow(user1, user2) + {:ok, user2} = Pleroma.User.follow(user2, user1) + + {:ok, activity} = + Pleroma.Web.CommonAPI.post(user1, %{ + "status" => "Some private post", + "visibility" => "private" + }) + + {:ok, activity_public} = + Pleroma.Web.CommonAPI.post(user2, %{ + "status" => "Some public reply", + "in_reply_to_status_id" => activity.id + }) + + Task.async_stream( + 1..opts[:non_visible_posts_max], + fn _ -> do_generate_non_visible_post(users, activity_public) end, + max_concurrency: 10, + timeout: 30_000 + ) + end + + defp do_generate_non_visible_post(users, activity) do + visibility = Enum.random(["private", "public"]) + + Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + "visibility" => visibility, + "status" => "Some #{visibility} reply", + "in_reply_to_status_id" => activity.id + }) + end end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 9ed30db7e..d17cf0b3e 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -21,31 +21,38 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do - `--activities NUMBER` - number of activities to generate (default: 20000) """ - @aliases [u: :users, a: :activities, d: :delete] - @switches [users: :integer, activities: :integer, delete: :boolean] + @aliases [u: :users, a: :activities] + @switches [ + users: :integer, + activities: :integer, + dms: :integer, + thread_length: :integer, + non_visible_posts: :integer + ] @users_default 20_000 @activities_default 50_000 + @dms_default 50_000 + @thread_length_default 2_000 + @non_visible_posts_default 2_000 def run(args) do - {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) start_pleroma() + {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) - current_max = Keyword.get(opts, :users, @users_default) + users_max = Keyword.get(opts, :users, @users_default) activities_max = Keyword.get(opts, :activities, @activities_default) + dms_max = Keyword.get(opts, :dms, @dms_default) + long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default) + non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default) - {users_min, users_max} = - if opts[:delete] do - clean_tables() - {1, current_max} - else - current_count = Repo.aggregate(from(u in User), :count, :id) + 1 - {current_count, current_max + current_count} - end + clean_tables() opts = - Keyword.put(opts, :users_min, users_min) - |> Keyword.put(:users_max, users_max) + Keyword.put(opts, :users_max, users_max) |> Keyword.put(:activities_max, activities_max) + |> Keyword.put(:dms_max, dms_max) + |> Keyword.put(:long_thread_length, long_thread_length) + |> Keyword.put(:non_visible_posts_max, non_visible_posts) generate_users(opts) @@ -76,6 +83,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do generate_activities(users, Keyword.put(opts, :mention, user)) + generate_dms(user, users, opts) + + {:ok, activity} = generate_long_thread(user, users, opts) + + generate_private_thread(users, opts) + # generate_replies(user, users, activities) # activity = Enum.random(activities) @@ -86,9 +99,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + fetch_user(user) query_timelines(user) query_notifications(user) - # query_long_thread(user, activity) + query_dms(user) + query_long_thread(user, activity) + query_timelines(user) end defp clean_tables do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index eeb826814..f2b322314 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -249,6 +249,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # only accept false as false value local = !(params[:local] == false) published = params[:published] + quick_insert? = Pleroma.Config.get([:env]) == :benchmark with create_data <- make_create_data( @@ -259,12 +260,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), _ <- increase_poll_votes_if_vote(create_data), + {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, # Changing note count prior to enqueuing federation task in order to avoid # race conditions on updating user.info {:ok, _actor} <- increase_note_count_if_public(actor, activity), :ok <- maybe_federate(activity) do {:ok, activity} else + {:quick_insert, true, activity} -> + {:ok, activity} + {:fake, true, activity} -> {:ok, activity} From a1125bd5644a5819d1cfbbc8f6b3bb2dadbbc38a Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 5 Sep 2019 16:01:52 +0300 Subject: [PATCH 03/84] generatoin and fetching --- lib/load_testing/fetcher.ex | 126 +++++++++++++++- lib/load_testing/generator.ex | 203 +++++++++++++++++++------- lib/load_testing/helper.ex | 5 - lib/mix/tasks/pleroma/load_testing.ex | 49 +++---- 4 files changed, 290 insertions(+), 93 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index ed744db9b..825f921e6 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -51,10 +51,52 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end, "User mastodon public timeline" => fn -> - ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_public_timeline_params + ) end, "User mastodon federated public timeline" => fn -> - ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params) + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_federated_timeline_params + ) + end + }) + + home_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( + [user.ap_id | user.following], + home_timeline_params + ) + + public_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + + public_federated_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_federated_timeline_params + ) + + Benchee.run(%{ + "Rendering home timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: home_activities, + for: user, + as: :activity + }) + end, + "Rendering public timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: public_activities, + for: user, + as: :activity + }) + end, + "Rendering public federated timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: public_federated_activities, + for: user, + as: :activity + }) end }) end @@ -72,6 +114,27 @@ defmodule Pleroma.LoadTesting.Fetcher do Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) end }) + + without_muted_notifications = + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) + + with_muted_notifications = + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) + + Benchee.run(%{ + "Render notifications without muted" => fn -> + Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ + notifications: without_muted_notifications, + for: user + }) + end, + "Render notifications with muted" => fn -> + Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ + notifications: with_muted_notifications, + for: user + }) + end + }) end def query_dms(user) do @@ -96,13 +159,40 @@ defmodule Pleroma.LoadTesting.Fetcher do |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) end }) + + dms_with_muted = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(params) + + dms_without_muted = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) + + Benchee.run(%{ + "Rendering dms with muted" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: dms_with_muted, + for: user, + as: :activity + }) + end, + "Rendering dms without muted" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: dms_without_muted, + for: user, + as: :activity + }) + end + }) end def query_long_thread(user, activity) do IO.puts("\n=================================") Benchee.run(%{ - "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end, + "Fetch main post" => fn -> + Pleroma.Activity.get_by_id_with_object(activity.id) + end, "Fetch context of main post" => fn -> Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( activity.data["context"], @@ -114,5 +204,35 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end }) + + activity = Pleroma.Activity.get_by_id_with_object(activity.id) + + context = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( + activity.data["context"], + %{ + "blocking_user" => user, + "user" => user, + "exclude_id" => activity.id + } + ) + + Benchee.run(%{ + "Render status" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{ + activity: activity, + for: user + }) + end, + "Render context ancestors" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render( + "index.json", + for: user, + activities: context, + as: :activity + ) + |> Enum.reverse() + end + }) end end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 7f50ee68e..6df518aba 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -1,9 +1,11 @@ defmodule Pleroma.LoadTesting.Generator do use Pleroma.LoadTesting.Helper + alias Pleroma.Web.CommonAPI def generate_users(opts) do IO.puts("Starting generating #{opts[:users_max]} users...") {time, _} = :timer.tc(fn -> do_generate_users(opts) end) + IO.puts("Inserting users take #{to_sec(time)} sec.\n") end @@ -37,34 +39,111 @@ defmodule Pleroma.LoadTesting.Generator do following: [User.ap_id(user)] } - Pleroma.Repo.insert!(user) + Repo.insert!(user) end - def generate_activities(users, opts) do - IO.puts("Starting generating #{opts[:activities_max]} activities...") - {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end) - IO.puts("Inserting activities take #{to_sec(time)} sec.\n") + def generate_activities(user, users) do + do_generate_activities(user, users) end - defp do_generate_activities(users, opts) do - Task.async_stream( - 1..opts[:activities_max], - fn _ -> - do_generate_activity(users, opts) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() + defp do_generate_activities(user, users) do + IO.puts("Starting generating 20000 common activities...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..20_000, + fn _ -> + do_generate_activity([user | users]) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting common activities take #{to_sec(time)} sec.\n") + + IO.puts("Starting generating 20000 activities with mentions...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..20_000, + fn _ -> + do_generate_activity_with_mention(user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n") + + IO.puts("Starting generating 10000 activities with threads...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..10_000, + fn _ -> + do_generate_threads([user | users]) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n") end - defp do_generate_activity(users, opts) do - status = - if opts[:mention], - do: "some status with @#{opts[:mention].nickname}", - else: "some status" + defp do_generate_activity(users) do + post = %{ + "status" => "Some status without mention with random user" + } - Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) + CommonAPI.post(Enum.random(users), post) + end + + defp do_generate_activity_with_mention(user, users) do + mentions_cnt = Enum.random([2, 3, 4, 5]) + with_user = Enum.random([true, false]) + users = Enum.shuffle(users) + mentions_users = Enum.take(users, mentions_cnt) + mentions_users = if with_user, do: [user | mentions_users], else: mentions_users + + mentions_str = + Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ") + + post = %{ + "status" => mentions_str <> "some status with mentions random users" + } + + CommonAPI.post(Enum.random(users), post) + end + + defp do_generate_threads(users) do + thread_length = Enum.random([2, 3, 4, 5]) + actor = Enum.random(users) + + post = %{ + "status" => "Start of the thread" + } + + {:ok, activity} = CommonAPI.post(actor, post) + + Enum.each(1..thread_length, fn _ -> + user = Enum.random(users) + + post = %{ + "status" => "@#{actor.nickname} reply to thread", + "in_reply_to_status_id" => activity.id + } + + CommonAPI.post(user, post) + end) end def generate_dms(user, users, opts) do @@ -91,22 +170,21 @@ defmodule Pleroma.LoadTesting.Generator do "visibility" => "direct" } - Pleroma.Web.CommonAPI.post(Enum.random(users), post) + CommonAPI.post(Enum.random(users), post) end def generate_long_thread(user, users, opts) do - IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies") + IO.puts("Starting generating long thread with #{opts[:thread_length]} replies") {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end) IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") {:ok, activity} end defp do_generate_long_thread(user, users, opts) do - {:ok, %{id: id} = activity} = - Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"}) + {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"}) Task.async_stream( - 1..opts[:long_thread_length], + 1..opts[:thread_length], fn _ -> do_generate_thread(users, id) end, max_concurrency: 10, timeout: 30_000 @@ -117,50 +195,63 @@ defmodule Pleroma.LoadTesting.Generator do end defp do_generate_thread(users, activity_id) do - Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + CommonAPI.post(Enum.random(users), %{ "status" => "reply to main post", "in_reply_to_status_id" => activity_id }) end - def generate_private_thread(users, opts) do - IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies") - {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end) - IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + def generate_non_visible_message(user, users) do + IO.puts("Starting generating 1000 non visible posts") + + {time, _} = + :timer.tc(fn -> + do_generate_non_visible_posts(user, users) + end) + + IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n") end - defp do_generate_non_visible_posts(users, opts) do - [user1, user2] = Enum.take(users, 2) - {:ok, user1} = Pleroma.User.follow(user1, user2) - {:ok, user2} = Pleroma.User.follow(user2, user1) + defp do_generate_non_visible_posts(user, users) do + [not_friend | users] = users - {:ok, activity} = - Pleroma.Web.CommonAPI.post(user1, %{ - "status" => "Some private post", - "visibility" => "private" - }) + make_friends(user, users) - {:ok, activity_public} = - Pleroma.Web.CommonAPI.post(user2, %{ - "status" => "Some public reply", - "in_reply_to_status_id" => activity.id - }) - - Task.async_stream( - 1..opts[:non_visible_posts_max], - fn _ -> do_generate_non_visible_post(users, activity_public) end, + Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end, max_concurrency: 10, timeout: 30_000 ) + |> Stream.run() end - defp do_generate_non_visible_post(users, activity) do - visibility = Enum.random(["private", "public"]) + defp make_friends(_user, []), do: nil - Pleroma.Web.CommonAPI.post(Enum.random(users), %{ - "visibility" => visibility, - "status" => "Some #{visibility} reply", - "in_reply_to_status_id" => activity.id - }) + defp make_friends(user, [friend | users]) do + {:ok, _} = User.follow(user, friend) + {:ok, _} = User.follow(friend, user) + make_friends(user, users) + end + + defp do_generate_non_visible_post(not_friend, users) do + post = %{ + "status" => "some non visible post", + "visibility" => "private" + } + + {:ok, activity} = CommonAPI.post(not_friend, post) + + thread_length = Enum.random([2, 3, 4, 5]) + + Enum.each(1..thread_length, fn _ -> + user = Enum.random(users) + + post = %{ + "status" => "@#{not_friend.nickname} reply to non visible post", + "in_reply_to_status_id" => activity.id, + "visibility" => "private" + } + + CommonAPI.post(user, post) + end) end end diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex index 338dba323..47b25c65f 100644 --- a/lib/load_testing/helper.ex +++ b/lib/load_testing/helper.ex @@ -2,13 +2,8 @@ defmodule Pleroma.LoadTesting.Helper do defmacro __using__(_) do quote do import Ecto.Query - alias Pleroma.Activity - alias Pleroma.Notification - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.ActivityPub - alias Pleroma.Web.CommonAPI defp to_sec(microseconds), do: microseconds / 1_000_000 end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index d17cf0b3e..83e531abf 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -13,46 +13,37 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do - activities with notifications ## Generate data - MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000 - MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000 + MIX_ENV=benchmark mix pleroma.load_testing --users 10000 + MIX_ENV=benchmark mix pleroma.load_testing -u 10000 Options: - `--users NUMBER` - number of users to generate (default: 10000) - - `--activities NUMBER` - number of activities to generate (default: 20000) """ - @aliases [u: :users, a: :activities] + @aliases [u: :users, d: :dms, t: :thread_length] @switches [ users: :integer, - activities: :integer, dms: :integer, - thread_length: :integer, - non_visible_posts: :integer + thread_length: :integer ] @users_default 20_000 - @activities_default 50_000 - @dms_default 50_000 + @dms_default 20_000 @thread_length_default 2_000 - @non_visible_posts_default 2_000 def run(args) do start_pleroma() {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) users_max = Keyword.get(opts, :users, @users_default) - activities_max = Keyword.get(opts, :activities, @activities_default) dms_max = Keyword.get(opts, :dms, @dms_default) - long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default) - non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default) + thread_length = Keyword.get(opts, :thread_length, @thread_length_default) clean_tables() opts = Keyword.put(opts, :users_max, users_max) - |> Keyword.put(:activities_max, activities_max) |> Keyword.put(:dms_max, dms_max) - |> Keyword.put(:long_thread_length, long_thread_length) - |> Keyword.put(:non_visible_posts_max, non_visible_posts) + |> Keyword.put(:thread_length, thread_length) generate_users(opts) @@ -60,7 +51,9 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do IO.puts("Fetching main user...") {time, user} = - :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end) + :timer.tc(fn -> + Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) + end) IO.puts("Fetching main user take #{to_sec(time)} sec.\n") @@ -79,25 +72,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do IO.puts("Fetching users take #{to_sec(time)} sec.\n") - generate_activities(users, opts) - - generate_activities(users, Keyword.put(opts, :mention, user)) + generate_activities(user, users) generate_dms(user, users, opts) {:ok, activity} = generate_long_thread(user, users, opts) - generate_private_thread(users, opts) - - # generate_replies(user, users, activities) - - # activity = Enum.random(activities) - # generate_long_thread(user, users, activity) + generate_non_visible_message(user, users) IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}") - IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}") - IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") - IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + + IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}") + + IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}") + + IO.puts( + "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}" + ) fetch_user(user) query_timelines(user) From 252e5db45c5d9d63116e5a24cc35c26f91ec7de4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 16:37:18 +0300 Subject: [PATCH 04/84] docs fixes --- lib/load_testing/fetcher.ex | 9 --------- lib/load_testing/generator.ex | 3 ++- lib/mix/tasks/pleroma/load_testing.ex | 16 +++++++++++----- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index 825f921e6..b92b6e04f 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -2,8 +2,6 @@ defmodule Pleroma.LoadTesting.Fetcher do use Pleroma.LoadTesting.Helper def fetch_user(user) do - IO.puts("=================================") - Benchee.run(%{ "By id" => fn -> Repo.get_by(User, id: user.id) end, "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, @@ -13,8 +11,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_timelines(user) do - IO.puts("\n=================================") - home_timeline_params = %{ "count" => 20, "with_muted" => true, @@ -102,7 +98,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_notifications(user) do - IO.puts("\n=================================") without_muted_params = %{"count" => "20", "with_muted" => "false"} with_muted_params = %{"count" => "20", "with_muted" => "true"} @@ -138,8 +133,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_dms(user) do - IO.puts("\n=================================") - params = %{ "count" => "20", "with_muted" => "true", @@ -187,8 +180,6 @@ defmodule Pleroma.LoadTesting.Fetcher do end def query_long_thread(user, activity) do - IO.puts("\n=================================") - Benchee.run(%{ "Fetch main post" => fn -> Pleroma.Activity.get_by_id_with_object(activity.id) diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 6df518aba..7ad68dcc1 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do nickname: "nick#{i}", password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), bio: "Tester Number #{i}", - info: %{} + info: %{}, + local: Enum.random([true, false]) } user = %{ diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 83e531abf..4fed6bf74 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -5,19 +5,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do import Pleroma.LoadTesting.Generator import Pleroma.LoadTesting.Fetcher - # tODO: remove autovacuum worker until generation is not ended @shortdoc "Factory for generation data" @moduledoc """ Generates data like: - users - activities with notifications + - direct messages + - long thread + - non visible posts ## Generate data - MIX_ENV=benchmark mix pleroma.load_testing --users 10000 - MIX_ENV=benchmark mix pleroma.load_testing -u 10000 + MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000 + MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000 Options: - - `--users NUMBER` - number of users to generate (default: 10000) + - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u` + - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d` + - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t` """ @aliases [u: :users, d: :dms, t: :thread_length] @@ -32,6 +36,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do def run(args) do start_pleroma() + Pleroma.Config.put([:instance, :skip_thread_containment], true) {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) users_max = Keyword.get(opts, :users, @users_default) @@ -95,11 +100,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do query_notifications(user) query_dms(user) query_long_thread(user, activity) + Pleroma.Config.put([:instance, :skip_thread_containment], false) query_timelines(user) end defp clean_tables do - IO.puts("\n\nDeleting old data...\n") + IO.puts("Deleting old data...\n") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") From b3f6f6a4091c048813e367f6a9414eba6fce5109 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 20:14:02 +0300 Subject: [PATCH 05/84] generating remote users --- lib/load_testing/fetcher.ex | 2 +- lib/load_testing/generator.ex | 34 ++++++++++++++++++++++++++-------- lib/pleroma/user.ex | 4 +++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index b92b6e04f..0ff2f28c0 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -215,7 +215,7 @@ defmodule Pleroma.LoadTesting.Fetcher do for: user }) end, - "Render context ancestors" => fn -> + "Render context" => fn -> Pleroma.Web.MastodonAPI.StatusView.render( "index.json", for: user, diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 7ad68dcc1..61e3d8686 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -22,6 +22,8 @@ defmodule Pleroma.LoadTesting.Generator do end defp generate_user_data(i) do + remote = Enum.random([true, false]) + user = %User{ name: "Test テスト User #{i}", email: "user#{i}@example.com", @@ -29,16 +31,32 @@ defmodule Pleroma.LoadTesting.Generator do password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), bio: "Tester Number #{i}", info: %{}, - local: Enum.random([true, false]) + local: remote } - user = %{ - user - | ap_id: User.ap_id(user), - follower_address: User.ap_followers(user), - following_address: User.ap_following(user), - following: [User.ap_id(user)] - } + user_urls = + if remote do + base_url = + Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"]) + + ap_id = "#{base_url}/users/#{user.nickname}" + + %{ + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + following: [ap_id] + } + else + %{ + ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user), + following: [User.ap_id(user)] + } + end + + user = Map.merge(user, user_urls) Repo.insert!(user) end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3aa245f2a..37e59a651 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -429,7 +429,9 @@ defmodule Pleroma.User do {:error, "Could not follow user: #{followed.nickname} blocked you."} true -> - if !followed.local && follower.local && !ap_enabled?(followed) do + benchmark? = Pleroma.Config.get([:env]) == :benchmark + + if !followed.local && follower.local && !ap_enabled?(followed) && !benchmark? do Websub.subscribe(follower, followed) end From 924d7e6aa60095e27c15b4a5f473a192ab2b42ef Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 19 Sep 2019 12:59:36 +0300 Subject: [PATCH 06/84] generating remote activities --- lib/load_testing/generator.ex | 78 ++++++++++++++++++++++++++- lib/mix/tasks/pleroma/load_testing.ex | 29 ++++++++-- 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 61e3d8686..5c5a5c122 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do name: "Test テスト User #{i}", email: "user#{i}@example.com", nickname: "nick#{i}", - password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + password_hash: + "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg", bio: "Tester Number #{i}", info: %{}, local: remote @@ -165,6 +166,81 @@ defmodule Pleroma.LoadTesting.Generator do end) end + def generate_remote_activities(user, users) do + do_generate_remote_activities(user, users) + end + + defp do_generate_remote_activities(user, users) do + IO.puts("Starting generating 10000 remote activities...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..10_000, + fn i -> + do_generate_remote_activity(i, user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n") + end + + defp do_generate_remote_activity(i, user, users) do + actor = Enum.random(users) + %{host: host} = URI.parse(actor.ap_id) + date = Date.utc_today() + datetime = DateTime.utc_now() + + map = %{ + "actor" => actor.ap_id, + "cc" => [actor.follower_address, user.ap_id], + "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "id" => actor.ap_id <> "/statuses/#{i}/activity", + "object" => %{ + "actor" => actor.ap_id, + "atomUri" => actor.ap_id <> "/statuses/#{i}", + "attachment" => [], + "attributedTo" => actor.ap_id, + "bcc" => [], + "bto" => [], + "cc" => [actor.follower_address, user.ap_id], + "content" => + "

+ user.ap_id <> + "\" class=\"u-url mention\">@" <> user.nickname <> "

", + "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "conversation" => + "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "emoji" => %{}, + "id" => actor.ap_id <> "/statuses/#{i}", + "inReplyTo" => nil, + "inReplyToAtomUri" => nil, + "published" => datetime, + "sensitive" => true, + "summary" => "cw", + "tag" => [ + %{ + "href" => user.ap_id, + "name" => "@#{user.nickname}@#{host}", + "type" => "Mention" + } + ], + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Note", + "url" => "http://#{host}/@#{actor.nickname}/#{i}" + }, + "published" => datetime, + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create" + } + + Pleroma.Web.ActivityPub.ActivityPub.insert(map, false) + end + def generate_dms(user, users, opts) do IO.puts("Starting generating #{opts[:dms_max]} DMs") {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end) diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 4fed6bf74..a7057395d 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -53,32 +53,53 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do generate_users(opts) # main user for queries - IO.puts("Fetching main user...") + IO.puts("Fetching local main user...") {time, user} = :timer.tc(fn -> - Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) + Repo.one( + from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1) + ) end) IO.puts("Fetching main user take #{to_sec(time)} sec.\n") - IO.puts("Fetching users...") + IO.puts("Fetching local users...") {time, users} = :timer.tc(fn -> Repo.all( from(u in User, where: u.id != ^user.id, + where: u.local == true, order_by: fragment("RANDOM()"), limit: 10 ) ) end) - IO.puts("Fetching users take #{to_sec(time)} sec.\n") + IO.puts("Fetching local users take #{to_sec(time)} sec.\n") + + IO.puts("Fetching remote users...") + + {time, remote_users} = + :timer.tc(fn -> + Repo.all( + from(u in User, + where: u.id != ^user.id, + where: u.local == false, + order_by: fragment("RANDOM()"), + limit: 10 + ) + ) + end) + + IO.puts("Fetching remote users take #{to_sec(time)} sec.\n") generate_activities(user, users) + generate_remote_activities(user, remote_users) + generate_dms(user, users, opts) {:ok, activity} = generate_long_thread(user, users, opts) From 1d285e6fada841b6959b6516aca1c84d1a2c9b93 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 19 Sep 2019 14:02:27 +0300 Subject: [PATCH 07/84] moving to separate dir --- {lib => benchmarks}/load_testing/fetcher.ex | 0 {lib => benchmarks}/load_testing/generator.ex | 0 {lib => benchmarks}/load_testing/helper.ex | 0 {lib => benchmarks}/mix/tasks/pleroma/load_testing.ex | 4 ++-- mix.exs | 1 + 5 files changed, 3 insertions(+), 2 deletions(-) rename {lib => benchmarks}/load_testing/fetcher.ex (100%) rename {lib => benchmarks}/load_testing/generator.ex (100%) rename {lib => benchmarks}/load_testing/helper.ex (100%) rename {lib => benchmarks}/mix/tasks/pleroma/load_testing.ex (98%) diff --git a/lib/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex similarity index 100% rename from lib/load_testing/fetcher.ex rename to benchmarks/load_testing/fetcher.ex diff --git a/lib/load_testing/generator.ex b/benchmarks/load_testing/generator.ex similarity index 100% rename from lib/load_testing/generator.ex rename to benchmarks/load_testing/generator.ex diff --git a/lib/load_testing/helper.ex b/benchmarks/load_testing/helper.ex similarity index 100% rename from lib/load_testing/helper.ex rename to benchmarks/load_testing/helper.ex diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex similarity index 98% rename from lib/mix/tasks/pleroma/load_testing.ex rename to benchmarks/mix/tasks/pleroma/load_testing.ex index a7057395d..7b2293acc 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -8,8 +8,8 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do @shortdoc "Factory for generation data" @moduledoc """ Generates data like: - - users - - activities with notifications + - local/remote users + - local/remote activities with notifications - direct messages - long thread - non visible posts diff --git a/mix.exs b/mix.exs index 3170d6f2d..b0f2c4afb 100644 --- a/mix.exs +++ b/mix.exs @@ -69,6 +69,7 @@ defmodule Pleroma.Mixfile do end # Specifies which paths to compile per environment. + defp elixirc_paths(:benchmark), do: ["lib", "benchmarks"] defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] From 32808208bef12996f819d21ca2635837aff98cbf Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 8 Oct 2019 15:16:39 +0300 Subject: [PATCH 08/84] Enable `mix format` for migrations --- .formatter.exs | 2 +- .../migrations/20170320193800_create_user.exs | 11 ++- .../20170321074828_create_activity.exs | 5 +- .../20170321074832_create_object.exs | 3 +- ...0321133335_add_following_list_to_users.exs | 2 +- .../20170321143152_add_ap_id_to_users.exs | 2 +- .../20170330153447_add_index_to_objects.exs | 2 +- ...add_unique_index_to_email_and_nickname.exs | 4 +- ...70416122418_add_avatar_object_to_users.exs | 2 +- ...143_create_webssub_server_subscription.exs | 10 +-- .../20170423154511_add_fields_to_users.exs | 4 +- ...4155_create_websub_client_subscription.exs | 10 +-- .../20170427054757_add_user_and_hub.exs | 4 +- ...nts_to_activities_and_objects_part_two.exs | 14 ++- ...02083023_add_local_field_to_activities.exs | 4 +- ...0170506222027_add_unique_index_to_apid.exs | 2 +- .../migrations/20170529093232_longer_bios.exs | 5 +- ...20170620095947_remove_activities_index.exs | 2 +- ...420_add_object_activity_index_part_two.exs | 13 ++- ...0701142005_add_actor_index_to_activity.exs | 4 +- ...719152213_add_follower_address_to_user.exs | 4 +- .../20170906120646_add_mastodon_apps.exs | 12 +-- ...906143140_create_o_auth_authorizations.exs | 10 +-- .../20170906152508_create_o_auth_token.exs | 10 +-- .../20170911123607_create_notifications.exs | 8 +- .../20170912114248_add_context_index.exs | 7 +- ...0916090107_add_fts_index_to_activities.exs | 8 +- .../20170917120416_add_tag_index.exs | 8 +- ...019141706_create_password_reset_tokens.exs | 6 +- ...035_add_second_object_index_to_activty.exs | 14 ++- .../20171024090137_drop_object_index.exs | 2 +- .../20171024121413_add_object_actor_index.exs | 7 +- .../20171109091239_add_actor_to_activity.exs | 9 +- .../20171109114020_fill_actor_field.exs | 9 +- ...109141309_add_sort_index_to_activities.exs | 2 +- ...20171130135819_add_local_index_to_user.exs | 2 +- ...212163643_add_recipients_to_activities.exs | 4 +- ...12164525_fill_recipients_in_activities.exs | 12 ++- ...21210540_make_following_postgres_array.exs | 11 +-- ...51_add_follower_address_index_to_users.exs | 4 +- ...7174350_drop_local_index_on_activities.exs | 2 +- ...180327175831_actually_drop_local_index.exs | 4 +- .../20180429094642_create_lists.exs | 8 +- .../20180513104714_modify_activity_index.exs | 4 +- .../20180516144508_add_trigram_extension.exs | 11 ++- ...180516154905_create_user_trigram_index.exs | 4 +- .../20180530123448_add_list_follow_index.exs | 2 +- ...3637_create_apid_host_extraction_index.exs | 7 +- ...180612110515_create_user_invite_tokens.exs | 4 +- ...40_create_activities_in_reply_to_index.exs | 7 +- .../20180813003722_create_filters.exs | 21 +++-- ...ipients_to_and_cc_fields_to_activities.exs | 8 +- ...829182612_activities_add_to_cc_indices.exs | 9 +- ...ients_to_and_cc_fields_from_activities.exs | 8 +- ...903114437_users_add_is_moderator_index.exs | 4 +- ...180918182427_create_push_subscriptions.exs | 14 +-- ...0919060348_users_add_last_refreshed_at.exs | 2 +- .../20181206125616_add_tags_to_users.exs | 4 +- .../20181214121049_add_bookmarks_to_users.exs | 2 +- ...18172826_users_and_activities_flake_id.exs | 85 +++++++++++-------- ...20190109152453_add_visibility_function.exs | 4 +- .../20190115085500_create_user_fts_index.exs | 24 +++--- ...90122153157_update_activity_visibility.exs | 2 - ...0190123092341_users_add_is_admin_index.exs | 4 +- .../20190123125546_create_instances.exs | 8 +- ...31141_update_activity_visibility_again.exs | 3 - ...90127151220_add_activities_likes_index.exs | 8 +- .../20190203185340_split_hide_network.exs | 14 ++- .../20190205114625_create_thread_mutes.exs | 6 +- ...08131753_add_scopes_to_o_auth_entities.exs | 2 +- ...03_change_apps_scopes_to_varchar_array.exs | 18 ++-- ..._data_migration_populate_o_auth_scopes.exs | 2 +- ...104808_data_migration_normalize_scopes.exs | 2 +- ...0190301101154_add_default_tags_to_user.exs | 2 +- ...190303120636_update_user_note_counters.exs | 8 +- .../20190315101315_create_registrations.exs | 14 +-- ...325185009_create_notification_id_index.exs | 2 +- ...0190405160700_add_index_on_subscribers.exs | 8 +- .../20190408123347_create_conversations.exs | 6 +- ...859_add_participation_updated_at_index.exs | 2 +- ...120_add_index_on_user_info_deactivated.exs | 4 +- .../20190414125034_migrate_old_bookmarks.exs | 4 +- ...0190501125843_add_fts_index_to_objects.exs | 15 +++- ...511191044_set_default_state_to_reports.exs | 8 +- ...809_change_hide_column_in_filter_table.exs | 4 +- ...603162018_add_object_in_reply_to_index.exs | 2 +- ...0190603173419_add_tag_index_to_objects.exs | 7 +- ...2024_copy_muted_to_muted_notifications.exs | 4 +- ...ersation_participation_recipient_ships.exs | 4 +- ...90823000549_add_likes_index_to_objects.exs | 2 +- .../20190912065617_create_deliveries.exs | 3 +- ...0929201536_drop_subscription_if_exists.exs | 2 +- .../20191006123824_add_keys_column.exs | 2 +- ...006135457_move_keys_to_separate_column.exs | 5 +- 94 files changed, 405 insertions(+), 264 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index 2bed17cc0..7fa95a619 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,3 +1,3 @@ [ - inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"] + inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}", "priv/repo/migrations/*.exs"] ] diff --git a/priv/repo/migrations/20170320193800_create_user.exs b/priv/repo/migrations/20170320193800_create_user.exs index 089964a26..e5f6ac52e 100644 --- a/priv/repo/migrations/20170320193800_create_user.exs +++ b/priv/repo/migrations/20170320193800_create_user.exs @@ -3,14 +3,13 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.User do def change do create_if_not_exists table(:users) do - add :email, :string - add :password_hash, :string - add :name, :string - add :nickname, :string - add :bio, :string + add(:email, :string) + add(:password_hash, :string) + add(:name, :string) + add(:nickname, :string) + add(:bio, :string) timestamps() end - end end diff --git a/priv/repo/migrations/20170321074828_create_activity.exs b/priv/repo/migrations/20170321074828_create_activity.exs index f5c872721..ab00a47c3 100644 --- a/priv/repo/migrations/20170321074828_create_activity.exs +++ b/priv/repo/migrations/20170321074828_create_activity.exs @@ -3,12 +3,11 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.Activity do def change do create_if_not_exists table(:activities) do - add :data, :map + add(:data, :map) timestamps() end - create_if_not_exists index(:activities, [:data], using: :gin) - + create_if_not_exists(index(:activities, [:data], using: :gin)) end end diff --git a/priv/repo/migrations/20170321074832_create_object.exs b/priv/repo/migrations/20170321074832_create_object.exs index b184672ad..c5a59ecba 100644 --- a/priv/repo/migrations/20170321074832_create_object.exs +++ b/priv/repo/migrations/20170321074832_create_object.exs @@ -3,10 +3,9 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.Object do def change do create_if_not_exists table(:objects) do - add :data, :map + add(:data, :map) timestamps() end - end end diff --git a/priv/repo/migrations/20170321133335_add_following_list_to_users.exs b/priv/repo/migrations/20170321133335_add_following_list_to_users.exs index b02c1272c..8dd83c3f8 100644 --- a/priv/repo/migrations/20170321133335_add_following_list_to_users.exs +++ b/priv/repo/migrations/20170321133335_add_following_list_to_users.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.AddFollowingListToUsers do def change do alter table(:users) do - add :following, :map + add(:following, :map) end end end diff --git a/priv/repo/migrations/20170321143152_add_ap_id_to_users.exs b/priv/repo/migrations/20170321143152_add_ap_id_to_users.exs index e6f0366b5..38ceb87fd 100644 --- a/priv/repo/migrations/20170321143152_add_ap_id_to_users.exs +++ b/priv/repo/migrations/20170321143152_add_ap_id_to_users.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.AddApIdToUsers do def change do alter table(:users) do - add :ap_id, :string + add(:ap_id, :string) end end end diff --git a/priv/repo/migrations/20170330153447_add_index_to_objects.exs b/priv/repo/migrations/20170330153447_add_index_to_objects.exs index 25e308533..93198b462 100644 --- a/priv/repo/migrations/20170330153447_add_index_to_objects.exs +++ b/priv/repo/migrations/20170330153447_add_index_to_objects.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexToObjects do use Ecto.Migration def change do - create_if_not_exists index(:objects, [:data], using: :gin) + create_if_not_exists(index(:objects, [:data], using: :gin)) end end diff --git a/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs b/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs index 42da88954..b18c67dcb 100644 --- a/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs +++ b/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddUniqueIndexToEmailAndNickname do use Ecto.Migration def change do - create_if_not_exists unique_index(:users, [:email]) - create_if_not_exists unique_index(:users, [:nickname]) + create_if_not_exists(unique_index(:users, [:email])) + create_if_not_exists(unique_index(:users, [:nickname])) end end diff --git a/priv/repo/migrations/20170416122418_add_avatar_object_to_users.exs b/priv/repo/migrations/20170416122418_add_avatar_object_to_users.exs index b6d8742dc..e88752c30 100644 --- a/priv/repo/migrations/20170416122418_add_avatar_object_to_users.exs +++ b/priv/repo/migrations/20170416122418_add_avatar_object_to_users.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.AddAvatarObjectToUsers do def change do alter table(:users) do - add :avatar, :map + add(:avatar, :map) end end end diff --git a/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs b/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs index 243280378..3d94e4ee7 100644 --- a/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs +++ b/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs @@ -3,11 +3,11 @@ defmodule Pleroma.Repo.Migrations.CreateWebsubServerSubscription do def change do create_if_not_exists table(:websub_server_subscriptions) do - add :topic, :string - add :callback, :string - add :secret, :string - add :valid_until, :naive_datetime - add :state, :string + add(:topic, :string) + add(:callback, :string) + add(:secret, :string) + add(:valid_until, :naive_datetime) + add(:state, :string) timestamps() end diff --git a/priv/repo/migrations/20170423154511_add_fields_to_users.exs b/priv/repo/migrations/20170423154511_add_fields_to_users.exs index 84de74bc4..a079c73bd 100644 --- a/priv/repo/migrations/20170423154511_add_fields_to_users.exs +++ b/priv/repo/migrations/20170423154511_add_fields_to_users.exs @@ -3,8 +3,8 @@ defmodule Pleroma.Repo.Migrations.AddFieldsToUsers do def change do alter table(:users) do - add :local, :boolean, default: true - add :info, :map + add(:local, :boolean, default: true) + add(:info, :map) end end end diff --git a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs index 4b79d7506..d020614e1 100644 --- a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs +++ b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs @@ -3,11 +3,11 @@ defmodule Pleroma.Repo.Migrations.CreateWebsubClientSubscription do def change do create_if_not_exists table(:websub_client_subscriptions) do - add :topic, :string - add :secret, :string - add :valid_until, :naive_datetime_usec - add :state, :string - add :subscribers, :map + add(:topic, :string) + add(:secret, :string) + add(:valid_until, :naive_datetime_usec) + add(:state, :string) + add(:subscribers, :map) timestamps() end diff --git a/priv/repo/migrations/20170427054757_add_user_and_hub.exs b/priv/repo/migrations/20170427054757_add_user_and_hub.exs index 4f9a520bd..f33a8572f 100644 --- a/priv/repo/migrations/20170427054757_add_user_and_hub.exs +++ b/priv/repo/migrations/20170427054757_add_user_and_hub.exs @@ -3,8 +3,8 @@ defmodule Pleroma.Repo.Migrations.AddUserAndHub do def change do alter table(:websub_client_subscriptions) do - add :hub, :string - add :user_id, references(:users) + add(:hub, :string) + add(:user_id, references(:users)) end end end diff --git a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs index f5e5cd269..ecc7c23cc 100644 --- a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs +++ b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs @@ -2,10 +2,16 @@ defmodule Pleroma.Repo.Migrations.AddIdContraintsToActivitiesAndObjectsPartTwo d use Ecto.Migration def up do - drop_if_exists index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index) - drop_if_exists index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index) - create_if_not_exists unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) - create_if_not_exists unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) + drop_if_exists(index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index)) + drop_if_exists(index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index)) + + create_if_not_exists( + unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) + ) + + create_if_not_exists( + unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) + ) end def down, do: :ok diff --git a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs index cebc11d21..6b61bd464 100644 --- a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs +++ b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs @@ -3,9 +3,9 @@ defmodule Pleroma.Repo.Migrations.AddLocalFieldToActivities do def change do alter table(:activities) do - add :local, :boolean, default: true + add(:local, :boolean, default: true) end - create_if_not_exists index(:activities, [:local]) + create_if_not_exists(index(:activities, [:local])) end end diff --git a/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs index 1b7e33b70..80f50029a 100644 --- a/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs +++ b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAPID do use Ecto.Migration def change do - create_if_not_exists unique_index(:users, [:ap_id]) + create_if_not_exists(unique_index(:users, [:ap_id])) end end diff --git a/priv/repo/migrations/20170529093232_longer_bios.exs b/priv/repo/migrations/20170529093232_longer_bios.exs index 9188f4bee..e25e5e144 100644 --- a/priv/repo/migrations/20170529093232_longer_bios.exs +++ b/priv/repo/migrations/20170529093232_longer_bios.exs @@ -3,14 +3,13 @@ defmodule Pleroma.Repo.Migrations.LongerBios do def up do alter table(:users) do - modify :bio, :text + modify(:bio, :text) end end def down do alter table(:users) do - modify :bio, :string + modify(:bio, :string) end end - end diff --git a/priv/repo/migrations/20170620095947_remove_activities_index.exs b/priv/repo/migrations/20170620095947_remove_activities_index.exs index e7d41eac4..ea3d4a509 100644 --- a/priv/repo/migrations/20170620095947_remove_activities_index.exs +++ b/priv/repo/migrations/20170620095947_remove_activities_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.RemoveActivitiesIndex do use Ecto.Migration def change do - drop_if_exists index(:activities, [:data]) + drop_if_exists(index(:activities, [:data])) end end diff --git a/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs b/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs index c95218fad..c015afbe1 100644 --- a/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs +++ b/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs @@ -2,7 +2,16 @@ defmodule Pleroma.Repo.Migrations.AddObjectActivityIndexPartTwo do use Ecto.Migration def change do - drop_if_exists index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) - create_if_not_exists index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) + drop_if_exists( + index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], + name: :activities_create_objects_index + ) + ) + + create_if_not_exists( + index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], + name: :activities_create_objects_index + ) + ) end end diff --git a/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs b/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs index 807fe3728..220c48abd 100644 --- a/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs +++ b/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs @@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.AddActorIndexToActivity do use Ecto.Migration def change do - create_if_not_exists index(:activities, ["(data->>'actor')", "inserted_at desc"], name: :activities_actor_index) + create_if_not_exists( + index(:activities, ["(data->>'actor')", "inserted_at desc"], name: :activities_actor_index) + ) end end diff --git a/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs b/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs index 591164be5..be5eca36c 100644 --- a/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs +++ b/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs @@ -3,13 +3,13 @@ defmodule Pleroma.Repo.Migrations.AddFollowerAddressToUser do def up do alter table(:users) do - add :follower_address, :string, unique: true + add(:follower_address, :string, unique: true) end end def down do alter table(:users) do - remove :follower_address + remove(:follower_address) end end end diff --git a/priv/repo/migrations/20170906120646_add_mastodon_apps.exs b/priv/repo/migrations/20170906120646_add_mastodon_apps.exs index ccd5e3fe2..0e01625ff 100644 --- a/priv/repo/migrations/20170906120646_add_mastodon_apps.exs +++ b/priv/repo/migrations/20170906120646_add_mastodon_apps.exs @@ -3,12 +3,12 @@ defmodule Pleroma.Repo.Migrations.AddMastodonApps do def change do create_if_not_exists table(:apps) do - add :client_name, :string - add :redirect_uris, :string - add :scopes, :string - add :website, :string - add :client_id, :string - add :client_secret, :string + add(:client_name, :string) + add(:redirect_uris, :string) + add(:scopes, :string) + add(:website, :string) + add(:client_id, :string) + add(:client_secret, :string) timestamps() end diff --git a/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs b/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs index 63b25c537..9af8315a8 100644 --- a/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs +++ b/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs @@ -3,11 +3,11 @@ defmodule Pleroma.Repo.Migrations.CreateOAuthAuthorizations do def change do create_if_not_exists table(:oauth_authorizations) do - add :app_id, references(:apps) - add :user_id, references(:users) - add :token, :string - add :valid_until, :naive_datetime_usec - add :used, :boolean, default: false + add(:app_id, references(:apps)) + add(:user_id, references(:users)) + add(:token, :string) + add(:valid_until, :naive_datetime_usec) + add(:used, :boolean, default: false) timestamps() end diff --git a/priv/repo/migrations/20170906152508_create_o_auth_token.exs b/priv/repo/migrations/20170906152508_create_o_auth_token.exs index 08471bbf8..bfad98b76 100644 --- a/priv/repo/migrations/20170906152508_create_o_auth_token.exs +++ b/priv/repo/migrations/20170906152508_create_o_auth_token.exs @@ -3,11 +3,11 @@ defmodule Pleroma.Repo.Migrations.CreateOAuthToken do def change do create_if_not_exists table(:oauth_tokens) do - add :app_id, references(:apps) - add :user_id, references(:users) - add :token, :string - add :refresh_token, :string - add :valid_until, :naive_datetime_usec + add(:app_id, references(:apps)) + add(:user_id, references(:users)) + add(:token, :string) + add(:refresh_token, :string) + add(:valid_until, :naive_datetime_usec) timestamps() end diff --git a/priv/repo/migrations/20170911123607_create_notifications.exs b/priv/repo/migrations/20170911123607_create_notifications.exs index 50de9c5f1..36facc5a0 100644 --- a/priv/repo/migrations/20170911123607_create_notifications.exs +++ b/priv/repo/migrations/20170911123607_create_notifications.exs @@ -3,13 +3,13 @@ defmodule Pleroma.Repo.Migrations.CreateNotifications do def change do create_if_not_exists table(:notifications) do - add :user_id, references(:users, on_delete: :delete_all) - add :activity_id, references(:activities, on_delete: :delete_all) - add :seen, :boolean, default: false + add(:user_id, references(:users, on_delete: :delete_all)) + add(:activity_id, references(:activities, on_delete: :delete_all)) + add(:seen, :boolean, default: false) timestamps() end - create_if_not_exists index(:notifications, [:user_id]) + create_if_not_exists(index(:notifications, [:user_id])) end end diff --git a/priv/repo/migrations/20170912114248_add_context_index.exs b/priv/repo/migrations/20170912114248_add_context_index.exs index 83c585136..400a432ff 100644 --- a/priv/repo/migrations/20170912114248_add_context_index.exs +++ b/priv/repo/migrations/20170912114248_add_context_index.exs @@ -3,6 +3,11 @@ defmodule Pleroma.Repo.Migrations.AddContextIndex do @disable_ddl_transaction true def change do - create index(:activities, ["(data->>'type')", "(data->>'context')"], name: :activities_context_index, concurrently: true) + create( + index(:activities, ["(data->>'type')", "(data->>'context')"], + name: :activities_context_index, + concurrently: true + ) + ) end end diff --git a/priv/repo/migrations/20170916090107_add_fts_index_to_activities.exs b/priv/repo/migrations/20170916090107_add_fts_index_to_activities.exs index c17da8309..717e25412 100644 --- a/priv/repo/migrations/20170916090107_add_fts_index_to_activities.exs +++ b/priv/repo/migrations/20170916090107_add_fts_index_to_activities.exs @@ -3,6 +3,12 @@ defmodule Pleroma.Repo.Migrations.AddFTSIndexToActivities do @disable_ddl_transaction true def change do - create index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], concurrently: true, using: :gin, name: :activities_fts) + create( + index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], + concurrently: true, + using: :gin, + name: :activities_fts + ) + ) end end diff --git a/priv/repo/migrations/20170917120416_add_tag_index.exs b/priv/repo/migrations/20170917120416_add_tag_index.exs index d9391dda9..c69e0ef8f 100644 --- a/priv/repo/migrations/20170917120416_add_tag_index.exs +++ b/priv/repo/migrations/20170917120416_add_tag_index.exs @@ -4,6 +4,12 @@ defmodule Pleroma.Repo.Migrations.AddTagIndex do @disable_ddl_transaction true def change do - create index(:activities, ["(data #> '{\"object\",\"tag\"}')"], concurrently: true, using: :gin, name: :activities_tags) + create( + index(:activities, ["(data #> '{\"object\",\"tag\"}')"], + concurrently: true, + using: :gin, + name: :activities_tags + ) + ) end end diff --git a/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs b/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs index dde0f945f..2be50d15e 100644 --- a/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs +++ b/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs @@ -3,9 +3,9 @@ defmodule Pleroma.Repo.Migrations.CreatePasswordResetTokens do def change do create_if_not_exists table(:password_reset_tokens) do - add :token, :string - add :user_id, references(:users) - add :used, :boolean, default: false + add(:token, :string) + add(:user_id, references(:users)) + add(:used, :boolean, default: false) timestamps() end diff --git a/priv/repo/migrations/20171023155035_add_second_object_index_to_activty.exs b/priv/repo/migrations/20171023155035_add_second_object_index_to_activty.exs index c6df53ec9..261940a96 100644 --- a/priv/repo/migrations/20171023155035_add_second_object_index_to_activty.exs +++ b/priv/repo/migrations/20171023155035_add_second_object_index_to_activty.exs @@ -4,7 +4,17 @@ defmodule Pleroma.Repo.Migrations.AddSecondObjectIndexToActivty do @disable_ddl_transaction true def change do - drop_if_exists index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) - create index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], name: :activities_create_objects_index, concurrently: true) + drop_if_exists( + index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], + name: :activities_create_objects_index + ) + ) + + create( + index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :activities_create_objects_index, + concurrently: true + ) + ) end end diff --git a/priv/repo/migrations/20171024090137_drop_object_index.exs b/priv/repo/migrations/20171024090137_drop_object_index.exs index 29b4c9333..d417577ae 100644 --- a/priv/repo/migrations/20171024090137_drop_object_index.exs +++ b/priv/repo/migrations/20171024090137_drop_object_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.DropObjectIndex do use Ecto.Migration def change do - drop_if_exists index(:objects, [:data], using: :gin) + drop_if_exists(index(:objects, [:data], using: :gin)) end end diff --git a/priv/repo/migrations/20171024121413_add_object_actor_index.exs b/priv/repo/migrations/20171024121413_add_object_actor_index.exs index 344c9c825..78084536c 100644 --- a/priv/repo/migrations/20171024121413_add_object_actor_index.exs +++ b/priv/repo/migrations/20171024121413_add_object_actor_index.exs @@ -4,6 +4,11 @@ defmodule Pleroma.Repo.Migrations.AddObjectActorIndex do @disable_ddl_transaction true def change do - create index(:objects, ["(data->>'actor')", "(data->>'type')"], concurrently: true, name: :objects_actor_type) + create( + index(:objects, ["(data->>'actor')", "(data->>'type')"], + concurrently: true, + name: :objects_actor_type + ) + ) end end diff --git a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs index fb5f80c98..91348f5c3 100644 --- a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs +++ b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs @@ -5,16 +5,17 @@ defmodule Pleroma.Repo.Migrations.AddActorToActivity do def up do alter table(:activities) do - add :actor, :string + add(:actor, :string) end - create index(:activities, [:actor, "id DESC NULLS LAST"], concurrently: true) + create(index(:activities, [:actor, "id DESC NULLS LAST"], concurrently: true)) end def down do - drop_if_exists index(:activities, [:actor, "id DESC NULLS LAST"]) + drop_if_exists(index(:activities, [:actor, "id DESC NULLS LAST"])) + alter table(:activities) do - remove :actor + remove(:actor) end end end diff --git a/priv/repo/migrations/20171109114020_fill_actor_field.exs b/priv/repo/migrations/20171109114020_fill_actor_field.exs index 255ca46d5..fb7eca692 100644 --- a/priv/repo/migrations/20171109114020_fill_actor_field.exs +++ b/priv/repo/migrations/20171109114020_fill_actor_field.exs @@ -5,17 +5,19 @@ defmodule Pleroma.Repo.Migrations.FillActorField do def up do max = Repo.aggregate(Activity, :max, :id) + if max do IO.puts("#{max} activities") - chunks = 0..(round(max / 10_000)) + chunks = 0..round(max / 10_000) - Enum.each(chunks, fn (i) -> + Enum.each(chunks, fn i -> min = i * 10_000 max = min + 10_000 + execute(""" update activities set actor = data->>'actor' where id > #{min} and id <= #{max}; """) - |> IO.inspect + |> IO.inspect() end) end end @@ -23,4 +25,3 @@ defmodule Pleroma.Repo.Migrations.FillActorField do def down do end end - diff --git a/priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs b/priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs index 2d21c56ca..37fb2ce32 100644 --- a/priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs +++ b/priv/repo/migrations/20171109141309_add_sort_index_to_activities.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddSortIndexToActivities do @disable_ddl_transaction true def change do - create index(:activities, ["id desc nulls last"], concurrently: true) + create(index(:activities, ["id desc nulls last"], concurrently: true)) end end diff --git a/priv/repo/migrations/20171130135819_add_local_index_to_user.exs b/priv/repo/migrations/20171130135819_add_local_index_to_user.exs index 3438bbbc4..76bf9584e 100644 --- a/priv/repo/migrations/20171130135819_add_local_index_to_user.exs +++ b/priv/repo/migrations/20171130135819_add_local_index_to_user.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddLocalIndexToUser do use Ecto.Migration def change do - create_if_not_exists index(:users, [:local]) + create_if_not_exists(index(:users, [:local])) end end diff --git a/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs b/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs index 4520b398e..6344fbeee 100644 --- a/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs +++ b/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs @@ -3,9 +3,9 @@ defmodule Pleroma.Repo.Migrations.AddRecipientsToActivities do def change do alter table(:activities) do - add :recipients, {:array, :string} + add(:recipients, {:array, :string}) end - create_if_not_exists index(:activities, [:recipients], using: :gin) + create_if_not_exists(index(:activities, [:recipients], using: :gin)) end end diff --git a/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs b/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs index 87de64ca5..6dfa93716 100644 --- a/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs +++ b/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs @@ -4,17 +4,21 @@ defmodule Pleroma.Repo.Migrations.FillRecipientsInActivities do def up do max = Repo.aggregate(Activity, :max, :id) + if max do IO.puts("#{max} activities") - chunks = 0..(round(max / 10_000)) + chunks = 0..round(max / 10_000) - Enum.each(chunks, fn (i) -> + Enum.each(chunks, fn i -> min = i * 10_000 max = min + 10_000 + execute(""" - update activities set recipients = array(select jsonb_array_elements_text(data->'to')) where id > #{min} and id <= #{max}; + update activities set recipients = array(select jsonb_array_elements_text(data->'to')) where id > #{ + min + } and id <= #{max}; """) - |> IO.inspect + |> IO.inspect() end) end end diff --git a/priv/repo/migrations/20180221210540_make_following_postgres_array.exs b/priv/repo/migrations/20180221210540_make_following_postgres_array.exs index 5a8f8f669..34e94fdc8 100644 --- a/priv/repo/migrations/20180221210540_make_following_postgres_array.exs +++ b/priv/repo/migrations/20180221210540_make_following_postgres_array.exs @@ -3,17 +3,18 @@ defmodule Pleroma.Repo.Migrations.MakeFollowingPostgresArray do def up do alter table(:users) do - add :following_temp, {:array, :string} + add(:following_temp, {:array, :string}) end - execute """ + execute(""" update users set following_temp = array(select jsonb_array_elements_text(following)); - """ + """) alter table(:users) do - remove :following + remove(:following) end - rename table(:users), :following_temp, to: :following + + rename(table(:users), :following_temp, to: :following) end def down, do: :ok diff --git a/priv/repo/migrations/20180325172351_add_follower_address_index_to_users.exs b/priv/repo/migrations/20180325172351_add_follower_address_index_to_users.exs index 234d33735..18b54411c 100644 --- a/priv/repo/migrations/20180325172351_add_follower_address_index_to_users.exs +++ b/priv/repo/migrations/20180325172351_add_follower_address_index_to_users.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.AddFollowerAddressIndexToUsers do @disable_ddl_transaction true def change do - create index(:users, [:follower_address], concurrently: true) - create index(:users, [:following], concurrently: true, using: :gin) + create(index(:users, [:follower_address], concurrently: true)) + create(index(:users, [:following], concurrently: true, using: :gin)) end end diff --git a/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs b/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs index 35c4ce62f..1574e0e00 100644 --- a/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs +++ b/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.DropLocalIndexOnActivities do use Ecto.Migration def change do - drop_if_exists index(:users, [:local]) + drop_if_exists(index(:users, [:local])) end end diff --git a/priv/repo/migrations/20180327175831_actually_drop_local_index.exs b/priv/repo/migrations/20180327175831_actually_drop_local_index.exs index 7556336ed..3d52c7c80 100644 --- a/priv/repo/migrations/20180327175831_actually_drop_local_index.exs +++ b/priv/repo/migrations/20180327175831_actually_drop_local_index.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.ActuallyDropLocalIndex do use Ecto.Migration def change do - create_if_not_exists index(:users, [:local]) - drop_if_exists index("activities", :local) + create_if_not_exists(index(:users, [:local])) + drop_if_exists(index("activities", :local)) end end diff --git a/priv/repo/migrations/20180429094642_create_lists.exs b/priv/repo/migrations/20180429094642_create_lists.exs index 9d3ce50b3..e1eb7e426 100644 --- a/priv/repo/migrations/20180429094642_create_lists.exs +++ b/priv/repo/migrations/20180429094642_create_lists.exs @@ -3,13 +3,13 @@ defmodule Pleroma.Repo.Migrations.CreateLists do def change do create_if_not_exists table(:lists) do - add :user_id, references(:users, on_delete: :delete_all) - add :title, :string - add :following, {:array, :string} + add(:user_id, references(:users, on_delete: :delete_all)) + add(:title, :string) + add(:following, {:array, :string}) timestamps() end - create_if_not_exists index(:lists, [:user_id]) + create_if_not_exists(index(:lists, [:user_id])) end end diff --git a/priv/repo/migrations/20180513104714_modify_activity_index.exs b/priv/repo/migrations/20180513104714_modify_activity_index.exs index 2df530839..ec0efa238 100644 --- a/priv/repo/migrations/20180513104714_modify_activity_index.exs +++ b/priv/repo/migrations/20180513104714_modify_activity_index.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.ModifyActivityIndex do @disable_ddl_transaction true def change do - create index(:activities, ["id desc nulls last", "local"], concurrently: true) - drop_if_exists index(:activities, ["id desc nulls last"]) + create(index(:activities, ["id desc nulls last", "local"], concurrently: true)) + drop_if_exists(index(:activities, ["id desc nulls last"])) end end diff --git a/priv/repo/migrations/20180516144508_add_trigram_extension.exs b/priv/repo/migrations/20180516144508_add_trigram_extension.exs index f2f0fca86..ff0710f84 100644 --- a/priv/repo/migrations/20180516144508_add_trigram_extension.exs +++ b/priv/repo/migrations/20180516144508_add_trigram_extension.exs @@ -4,8 +4,15 @@ defmodule Pleroma.Repo.Migrations.AddTrigramExtension do def up do Logger.warn("ATTENTION ATTENTION ATTENTION\n") - Logger.warn("This will try to create the pg_trgm extension on your database. If your database user does NOT have the necessary rights, you will have to do it manually and re-run the migrations.\nYou can probably do this by running the following:\n") - Logger.warn("sudo -u postgres psql pleroma_dev -c \"create extension if not exists pg_trgm\"\n") + + Logger.warn( + "This will try to create the pg_trgm extension on your database. If your database user does NOT have the necessary rights, you will have to do it manually and re-run the migrations.\nYou can probably do this by running the following:\n" + ) + + Logger.warn( + "sudo -u postgres psql pleroma_dev -c \"create extension if not exists pg_trgm\"\n" + ) + execute("create extension if not exists pg_trgm") end diff --git a/priv/repo/migrations/20180516154905_create_user_trigram_index.exs b/priv/repo/migrations/20180516154905_create_user_trigram_index.exs index 58622a87e..0713a7297 100644 --- a/priv/repo/migrations/20180516154905_create_user_trigram_index.exs +++ b/priv/repo/migrations/20180516154905_create_user_trigram_index.exs @@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.CreateUserTrigramIndex do use Ecto.Migration def change do - create_if_not_exists index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) + create_if_not_exists( + index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) + ) end end diff --git a/priv/repo/migrations/20180530123448_add_list_follow_index.exs b/priv/repo/migrations/20180530123448_add_list_follow_index.exs index 86b8de30a..57f8d478f 100644 --- a/priv/repo/migrations/20180530123448_add_list_follow_index.exs +++ b/priv/repo/migrations/20180530123448_add_list_follow_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddListFollowIndex do use Ecto.Migration def change do - create_if_not_exists index(:lists, [:following]) + create_if_not_exists(index(:lists, [:following])) end end diff --git a/priv/repo/migrations/20180606173637_create_apid_host_extraction_index.exs b/priv/repo/migrations/20180606173637_create_apid_host_extraction_index.exs index 9831a1b82..07b3f2875 100644 --- a/priv/repo/migrations/20180606173637_create_apid_host_extraction_index.exs +++ b/priv/repo/migrations/20180606173637_create_apid_host_extraction_index.exs @@ -3,6 +3,11 @@ defmodule Pleroma.Repo.Migrations.CreateApidHostExtractionIndex do @disable_ddl_transaction true def change do - create index(:activities, ["(split_part(actor, '/', 3))"], concurrently: true, name: :activities_hosts) + create( + index(:activities, ["(split_part(actor, '/', 3))"], + concurrently: true, + name: :activities_hosts + ) + ) end end diff --git a/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs b/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs index faee379f0..a75ff2a51 100644 --- a/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs +++ b/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs @@ -3,8 +3,8 @@ defmodule Pleroma.Repo.Migrations.CreateUserInviteTokens do def change do create_if_not_exists table(:user_invite_tokens) do - add :token, :string - add :used, :boolean, default: false + add(:token, :string) + add(:used, :boolean, default: false) timestamps() end diff --git a/priv/repo/migrations/20180617221540_create_activities_in_reply_to_index.exs b/priv/repo/migrations/20180617221540_create_activities_in_reply_to_index.exs index 1fee6fd7a..c8a0e60a0 100644 --- a/priv/repo/migrations/20180617221540_create_activities_in_reply_to_index.exs +++ b/priv/repo/migrations/20180617221540_create_activities_in_reply_to_index.exs @@ -3,6 +3,11 @@ defmodule Pleroma.Repo.Migrations.CreateActivitiesInReplyToIndex do @disable_ddl_transaction true def change do - create index(:activities, ["(data->'object'->>'inReplyTo')"], concurrently: true, name: :activities_in_reply_to) + create( + index(:activities, ["(data->'object'->>'inReplyTo')"], + concurrently: true, + name: :activities_in_reply_to + ) + ) end end diff --git a/priv/repo/migrations/20180813003722_create_filters.exs b/priv/repo/migrations/20180813003722_create_filters.exs index 541cf46a1..7803558df 100644 --- a/priv/repo/migrations/20180813003722_create_filters.exs +++ b/priv/repo/migrations/20180813003722_create_filters.exs @@ -3,18 +3,21 @@ defmodule Pleroma.Repo.Migrations.CreateFilters do def change do create_if_not_exists table(:filters) do - add :user_id, references(:users, on_delete: :delete_all) - add :filter_id, :integer - add :hide, :boolean - add :phrase, :string - add :context, {:array, :string} - add :expires_at, :utc_datetime - add :whole_word, :boolean + add(:user_id, references(:users, on_delete: :delete_all)) + add(:filter_id, :integer) + add(:hide, :boolean) + add(:phrase, :string) + add(:context, {:array, :string}) + add(:expires_at, :utc_datetime) + add(:whole_word, :boolean) timestamps() end - create_if_not_exists index(:filters, [:user_id]) - create_if_not_exists index(:filters, [:phrase], where: "hide = true", name: :hided_phrases_index) + create_if_not_exists(index(:filters, [:user_id])) + + create_if_not_exists( + index(:filters, [:phrase], where: "hide = true", name: :hided_phrases_index) + ) end end diff --git a/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs b/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs index af9d521c0..481986039 100644 --- a/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs +++ b/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs @@ -3,11 +3,11 @@ defmodule Pleroma.Repo.Migrations.AddRecipientsToAndCcFieldsToActivities do def change do alter table(:activities) do - add :recipients_to, {:array, :string} - add :recipients_cc, {:array, :string} + add(:recipients_to, {:array, :string}) + add(:recipients_cc, {:array, :string}) end - create_if_not_exists index(:activities, [:recipients_to], using: :gin) - create_if_not_exists index(:activities, [:recipients_cc], using: :gin) + create_if_not_exists(index(:activities, [:recipients_to], using: :gin)) + create_if_not_exists(index(:activities, [:recipients_cc], using: :gin)) end end diff --git a/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs b/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs index 9d31f6779..1f9f97861 100644 --- a/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs +++ b/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs @@ -2,7 +2,12 @@ defmodule Pleroma.Repo.Migrations.ActivitiesAddToCcIndices do use Ecto.Migration def change do - create_if_not_exists index(:activities, ["(data->'to')"], name: :activities_to_index, using: :gin) - create_if_not_exists index(:activities, ["(data->'cc')"], name: :activities_cc_index, using: :gin) + create_if_not_exists( + index(:activities, ["(data->'to')"], name: :activities_to_index, using: :gin) + ) + + create_if_not_exists( + index(:activities, ["(data->'cc')"], name: :activities_cc_index, using: :gin) + ) end end diff --git a/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs b/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs index 017ef161f..65576b8d5 100644 --- a/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs +++ b/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs @@ -3,15 +3,15 @@ defmodule Pleroma.Repo.Migrations.RemoveRecipientsToAndCcFieldsFromActivities do def up do alter table(:activities) do - remove :recipients_to - remove :recipients_cc + remove(:recipients_to) + remove(:recipients_cc) end end def down do alter table(:activities) do - add :recipients_to, {:array, :string} - add :recipients_cc, {:array, :string} + add(:recipients_to, {:array, :string}) + add(:recipients_cc, {:array, :string}) end end end diff --git a/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs b/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs index adce28bdf..cbe79de05 100644 --- a/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs +++ b/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs @@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.UsersAddIsModeratorIndex do use Ecto.Migration def change do - create_if_not_exists index(:users, ["(info->'is_moderator')"], name: :users_is_moderator_index, using: :gin) + create_if_not_exists( + index(:users, ["(info->'is_moderator')"], name: :users_is_moderator_index, using: :gin) + ) end end diff --git a/priv/repo/migrations/20180918182427_create_push_subscriptions.exs b/priv/repo/migrations/20180918182427_create_push_subscriptions.exs index 36bdf322a..c1b55d018 100644 --- a/priv/repo/migrations/20180918182427_create_push_subscriptions.exs +++ b/priv/repo/migrations/20180918182427_create_push_subscriptions.exs @@ -3,16 +3,16 @@ defmodule Pleroma.Repo.Migrations.CreatePushSubscriptions do def change do create_if_not_exists table("push_subscriptions") do - add :user_id, references("users", on_delete: :delete_all) - add :token_id, references("oauth_tokens", on_delete: :delete_all) - add :endpoint, :string - add :key_p256dh, :string - add :key_auth, :string - add :data, :map + add(:user_id, references("users", on_delete: :delete_all)) + add(:token_id, references("oauth_tokens", on_delete: :delete_all)) + add(:endpoint, :string) + add(:key_p256dh, :string) + add(:key_auth, :string) + add(:data, :map) timestamps() end - create_if_not_exists index("push_subscriptions", [:user_id, :token_id], unique: true) + create_if_not_exists(index("push_subscriptions", [:user_id, :token_id], unique: true)) end end diff --git a/priv/repo/migrations/20180919060348_users_add_last_refreshed_at.exs b/priv/repo/migrations/20180919060348_users_add_last_refreshed_at.exs index 815177e05..16605cf7b 100644 --- a/priv/repo/migrations/20180919060348_users_add_last_refreshed_at.exs +++ b/priv/repo/migrations/20180919060348_users_add_last_refreshed_at.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.UsersAddLastRefreshedAt do def change do alter table(:users) do - add :last_refreshed_at, :naive_datetime_usec + add(:last_refreshed_at, :naive_datetime_usec) end end end diff --git a/priv/repo/migrations/20181206125616_add_tags_to_users.exs b/priv/repo/migrations/20181206125616_add_tags_to_users.exs index 7d42a0fba..a46c0fc35 100644 --- a/priv/repo/migrations/20181206125616_add_tags_to_users.exs +++ b/priv/repo/migrations/20181206125616_add_tags_to_users.exs @@ -3,9 +3,9 @@ defmodule Pleroma.Repo.Migrations.AddTagsToUsers do def change do alter table(:users) do - add :tags, {:array, :string} + add(:tags, {:array, :string}) end - create_if_not_exists index(:users, [:tags], using: :gin) + create_if_not_exists(index(:users, [:tags], using: :gin)) end end diff --git a/priv/repo/migrations/20181214121049_add_bookmarks_to_users.exs b/priv/repo/migrations/20181214121049_add_bookmarks_to_users.exs index 55e97ae0e..6228f1bad 100644 --- a/priv/repo/migrations/20181214121049_add_bookmarks_to_users.exs +++ b/priv/repo/migrations/20181214121049_add_bookmarks_to_users.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.AddBookmarksToUsers do def change do alter table(:users) do - add :bookmarks, {:array, :string}, null: false, default: [] + add(:bookmarks, {:array, :string}, null: false, default: []) end end end diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index a5b4c543d..c58d829af 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -16,32 +16,34 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do # Old serial int ids are transformed to 128bits with extra padding. # The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility # with previously issued ids. - #execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" - #execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + # execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" + # execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" clippy = start_clippy_heartbeats() # Lock both tables to avoid a running server to meddling with our transaction - execute "LOCK TABLE activities;" - execute "LOCK TABLE users;" + execute("LOCK TABLE activities;") + execute("LOCK TABLE users;") - execute """ + execute(""" ALTER TABLE activities DROP CONSTRAINT activities_pkey CASCADE, ALTER COLUMN id DROP default, ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid), ADD PRIMARY KEY (id); - """ + """) - execute """ + execute(""" ALTER TABLE users DROP CONSTRAINT users_pkey CASCADE, ALTER COLUMN id DROP default, ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid), ADD PRIMARY KEY (id); - """ + """) - execute "UPDATE users SET info = jsonb_set(info, '{pinned_activities}', array_to_json(ARRAY(select jsonb_array_elements_text(info->'pinned_activities')))::jsonb);" + execute( + "UPDATE users SET info = jsonb_set(info, '{pinned_activities}', array_to_json(ARRAY(select jsonb_array_elements_text(info->'pinned_activities')))::jsonb);" + ) # Fkeys: # Activities - Referenced by: @@ -56,18 +58,19 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do # TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE # TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) - execute """ + execute(""" ALTER TABLE notifications ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid), ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE; - """ + """) - for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do - execute """ + for table <- + ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do + execute(""" ALTER TABLE #{table} ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid), ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; - """ + """) end flush() @@ -78,41 +81,50 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do def down, do: :ok defp start_clippy_heartbeats() do - count = from(a in "activities", select: count(a.id)) |> Repo.one! + count = from(a in "activities", select: count(a.id)) |> Repo.one!() if count > 5000 do heartbeat_interval = :timer.minutes(2) + :timer.seconds(30) - all_tips = Clippy.tips() ++ [ - "The migration is still running, maybe it's time for another “tea”?", - "Happy rabbits practice a cute behavior known as a\n“binky:” they jump up in the air\nand twist\nand spin around!", - "Nothing and everything.\n\nI still work.", - "Pleroma runs on a Raspberry Pi!\n\n … but this migration will take forever if you\nactually run on a raspberry pi", - "Status? Stati? Post? Note? Toot?\nRepeat? Reboost? Boost? Retweet? Retoot??\n\nI-I'm confused.", - ] - heartbeat = fn(heartbeat, runs, all_tips, tips) -> - tips = if Integer.is_even(runs) do - tips = if tips == [], do: all_tips, else: tips - [tip | tips] = Enum.shuffle(tips) - Clippy.puts(tip) - tips - else - IO.puts "\n -- #{DateTime.to_string(DateTime.utc_now())} Migration still running, please wait…\n" - tips - end + all_tips = + Clippy.tips() ++ + [ + "The migration is still running, maybe it's time for another “tea”?", + "Happy rabbits practice a cute behavior known as a\n“binky:” they jump up in the air\nand twist\nand spin around!", + "Nothing and everything.\n\nI still work.", + "Pleroma runs on a Raspberry Pi!\n\n … but this migration will take forever if you\nactually run on a raspberry pi", + "Status? Stati? Post? Note? Toot?\nRepeat? Reboost? Boost? Retweet? Retoot??\n\nI-I'm confused." + ] + + heartbeat = fn heartbeat, runs, all_tips, tips -> + tips = + if Integer.is_even(runs) do + tips = if tips == [], do: all_tips, else: tips + [tip | tips] = Enum.shuffle(tips) + Clippy.puts(tip) + tips + else + IO.puts( + "\n -- #{DateTime.to_string(DateTime.utc_now())} Migration still running, please wait…\n" + ) + + tips + end + :timer.sleep(heartbeat_interval) heartbeat.(heartbeat, runs + 1, all_tips, tips) end - Clippy.puts [ + Clippy.puts([ [:red, :bright, "It looks like you are running an older instance!"], [""], [:bright, "This migration may take a long time", :reset, " -- so you probably should"], ["go drink a cofe, or a tea, or a beer, a whiskey, a vodka,"], ["while it runs to deal with your temporary fediverse pause!"] - ] + ]) + :timer.sleep(heartbeat_interval) - spawn_link(fn() -> heartbeat.(heartbeat, 1, all_tips, []) end) + spawn_link(fn -> heartbeat.(heartbeat, 1, all_tips, []) end) end end @@ -120,8 +132,7 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do if pid do Process.unlink(pid) Process.exit(pid, :kill) - Clippy.puts [[:green, :bright, "Hurray!!", "", "", "Migration completed!"]] + Clippy.puts([[:green, :bright, "Hurray!!", "", "", "Migration completed!"]]) end end - end diff --git a/priv/repo/migrations/20190109152453_add_visibility_function.exs b/priv/repo/migrations/20190109152453_add_visibility_function.exs index b6a4e752b..43d1074aa 100644 --- a/priv/repo/migrations/20190109152453_add_visibility_function.exs +++ b/priv/repo/migrations/20190109152453_add_visibility_function.exs @@ -43,6 +43,8 @@ defmodule Pleroma.Repo.Migrations.AddVisibilityFunction do ) ) - execute("drop function if exists activity_visibility(actor varchar, recipients varchar[], data jsonb)") + execute( + "drop function if exists activity_visibility(actor varchar, recipients varchar[], data jsonb)" + ) end end diff --git a/priv/repo/migrations/20190115085500_create_user_fts_index.exs b/priv/repo/migrations/20190115085500_create_user_fts_index.exs index cff975318..0c0c512d3 100644 --- a/priv/repo/migrations/20190115085500_create_user_fts_index.exs +++ b/priv/repo/migrations/20190115085500_create_user_fts_index.exs @@ -2,16 +2,18 @@ defmodule Pleroma.Repo.Migrations.CreateUserFtsIndex do use Ecto.Migration def change do - create_if_not_exists index( - :users, - [ - """ - (setweight(to_tsvector('simple', regexp_replace(nickname, '\\W', ' ', 'g')), 'A') || - setweight(to_tsvector('simple', regexp_replace(coalesce(name, ''), '\\W', ' ', 'g')), 'B')) - """ - ], - name: :users_fts_index, - using: :gin - ) + create_if_not_exists( + index( + :users, + [ + """ + (setweight(to_tsvector('simple', regexp_replace(nickname, '\\W', ' ', 'g')), 'A') || + setweight(to_tsvector('simple', regexp_replace(coalesce(name, ''), '\\W', ' ', 'g')), 'B')) + """ + ], + name: :users_fts_index, + using: :gin + ) + ) end end diff --git a/priv/repo/migrations/20190122153157_update_activity_visibility.exs b/priv/repo/migrations/20190122153157_update_activity_visibility.exs index 30075137c..9e29571ee 100644 --- a/priv/repo/migrations/20190122153157_update_activity_visibility.exs +++ b/priv/repo/migrations/20190122153157_update_activity_visibility.exs @@ -27,10 +27,8 @@ defmodule Pleroma.Repo.Migrations.UpdateActivityVisibility do """ execute(definition) - end def down do - end end diff --git a/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs b/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs index 25f248c59..f42d46427 100644 --- a/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs +++ b/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs @@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.UsersAddIsAdminIndex do use Ecto.Migration def change do - create_if_not_exists(index(:users, ["(info->'is_admin')"], name: :users_is_admin_index, using: :gin)) + create_if_not_exists( + index(:users, ["(info->'is_admin')"], name: :users_is_admin_index, using: :gin) + ) end end diff --git a/priv/repo/migrations/20190123125546_create_instances.exs b/priv/repo/migrations/20190123125546_create_instances.exs index a9b356bc3..9438736ba 100644 --- a/priv/repo/migrations/20190123125546_create_instances.exs +++ b/priv/repo/migrations/20190123125546_create_instances.exs @@ -3,13 +3,13 @@ defmodule Pleroma.Repo.Migrations.CreateInstances do def change do create_if_not_exists table(:instances) do - add :host, :string - add :unreachable_since, :naive_datetime_usec + add(:host, :string) + add(:unreachable_since, :naive_datetime_usec) timestamps() end - create_if_not_exists unique_index(:instances, [:host]) - create_if_not_exists index(:instances, [:unreachable_since]) + create_if_not_exists(unique_index(:instances, [:host])) + create_if_not_exists(index(:instances, [:unreachable_since])) end end diff --git a/priv/repo/migrations/20190124131141_update_activity_visibility_again.exs b/priv/repo/migrations/20190124131141_update_activity_visibility_again.exs index 0519a5143..a42e4cad9 100644 --- a/priv/repo/migrations/20190124131141_update_activity_visibility_again.exs +++ b/priv/repo/migrations/20190124131141_update_activity_visibility_again.exs @@ -27,11 +27,8 @@ defmodule Pleroma.Repo.Migrations.UpdateActivityVisibilityAgain do """ execute(definition) - end def down do - end - end diff --git a/priv/repo/migrations/20190127151220_add_activities_likes_index.exs b/priv/repo/migrations/20190127151220_add_activities_likes_index.exs index b1822d265..115b12491 100644 --- a/priv/repo/migrations/20190127151220_add_activities_likes_index.exs +++ b/priv/repo/migrations/20190127151220_add_activities_likes_index.exs @@ -3,6 +3,12 @@ defmodule Pleroma.Repo.Migrations.AddActivitiesLikesIndex do @disable_ddl_transaction true def change do - create index(:activities, ["((data #> '{\"object\",\"likes\"}'))"], concurrently: true, name: :activities_likes, using: :gin) + create( + index(:activities, ["((data #> '{\"object\",\"likes\"}'))"], + concurrently: true, + name: :activities_likes, + using: :gin + ) + ) end end diff --git a/priv/repo/migrations/20190203185340_split_hide_network.exs b/priv/repo/migrations/20190203185340_split_hide_network.exs index 8b7a9151b..fb677f68a 100644 --- a/priv/repo/migrations/20190203185340_split_hide_network.exs +++ b/priv/repo/migrations/20190203185340_split_hide_network.exs @@ -2,9 +2,17 @@ defmodule Pleroma.Repo.Migrations.SplitHideNetwork do use Ecto.Migration def up do - execute("UPDATE users SET info = jsonb_set(info, '{hide_network}'::text[], 'false'::jsonb) WHERE NOT(info::jsonb ? 'hide_network') AND local") - execute("UPDATE users SET info = jsonb_set(info, '{hide_followings}'::text[], info->'hide_network') WHERE local") - execute("UPDATE users SET info = jsonb_set(info, '{hide_followers}'::text[], info->'hide_network') WHERE local") + execute( + "UPDATE users SET info = jsonb_set(info, '{hide_network}'::text[], 'false'::jsonb) WHERE NOT(info::jsonb ? 'hide_network') AND local" + ) + + execute( + "UPDATE users SET info = jsonb_set(info, '{hide_followings}'::text[], info->'hide_network') WHERE local" + ) + + execute( + "UPDATE users SET info = jsonb_set(info, '{hide_followers}'::text[], info->'hide_network') WHERE local" + ) end def down do diff --git a/priv/repo/migrations/20190205114625_create_thread_mutes.exs b/priv/repo/migrations/20190205114625_create_thread_mutes.exs index baaf07253..df9eb7677 100644 --- a/priv/repo/migrations/20190205114625_create_thread_mutes.exs +++ b/priv/repo/migrations/20190205114625_create_thread_mutes.exs @@ -3,10 +3,10 @@ defmodule Pleroma.Repo.Migrations.CreateThreadMutes do def change do create_if_not_exists table(:thread_mutes) do - add :user_id, references(:users, type: :uuid, on_delete: :delete_all) - add :context, :string + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:context, :string) end - create_if_not_exists unique_index(:thread_mutes, [:user_id, :context], name: :unique_index) + create_if_not_exists(unique_index(:thread_mutes, [:user_id, :context], name: :unique_index)) end end diff --git a/priv/repo/migrations/20190208131753_add_scopes_to_o_auth_entities.exs b/priv/repo/migrations/20190208131753_add_scopes_to_o_auth_entities.exs index 4efbebc4d..ad93bfce2 100644 --- a/priv/repo/migrations/20190208131753_add_scopes_to_o_auth_entities.exs +++ b/priv/repo/migrations/20190208131753_add_scopes_to_o_auth_entities.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.AddScopeSToOAuthEntities do def change do for t <- [:oauth_authorizations, :oauth_tokens] do alter table(t) do - add :scopes, {:array, :string}, default: [], null: false + add(:scopes, {:array, :string}, default: [], null: false) end end end diff --git a/priv/repo/migrations/20190213185503_change_apps_scopes_to_varchar_array.exs b/priv/repo/migrations/20190213185503_change_apps_scopes_to_varchar_array.exs index 72decd401..eb6fcb012 100644 --- a/priv/repo/migrations/20190213185503_change_apps_scopes_to_varchar_array.exs +++ b/priv/repo/migrations/20190213185503_change_apps_scopes_to_varchar_array.exs @@ -4,14 +4,20 @@ defmodule Pleroma.Repo.Migrations.ChangeAppsScopesToVarcharArray do @alter_apps_scopes "ALTER TABLE apps ALTER COLUMN scopes" def up do - execute "#{@alter_apps_scopes} TYPE varchar(255)[] USING string_to_array(scopes, ',')::varchar(255)[];" - execute "#{@alter_apps_scopes} SET DEFAULT ARRAY[]::character varying[];" - execute "#{@alter_apps_scopes} SET NOT NULL;" + execute( + "#{@alter_apps_scopes} TYPE varchar(255)[] USING string_to_array(scopes, ',')::varchar(255)[];" + ) + + execute("#{@alter_apps_scopes} SET DEFAULT ARRAY[]::character varying[];") + execute("#{@alter_apps_scopes} SET NOT NULL;") end def down do - execute "#{@alter_apps_scopes} DROP NOT NULL;" - execute "#{@alter_apps_scopes} DROP DEFAULT;" - execute "#{@alter_apps_scopes} TYPE varchar(255) USING array_to_string(scopes, ',')::varchar(255);" + execute("#{@alter_apps_scopes} DROP NOT NULL;") + execute("#{@alter_apps_scopes} DROP DEFAULT;") + + execute( + "#{@alter_apps_scopes} TYPE varchar(255) USING array_to_string(scopes, ',')::varchar(255);" + ) end end diff --git a/priv/repo/migrations/20190213185600_data_migration_populate_o_auth_scopes.exs b/priv/repo/migrations/20190213185600_data_migration_populate_o_auth_scopes.exs index 7afbcbd76..ef5b35125 100644 --- a/priv/repo/migrations/20190213185600_data_migration_populate_o_auth_scopes.exs +++ b/priv/repo/migrations/20190213185600_data_migration_populate_o_auth_scopes.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.DataMigrationPopulateOAuthScopes do def up do for t <- [:oauth_authorizations, :oauth_tokens] do - execute "UPDATE #{t} SET scopes = apps.scopes FROM apps WHERE #{t}.app_id = apps.id;" + execute("UPDATE #{t} SET scopes = apps.scopes FROM apps WHERE #{t}.app_id = apps.id;") end end diff --git a/priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs b/priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs index d44e5096b..92ab9bd2c 100644 --- a/priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs +++ b/priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.DataMigrationNormalizeScopes do def up do for t <- [:apps, :oauth_authorizations, :oauth_tokens] do - execute "UPDATE #{t} SET scopes = string_to_array(array_to_string(scopes, ' '), ' ');" + execute("UPDATE #{t} SET scopes = string_to_array(array_to_string(scopes, ' '), ' ');") end end diff --git a/priv/repo/migrations/20190301101154_add_default_tags_to_user.exs b/priv/repo/migrations/20190301101154_add_default_tags_to_user.exs index faeb8f1c6..ea0947852 100644 --- a/priv/repo/migrations/20190301101154_add_default_tags_to_user.exs +++ b/priv/repo/migrations/20190301101154_add_default_tags_to_user.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddDefaultTagsToUser do use Ecto.Migration def up do - execute "UPDATE users SET tags = array[]::varchar[] where tags IS NULL" + execute("UPDATE users SET tags = array[]::varchar[] where tags IS NULL") end def down, do: :noop diff --git a/priv/repo/migrations/20190303120636_update_user_note_counters.exs b/priv/repo/migrations/20190303120636_update_user_note_counters.exs index 54e68f7c9..95dbd012f 100644 --- a/priv/repo/migrations/20190303120636_update_user_note_counters.exs +++ b/priv/repo/migrations/20190303120636_update_user_note_counters.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.UpdateUserNoteCounters do @public "https://www.w3.org/ns/activitystreams#Public" def up do - execute """ + execute(""" WITH public_note_count AS ( SELECT data->>'actor' AS actor, @@ -19,11 +19,11 @@ defmodule Pleroma.Repo.Migrations.UpdateUserNoteCounters do SET "info" = jsonb_set(u.info, '{note_count}', o.count::varchar::jsonb, true) FROM public_note_count AS o WHERE u.ap_id = o.actor - """ + """) end def down do - execute """ + execute(""" WITH public_note_count AS ( SELECT data->>'actor' AS actor, @@ -36,6 +36,6 @@ defmodule Pleroma.Repo.Migrations.UpdateUserNoteCounters do SET "info" = jsonb_set(u.info, '{note_count}', o.count::varchar::jsonb, true) FROM public_note_count AS o WHERE u.ap_id = o.actor - """ + """) end end diff --git a/priv/repo/migrations/20190315101315_create_registrations.exs b/priv/repo/migrations/20190315101315_create_registrations.exs index 34a390a93..d705a499e 100644 --- a/priv/repo/migrations/20190315101315_create_registrations.exs +++ b/priv/repo/migrations/20190315101315_create_registrations.exs @@ -3,16 +3,16 @@ defmodule Pleroma.Repo.Migrations.CreateRegistrations do def change do create_if_not_exists table(:registrations, primary_key: false) do - add :id, :uuid, primary_key: true - add :user_id, references(:users, type: :uuid, on_delete: :delete_all) - add :provider, :string - add :uid, :string - add :info, :map, default: %{} + add(:id, :uuid, primary_key: true) + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:provider, :string) + add(:uid, :string) + add(:info, :map, default: %{}) timestamps() end - create_if_not_exists unique_index(:registrations, [:provider, :uid]) - create_if_not_exists unique_index(:registrations, [:user_id, :provider, :uid]) + create_if_not_exists(unique_index(:registrations, [:provider, :uid])) + create_if_not_exists(unique_index(:registrations, [:user_id, :provider, :uid])) end end diff --git a/priv/repo/migrations/20190325185009_create_notification_id_index.exs b/priv/repo/migrations/20190325185009_create_notification_id_index.exs index 01cb30559..7209c16a9 100644 --- a/priv/repo/migrations/20190325185009_create_notification_id_index.exs +++ b/priv/repo/migrations/20190325185009_create_notification_id_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.CreateNotificationIdIndex do use Ecto.Migration def change do - create_if_not_exists index(:notifications, ["id desc nulls last"]) + create_if_not_exists(index(:notifications, ["id desc nulls last"])) end end diff --git a/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs b/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs index 460dafb1b..bbf47f72c 100644 --- a/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs +++ b/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs @@ -3,6 +3,12 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnSubscribers do @disable_ddl_transaction true def change do - create index(:users, ["(info->'subscribers')"], name: :users_subscribers_index, using: :gin, concurrently: true) + create( + index(:users, ["(info->'subscribers')"], + name: :users_subscribers_index, + using: :gin, + concurrently: true + ) + ) end end diff --git a/priv/repo/migrations/20190408123347_create_conversations.exs b/priv/repo/migrations/20190408123347_create_conversations.exs index 7b7d89da7..d75459e82 100644 --- a/priv/repo/migrations/20190408123347_create_conversations.exs +++ b/priv/repo/migrations/20190408123347_create_conversations.exs @@ -19,8 +19,8 @@ defmodule Pleroma.Repo.Migrations.CreateConversations do timestamps() end - create_if_not_exists index(:conversation_participations, [:conversation_id]) - create_if_not_exists unique_index(:conversation_participations, [:user_id, :conversation_id]) - create_if_not_exists unique_index(:conversations, [:ap_id]) + create_if_not_exists(index(:conversation_participations, [:conversation_id])) + create_if_not_exists(unique_index(:conversation_participations, [:user_id, :conversation_id])) + create_if_not_exists(unique_index(:conversations, [:ap_id])) end end diff --git a/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs index b5ca2fc0f..e22c6e57d 100644 --- a/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs +++ b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddParticipationUpdatedAtIndex do use Ecto.Migration def change do - create_if_not_exists index(:conversation_participations, ["updated_at desc"]) + create_if_not_exists(index(:conversation_participations, ["updated_at desc"])) end end diff --git a/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs b/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs index c19427f12..374e2323d 100644 --- a/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs +++ b/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs @@ -2,6 +2,8 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnUserInfoDeactivated do use Ecto.Migration def change do - create_if_not_exists(index(:users, ["(info->'deactivated')"], name: :users_deactivated_index, using: :gin)) + create_if_not_exists( + index(:users, ["(info->'deactivated')"], name: :users_deactivated_index, using: :gin) + ) end end diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs index ce4590954..f3928a149 100644 --- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs +++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs @@ -18,7 +18,7 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do |> Enum.each(fn %{id: user_id, bookmarks: bookmarks} -> Enum.each(bookmarks, fn ap_id -> activity = Activity.get_create_by_object_ap_id(ap_id) - unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id) + unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id) end) end) @@ -29,7 +29,7 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do def down do alter table(:users) do - add :bookmarks, {:array, :string}, null: false, default: [] + add(:bookmarks, {:array, :string}, null: false, default: []) end end end diff --git a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs index d4de51691..41630bace 100644 --- a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs +++ b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs @@ -2,7 +2,18 @@ defmodule Pleroma.Repo.Migrations.AddFTSIndexToObjects do use Ecto.Migration def change do - drop_if_exists index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], using: :gin, name: :activities_fts) - create_if_not_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + drop_if_exists( + index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], + using: :gin, + name: :activities_fts + ) + ) + + create_if_not_exists( + index(:objects, ["(to_tsvector('english', data->>'content'))"], + using: :gin, + name: :objects_fts + ) + ) end end diff --git a/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs b/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs index 0d3d253b6..ab1351d56 100644 --- a/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs +++ b/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs @@ -2,18 +2,18 @@ defmodule Pleroma.Repo.Migrations.SetDefaultStateToReports do use Ecto.Migration def up do - execute """ + execute(""" UPDATE activities AS a SET data = jsonb_set(data, '{state}', '"open"', true) WHERE data->>'type' = 'Flag' - """ + """) end def down do - execute """ + execute(""" UPDATE activities AS a SET data = data #- '{state}' WHERE data->>'type' = 'Flag' - """ + """) end end diff --git a/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs b/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs index 246b70cfb..8135ab178 100644 --- a/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs +++ b/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs @@ -3,13 +3,13 @@ defmodule Pleroma.Repo.Migrations.ChangeHideColumnInFilterTable do def up do alter table(:filters) do - modify :hide, :boolean, default: false + modify(:hide, :boolean, default: false) end end def down do alter table(:filters) do - modify :hide, :boolean + modify(:hide, :boolean) end end end diff --git a/priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs b/priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs index df4ac7782..faed5e31b 100644 --- a/priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs +++ b/priv/repo/migrations/20190603162018_add_object_in_reply_to_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddObjectInReplyToIndex do use Ecto.Migration def change do - create index(:objects, ["(data->>'inReplyTo')"], name: :objects_in_reply_to_index) + create(index(:objects, ["(data->>'inReplyTo')"], name: :objects_in_reply_to_index)) end end diff --git a/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs index 93d57a249..9ba95917c 100644 --- a/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs +++ b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs @@ -2,7 +2,10 @@ defmodule Pleroma.Repo.Migrations.AddTagIndexToObjects do use Ecto.Migration def change do - drop_if_exists index(:activities, ["(data #> '{\"object\",\"tag\"}')"], using: :gin, name: :activities_tags) - create_if_not_exists index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags) + drop_if_exists( + index(:activities, ["(data #> '{\"object\",\"tag\"}')"], using: :gin, name: :activities_tags) + ) + + create_if_not_exists(index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags)) end end diff --git a/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs b/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs index b717cab2e..bc4e828cc 100644 --- a/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs +++ b/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs @@ -3,6 +3,8 @@ defmodule Pleroma.Repo.Migrations.CopyMutedToMutedNotifications do alias Pleroma.User def change do - execute("update users set info = jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true") + execute( + "update users set info = jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true" + ) end end diff --git a/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs b/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs index c6e3469d5..59cbe25ad 100644 --- a/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs +++ b/priv/repo/migrations/20190801154554_create_conversation_participation_recipient_ships.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Repo.Migrations.CreateConversationParticipationRecipientShips add(:participation_id, references(:conversation_participations, on_delete: :delete_all)) end - create_if_not_exists index(:conversation_participation_recipient_ships, [:user_id]) - create_if_not_exists index(:conversation_participation_recipient_ships, [:participation_id]) + create_if_not_exists(index(:conversation_participation_recipient_ships, [:user_id])) + create_if_not_exists(index(:conversation_participation_recipient_ships, [:participation_id])) end end diff --git a/priv/repo/migrations/20190823000549_add_likes_index_to_objects.exs b/priv/repo/migrations/20190823000549_add_likes_index_to_objects.exs index 13f3d6e83..c410dcdc2 100644 --- a/priv/repo/migrations/20190823000549_add_likes_index_to_objects.exs +++ b/priv/repo/migrations/20190823000549_add_likes_index_to_objects.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddLikesIndexToObjects do use Ecto.Migration def change do - create_if_not_exists index(:objects, ["(data->'likes')"], using: :gin, name: :objects_likes) + create_if_not_exists(index(:objects, ["(data->'likes')"], using: :gin, name: :objects_likes)) end end diff --git a/priv/repo/migrations/20190912065617_create_deliveries.exs b/priv/repo/migrations/20190912065617_create_deliveries.exs index 79071a799..ac2832a77 100644 --- a/priv/repo/migrations/20190912065617_create_deliveries.exs +++ b/priv/repo/migrations/20190912065617_create_deliveries.exs @@ -6,7 +6,8 @@ defmodule Pleroma.Repo.Migrations.CreateDeliveries do add(:object_id, references(:objects, type: :id), null: false) add(:user_id, references(:users, type: :uuid, on_delete: :delete_all), null: false) end - create_if_not_exists index(:deliveries, :object_id, name: :deliveries_object_id) + + create_if_not_exists(index(:deliveries, :object_id, name: :deliveries_object_id)) create_if_not_exists(unique_index(:deliveries, [:user_id, :object_id])) end end diff --git a/priv/repo/migrations/20190929201536_drop_subscription_if_exists.exs b/priv/repo/migrations/20190929201536_drop_subscription_if_exists.exs index bbf70f78b..8bd2a98bb 100644 --- a/priv/repo/migrations/20190929201536_drop_subscription_if_exists.exs +++ b/priv/repo/migrations/20190929201536_drop_subscription_if_exists.exs @@ -2,7 +2,6 @@ defmodule Pleroma.Repo.Migrations.DropSubscriptionIfExists do use Ecto.Migration def change do - end def up do @@ -10,6 +9,7 @@ defmodule Pleroma.Repo.Migrations.DropSubscriptionIfExists do drop_if_exists(index(:subscription_notifications, ["id desc nulls last"])) drop_if_exists(table(:subscription_notifications)) end + def down do :ok end diff --git a/priv/repo/migrations/20191006123824_add_keys_column.exs b/priv/repo/migrations/20191006123824_add_keys_column.exs index b6c615646..4114ba416 100644 --- a/priv/repo/migrations/20191006123824_add_keys_column.exs +++ b/priv/repo/migrations/20191006123824_add_keys_column.exs @@ -3,7 +3,7 @@ defmodule Pleroma.Repo.Migrations.AddKeysColumn do def change do alter table("users") do - add_if_not_exists :keys, :text + add_if_not_exists(:keys, :text) end end end diff --git a/priv/repo/migrations/20191006135457_move_keys_to_separate_column.exs b/priv/repo/migrations/20191006135457_move_keys_to_separate_column.exs index 504dde53a..cb8d4ae9e 100644 --- a/priv/repo/migrations/20191006135457_move_keys_to_separate_column.exs +++ b/priv/repo/migrations/20191006135457_move_keys_to_separate_column.exs @@ -2,6 +2,9 @@ defmodule Pleroma.Repo.Migrations.MoveKeysToSeparateColumn do use Ecto.Migration def change do - execute("update users set keys = info->>'keys' where local", "update users set info = jsonb_set(info, '{keys}'::text[], to_jsonb(keys)) where local") + execute( + "update users set keys = info->>'keys' where local", + "update users set info = jsonb_set(info, '{keys}'::text[], to_jsonb(keys)) where local" + ) end end From 580a67f5ca152fa4454fd9d8ab2e2a5a55af1a2a Mon Sep 17 00:00:00 2001 From: eugenijm Date: Wed, 9 Oct 2019 06:51:14 +0300 Subject: [PATCH 09/84] Mastodon API: Return `pleroma.direct_conversation_id` when viewing a status (`GET /api/v1/statuses/:id`) --- CHANGELOG.md | 1 + .../controllers/status_controller.ex | 6 +++++- .../controllers/status_controller_test.exs | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b24db7f4..da921b98e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Return link alongside with token on password reset - MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities - OStatus: Extract RSS functionality +- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) ### Fixed - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 0c16e9b0f..e5d016f63 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -167,7 +167,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do def show(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do - try_render(conn, "show.json", activity: activity, for: user) + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true + ) end end diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index a4bbfe055..2de2725e0 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do alias Pleroma.Activity alias Pleroma.ActivityExpiration alias Pleroma.Config + alias Pleroma.Conversation.Participation alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity @@ -465,6 +466,24 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do assert id == to_string(activity.id) end + test "get a direct status", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "@#{other_user.nickname}", "visibility" => "direct"}) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/statuses/#{activity.id}") + + [participation] = Participation.for_user(user) + + res = json_response(conn, 200) + assert res["pleroma"]["direct_conversation_id"] == participation.id + end + test "get statuses by IDs", %{conn: conn} do %{id: id1} = insert(:note_activity) %{id: id2} = insert(:note_activity) From d537bfd4e1e3119188519a4b1839fdbe3f6fad39 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 9 Oct 2019 13:11:57 +0700 Subject: [PATCH 10/84] Add a task to re-count statuses for all users --- CHANGELOG.md | 1 + lib/mix/tasks/pleroma/count_statuses.ex | 22 ++++++++++++++ test/tasks/count_statuses_test.exs | 39 +++++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 lib/mix/tasks/pleroma/count_statuses.ex create mode 100644 test/tasks/count_statuses_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b24db7f4..759745dd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: support for hierarchical permissions / [Mastodon 2.4.3 OAuth permissions](https://docs.joinmastodon.org/api/permissions/) - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed +- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/lib/mix/tasks/pleroma/count_statuses.ex b/lib/mix/tasks/pleroma/count_statuses.ex new file mode 100644 index 000000000..e1e8195dd --- /dev/null +++ b/lib/mix/tasks/pleroma/count_statuses.ex @@ -0,0 +1,22 @@ +defmodule Mix.Tasks.Pleroma.CountStatuses do + @shortdoc "Re-counts statuses for all users" + + use Mix.Task + alias Pleroma.User + import Ecto.Query + + def run([]) do + Mix.Pleroma.start_pleroma() + + stream = + User + |> where(local: true) + |> Pleroma.Repo.stream() + + Pleroma.Repo.transaction(fn -> + Enum.each(stream, &User.update_note_count/1) + end) + + Mix.Pleroma.shell_info("Done") + end +end diff --git a/test/tasks/count_statuses_test.exs b/test/tasks/count_statuses_test.exs new file mode 100644 index 000000000..6035da3c3 --- /dev/null +++ b/test/tasks/count_statuses_test.exs @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.CountStatusesTest do + use Pleroma.DataCase + + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + import ExUnit.CaptureIO, only: [capture_io: 1] + import Pleroma.Factory + + test "counts statuses" do + user = insert(:user) + {:ok, _} = CommonAPI.post(user, %{"status" => "test"}) + {:ok, _} = CommonAPI.post(user, %{"status" => "test2"}) + + user2 = insert(:user) + {:ok, _} = CommonAPI.post(user2, %{"status" => "test3"}) + + user = refresh_record(user) + user2 = refresh_record(user2) + + assert %{info: %{note_count: 2}} = user + assert %{info: %{note_count: 1}} = user2 + + {:ok, user} = User.update_info(user, &User.Info.set_note_count(&1, 0)) + {:ok, user2} = User.update_info(user2, &User.Info.set_note_count(&1, 0)) + + assert %{info: %{note_count: 0}} = user + assert %{info: %{note_count: 0}} = user2 + + assert capture_io(fn -> Mix.Tasks.Pleroma.CountStatuses.run([]) end) == "Done\n" + + assert %{info: %{note_count: 2}} = refresh_record(user) + assert %{info: %{note_count: 1}} = refresh_record(user2) + end +end From c1bae0137666505c16482fd22cb3a14bed7a4499 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 9 Oct 2019 16:03:45 +0200 Subject: [PATCH 11/84] Search: Add tests for unicode searches. --- test/activity_test.exs | 18 +++++++++++++++++- .../controllers/search_controller_test.exs | 15 +++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/test/activity_test.exs b/test/activity_test.exs index 95d9341c4..e7ea2bd5e 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -126,9 +126,25 @@ defmodule Pleroma.ActivityTest do } {:ok, local_activity} = Pleroma.Web.CommonAPI.post(user, %{"status" => "find me!"}) + {:ok, japanese_activity} = Pleroma.Web.CommonAPI.post(user, %{"status" => "更新情報"}) {:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params) {:ok, remote_activity} = ObanHelpers.perform(job) - %{local_activity: local_activity, remote_activity: remote_activity, user: user} + + %{ + japanese_activity: japanese_activity, + local_activity: local_activity, + remote_activity: remote_activity, + user: user + } + end + + test "finds utf8 text in statuses", %{ + japanese_activity: japanese_activity, + user: user + } do + activities = Activity.search(user, "更新情報") + + assert [^japanese_activity] = activities end test "find local and remote statuses for authenticated users", %{ diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index 49c79ff0a..0ca896e01 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -40,7 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do test "search", %{conn: conn} do user = insert(:user) user_two = insert(:user, %{nickname: "shp@shitposter.club"}) - user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"}) + user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu 天子"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "This is about 2hu private"}) @@ -52,9 +52,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do {:ok, _} = CommonAPI.post(user_two, %{"status" => "This isn't"}) - conn = get(conn, "/api/v2/search", %{"q" => "2hu #private"}) - - assert results = json_response(conn, 200) + results = + get(conn, "/api/v2/search", %{"q" => "2hu #private"}) + |> json_response(200) [account | _] = results["accounts"] assert account["id"] == to_string(user_three.id) @@ -65,6 +65,13 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do [status] = results["statuses"] assert status["id"] == to_string(activity.id) + + results = + get(conn, "/api/v2/search", %{"q" => "天子"}) + |> json_response(200) + + [account] == results["accounts"] + assert account["id"] == to_string(user_three.id) end end From ad42837244ba4c945b76c5addaffe47353cf62a8 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 9 Oct 2019 17:03:54 +0300 Subject: [PATCH 12/84] Ability to toggle activation status and permission group for a group of users --- CHANGELOG.md | 3 + docs/API/admin_api.md | 49 +++++-- lib/pleroma/moderation_log.ex | 60 ++++----- lib/pleroma/user.ex | 16 ++- .../web/admin_api/admin_api_controller.ex | 95 ++++++------- .../web/admin_api/views/account_view.ex | 6 + lib/pleroma/web/router.ex | 14 +- test/moderation_log_test.exs | 4 +- .../admin_api/admin_api_controller_test.exs | 125 +++++++----------- 9 files changed, 189 insertions(+), 183 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b24db7f4..584c917f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset +- **Breaking:** Admin API: `/users/:nickname/toggle_activation` endpoint was split into two: `/users/activate`, `/users/deactivate`, both accept `nicknames` array +- **Breaking:** Admin API: `POST /users/permission_group/:permission_group` / `DELETE /users/permission_group/:permission_group` now accept `nicknames` array - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports @@ -40,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - **Breaking:** GNU Social API with Qvitter extensions support +- **Breaking:** Admin API: `/users/:nickname/activation_status` was removed in favor of `/users/activate`, `/users/deactivate` - Emoji: Remove longfox emojis. - Remove `Reply-To` header from report emails for admins. diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index ee9e68cb1..55f8749e1 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -154,31 +154,62 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## `POST /api/pleroma/admin/users/permission_group/:permission_group` + ### Add user in permission group -- Method: `POST` -- Params: none +- Params: + - `nicknames`: nicknames array - Response: - On failure: `{"error": "…"}` - On success: JSON of the `user.info` +## `DELETE /api/pleroma/admin/users/permission_group/:permission_group` + ### Remove user from permission group -- Method: `DELETE` -- Params: none +- Params: + - `nicknames`: nicknames array - Response: - On failure: `{"error": "…"}` - On success: JSON of the `user.info` - Note: An admin cannot revoke their own admin status. -## `/api/pleroma/admin/users/:nickname/activation_status` +## `PATCH /api/pleroma/admin/users/activate` -### Active or deactivate a user +### Activate user -- Method: `PUT` - Params: - - `nickname` - - `status` BOOLEAN field, false value means deactivation. + - `nicknames`: nicknames array +- Response: + +```json +{ + users: [ + { + // user object + } + ] +} +``` + +## `PATCH /api/pleroma/admin/users/deactivate` + +### Deactivate user + +- Params: + - `nicknames`: nicknames array +- Response: + +```json +{ + users: [ + { + // user object + } + ] +} +``` ## `/api/pleroma/admin/users/:nickname_or_id` diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 352cad433..42649ff02 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -86,18 +86,18 @@ defmodule Pleroma.ModerationLog do parsed_datetime end - @spec insert_log(%{actor: User, subject: User, action: String.t(), permission: String.t()}) :: + @spec insert_log(%{actor: User, subject: [User], action: String.t(), permission: String.t()}) :: {:ok, ModerationLog} | {:error, any} def insert_log(%{ actor: %User{} = actor, - subject: %User{} = subject, + subject: subjects, action: action, permission: permission }) do %ModerationLog{ data: %{ "actor" => user_to_map(actor), - "subject" => user_to_map(subject), + "subject" => user_to_map(subjects), "action" => action, "permission" => permission, "message" => "" @@ -303,13 +303,16 @@ defmodule Pleroma.ModerationLog do end @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any} - defp insert_log_entry_with_message(entry) do entry.data["message"] |> put_in(get_log_entry_message(entry)) |> Repo.insert() end + defp user_to_map(users) when is_list(users) do + users |> Enum.map(&user_to_map/1) + end + defp user_to_map(%User{} = user) do user |> Map.from_struct() @@ -363,12 +366,7 @@ defmodule Pleroma.ModerationLog do "subjects" => subjects } }) do - nicknames = - subjects - |> Enum.map(&"@#{&1["nickname"]}") - |> Enum.join(", ") - - "@#{actor_nickname} created users: #{nicknames}" + "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -376,10 +374,10 @@ defmodule Pleroma.ModerationLog do data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "activate", - "subject" => %{"nickname" => subject_nickname, "type" => "user"} + "subject" => users } }) do - "@#{actor_nickname} activated user @#{subject_nickname}" + "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -387,10 +385,10 @@ defmodule Pleroma.ModerationLog do data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "deactivate", - "subject" => %{"nickname" => subject_nickname, "type" => "user"} + "subject" => users } }) do - "@#{actor_nickname} deactivated user @#{subject_nickname}" + "@#{actor_nickname} deactivated users: #{users_to_nicknames_string(users)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -402,14 +400,9 @@ defmodule Pleroma.ModerationLog do "action" => "tag" } }) do - nicknames_string = - nicknames - |> Enum.map(&"@#{&1}") - |> Enum.join(", ") - tags_string = tags |> Enum.join(", ") - "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_string}" + "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_to_string(nicknames)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -421,14 +414,9 @@ defmodule Pleroma.ModerationLog do "action" => "untag" } }) do - nicknames_string = - nicknames - |> Enum.map(&"@#{&1}") - |> Enum.join(", ") - tags_string = tags |> Enum.join(", ") - "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_string}" + "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -436,11 +424,11 @@ defmodule Pleroma.ModerationLog do data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "grant", - "subject" => %{"nickname" => subject_nickname}, + "subject" => users, "permission" => permission } }) do - "@#{actor_nickname} made @#{subject_nickname} #{permission}" + "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -448,11 +436,11 @@ defmodule Pleroma.ModerationLog do data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "revoke", - "subject" => %{"nickname" => subject_nickname}, + "subject" => users, "permission" => permission } }) do - "@#{actor_nickname} revoked #{permission} role from @#{subject_nickname}" + "@#{actor_nickname} revoked #{permission} role from #{users_to_nicknames_string(users)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -551,4 +539,16 @@ defmodule Pleroma.ModerationLog do }) do "@#{actor_nickname} deleted status ##{subject_id}" end + + defp nicknames_to_string(nicknames) do + nicknames + |> Enum.map(&"@#{&1}") + |> Enum.join(", ") + end + + defp users_to_nicknames_string(users) do + users + |> Enum.map(&"@#{&1["nickname"]}") + |> Enum.join(", ") + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2cfb13a8c..a76a5ad70 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1059,7 +1059,15 @@ defmodule Pleroma.User do BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status}) end - def deactivate(%User{} = user, status \\ true) do + def deactivate(user, status \\ true) + + def deactivate(users, status) when is_list(users) do + Repo.transaction(fn -> + for user <- users, do: deactivate(user, status) + end) + end + + def deactivate(%User{} = user, status) do with {:ok, user} <- update_info(user, &User.Info.set_activation_status(&1, status)) do Enum.each(get_followers(user), &invalidate_cache/1) Enum.each(get_friends(user), &update_follower_count/1) @@ -1625,6 +1633,12 @@ defmodule Pleroma.User do `fun` is called with the `user.info`. """ + def update_info(users, fun) when is_list(users) do + Repo.transaction(fn -> + for user <- users, do: update_info(user, fun) + end) + end + def update_info(user, fun) do user |> change_info(fun) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 513bae800..d825a5d28 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -231,22 +231,34 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end - def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do - user = User.get_cached_by_nickname(nickname) - - {:ok, updated_user} = User.deactivate(user, !user.info.deactivated) - - action = if user.info.deactivated, do: "activate", else: "deactivate" + def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.deactivate(users, false) ModerationLog.insert_log(%{ actor: admin, - subject: user, - action: action + subject: users, + action: "activate" }) conn |> put_view(AccountView) - |> render("show.json", %{user: updated_user}) + |> render("index.json", %{users: Keyword.values(updated_users)}) + end + + def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.deactivate(users, true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "deactivate" + }) + + conn + |> put_view(AccountView) + |> render("index.json", %{users: Keyword.values(updated_users)}) end def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do @@ -315,20 +327,19 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def right_add(%{assigns: %{user: admin}} = conn, %{ "permission_group" => permission_group, - "nickname" => nickname + "nicknames" => nicknames }) when permission_group in ["moderator", "admin"] do info = Map.put(%{}, "is_" <> permission_group, true) - {:ok, user} = - nickname - |> User.get_cached_by_nickname() - |> User.update_info(&User.Info.admin_api_update(&1, info)) + users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) + + User.update_info(users, &User.Info.admin_api_update(&1, info)) ModerationLog.insert_log(%{ action: "grant", actor: admin, - subject: user, + subject: users, permission: permission_group }) @@ -349,58 +360,38 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) end - def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do - render_error(conn, :forbidden, "You can't revoke your own admin status.") - end - def right_delete( - %{assigns: %{user: admin}} = conn, + %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn, %{ "permission_group" => permission_group, - "nickname" => nickname + "nicknames" => nicknames } ) when permission_group in ["moderator", "admin"] do - info = Map.put(%{}, "is_" <> permission_group, false) + with false <- Enum.member?(nicknames, admin_nickname) do + info = Map.put(%{}, "is_" <> permission_group, false) - {:ok, user} = - nickname - |> User.get_cached_by_nickname() - |> User.update_info(&User.Info.admin_api_update(&1, info)) + users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) - ModerationLog.insert_log(%{ - action: "revoke", - actor: admin, - subject: user, - permission: permission_group - }) + User.update_info(users, &User.Info.admin_api_update(&1, info)) - json(conn, info) + ModerationLog.insert_log(%{ + action: "revoke", + actor: admin, + subject: users, + permission: permission_group + }) + + json(conn, info) + else + _ -> render_error(conn, :forbidden, "You can't revoke your own admin/moderator status.") + end end def right_delete(conn, _) do render_error(conn, :not_found, "No such permission_group") end - def set_activation_status(%{assigns: %{user: admin}} = conn, %{ - "nickname" => nickname, - "status" => status - }) do - with {:ok, status} <- Ecto.Type.cast(:boolean, status), - %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, _} <- User.deactivate(user, !status) do - action = if(user.info.deactivated, do: "activate", else: "deactivate") - - ModerationLog.insert_log(%{ - actor: admin, - subject: user, - action: action - }) - - json_response(conn, :no_content, "") - end - end - def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do with {:ok, _message} <- Relay.follow(target) do ModerationLog.insert_log(%{ diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index a96affd40..441269162 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -19,6 +19,12 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end + def render("index.json", %{users: users}) do + %{ + users: render_many(users, AccountView, "show.json", as: :user) + } + end + def render("show.json", %{user: user}) do avatar = User.avatar_url(user) |> MediaProxy.url() display_name = HTML.strip_tags(user.name || user.nickname) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ae799b8ac..894375357 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -136,21 +136,15 @@ defmodule Pleroma.Web.Router do delete("/users", AdminAPIController, :user_delete) post("/users", AdminAPIController, :users_create) - patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) + patch("/users/activate", AdminAPIController, :user_activate) + patch("/users/deactivate", AdminAPIController, :user_deactivate) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) get("/users/:nickname/permission_group", AdminAPIController, :right_get) get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get) - post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add) - - delete( - "/users/:nickname/permission_group/:permission_group", - AdminAPIController, - :right_delete - ) - - put("/users/:nickname/activation_status", AdminAPIController, :set_activation_status) + post("/users/permission_group/:permission_group", AdminAPIController, :right_add) + delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete) post("/relay", AdminAPIController, :relay_follow) delete("/relay", AdminAPIController, :relay_unfollow) diff --git a/test/moderation_log_test.exs b/test/moderation_log_test.exs index a39a00e02..ead97e948 100644 --- a/test/moderation_log_test.exs +++ b/test/moderation_log_test.exs @@ -128,7 +128,7 @@ defmodule Pleroma.ModerationLogTest do {:ok, _} = ModerationLog.insert_log(%{ actor: moderator, - subject: subject1, + subject: [subject1], action: "grant", permission: "moderator" }) @@ -142,7 +142,7 @@ defmodule Pleroma.ModerationLogTest do {:ok, _} = ModerationLog.insert_log(%{ actor: moderator, - subject: subject1, + subject: [subject1], action: "revoke", permission: "moderator" }) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index b5c355e66..c57c71203 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -386,13 +386,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do test "/:right POST, can add to a permission group" do admin = insert(:user, info: %{is_admin: true}) - user = insert(:user) + user_one = insert(:user) + user_two = insert(:user) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + |> post("/api/pleroma/admin/users/permission_group/admin", %{ + nicknames: [user_one.nickname, user_two.nickname] + }) assert json_response(conn, 200) == %{ "is_admin" => true @@ -401,18 +404,21 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do log_entry = Repo.one(ModerationLog) assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} made @#{user.nickname} admin" + "@#{admin.nickname} made @#{user_one.nickname}, @#{user_two.nickname} admin" end test "/:right DELETE, can remove from a permission group" do admin = insert(:user, info: %{is_admin: true}) - user = insert(:user, info: %{is_admin: true}) + user_one = insert(:user, info: %{is_admin: true}) + user_two = insert(:user, info: %{is_admin: true}) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> delete("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + |> delete("/api/pleroma/admin/users/permission_group/admin", %{ + nicknames: [user_one.nickname, user_two.nickname] + }) assert json_response(conn, 200) == %{ "is_admin" => false @@ -421,65 +427,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do log_entry = Repo.one(ModerationLog) assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} revoked admin role from @#{user.nickname}" - end - end - - describe "PUT /api/pleroma/admin/users/:nickname/activation_status" do - setup %{conn: conn} do - admin = insert(:user, info: %{is_admin: true}) - - conn = - conn - |> assign(:user, admin) - |> put_req_header("accept", "application/json") - - %{conn: conn, admin: admin} - end - - test "deactivates the user", %{conn: conn, admin: admin} do - user = insert(:user) - - conn = - conn - |> put("/api/pleroma/admin/users/#{user.nickname}/activation_status", %{status: false}) - - user = User.get_cached_by_id(user.id) - assert user.info.deactivated == true - assert json_response(conn, :no_content) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deactivated user @#{user.nickname}" - end - - test "activates the user", %{conn: conn, admin: admin} do - user = insert(:user, info: %{deactivated: true}) - - conn = - conn - |> put("/api/pleroma/admin/users/#{user.nickname}/activation_status", %{status: true}) - - user = User.get_cached_by_id(user.id) - assert user.info.deactivated == false - assert json_response(conn, :no_content) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} activated user @#{user.nickname}" - end - - test "returns 403 when requested by a non-admin", %{conn: conn} do - user = insert(:user) - - conn = - conn - |> assign(:user, user) - |> put("/api/pleroma/admin/users/#{user.nickname}/activation_status", %{status: false}) - - assert json_response(conn, :forbidden) + "@#{admin.nickname} revoked admin role from @#{user_one.nickname}, @#{ + user_two.nickname + }" end end @@ -1029,31 +979,48 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end end - test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do + test "PATCH /api/pleroma/admin/users/activate" do admin = insert(:user, info: %{is_admin: true}) - user = insert(:user) + user_one = insert(:user, info: %{deactivated: true}) + user_two = insert(:user, info: %{deactivated: true}) conn = build_conn() |> assign(:user, admin) - |> patch("/api/pleroma/admin/users/#{user.nickname}/toggle_activation") + |> patch( + "/api/pleroma/admin/users/activate", + %{nicknames: [user_one.nickname, user_two.nickname]} + ) - assert json_response(conn, 200) == - %{ - "deactivated" => !user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) - } + response = json_response(conn, 200) + assert Enum.map(response["users"], & &1["deactivated"]) == [false, false] log_entry = Repo.one(ModerationLog) assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deactivated user @#{user.nickname}" + "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}" + end + + test "PATCH /api/pleroma/admin/users/deactivate" do + admin = insert(:user, info: %{is_admin: true}) + user_one = insert(:user, info: %{deactivated: false}) + user_two = insert(:user, info: %{deactivated: false}) + + conn = + build_conn() + |> assign(:user, admin) + |> patch( + "/api/pleroma/admin/users/deactivate", + %{nicknames: [user_one.nickname, user_two.nickname]} + ) + + response = json_response(conn, 200) + assert Enum.map(response["users"], & &1["deactivated"]) == [true, true] + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}" end describe "POST /api/pleroma/admin/users/invite_token" do From dc6d20b68d3940b9537981a2556980a6ee3a43e3 Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Wed, 9 Oct 2019 16:32:28 +0000 Subject: [PATCH 13/84] provide mocks for webfinger endpoints --- test/fixtures/tesla_mock/mstdn.jp_host_meta | 4 + .../pekorino@pawoo.net_host_meta.json | 12 ++ test/fixtures/tesla_mock/sdf.org_host_meta | 4 + .../snowdusk@sdf.org_host_meta.json | 12 ++ test/fixtures/tesla_mock/soykaf.com_host_meta | 4 + .../stopwatchingus-heidelberg.de_host_meta | 31 ++++ .../fixtures/tesla_mock/xn--q9jyb4c_host_meta | 4 + test/support/http_request_mock.ex | 175 ++++++++++++++++++ 8 files changed, 246 insertions(+) create mode 100644 test/fixtures/tesla_mock/mstdn.jp_host_meta create mode 100644 test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json create mode 100644 test/fixtures/tesla_mock/sdf.org_host_meta create mode 100644 test/fixtures/tesla_mock/snowdusk@sdf.org_host_meta.json create mode 100644 test/fixtures/tesla_mock/soykaf.com_host_meta create mode 100644 test/fixtures/tesla_mock/stopwatchingus-heidelberg.de_host_meta create mode 100644 test/fixtures/tesla_mock/xn--q9jyb4c_host_meta diff --git a/test/fixtures/tesla_mock/mstdn.jp_host_meta b/test/fixtures/tesla_mock/mstdn.jp_host_meta new file mode 100644 index 000000000..e76ddd47f --- /dev/null +++ b/test/fixtures/tesla_mock/mstdn.jp_host_meta @@ -0,0 +1,4 @@ + + + + diff --git a/test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json b/test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json new file mode 100644 index 000000000..3757c0dad --- /dev/null +++ b/test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json @@ -0,0 +1,12 @@ +{ + "subject":"acct:pekorino@pawoo.net", + "aliases":["https://pawoo.net/@pekorino","https://pawoo.net/users/pekorino"], + "links":[ + {"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"https://pawoo.net/@pekorino"}, + {"rel":"http://schemas.google.com/g/2010#updates-from","type":"application/atom+xml","href":"https://pawoo.net/users/pekorino.atom"}, + {"rel":"self","type":"application/activity+json","href":"https://pawoo.net/users/pekorino"}, + {"rel":"salmon","href":"https://pawoo.net/api/salmon/128378"}, + {"rel":"magic-public-key","href":"data:application/magic-public-key,RSA.1x8XXmBqzyb-QRkfUKxKPd7Ac2KbaFhdKy2FkJY64G-ifga-BppzEb62Q5TdkRdVKdHjh5qI7A1Hk3KfnNQcNWqqak-jxII_txC2grbWpp7v-boceD2pnzdVK5l-RR-9wEwxcoCUeRWS1Ak6DStqE5tFQOAK4IIGQB-thSQGlU75KZ-2080fPA3Xc_ycH3_eB4YqawSxXrh6IeScMevN0YHSF84GAcvhXmwLKZRugiF6nYrknbPEe_niIOmN8hhEXLN9_4kDcH83hkVZd5VXssRrxqDhtokx9emvTHkA7sY1AjYeehTPZErlV74GN-kFYLeI6DluXoSI2sX1QcS08w==.AQAB"}, + {"rel":"http://ostatus.org/schema/1.0/subscribe","template":"https://pawoo.net/authorize_follow?acct={uri}"} + ] +} diff --git a/test/fixtures/tesla_mock/sdf.org_host_meta b/test/fixtures/tesla_mock/sdf.org_host_meta new file mode 100644 index 000000000..0ffc4f096 --- /dev/null +++ b/test/fixtures/tesla_mock/sdf.org_host_meta @@ -0,0 +1,4 @@ + + + + diff --git a/test/fixtures/tesla_mock/snowdusk@sdf.org_host_meta.json b/test/fixtures/tesla_mock/snowdusk@sdf.org_host_meta.json new file mode 100644 index 000000000..273fc3804 --- /dev/null +++ b/test/fixtures/tesla_mock/snowdusk@sdf.org_host_meta.json @@ -0,0 +1,12 @@ +{ + "subject":"acct:snowdusk@mastodon.sdf.org", + "aliases":["https://mastodon.sdf.org/@snowdusk","https://mastodon.sdf.org/users/snowdusk"], + "links":[ + {"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"https://mastodon.sdf.org/@snowdusk"}, + {"rel":"http://schemas.google.com/g/2010#updates-from","type":"application/atom+xml","href":"https://mastodon.sdf.org/users/snowdusk.atom"}, + {"rel":"self","type":"application/activity+json","href":"https://mastodon.sdf.org/users/snowdusk"}, + {"rel":"salmon","href":"https://mastodon.sdf.org/api/salmon/2"}, + {"rel":"magic-public-key","href":"data:application/magic-public-key,RSA.k4_Hr0WQUHumAD4uwWIz7OybovIKgIuanbXhX5pl7oGyb2TuifBf3nAqEhD6eLSo6-_6160L4BvPPV_l_6rlZEi6_nbeJUgVkayZgcZN3oou3IErSt8L0IbUdWT5s4fWM2zpkndLCkVbeeNQ3DOBccvJw7iA_QNTao8wr3ILvQaKEDnf-H5QBd9Tj3seyo4-7E0e6wCKOH_uBm8pSRgpdMdl2CehiFzaABBkmCeUKH-buU7iNQGi0fsV5VIHn6zffrv6p0EVNkjTDi1vTmmfrp9W0mcKZJ9DtvdehOKSgh3J7Mem-ILbPy6FSL2Oi6Ekj_Wh4M8Ie-YKuxI3N_0Baw==.AQAB"}, + {"rel":"http://ostatus.org/schema/1.0/subscribe","template":"https://mastodon.sdf.org/authorize_interaction?uri={uri}"} + ] +} diff --git a/test/fixtures/tesla_mock/soykaf.com_host_meta b/test/fixtures/tesla_mock/soykaf.com_host_meta new file mode 100644 index 000000000..99d552d32 --- /dev/null +++ b/test/fixtures/tesla_mock/soykaf.com_host_meta @@ -0,0 +1,4 @@ + + + + diff --git a/test/fixtures/tesla_mock/stopwatchingus-heidelberg.de_host_meta b/test/fixtures/tesla_mock/stopwatchingus-heidelberg.de_host_meta new file mode 100644 index 000000000..481cfec8d --- /dev/null +++ b/test/fixtures/tesla_mock/stopwatchingus-heidelberg.de_host_meta @@ -0,0 +1,31 @@ +{ + "links":[ + { + "rel":"lrdd", + "type":"application\/jrd+json", + "template":"https:\/\/social.stopwatchingus-heidelberg.de\/.well-known\/webfinger?resource={uri}" + }, + { + "rel":"lrdd", + "type":"application\/json", + "template":"https:\/\/social.stopwatchingus-heidelberg.de\/.well-known\/webfinger?resource={uri}" + }, + { + "rel":"lrdd", + "type":"application\/xrd+xml", + "template":"https:\/\/social.stopwatchingus-heidelberg.de\/.well-known\/webfinger?resource={uri}" + }, + { + "rel":"http:\/\/apinamespace.org\/oauth\/access_token", + "href":"https:\/\/social.stopwatchingus-heidelberg.de\/api\/oauth\/access_token" + }, + { + "rel":"http:\/\/apinamespace.org\/oauth\/request_token", + "href":"https:\/\/social.stopwatchingus-heidelberg.de\/api\/oauth\/request_token" + }, + { + "rel":"http:\/\/apinamespace.org\/oauth\/authorize", + "href":"https:\/\/social.stopwatchingus-heidelberg.de\/api\/oauth\/authorize" + } + ] +} diff --git a/test/fixtures/tesla_mock/xn--q9jyb4c_host_meta b/test/fixtures/tesla_mock/xn--q9jyb4c_host_meta new file mode 100644 index 000000000..45d260e55 --- /dev/null +++ b/test/fixtures/tesla_mock/xn--q9jyb4c_host_meta @@ -0,0 +1,4 @@ + + + + diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index b825a9307..4feb57f3a 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -344,6 +344,181 @@ defmodule HttpRequestMock do {:error, :nxdomain} end + def get("http://osada.macgirvin.com/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 404, + body: "" + }} + end + + def get("https://osada.macgirvin.com/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 404, + body: "" + }} + end + + def get("http://mastodon.sdf.org/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/sdf.org_host_meta") + }} + end + + def get("https://mastodon.sdf.org/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/sdf.org_host_meta") + }} + end + + def get( + "https://mastodon.sdf.org/.well-known/webfinger?resource=https://mastodon.sdf.org/users/snowdusk", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/snowdusk@sdf.org_host_meta.json") + }} + end + + def get("http://mstdn.jp/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mstdn.jp_host_meta") + }} + end + + def get("https://mstdn.jp/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mstdn.jp_host_meta") + }} + end + + def get("https://mstdn.jp/.well-known/webfinger?resource=kpherox@mstdn.jp", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/kpherox@mstdn.jp.xml") + }} + end + + def get("http://mamot.fr/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mamot.fr_host_meta") + }} + end + + def get("https://mamot.fr/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mamot.fr_host_meta") + }} + end + + def get( + "https://mamot.fr/.well-known/webfinger?resource=https://mamot.fr/users/Skruyb", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/skruyb@mamot.fr.atom") + }} + end + + def get("http://pawoo.net/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta") + }} + end + + def get("https://pawoo.net/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta") + }} + end + + def get( + "https://pawoo.net/.well-known/webfinger?resource=https://pawoo.net/users/pekorino", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json") + }} + end + + def get("http://zetsubou.xn--q9jyb4c/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/xn--q9jyb4c_host_meta") + }} + end + + def get("https://zetsubou.xn--q9jyb4c/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/xn--q9jyb4c_host_meta") + }} + end + + def get("http://pleroma.soykaf.com/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/soykaf.com_host_meta") + }} + end + + def get("https://pleroma.soykaf.com/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/soykaf.com_host_meta") + }} + end + + def get("http://social.stopwatchingus-heidelberg.de/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/stopwatchingus-heidelberg.de_host_meta") + }} + end + + def get("https://social.stopwatchingus-heidelberg.de/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/stopwatchingus-heidelberg.de_host_meta") + }} + end + def get( "http://mastodon.example.org/@admin/99541947525187367", _, From 02f8e2a8ab65c3e8497bab4576ce4e75f8df3217 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 14:24:54 +0200 Subject: [PATCH 14/84] Gitlab: Run benchmark in CI. --- .gitlab-ci.yml | 14 ++++++++++++++ config/benchmark.exs | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0c540b16..09684df02 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ cache: stages: - build - test + - benchmark - deploy - release @@ -44,6 +45,19 @@ docs-build: paths: - priv/static/doc +benchmark: + stage: benchmark + variables: + MIX_ENV: benchmark + services: + - name: lainsoykaf/postgres-with-rum + alias: postgres + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + script: + - mix deps.get + - mix ecto.create + - mix ecto.migrate + - mix pleroma.benchmark unit-testing: stage: test diff --git a/config/benchmark.exs b/config/benchmark.exs index b4f5dbdbd..62ba42e28 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -29,7 +29,8 @@ config :pleroma, :instance, email: "admin@example.com", notify_email: "noreply@example.com", skip_thread_containment: false, - federating: false + federating: false, + external_user_synchronization: false config :pleroma, :activitypub, sign_object_fetches: false From 2629493804bb5522903434cef7fe70510725f0c1 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:01:06 +0200 Subject: [PATCH 15/84] Benchmark config: Database adjustments. --- config/benchmark.exs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/config/benchmark.exs b/config/benchmark.exs index 62ba42e28..289a0e317 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -37,16 +37,11 @@ config :pleroma, :activitypub, sign_object_fetches: false # Configure your database config :pleroma, Pleroma.Repo, adapter: Ecto.Adapters.Postgres, - username: System.get_env("DB_USER") || "postgres", - database: System.get_env("DB_DATABASE") || "pleroma_test", + username: "postgres", + password: "postgres", + database: "pleroma_test", hostname: System.get_env("DB_HOST") || "localhost", - # username: "pleroma", - password: - System.get_env("DB_PASS") || - "cAUrGezwXjRwd/lIPzZAcwjb/hiZiGi3FIaSGy9l/XsTcGA61FMy7eCBiRcg1DyQ", - # password: "", - pool_size: 10, - timeout: 180_000 + pool: Ecto.Adapters.SQL.Sandbox # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 From b9b00e6e96b458d9c951ce6505ae903fe47ffa57 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:31:32 +0200 Subject: [PATCH 16/84] Gitlab CI: Run correct benchmark task. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 460c1311f..7c3ca5ee0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ benchmark: - mix deps.get - mix ecto.create - mix ecto.migrate - - mix pleroma.benchmark + - mix pleroma.load_testing unit-testing: stage: test From ecd7ac855bfc6a8e2f95c099b442d9f0f9117e6b Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:42:54 +0200 Subject: [PATCH 17/84] Benchmark config: Don't use the sql sandbox. --- config/benchmark.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/benchmark.exs b/config/benchmark.exs index 289a0e317..dd99cf5fd 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -41,7 +41,7 @@ config :pleroma, Pleroma.Repo, password: "postgres", database: "pleroma_test", hostname: System.get_env("DB_HOST") || "localhost", - pool: Ecto.Adapters.SQL.Sandbox + pool_size: 10 # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 From 29647dfd09a78fa66a605ff97119a21779d15020 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 17:17:33 +0200 Subject: [PATCH 18/84] Transmogrifier: Save correct ids for incoming deletes. --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++++-- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++-- test/web/activity_pub/transmogrifier_test.exs | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9f29087df..3a0c382ca 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -409,7 +409,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do + local = Keyword.get(options, :local, true) + activity_id = Keyword.get(options, :activity_id, nil) + user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -420,7 +423,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "object" => id, "to" => to, "deleted_activity_id" => activity && activity.id - }, + } + |> maybe_put("id", activity_id), {:ok, activity} <- insert(data, local, false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 872ed0eb2..620b54d3b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -637,7 +637,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # an error or a tombstone. This would allow us to verify that a deletion actually took # place. def handle_incoming( - %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data, + %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => id} = data, _options ) do object_id = Utils.get_ap_id(object_id) @@ -646,7 +646,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, false) do + {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do {:ok, activity} else nil -> diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 50c0bfb84..4f9c4af33 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -696,8 +696,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Map.put("object", object) |> Map.put("actor", activity.data["actor"]) - {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{local: false, data: %{"id" => id}}} = Transmogrifier.handle_incoming(data) + assert id == data["id"] refute Activity.get_by_id(activity.id) end From fb5dce481c8225579648ec52379a8f5611686196 Mon Sep 17 00:00:00 2001 From: yalh76 Date: Thu, 10 Oct 2019 20:05:54 +0000 Subject: [PATCH 19/84] Fix https://git.pleroma.social/pleroma/pleroma/issues/1289 --- config/releases.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/releases.exs b/config/releases.exs index 98c5ceccd..36c493673 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -1,6 +1,6 @@ import Config -config :pleroma, :instance, static_dir: "/var/lib/pleroma/static" +config :pleroma, :instance, static: "/var/lib/pleroma/static" config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads" config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" From f5104f36bbec7d49d4ff5acee4b9d28223c6474d Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 11 Oct 2019 00:24:31 +0300 Subject: [PATCH 20/84] Deprecate /api/pleroma/admin/users/:nickname/toggle_activation instead of deleting it --- CHANGELOG.md | 3 +-- docs/API/admin_api.md | 8 ++++++ .../web/admin_api/admin_api_controller.ex | 20 ++++++++++++++ lib/pleroma/web/router.ex | 1 + .../admin_api/admin_api_controller_test.exs | 27 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 584c917f4..c996e7476 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: support for hierarchical permissions / [Mastodon 2.4.3 OAuth permissions](https://docs.joinmastodon.org/api/permissions/) - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed +- Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset -- **Breaking:** Admin API: `/users/:nickname/toggle_activation` endpoint was split into two: `/users/activate`, `/users/deactivate`, both accept `nicknames` array - **Breaking:** Admin API: `POST /users/permission_group/:permission_group` / `DELETE /users/permission_group/:permission_group` now accept `nicknames` array - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler @@ -42,7 +42,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - **Breaking:** GNU Social API with Qvitter extensions support -- **Breaking:** Admin API: `/users/:nickname/activation_status` was removed in favor of `/users/activate`, `/users/deactivate` - Emoji: Remove longfox emojis. - Remove `Reply-To` header from report emails for admins. diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 55f8749e1..b0b827960 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -211,6 +211,14 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## DEPRECATED `PATCH /api/pleroma/admin/users/:nickname/activation_status` + +### Active or deactivate a user + +- Params: + - `nickname` + - `status` BOOLEAN field, false value means deactivation. + ## `/api/pleroma/admin/users/:nickname_or_id` ### Retrive the details of a user diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index d825a5d28..5b513bd7c 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -46,6 +46,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do :user_delete, :users_create, :user_toggle_activation, + :user_activate, + :user_deactivate, :tag_users, :untag_users, :right_add, @@ -231,6 +233,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end + def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + user = User.get_cached_by_nickname(nickname) + + {:ok, updated_user} = User.deactivate(user, !user.info.deactivated) + + action = if user.info.deactivated, do: "activate", else: "deactivate" + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: action + }) + + conn + |> put_view(AccountView) + |> render("show.json", %{user: updated_user}) + end + def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do users = Enum.map(nicknames, &User.get_cached_by_nickname/1) {:ok, updated_users} = User.deactivate(users, false) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 894375357..a79df51a2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -136,6 +136,7 @@ defmodule Pleroma.Web.Router do delete("/users", AdminAPIController, :user_delete) post("/users", AdminAPIController, :users_create) + patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) patch("/users/activate", AdminAPIController, :user_activate) patch("/users/deactivate", AdminAPIController, :user_deactivate) put("/users/tag", AdminAPIController, :tag_users) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c57c71203..c1b325a24 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1023,6 +1023,33 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}" end + test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> patch("/api/pleroma/admin/users/#{user.nickname}/toggle_activation") + + assert json_response(conn, 200) == + %{ + "deactivated" => !user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => true, + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) + } + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deactivated users: @#{user.nickname}" + end + describe "POST /api/pleroma/admin/users/invite_token" do setup do admin = insert(:user, info: %{is_admin: true}) From 9b963064eb4c8cb740ffa128f491f2ee581fdb8b Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 11:25:45 +0200 Subject: [PATCH 21/84] Transmogrifier: Actually store who deleted a note. --- lib/pleroma/web/activity_pub/activity_pub.ex | 18 ++++++++++-------- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 ++- test/web/activity_pub/transmogrifier_test.exs | 7 +++++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3a0c382ca..f78574455 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -412,19 +412,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do local = Keyword.get(options, :local, true) activity_id = Keyword.get(options, :activity_id, nil) + actor = Keyword.get(options, :actor, actor) user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) with {:ok, object, activity} <- Object.delete(object), - data <- %{ - "type" => "Delete", - "actor" => actor, - "object" => id, - "to" => to, - "deleted_activity_id" => activity && activity.id - } - |> maybe_put("id", activity_id), + data <- + %{ + "type" => "Delete", + "actor" => actor, + "object" => id, + "to" => to, + "deleted_activity_id" => activity && activity.id + } + |> maybe_put("id", activity_id), {:ok, activity} <- insert(data, local, false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 620b54d3b..64fbb9fa4 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -646,7 +646,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do + {:ok, activity} <- + ActivityPub.delete(object, local: false, activity_id: id, actor: actor.ap_id) do {:ok, activity} else nil -> diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 4f9c4af33..9f98ec708 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -682,6 +682,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do test "it works for incoming deletes" do activity = insert(:note_activity) + deleting_user = insert(:user) data = File.read!("test/fixtures/mastodon-delete.json") @@ -694,12 +695,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = data |> Map.put("object", object) - |> Map.put("actor", activity.data["actor"]) + |> Map.put("actor", deleting_user.ap_id) - {:ok, %Activity{local: false, data: %{"id" => id}}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{actor: actor, local: false, data: %{"id" => id}}} = + Transmogrifier.handle_incoming(data) assert id == data["id"] refute Activity.get_by_id(activity.id) + assert actor == deleting_user.ap_id end test "it fails for incoming deletes with spoofed origin" do From 37812740c4c03f0a79d17ff186db644a60414ff7 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 11:48:58 +0200 Subject: [PATCH 22/84] Transmogrifier: Correctly save incoming ids for Accept/Reject. --- lib/pleroma/web/activity_pub/activity_pub.ex | 25 +++++++++---------- .../web/activity_pub/transmogrifier.ex | 10 +++++--- test/web/activity_pub/transmogrifier_test.exs | 3 +++ 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f78574455..364452b5d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -269,22 +269,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def accept(%{to: to, actor: actor, object: object} = params) do - # only accept false as false value - local = !(params[:local] == false) - - with data <- %{"to" => to, "type" => "Accept", "actor" => actor.ap_id, "object" => object}, - {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do - {:ok, activity} - end + def accept(params) do + accept_or_reject("Accept", params) end - def reject(%{to: to, actor: actor, object: object} = params) do - # only accept false as false value - local = !(params[:local] == false) + def reject(params) do + accept_or_reject("Reject", params) + end - with data <- %{"to" => to, "type" => "Reject", "actor" => actor.ap_id, "object" => object}, + def accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do + local = Map.get(params, :local, true) + activity_id = Map.get(params, :activity_id, nil) + + with data <- + %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object} + |> Utils.maybe_put("id", activity_id), {:ok, activity} <- insert(data, local), :ok <- maybe_federate(activity) do {:ok, activity} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 64fbb9fa4..b56343beb 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -514,7 +514,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => id} = data, _options ) do with actor <- Containment.get_actor(data), @@ -528,7 +528,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do type: "Accept", actor: followed, object: follow_activity.data["id"], - local: false + local: false, + activity_id: id }) else _e -> :error @@ -536,7 +537,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => id} = data, _options ) do with actor <- Containment.get_actor(data), @@ -550,7 +551,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do type: "Reject", actor: followed, object: follow_activity.data["id"], - local: false + local: false, + activity_id: id }) do User.unfollow(follower, followed) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 9f98ec708..6c35a6f4d 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -909,6 +909,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["object"] == follow_activity.data["id"] + assert activity.data["id"] == accept_data["id"] + follower = User.get_cached_by_id(follower.id) assert User.following?(follower, followed) == true @@ -1013,6 +1015,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, activity} = Transmogrifier.handle_incoming(reject_data) refute activity.local + assert activity.data["id"] == reject_data["id"] follower = User.get_cached_by_id(follower.id) From b750135acd9d1f8444f85ea5e0345483599daed9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 11 Oct 2019 13:22:21 +0300 Subject: [PATCH 23/84] pleroma_ctl: Make it possible to specify branches/flavours when updating --- rel/files/bin/pleroma_ctl | 94 +++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index e731d20eb..f767fe134 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -37,29 +37,60 @@ detect_branch() { elif [ "$branch" = "" ]; then echo "master" else - # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — - # if supporting releases for more branches, need to ensure they contain only these symbols. + # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — + # if supporting releases for more branches, need to ensure they contain only these symbols. echo "Releases are built only for master and develop branches" >&2 exit 1 fi } update() { set -e + NO_RM=false + + while echo "$1" | grep "^-" >/dev/null; do + case "$1" in + --zip-url) + FULL_URI="$2" + shift 2 + ;; + --no-rm) + NO_RM=true + shift + ;; + --flavour) + FLAVOUR="$2" + shift 2 + ;; + --branch) + BRANCH="$2" + shift 2 + ;; + --tmp-dir) + TMP_DIR="$2" + shift 2 + ;; + -*) + echo "invalid option: $1" 1>&2 + shift + ;; + esac + done + RELEASE_ROOT=$(dirname "$SCRIPTPATH") - uri="${PLEROMA_CTL_URI:-https://git.pleroma.social}" - project_id="${PLEROMA_CTL_PROJECT_ID:-2}" - project_branch="$(detect_branch)" - flavour="${PLEROMA_CTL_FLAVOUR:-$(detect_flavour)}" - echo "Detected flavour: $flavour" - tmp="${PLEROMA_CTL_TMP_DIR:-/tmp}" + uri="https://git.pleroma.social" + project_id="2" + project_branch="${BRANCH:-$(detect_branch)}" + flavour="${FLAVOUR:-$(detect_flavour)}" + tmp="${TMP_DIR:-/tmp}" artifact="$tmp/pleroma.zip" - full_uri="${uri}/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=${flavour}" + full_uri="${FULL_URI:-${uri}/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=${flavour}}" echo "Downloading the artifact from ${full_uri} to ${artifact}" curl "$full_uri" -o "${artifact}" echo "Unpacking ${artifact} to ${tmp}" unzip -q "$artifact" -d "$tmp" echo "Copying files over to $RELEASE_ROOT" - if [ "$1" != "--no-rm" ]; then + if [ "$NO_RM" = false ]; then + echo "Removing files from the previous release" rm -r "${RELEASE_ROOT:-?}"/* fi cp -rf "$tmp/release"/* "$RELEASE_ROOT" @@ -86,36 +117,41 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then Rollback database migrations (needs to be done before downgrading) update [OPTIONS] - Update the instance using the latest CI artifact for the current branch. - - The only supported option is --no-rm, when set the script won't delete the whole directory, but - just force copy over files from the new release. This wastes more space, but may be useful if - some files are stored inside of the release directories (although you really shouldn't store them - there), or if you want to be able to quickly revert a broken update. - - The script will try to detect your architecture and ABI and set a flavour automatically, - but if it is wrong, you can overwrite it by setting PLEROMA_CTL_FLAVOUR to the desired flavour. - - By default the artifact will be downloaded from https://git.pleroma.social for pleroma/pleroma (project id: 2) - to /tmp/, you can overwrite these settings by setting PLEROMA_CTL_URI, PLEROMA_CTL_PROJECT_ID and PLEROMA_CTL_TMP_DIR - respectively. + Update the instance. + Options: + --branch Update to a specified branch, instead of the latest version of the current one. + --flavour Update to a specified flavour (CPU architecture+libc), instead of the current one. + --zip-url Get the release from a specified url. If set, renders the previous 2 options inactive. + --no-rm Do not erase previous release's files. + --tmp-dir Download the temporary files to a specified directory. and any mix tasks under Pleroma namespace, for example \`mix pleroma.user COMMAND\` is equivalent to \`$(basename "$0") user COMMAND\` By default pleroma_ctl will try calling into a running instance to execute non migration-related commands, - if for some reason this is undesired, set PLEROMA_CTL_RPC_DISABLED environment variable + if for some reason this is undesired, set PLEROMA_CTL_RPC_DISABLED environment variable. + " else SCRIPT=$(readlink -f "$0") SCRIPTPATH=$(dirname "$SCRIPT") - if [ "$1" = "update" ]; then - update "$2" - elif [ "$1" = "migrate" ] || [ "$1" = "rollback" ] || [ "$1" = "create" ] || [ "$1 $2" = "instance gen" ] || [ -n "$PLEROMA_CTL_RPC_DISABLED" ]; then - "$SCRIPTPATH"/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")' + FULL_ARGS="$*" + + ACTION="$1" + shift + + if [ "$(echo \"$1\" | grep \"^-\" >/dev/null)" = false ]; then + SUBACTION="$1" + shift + fi + + if [ "$ACTION" = "update" ]; then + update "$@" + elif [ "$ACTION" = "migrate" ] || [ "$ACTION" = "rollback" ] || [ "$ACTION" = "create" ] || [ "$ACTION $SUBACTION" = "instance gen" ] || [ "$PLEROMA_CTL_RPC_DISABLED" = true ]; then + "$SCRIPTPATH"/pleroma eval 'Pleroma.ReleaseTasks.run("'"$FULL_ARGS"'")' else - "$SCRIPTPATH"/pleroma rpc 'Pleroma.ReleaseTasks.run("'"$*"'")' + "$SCRIPTPATH"/pleroma rpc 'Pleroma.ReleaseTasks.run("'"$FULL_ARGS"'")' fi fi From cb8492962e2a765090dbce82b8fc9f000d49bfa1 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 12:41:44 +0200 Subject: [PATCH 24/84] SearchController: Fix test. Turns out you can't actually find the user with this. --- .../mastodon_api/controllers/search_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index 0ca896e01..ee413eef7 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -40,9 +40,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do test "search", %{conn: conn} do user = insert(:user) user_two = insert(:user, %{nickname: "shp@shitposter.club"}) - user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu 天子"}) + user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"}) - {:ok, activity} = CommonAPI.post(user, %{"status" => "This is about 2hu private"}) + {:ok, activity} = CommonAPI.post(user, %{"status" => "This is about 2hu private 天子"}) {:ok, _activity} = CommonAPI.post(user, %{ @@ -70,8 +70,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do get(conn, "/api/v2/search", %{"q" => "天子"}) |> json_response(200) - [account] == results["accounts"] - assert account["id"] == to_string(user_three.id) + [status] = results["statuses"] + assert status["id"] == to_string(activity.id) end end From 422aa6befee0cbcbf71d3cacd96d90c1be3263ba Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 12:53:09 +0200 Subject: [PATCH 25/84] Ostatus DeleteHandler: Fix for new option format. --- lib/pleroma/web/ostatus/handlers/delete_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index b2f9f3946..ac2dc115c 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, false) do + {:ok, delete} <- ActivityPub.delete(object, local: false) do delete end end From aaa4252f416fbad099f95232de4cf6eab11dd7d2 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 11 Oct 2019 15:58:45 +0300 Subject: [PATCH 26/84] Deprecate POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group instead of deleting it --- CHANGELOG.md | 3 +- docs/API/admin_api.md | 23 ++++++- .../web/admin_api/admin_api_controller.ex | 61 ++++++++++++++++++- lib/pleroma/web/router.ex | 18 +++++- .../admin_api/admin_api_controller_test.exs | 40 ++++++++++++ 5 files changed, 137 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c996e7476..f06ad365d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed - Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array +- Admin API: `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group` / `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST /api/pleroma/admin/users/permission_group/:permission_group` / `DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array) + ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset -- **Breaking:** Admin API: `POST /users/permission_group/:permission_group` / `DELETE /users/permission_group/:permission_group` now accept `nicknames` array - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index b0b827960..2c8237b57 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -154,9 +154,18 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## DEPRECATED `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group` + +### Add user to permission group + +- Params: none +- Response: + - On failure: `{"error": "…"}` + - On success: JSON of the `user.info` + ## `POST /api/pleroma/admin/users/permission_group/:permission_group` -### Add user in permission group +### Add users to permission group - Params: - `nicknames`: nicknames array @@ -164,10 +173,20 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - On failure: `{"error": "…"}` - On success: JSON of the `user.info` -## `DELETE /api/pleroma/admin/users/permission_group/:permission_group` +## DEPRECATED `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` ### Remove user from permission group +- Params: none +- Response: + - On failure: `{"error": "…"}` + - On success: JSON of the `user.info` +- Note: An admin cannot revoke their own admin status. + +## `DELETE /api/pleroma/admin/users/permission_group/:permission_group` + +### Remove users from permission group + - Params: - `nicknames`: nicknames array - Response: diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 5b513bd7c..33e2180ec 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -345,7 +345,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> Enum.into(%{}, &{&1, true}) end - def right_add(%{assigns: %{user: admin}} = conn, %{ + def right_add_multiple(%{assigns: %{user: admin}} = conn, %{ "permission_group" => permission_group, "nicknames" => nicknames }) @@ -366,6 +366,32 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do json(conn, info) end + def right_add_multiple(conn, _) do + render_error(conn, :not_found, "No such permission_group") + end + + def right_add(%{assigns: %{user: admin}} = conn, %{ + "permission_group" => permission_group, + "nickname" => nickname + }) + when permission_group in ["moderator", "admin"] do + info = Map.put(%{}, "is_" <> permission_group, true) + + {:ok, user} = + nickname + |> User.get_cached_by_nickname() + |> User.update_info(&User.Info.admin_api_update(&1, info)) + + ModerationLog.insert_log(%{ + action: "grant", + actor: admin, + subject: [user], + permission: permission_group + }) + + json(conn, info) + end + def right_add(conn, _) do render_error(conn, :not_found, "No such permission_group") end @@ -380,7 +406,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) end - def right_delete( + def right_delete_multiple( %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn, %{ "permission_group" => permission_group, @@ -408,10 +434,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end - def right_delete(conn, _) do + def right_delete_multiple(conn, _) do render_error(conn, :not_found, "No such permission_group") end + def right_delete( + %{assigns: %{user: admin}} = conn, + %{ + "permission_group" => permission_group, + "nickname" => nickname + } + ) + when permission_group in ["moderator", "admin"] do + info = Map.put(%{}, "is_" <> permission_group, false) + + {:ok, user} = + nickname + |> User.get_cached_by_nickname() + |> User.update_info(&User.Info.admin_api_update(&1, info)) + + ModerationLog.insert_log(%{ + action: "revoke", + actor: admin, + subject: [user], + permission: permission_group + }) + + json(conn, info) + end + + def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do + render_error(conn, :forbidden, "You can't revoke your own admin status.") + end + def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do with {:ok, _message} <- Relay.follow(target) do ModerationLog.insert_log(%{ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a79df51a2..80651f3ff 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -144,8 +144,22 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/permission_group", AdminAPIController, :right_get) get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get) - post("/users/permission_group/:permission_group", AdminAPIController, :right_add) - delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete) + + post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add) + + delete( + "/users/:nickname/permission_group/:permission_group", + AdminAPIController, + :right_delete + ) + + post("/users/permission_group/:permission_group", AdminAPIController, :right_add_multiple) + + delete( + "/users/permission_group/:permission_group", + AdminAPIController, + :right_delete_multiple + ) post("/relay", AdminAPIController, :relay_follow) delete("/relay", AdminAPIController, :relay_unfollow) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c1b325a24..81dc5d101 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -385,6 +385,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end test "/:right POST, can add to a permission group" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + + assert json_response(conn, 200) == %{ + "is_admin" => true + } + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} made @#{user.nickname} admin" + end + + test "/:right POST, can add to a permission group (multiple)" do admin = insert(:user, info: %{is_admin: true}) user_one = insert(:user) user_two = insert(:user) @@ -408,6 +428,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end test "/:right DELETE, can remove from a permission group" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user, info: %{is_admin: true}) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> delete("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + + assert json_response(conn, 200) == %{ + "is_admin" => false + } + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} revoked admin role from @#{user.nickname}" + end + + test "/:right DELETE, can remove from a permission group (multiple)" do admin = insert(:user, info: %{is_admin: true}) user_one = insert(:user, info: %{is_admin: true}) user_two = insert(:user, info: %{is_admin: true}) From cc6875b582df49d2cb780e0940b85d5b04fe0e74 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 11 Oct 2019 19:12:29 +0300 Subject: [PATCH 27/84] Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays --- CHANGELOG.md | 1 + docs/API/admin_api.md | 8 ++ lib/mix/tasks/pleroma/relay.ex | 10 +-- lib/pleroma/web/activity_pub/relay.ex | 14 ++++ .../web/admin_api/admin_api_controller.ex | 10 +++ lib/pleroma/web/router.ex | 1 + .../admin_api/admin_api_controller_test.exs | 74 +++++++++++++++++++ .../controllers/search_controller_test.exs | 2 +- 8 files changed, 112 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd06ec866..2e7817a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) +- Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index ee9e68cb1..1c5c8afa4 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -222,6 +222,14 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Response: - On success: URL of the unfollowed relay +## `GET /api/pleroma/admin/relay` + +### List Relays + +- Params: none +- Response: + - On success: JSON array of relays + ## `/api/pleroma/admin/users/invite_token` ### Create an account registration invite token diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index d7a7b599f..7ef5f9678 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -5,7 +5,6 @@ defmodule Mix.Tasks.Pleroma.Relay do use Mix.Task import Mix.Pleroma - alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay @shortdoc "Manages remote relays" @@ -36,13 +35,10 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["list"]) do start_pleroma() - with %User{following: following} = _user <- Relay.get_actor() do - following - |> Enum.map(fn entry -> URI.parse(entry).host end) - |> Enum.uniq() - |> Enum.each(&shell_info(&1)) + with {:ok, list} <- Relay.list() do + list |> Enum.each(&shell_info(&1)) else - e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") + {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") end end end diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index c2ac38907..03fc434a9 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -51,6 +51,20 @@ defmodule Pleroma.Web.ActivityPub.Relay do def publish(_), do: {:error, "Not implemented"} + @spec list() :: {:ok, [String.t()]} | {:error, any()} + def list do + with %User{following: following} = _user <- get_actor() do + list = + following + |> Enum.map(fn entry -> URI.parse(entry).host end) + |> Enum.uniq() + + {:ok, list} + else + error -> format_error(error) + end + end + defp format_error({:error, error}), do: format_error(error) defp format_error(error) do diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 513bae800..24dda75a9 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -401,6 +401,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end + def relay_list(conn, _params) do + with {:ok, list} <- Relay.list() do + json(conn, %{relays: list}) + else + _ -> + conn + |> put_status(500) + end + end + def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do with {:ok, _message} <- Relay.follow(target) do ModerationLog.insert_log(%{ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ae799b8ac..8cc967af9 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -152,6 +152,7 @@ defmodule Pleroma.Web.Router do put("/users/:nickname/activation_status", AdminAPIController, :set_activation_status) + get("/relay", AdminAPIController, :relay_list) post("/relay", AdminAPIController, :relay_follow) delete("/relay", AdminAPIController, :relay_unfollow) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index b5c355e66..d7dce2da4 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -17,6 +17,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do alias Pleroma.Web.MediaProxy import Pleroma.Factory + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + describe "/api/pleroma/admin/users" do test "Delete" do admin = insert(:user, info: %{is_admin: true}) @@ -2486,6 +2492,74 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert User.get_by_id(user.id).info.password_reset_pending == true end end + + describe "relays" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin), admin: admin} + end + + test "POST /relay", %{admin: admin} do + conn = + build_conn() + |> assign(:user, admin) + |> post("/api/pleroma/admin/relay", %{ + relay_url: "http://mastodon.example.org/users/admin" + }) + + assert json_response(conn, 200) == "http://mastodon.example.org/users/admin" + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} followed relay: http://mastodon.example.org/users/admin" + end + + test "GET /relay", %{admin: admin} do + Pleroma.Web.ActivityPub.Relay.get_actor() + |> Ecto.Changeset.change( + following: [ + "http://test-app.com/user/test1", + "http://test-app.com/user/test1", + "http://test-app-42.com/user/test1" + ] + ) + |> Pleroma.User.update_and_set_cache() + + conn = + build_conn() + |> assign(:user, admin) + |> get("/api/pleroma/admin/relay") + + assert json_response(conn, 200)["relays"] -- ["test-app.com", "test-app-42.com"] == [] + end + + test "DELETE /relay", %{admin: admin} do + build_conn() + |> assign(:user, admin) + |> post("/api/pleroma/admin/relay", %{ + relay_url: "http://mastodon.example.org/users/admin" + }) + + conn = + build_conn() + |> assign(:user, admin) + |> delete("/api/pleroma/admin/relay", %{ + relay_url: "http://mastodon.example.org/users/admin" + }) + + assert json_response(conn, 200) == "http://mastodon.example.org/users/admin" + + [log_entry_one, log_entry_two] = Repo.all(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry_one) == + "@#{admin.nickname} followed relay: http://mastodon.example.org/users/admin" + + assert ModerationLog.get_log_entry_message(log_entry_two) == + "@#{admin.nickname} unfollowed relay: http://mastodon.example.org/users/admin" + end + end end # Needed for testing diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index 0ca896e01..62c3423f1 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -70,7 +70,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do get(conn, "/api/v2/search", %{"q" => "天子"}) |> json_response(200) - [account] == results["accounts"] + assert [account] == results["accounts"] assert account["id"] == to_string(user_three.id) end end From 751513b6df0c76ad14026c70e1bf2635053bfea4 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 11 Oct 2019 19:31:00 +0300 Subject: [PATCH 28/84] This line either causes a warning, or a failed test --- test/web/mastodon_api/controllers/search_controller_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index 62c3423f1..d0e1f60d3 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -70,7 +70,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do get(conn, "/api/v2/search", %{"q" => "天子"}) |> json_response(200) - assert [account] == results["accounts"] assert account["id"] == to_string(user_three.id) end end From 9bdbf0811b362d8eb9d2456e9c87c9dde1b35d9b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 11 Oct 2019 22:52:38 +0300 Subject: [PATCH 29/84] Make MediaProxy failure tracking less brutal The current failure tracking mechanism will never request anything that didn't respond with a success, 403, 404, or 5xx codes. This is causing issues when using in real fediverse because of weird status codes some software has and timeouts being frequent. This patch changes failure tracking mechanism to only never request the url again if it responded with 400, 204, or the body is too large, otherwise it can be re-requested in 60 seconds. --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 78144cae3..2ed719315 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -401,11 +401,9 @@ defmodule Pleroma.ReverseProxy do defp client, do: Pleroma.ReverseProxy.Client - defp track_failed_url(url, code, opts) do - code = to_string(code) - + defp track_failed_url(url, error, opts) do ttl = - if code in ["403", "404"] or String.starts_with?(code, "5") do + unless error in [:body_too_large, 400, 204] do Keyword.get(opts, :failed_request_ttl, @failed_request_ttl) else nil From a97b642289659a5fccb5943c54caa1ecdce5fd2f Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 8 Oct 2019 23:05:57 +0300 Subject: [PATCH 30/84] Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 7 ++ lib/pleroma/notification.ex | 108 +++++++++++++----- lib/pleroma/web/activity_pub/activity_pub.ex | 44 +++++++ lib/pleroma/web/mastodon_api/mastodon_api.ex | 1 + test/web/activity_pub/activity_pub_test.exs | 60 ++++++++++ .../controllers/account_controller_test.exs | 14 +++ .../notification_controller_test.exs | 51 +++++++++ .../controllers/timeline_controller_test.exs | 55 ++++++--- 9 files changed, 297 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd06ec866..e3839391b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) +- Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 21b297529..aca0f5e0e 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -13,6 +13,7 @@ Some apps operate under the assumption that no more than 4 attachments can be re ## Timelines Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users. +Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`. ## Statuses @@ -84,6 +85,12 @@ Has these additional fields under the `pleroma` object: - `is_seen`: true if the notification was read by the user +## GET `/api/v1/notifications` + +Accepts additional parameters: + +- `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`. + ## POST `/api/v1/statuses` Additional parameters can be added to the JSON body/Form data: diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index d94ae5971..d145f8d5b 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Notification do import Ecto.Query import Ecto.Changeset + require Logger @type t :: %__MODULE__{} @@ -34,43 +35,92 @@ defmodule Pleroma.Notification do end def for_user_query(user, opts \\ []) do - query = - Notification - |> where(user_id: ^user.id) + Notification + |> where(user_id: ^user.id) + |> where( + [n, a], + fragment( + "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')", + a.actor + ) + ) + |> join(:inner, [n], activity in assoc(n, :activity)) + |> join(:left, [n, a], object in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + object.data, + a.data + ) + ) + |> preload([n, a, o], activity: {a, object: o}) + |> exclude_muted(user, opts) + |> exclude_visibility(opts) + end + + defp exclude_muted(query, _, %{with_muted: true}) do + query + end + + defp exclude_muted(query, user, _opts) do + query + |> where([n, a], a.actor not in ^user.info.muted_notifications) + |> where([n, a], a.actor not in ^user.info.blocks) + |> where( + [n, a], + fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks + ) + |> join(:left, [n, a], tm in Pleroma.ThreadMute, + on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data) + ) + |> where([n, a, o, tm], is_nil(tm.user_id)) + end + + @valid_visibilities ~w[direct unlisted public private] + + defp exclude_visibility(query, %{exclude_visibilities: visibility}) + when is_list(visibility) do + if Enum.all?(visibility, &(&1 in @valid_visibilities)) do + query |> where( [n, a], - fragment( - "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')", - a.actor + not fragment( + "activity_visibility(?, ?, ?) = ANY (?)", + a.actor, + a.recipients, + a.data, + ^visibility ) ) - |> join(:inner, [n], activity in assoc(n, :activity)) - |> join(:left, [n, a], object in Object, - on: - fragment( - "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", - object.data, - a.data - ) - ) - |> preload([n, a, o], activity: {a, object: o}) - - if opts[:with_muted] do - query else - where(query, [n, a], a.actor not in ^user.info.muted_notifications) - |> where([n, a], a.actor not in ^user.info.blocks) - |> where( - [n, a], - fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks - ) - |> join(:left, [n, a], tm in Pleroma.ThreadMute, - on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data) - ) - |> where([n, a, o, tm], is_nil(tm.user_id)) + Logger.error("Could not exclude visibility to #{visibility}") + query end end + defp exclude_visibility(query, %{exclude_visibilities: visibility}) + when visibility in @valid_visibilities do + query + |> where( + [n, a], + not fragment( + "activity_visibility(?, ?, ?) = (?)", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) + end + + defp exclude_visibility(query, %{exclude_visibilities: visibility}) + when visibility not in @valid_visibilities do + Logger.error("Could not exclude visibility to #{visibility}") + query + end + + defp exclude_visibility(query, _visibility), do: query + def for_user(user, opts \\ %{}) do user |> for_user_query(opts) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 364452b5d..1d34c4d7e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -596,6 +596,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_visibility(query, _visibility), do: query + defp exclude_visibility(query, %{"exclude_visibilities" => visibility}) + when is_list(visibility) do + if Enum.all?(visibility, &(&1 in @valid_visibilities)) do + from( + a in query, + where: + not fragment( + "activity_visibility(?, ?, ?) = ANY (?)", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) + else + Logger.error("Could not exclude visibility to #{visibility}") + query + end + end + + defp exclude_visibility(query, %{"exclude_visibilities" => visibility}) + when visibility in @valid_visibilities do + from( + a in query, + where: + not fragment( + "activity_visibility(?, ?, ?) = ?", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) + end + + defp exclude_visibility(query, %{"exclude_visibilities" => visibility}) + when visibility not in @valid_visibilities do + Logger.error("Could not exclude visibility to #{visibility}") + query + end + + defp exclude_visibility(query, _visibility), do: query + defp restrict_thread_visibility(query, _, %{skip_thread_containment: true} = _), do: query @@ -960,6 +1003,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) + |> exclude_visibility(opts) end def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index ac01d1ff3..d875a5788 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -71,6 +71,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do defp cast_params(params) do param_types = %{ exclude_types: {:array, :string}, + exclude_visibilities: {:array, :string}, reblogs: :boolean, with_muted: :boolean } diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index c9f2a92e7..3a5a2f984 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -87,6 +87,66 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end end + describe "fetching excluded by visibility" do + test "it excludes by the appropriate visibility" do + user = insert(:user) + + {:ok, public_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"}) + + {:ok, direct_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + {:ok, unlisted_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "unlisted"}) + + {:ok, private_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "private"}) + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "direct", + "actor_id" => user.ap_id + }) + + assert public_activity in activities + assert unlisted_activity in activities + assert private_activity in activities + refute direct_activity in activities + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "unlisted", + "actor_id" => user.ap_id + }) + + assert public_activity in activities + refute unlisted_activity in activities + assert private_activity in activities + assert direct_activity in activities + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "private", + "actor_id" => user.ap_id + }) + + assert public_activity in activities + assert unlisted_activity in activities + refute private_activity in activities + assert direct_activity in activities + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "public", + "actor_id" => user.ap_id + }) + + refute public_activity in activities + assert unlisted_activity in activities + assert private_activity in activities + assert direct_activity in activities + end + end + describe "building a user from his ap id" do test "it returns a user" do user_id = "http://mastodon.example.org/users/admin" diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 6a59c3d94..745383757 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -237,6 +237,20 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert [%{"id" => id}] = json_response(conn, 200) assert id == to_string(post.id) end + + test "the user views their own timelines and excludes direct messages", %{conn: conn} do + user = insert(:user) + {:ok, public_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"}) + {:ok, _direct_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/accounts/#{user.id}/statuses", %{"exclude_visibilities" => ["direct"]}) + + assert [%{"id" => id}] = json_response(conn, 200) + assert id == to_string(public_activity.id) + end end describe "followers" do diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index e4137e92c..fa55a7cf9 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -137,6 +137,57 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert [%{"id" => ^notification3_id}, %{"id" => ^notification2_id}] = result end + test "filters notifications using exclude_visibilities", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, public_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "public"}) + + {:ok, direct_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "direct"}) + + {:ok, unlisted_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "unlisted"}) + + {:ok, private_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "private"}) + + conn = assign(conn, :user, user) + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["public", "unlisted", "private"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == direct_activity.id + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["public", "unlisted", "direct"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == private_activity.id + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["public", "private", "direct"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == unlisted_activity.id + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["unlisted", "private", "direct"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == public_activity.id + end + test "filters notifications using exclude_types", %{conn: conn} do user = insert(:user) other_user = insert(:user) diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs index d3652d964..fc45c25de 100644 --- a/test/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs @@ -20,27 +20,52 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do :ok end - test "the home timeline", %{conn: conn} do - user = insert(:user) - following = insert(:user) + describe "home" do + test "the home timeline", %{conn: conn} do + user = insert(:user) + following = insert(:user) - {:ok, _activity} = CommonAPI.post(following, %{"status" => "test"}) + {:ok, _activity} = CommonAPI.post(following, %{"status" => "test"}) - conn = - conn - |> assign(:user, user) - |> get("/api/v1/timelines/home") + conn = + conn + |> assign(:user, user) + |> get("/api/v1/timelines/home") - assert Enum.empty?(json_response(conn, :ok)) + assert Enum.empty?(json_response(conn, :ok)) - {:ok, user} = User.follow(user, following) + {:ok, user} = User.follow(user, following) - conn = - build_conn() - |> assign(:user, user) - |> get("/api/v1/timelines/home") + conn = + build_conn() + |> assign(:user, user) + |> get("/api/v1/timelines/home") - assert [%{"content" => "test"}] = json_response(conn, :ok) + assert [%{"content" => "test"}] = json_response(conn, :ok) + end + + test "the home timeline when the direct messages are excluded", %{conn: conn} do + user = insert(:user) + {:ok, public_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"}) + {:ok, direct_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + {:ok, unlisted_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "unlisted"}) + + {:ok, private_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "private"}) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/timelines/home", %{"exclude_visibilities" => ["direct"]}) + + assert status_ids = json_response(conn, :ok) |> Enum.map(& &1["id"]) + assert public_activity.id in status_ids + assert unlisted_activity.id in status_ids + assert private_activity.id in status_ids + refute direct_activity.id in status_ids + end end describe "public" do From 0e9243e8a28e957f2a9401eb153cc2b27723d0a3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:38:51 +0300 Subject: [PATCH 31/84] Preparations for renaming `master` to `stable` --- .gitlab-ci.yml | 5 +++-- docs/installation/alpine_linux_en.md | 2 +- docs/installation/arch_linux_en.md | 2 +- docs/installation/centos7_en.md | 2 +- docs/installation/debian_based_en.md | 2 +- docs/installation/debian_based_jp.md | 2 +- docs/installation/gentoo_en.md | 2 +- docs/installation/migrating_from_source_otp_en.md | 4 ++-- docs/installation/netbsd_en.md | 2 +- docs/installation/openbsd_en.md | 2 +- docs/installation/openbsd_fi.md | 2 +- docs/installation/otp_en.md | 2 +- mix.exs | 5 ++++- 13 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 748bec74a..36137b38e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,7 @@ docs-deploy: stage: deploy image: alpine:latest only: - - master@pleroma/pleroma + - stable@pleroma/pleroma - develop@pleroma/pleroma before_script: - apk add curl @@ -127,9 +127,10 @@ amd64: # TODO: Replace with upstream image when 1.9.0 comes out image: rinpatch/elixir:1.9.0-rc.0 only: &release-only - - master@pleroma/pleroma + - stable@pleroma/pleroma - develop@pleroma/pleroma - /^maint/.*$/@pleroma/pleroma + - /^release/.*$/@pleroma/pleroma artifacts: &release-artifacts name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME" paths: diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index f5d1fade1..2a9b8f6ff 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -91,7 +91,7 @@ sudo adduser -S -s /bin/false -h /opt/pleroma -H -G pleroma pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md index 58a8d023f..8370986ad 100644 --- a/docs/installation/arch_linux_en.md +++ b/docs/installation/arch_linux_en.md @@ -66,7 +66,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/centos7_en.md b/docs/installation/centos7_en.md index fe26339b5..ad4f58dc1 100644 --- a/docs/installation/centos7_en.md +++ b/docs/installation/centos7_en.md @@ -143,7 +143,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index e5f8dd670..fe2dbb92d 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -68,7 +68,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index e7d834b17..7aa0bcc24 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -68,7 +68,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ``` sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * 新しいディレクトリに移動します。 diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md index 55d677acf..1e61373cc 100644 --- a/docs/installation/gentoo_en.md +++ b/docs/installation/gentoo_en.md @@ -106,7 +106,7 @@ It is highly recommended you use your own fork for the `https://path/to/repo` pa ```shell pleroma$ cd ~ - pleroma$ git clone -b master https://path/to/repo + pleroma$ git clone -b stable https://path/to/repo ``` * Change to the new directory: diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index e204cb3ee..87568faad 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -96,9 +96,9 @@ rm -r ~pleroma/* export FLAVOUR="arm64-musl" # Clone the release build into a temporary directory and unpack it -# Replace `master` with `develop` if you want to run the develop branch +# Replace `stable` with `unstable` if you want to run the unstable branch su pleroma -s $SHELL -lc " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/master/download?job=$FLAVOUR' -o /tmp/pleroma.zip +curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip unzip /tmp/pleroma.zip -d /tmp/ " diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md index a096d5354..6a922a27e 100644 --- a/docs/installation/netbsd_en.md +++ b/docs/installation/netbsd_en.md @@ -58,7 +58,7 @@ Clone the repository: ``` $ cd /home/pleroma -$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git +$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git ``` Configure Pleroma. Note that you need a domain name at this point: diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index fcba38b2c..3585a326b 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -29,7 +29,7 @@ This creates a "pleroma" login class and sets higher values than default for dat Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): `useradd -m -L pleroma _pleroma` #### Clone pleroma's directory -Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b master https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide. +Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b stable https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide. #### Postgresql Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql: diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md index 39819a8c8..272273cff 100644 --- a/docs/installation/openbsd_fi.md +++ b/docs/installation/openbsd_fi.md @@ -44,7 +44,7 @@ Vaihda pleroma-käyttäjään ja mene kotihakemistoosi: Lataa pleroman lähdekoodi: -`$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git` +`$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git` `$ cd pleroma` diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index b4e5254cd..c028f4229 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -80,7 +80,7 @@ export FLAVOUR="arm64-musl" # Clone the release build into a temporary directory and unpack it su pleroma -s $SHELL -lc " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/master/download?job=$FLAVOUR' -o /tmp/pleroma.zip +curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip unzip /tmp/pleroma.zip -d /tmp/ " diff --git a/mix.exs b/mix.exs index 3a605b455..0ae9c3f05 100644 --- a/mix.exs +++ b/mix.exs @@ -220,7 +220,10 @@ defmodule Pleroma.Mixfile do with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, - true <- branch_name not in ["master", "HEAD"] do + true <- + !Enum.all?(["master", "HEAD", "release/", "stable"], fn name -> + name != branch_name + end) do branch_name = branch_name |> String.trim() From 58fea88564c8dcece007a547b186965963e4dcb4 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:39:56 +0300 Subject: [PATCH 32/84] Prepare pleroma_ctl for moving master to stable and relax the error message --- rel/files/bin/pleroma_ctl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index f767fe134..90f87a990 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -35,11 +35,11 @@ detect_branch() { if [ "$branch" = "develop" ]; then echo "develop" elif [ "$branch" = "" ]; then - echo "master" + echo "stable" else # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — # if supporting releases for more branches, need to ensure they contain only these symbols. - echo "Releases are built only for master and develop branches" >&2 + echo "Can't detect the branch automatically, please specify it by using the --branch option." >&2 exit 1 fi } From d1e969842d9f78c262504adc0a80b8025f46c6a9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:42:30 +0300 Subject: [PATCH 33/84] Add a changelog entry for branch-moving --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd06ec866..a9f124a8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Report emails now include functional links to profiles of remote user accounts ## [1.1.0] - 2019-??-?? +**Breaking:** The stable branch has been changed from `master` to `stable`, `master` now points to `release/1.0` ### Security - Mastodon API: respect post privacy in `/api/v1/statuses/:id/{favourited,reblogged}_by` From 17b8896c40fa1c4aaab96f82fd07d5f5f2698400 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:45:16 +0300 Subject: [PATCH 34/84] Correct version parser branch name detection --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 0ae9c3f05..329ce3d89 100644 --- a/mix.exs +++ b/mix.exs @@ -222,7 +222,7 @@ defmodule Pleroma.Mixfile do branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, true <- !Enum.all?(["master", "HEAD", "release/", "stable"], fn name -> - name != branch_name + String.starts_with?(name, branch_name) end) do branch_name = branch_name From 2f0ed5f6c4dcfb16de510851dbdf9886bc25f1d5 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 22:13:18 +0300 Subject: [PATCH 35/84] Fix hiding branch name logic --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 329ce3d89..6cf766c52 100644 --- a/mix.exs +++ b/mix.exs @@ -221,7 +221,7 @@ defmodule Pleroma.Mixfile do branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, true <- - !Enum.all?(["master", "HEAD", "release/", "stable"], fn name -> + !Enum.any?(["master", "HEAD", "release/", "stable"], fn name -> String.starts_with?(name, branch_name) end) do branch_name = From fbe5a00475331962d7112ed5a0d5cf8bb5bf3ece Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 30 Sep 2019 16:04:30 +0300 Subject: [PATCH 36/84] For pleroma.emoji downloaded packs, generate pack.json instead --- lib/mix/tasks/pleroma/emoji.ex | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 881a6f725..1470b9b72 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -158,19 +158,21 @@ defmodule Mix.Tasks.Pleroma.Emoji do file_list: files_to_unzip ) - IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name])) + IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name])) - emoji_txt_str = - Enum.map( - files, - fn {shortcode, path} -> - emojo_path = Path.join("/emoji/#{pack_name}", path) - "#{shortcode}, #{emojo_path}" - end - ) - |> Enum.join("\n") + pack_json = %{ + pack: %{ + "license" => pack["license"], + "homepage" => pack["homepage"], + "description" => pack["description"], + "fallback-src" => pack["src"], + "fallback-src-sha256" => pack["src_sha256"], + "share-files" => true + }, + files: files + } - File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str) + File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(pack_json, pretty: true)) else IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"])) end From 5bd0717de286a918e20c5ef2409e5f4cbcea8524 Mon Sep 17 00:00:00 2001 From: kPherox Date: Tue, 15 Oct 2019 19:10:22 +0900 Subject: [PATCH 37/84] Add `Sec-WebSocket-Protocol` to response header --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 3c26eb406..1a757363f 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -35,6 +35,13 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {_, stream} <- List.keyfind(params, "stream", 0), {:ok, user} <- allow_request(stream, [access_token, sec_websocket]), topic when is_binary(topic) <- expand_topic(stream, params) do + req = + if sec_websocket != nil do + :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req) + else + req + end + {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}} else {:error, code} -> From e7bb762ec256bb5dc607797c042aa9432a9aa993 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 15 Oct 2019 15:16:17 +0300 Subject: [PATCH 38/84] don't stream in benchmark env --- lib/pleroma/web/streamer/streamer.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index 8cf719277..2fc7ac8cf 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Web.Streamer do end end - defp handle_should_send(_) do - true - end + defp handle_should_send(:benchmark), do: false + + defp handle_should_send(_), do: true end From 8c0cfed825480ae6338fb1d183dee1440e0ee29c Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 15 Oct 2019 16:26:04 +0300 Subject: [PATCH 39/84] some fixes --- benchmarks/load_testing/fetcher.ex | 2 +- benchmarks/mix/tasks/pleroma/load_testing.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 0ff2f28c0..e378c51e7 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -210,7 +210,7 @@ defmodule Pleroma.LoadTesting.Fetcher do Benchee.run(%{ "Render status" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{ + Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{ activity: activity, for: user }) diff --git a/benchmarks/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex index 7b2293acc..4fa3eec49 100644 --- a/benchmarks/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -31,7 +31,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do thread_length: :integer ] @users_default 20_000 - @dms_default 20_000 + @dms_default 1_000 @thread_length_default 2_000 def run(args) do From c10ce113d487d71c4daa6fabcc641a5caa0d04cb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Oct 2019 12:52:47 +0300 Subject: [PATCH 40/84] User search: Remove trigram and refactor the module - Remove trigram as it tends to rank garbage results highly, resulting in it prioritized above fts, which gives actually decent results. ACKed by kaniini and lain on irc. - Remove a test for handling misspelled requests, since we no longer have trigram - Remove a test for searching users with `nil` display names, because it is unrealistic, we don't accept usernames that are not >1 char strings - Make rank boosting for followers/followees sane again, previous values resulted in garbage matches getting on top just because the users are followers/followees --- lib/pleroma/user/search.ex | 152 ++++++++++++------------------------- test/user_search_test.exs | 17 ----- 2 files changed, 50 insertions(+), 119 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 6fb2c2352..fb2f3fedb 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -4,7 +4,6 @@ defmodule Pleroma.User.Search do alias Pleroma.Pagination - alias Pleroma.Repo alias Pleroma.User import Ecto.Query @@ -23,18 +22,10 @@ defmodule Pleroma.User.Search do maybe_resolve(resolve, for_user, query_string) - {:ok, results} = - Repo.transaction(fn -> - Ecto.Adapters.SQL.query( - Repo, - "select set_limit(#{@similarity_threshold})", - [] - ) - - query_string - |> search_query(for_user, following) - |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset) - end) + results = + query_string + |> search_query(for_user, following) + |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset) results end @@ -56,15 +47,52 @@ defmodule Pleroma.User.Search do |> base_query(following) |> filter_blocked_user(for_user) |> filter_blocked_domains(for_user) - |> search_subqueries(query_string) - |> union_subqueries - |> distinct_query() - |> boost_search_rank_query(for_user) + |> fts_subquery(query_string) |> subquery() + |> where([u], u.search_rank > @similarity_threshold) + |> boost_search_rank(for_user) |> order_by(desc: :search_rank) |> maybe_restrict_local(for_user) end + @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/ + defp fts_subquery(query, query_string) do + {nickname_weight, name_weight} = + if String.match?(query_string, @nickname_regex) do + {"A", "B"} + else + {"B", "A"} + end + + query_string = to_tsquery(query_string) + + from( + u in query, + select_merge: %{ + search_rank: + fragment( + """ + ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?)) + """, + u.name, + ^name_weight, + u.nickname, + ^nickname_weight, + ^query_string + ) + } + ) + end + + defp to_tsquery(query_string) do + String.trim_trailing(query_string, "@" <> local_domain()) + |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ") + |> String.trim() + |> String.split() + |> Enum.map(&(&1 <> ":*")) + |> Enum.join(" | ") + end + defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) @@ -87,21 +115,6 @@ defmodule Pleroma.User.Search do defp filter_blocked_domains(query, _), do: query - defp union_subqueries({fts_subquery, trigram_subquery}) do - from(s in trigram_subquery, union_all: ^fts_subquery) - end - - defp search_subqueries(base_query, query_string) do - { - fts_search_subquery(base_query, query_string), - trigram_search_subquery(base_query, query_string) - } - end - - defp distinct_query(q) do - from(s in subquery(q), order_by: s.search_type, distinct: s.id) - end - defp maybe_resolve(true, user, query) do case {limit(), user} do {:all, _} -> :noop @@ -126,9 +139,9 @@ defmodule Pleroma.User.Search do defp restrict_local(q), do: where(q, [u], u.local == true) - defp boost_search_rank_query(query, nil), do: query + defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) - defp boost_search_rank_query(query, for_user) do + defp boost_search_rank(query, %User{} = for_user) do friends_ids = User.get_friends_ids(for_user) followers_ids = User.get_followers_ids(for_user) @@ -137,8 +150,8 @@ defmodule Pleroma.User.Search do search_rank: fragment( """ - CASE WHEN (?) THEN 0.5 + (?) * 1.3 - WHEN (?) THEN 0.5 + (?) * 1.2 + CASE WHEN (?) THEN (?) * 1.5 + WHEN (?) THEN (?) * 1.3 WHEN (?) THEN (?) * 1.1 ELSE (?) END """, @@ -154,70 +167,5 @@ defmodule Pleroma.User.Search do ) end - @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() - defp fts_search_subquery(query, term) do - processed_query = - String.trim_trailing(term, "@" <> local_domain()) - |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ") - |> String.trim() - |> String.split() - |> Enum.map(&(&1 <> ":*")) - |> Enum.join(" | ") - - from( - u in query, - select_merge: %{ - search_type: ^0, - search_rank: - fragment( - """ - ts_rank_cd( - setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || - setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'), - to_tsquery('simple', ?), - 32 - ) - """, - u.nickname, - u.name, - ^processed_query - ) - }, - where: - fragment( - """ - (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || - setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?) - """, - u.nickname, - u.name, - ^processed_query - ) - ) - |> User.restrict_deactivated() - end - - @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() - defp trigram_search_subquery(query, term) do - term = String.trim_trailing(term, "@" <> local_domain()) - - from( - u in query, - select_merge: %{ - # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason - search_type: fragment("?", 1), - search_rank: - fragment( - "similarity(?, trim(? || ' ' || coalesce(?, '')))", - ^term, - u.nickname, - u.name - ) - }, - where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term) - ) - |> User.restrict_deactivated() - end - - defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) + defp boost_search_rank(query, _for_user), do: query end diff --git a/test/user_search_test.exs b/test/user_search_test.exs index f7ab31287..e413f0893 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -74,12 +74,6 @@ defmodule Pleroma.UserSearchTest do assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) end - test "finds users, handling misspelled requests" do - u1 = insert(:user, %{name: "lain"}) - - assert [u1.id] == Enum.map(User.search("laiin"), & &1.id) - end - test "finds users, boosting ranks of friends and followers" do u1 = insert(:user) u2 = insert(:user, %{name: "Doe"}) @@ -163,17 +157,6 @@ defmodule Pleroma.UserSearchTest do Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end - test "finds a user whose name is nil" do - _user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"}) - user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"}) - - assert user_two == - User.search("lain@pleroma.soykaf.com") - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end - test "does not yield false-positive matches" do insert(:user, %{name: "John Doe"}) From 0a5175ecbb796cf3c192a42dc561debd73640a54 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Oct 2019 13:49:33 +0300 Subject: [PATCH 41/84] Order fts results by trigram --- lib/pleroma/user/search.ex | 48 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index fb2f3fedb..0d697fe3d 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -7,7 +7,6 @@ defmodule Pleroma.User.Search do alias Pleroma.User import Ecto.Query - @similarity_threshold 0.25 @limit 20 def search(query_string, opts \\ []) do @@ -47,16 +46,16 @@ defmodule Pleroma.User.Search do |> base_query(following) |> filter_blocked_user(for_user) |> filter_blocked_domains(for_user) - |> fts_subquery(query_string) - |> subquery() - |> where([u], u.search_rank > @similarity_threshold) + |> fts_search(query_string) + |> trigram_rank(query_string) |> boost_search_rank(for_user) + |> subquery() |> order_by(desc: :search_rank) |> maybe_restrict_local(for_user) end @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/ - defp fts_subquery(query, query_string) do + defp fts_search(query, query_string) do {nickname_weight, name_weight} = if String.match?(query_string, @nickname_regex) do {"A", "B"} @@ -68,19 +67,17 @@ defmodule Pleroma.User.Search do from( u in query, - select_merge: %{ - search_rank: - fragment( - """ - ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?)) - """, - u.name, - ^name_weight, - u.nickname, - ^nickname_weight, - ^query_string - ) - } + where: + fragment( + """ + (setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)) @@ to_tsquery('simple', ?) + """, + u.name, + ^name_weight, + u.nickname, + ^nickname_weight, + ^query_string + ) ) end @@ -93,6 +90,21 @@ defmodule Pleroma.User.Search do |> Enum.join(" | ") end + defp trigram_rank(query, query_string) do + from( + u in query, + select_merge: %{ + search_rank: + fragment( + "similarity(?, trim(? || ' ' || coalesce(?, '')))", + ^query_string, + u.nickname, + u.name + ) + } + ) + end + defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) From b8be6a4dc92ef067bc6844a89095986f07367fba Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Oct 2019 15:09:39 +0300 Subject: [PATCH 42/84] Remove a failing search test due to it being unrealistic While the case tested (`lain@ple`) gives wrong ordering due to using only trigram to order, it almost never happens in reality. In reality it would be either `lain` (in which case it's fine to list the user with `lain` in display name first), or `@lain@pleroma.soykaf.com`/`lain@pleroma.soykaf.com` (which is handled fine as well) --- test/user_search_test.exs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/user_search_test.exs b/test/user_search_test.exs index e413f0893..78a02d536 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -65,15 +65,6 @@ defmodule Pleroma.UserSearchTest do assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id) end - test "finds users, ranking by similarity" do - u1 = insert(:user, %{name: "lain"}) - _u2 = insert(:user, %{name: "ean"}) - u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"}) - u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"}) - - assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) - end - test "finds users, boosting ranks of friends and followers" do u1 = insert(:user) u2 = insert(:user, %{name: "Doe"}) From 359dd1890e6afcf80584021eaa2421336614dd07 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 17 Oct 2019 15:25:15 +0300 Subject: [PATCH 43/84] Mastodon API: Mark the conversation as read for the author when they send a new direct message --- CHANGELOG.md | 1 + lib/pleroma/conversation/participation.ex | 6 ++++ lib/pleroma/web/activity_pub/activity_pub.ex | 23 ++++++++---- test/conversation/participation_test.exs | 35 ++++++++++++++++++- test/web/activity_pub/activity_pub_test.exs | 21 +++++++++++ .../conversation_controller_test.exs | 14 +++++--- .../mastodon_api/views/account_view_test.exs | 4 +-- 7 files changed, 91 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24876d3f2..f9f84b056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities - OStatus: Extract RSS functionality - Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) +- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message ### Fixed - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ab81f3217..e17f49e58 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -48,6 +48,12 @@ defmodule Pleroma.Conversation.Participation do |> validate_required([:read]) end + def mark_as_read(%User{} = user, %Conversation{} = conversation) do + with %__MODULE__{} = participation <- for_user_and_conversation(user, conversation) do + mark_as_read(participation) + end + end + def mark_as_read(participation) do participation |> read_cng(%{read: true}) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 091ec2588..d391732a2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation + alias Pleroma.Conversation.Participation alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Object.Containment @@ -153,11 +154,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Notification.create_notifications(activity) - participations = - activity - |> Conversation.create_or_bump_for() - |> get_participations() - + conversation = create_or_bump_conversation(activity, map["actor"]) + participations = get_participations(conversation) stream_out(activity) stream_out_participations(participations) {:ok, activity} @@ -182,7 +180,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - defp get_participations({:ok, %{participations: participations}}), do: participations + defp create_or_bump_conversation(activity, actor) do + with {:ok, conversation} <- Conversation.create_or_bump_for(activity), + %User{} = user <- User.get_cached_by_ap_id(actor), + Participation.mark_as_read(user, conversation) do + {:ok, conversation} + end + end + + defp get_participations({:ok, conversation}) do + conversation + |> Repo.preload(:participations, force: true) + |> Map.get(:participations) + end + defp get_participations(_), do: [] def stream_out_participations(participations) do diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index f430bdf75..a5af0d1b2 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -23,6 +23,39 @@ defmodule Pleroma.Conversation.ParticipationTest do assert %Pleroma.Conversation{} = participation.conversation end + test "for a new conversation or a reply, it doesn't mark the author's participation as unread" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _} = + CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) + + user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(other_user.id) + + [%{read: true}] = Participation.for_user(user) + [%{read: false} = participation] = Participation.for_user(other_user) + + assert User.get_cached_by_id(user.id).info.unread_conversation_count == 0 + assert User.get_cached_by_id(other_user.id).info.unread_conversation_count == 1 + + {:ok, _} = + CommonAPI.post(other_user, %{ + "status" => "Hey @#{user.nickname}.", + "visibility" => "direct", + "in_reply_to_conversation_id" => participation.id + }) + + user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(other_user.id) + + [%{read: false}] = Participation.for_user(user) + [%{read: true}] = Participation.for_user(other_user) + + assert User.get_cached_by_id(user.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(other_user.id).info.unread_conversation_count == 0 + end + test "for a new conversation, it sets the recipents of the participation" do user = insert(:user) other_user = insert(:user) @@ -32,7 +65,7 @@ defmodule Pleroma.Conversation.ParticipationTest do CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) user = User.get_cached_by_id(user.id) - other_user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(other_user.id) [participation] = Participation.for_user(user) participation = Pleroma.Repo.preload(participation, :recipients) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 3a5a2f984..28a9b773c 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -41,6 +41,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert called(Pleroma.Web.Streamer.stream("participation", participations)) end end + + test "streams them out on activity creation" do + user_one = insert(:user) + user_two = insert(:user) + + with_mock Pleroma.Web.Streamer, + stream: fn _, _ -> nil end do + {:ok, activity} = + CommonAPI.post(user_one, %{ + "status" => "@#{user_two.nickname}", + "visibility" => "direct" + }) + + conversation = + activity.data["context"] + |> Pleroma.Conversation.get_for_ap_id() + |> Repo.preload(participations: :user) + + assert called(Pleroma.Web.Streamer.stream("participation", conversation.participations)) + end + end end describe "fetching restricted by visibility" do diff --git a/test/web/mastodon_api/controllers/conversation_controller_test.exs b/test/web/mastodon_api/controllers/conversation_controller_test.exs index a308a7620..d89a87179 100644 --- a/test/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/web/mastodon_api/controllers/conversation_controller_test.exs @@ -54,9 +54,9 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do assert user_two.id in account_ids assert user_three.id in account_ids assert is_binary(res_id) - assert unread == true + assert unread == false assert res_last_status["id"] == direct.id - assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 end test "updates the last_status on reply", %{conn: conn} do @@ -95,19 +95,23 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do "visibility" => "direct" }) + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 1 + [%{"id" => direct_conversation_id, "unread" => true}] = conn - |> assign(:user, user_one) + |> assign(:user, user_two) |> get("/api/v1/conversations") |> json_response(200) %{"unread" => false} = conn - |> assign(:user, user_one) + |> assign(:user, user_two) |> post("/api/v1/conversations/#{direct_conversation_id}/read") |> json_response(200) assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 # The conversation is marked as unread on reply {:ok, _} = @@ -124,6 +128,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do |> json_response(200) assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 # A reply doesn't increment the user's unread_conversation_count if the conversation is unread {:ok, _} = @@ -134,6 +139,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do }) assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 end test "(vanilla) Mastodon frontend behaviour", %{conn: conn} do diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index b7a4938a6..ad209b4a3 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -424,8 +424,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do other_user = insert(:user) {:ok, _activity} = - CommonAPI.post(user, %{ - "status" => "Hey @#{other_user.nickname}.", + CommonAPI.post(other_user, %{ + "status" => "Hey @#{user.nickname}.", "visibility" => "direct" }) From 733b73b71c2a13a2bfbf4c0a0d35ddfa140a1ce0 Mon Sep 17 00:00:00 2001 From: kaniini Date: Fri, 18 Oct 2019 04:36:37 +0000 Subject: [PATCH 44/84] Apply suggestion to lib/pleroma/web/mastodon_api/websocket_handler.ex --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 1a757363f..a400d1c8d 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -36,7 +36,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {:ok, user} <- allow_request(stream, [access_token, sec_websocket]), topic when is_binary(topic) <- expand_topic(stream, params) do req = - if sec_websocket != nil do + if sec_websocket do :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req) else req From 3c6fd0bb991c865aad2fb27bc798b2b3c3c248d2 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 18 Oct 2019 12:29:42 +0200 Subject: [PATCH 45/84] upload.ex: Remove deprecated configuration --- CHANGELOG.md | 3 +++ lib/pleroma/upload.ex | 33 +-------------------------------- 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f84b056..ee6b6c3a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array - Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body). +### Removed +- **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media` + ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 9f0adde5b..2e0986197 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -105,7 +105,7 @@ defmodule Pleroma.Upload do {Pleroma.Config.get!([:instance, :upload_limit]), "Document"} end - opts = %{ + %{ activity_type: Keyword.get(opts, :activity_type, activity_type), size_limit: Keyword.get(opts, :size_limit, size_limit), uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])), @@ -118,37 +118,6 @@ defmodule Pleroma.Upload do Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) ) } - - # TODO: 1.0+ : remove old config compatibility - opts = - if Pleroma.Config.get([__MODULE__, :strip_exif]) == true && - !Enum.member?(opts.filters, Pleroma.Upload.Filter.Mogrify) do - Logger.warn(""" - Pleroma: configuration `:instance, :strip_exif` is deprecated, please instead set: - - :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]] - - :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"] - """) - - Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"]) - Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify]) - else - opts - end - - if Pleroma.Config.get([:instance, :dedupe_media]) == true && - !Enum.member?(opts.filters, Pleroma.Upload.Filter.Dedupe) do - Logger.warn(""" - Pleroma: configuration `:instance, :dedupe_media` is deprecated, please instead set: - - :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Dedupe]] - """) - - Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Dedupe]) - else - opts - end end defp prepare_upload(%Plug.Upload{} = file, opts) do From 39e996528c1e7551675c0d0f140dcfa01e671004 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Oct 2019 14:11:30 +0300 Subject: [PATCH 46/84] Fix a migration wiping user info of users that don't have any mutes And introduce safe_jsonb_set --- lib/mix/tasks/pleroma/database.ex | 4 ++-- lib/pleroma/object.ex | 4 ++-- lib/pleroma/user.ex | 6 ++--- .../20190711042021_create_safe_jsonb_set.exs | 22 +++++++++++++++++++ ...2024_copy_muted_to_muted_notifications.exs | 2 +- 5 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index cfd9eeada..8a827ca80 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Database do Logger.info("Removing embedded objects") Repo.query!( - "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", + "update activities set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", [], timeout: :infinity ) @@ -126,7 +126,7 @@ defmodule Mix.Tasks.Pleroma.Database do set: [ data: fragment( - "jsonb_set(?, '{likes}', '[]'::jsonb, true)", + "safe_jsonb_set(?, '{likes}', '[]'::jsonb, true)", object.data ) ] diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index cdfbacb0e..d9b41d710 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -181,7 +181,7 @@ defmodule Pleroma.Object do data: fragment( """ - jsonb_set(?, '{repliesCount}', + safe_jsonb_set(?, '{repliesCount}', (coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true) """, o.data, @@ -204,7 +204,7 @@ defmodule Pleroma.Object do data: fragment( """ - jsonb_set(?, '{repliesCount}', + safe_jsonb_set(?, '{repliesCount}', (greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true) """, o.data, diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 917bcf2ef..d0d3e1f9a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -727,7 +727,7 @@ defmodule Pleroma.User do set: [ info: fragment( - "jsonb_set(?, '{note_count}', ((?->>'note_count')::int + 1)::varchar::jsonb, true)", + "safe_jsonb_set(?, '{note_count}', ((?->>'note_count')::int + 1)::varchar::jsonb, true)", u.info, u.info ) @@ -748,7 +748,7 @@ defmodule Pleroma.User do set: [ info: fragment( - "jsonb_set(?, '{note_count}', (greatest(0, (?->>'note_count')::int - 1))::varchar::jsonb, true)", + "safe_jsonb_set(?, '{note_count}', (greatest(0, (?->>'note_count')::int - 1))::varchar::jsonb, true)", u.info, u.info ) @@ -818,7 +818,7 @@ defmodule Pleroma.User do set: [ info: fragment( - "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)", + "safe_jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)", u.info, s.count ) diff --git a/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs b/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs new file mode 100644 index 000000000..2f336a5e8 --- /dev/null +++ b/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs @@ -0,0 +1,22 @@ +defmodule Pleroma.Repo.Migrations.CreateSafeJsonbSet do + use Ecto.Migration + alias Pleroma.User + + def change do + execute(""" + create or replace function safe_jsonb_set(target jsonb, path text[], new_value jsonb, create_missing boolean default true) returns jsonb as $$ + declare + result jsonb; + begin + result := jsonb_set(target, path, coalesce(new_value, 'null'::jsonb), create_missing); + if result is NULL then + raise 'jsonb_set tried to wipe the object, please report this incindent to Pleroma bug tracker. https://git.pleroma.social/pleroma/pleroma/issues/new'; + return target; + else + return result; + end if; + end; + $$ language plpgsql; + """) + end +end diff --git a/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs b/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs index bc4e828cc..a5eec848b 100644 --- a/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs +++ b/priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.CopyMutedToMutedNotifications do def change do execute( - "update users set info = jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true" + "update users set info = safe_jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true" ) end end From b085cd26602cca8fdb1343a2b72adb4e9575ab67 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Oct 2019 14:33:51 +0300 Subject: [PATCH 47/84] Add a test for safe_jsonb_set --- test/safe_jsonb_set_test.exs | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 test/safe_jsonb_set_test.exs diff --git a/test/safe_jsonb_set_test.exs b/test/safe_jsonb_set_test.exs new file mode 100644 index 000000000..748540570 --- /dev/null +++ b/test/safe_jsonb_set_test.exs @@ -0,0 +1,12 @@ +defmodule Pleroma.SafeJsonbSetTest do + use Pleroma.DataCase + + test "it doesn't wipe the object when asked to set the value to NULL" do + assert %{rows: [[%{"key" => "value", "test" => nil}]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select safe_jsonb_set('{\"key\": \"value\"}'::jsonb, '{test}', NULL);", + [] + ) + end +end From 09c1e7a82eae227191e91cd6f147c03c0be410aa Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Oct 2019 15:34:29 +0300 Subject: [PATCH 48/84] Sync the changelog from stable and collapse the API sections --- CHANGELOG.md | 107 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee6b6c3a0..002417f0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,38 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Removed +- **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media` + +### Changed +- **Breaking:** Elixir >=1.8 is now required (was >= 1.7) +- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) +- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler +- Enabled `:instance, extended_nickname_format` in the default config +- Add `rel="ugc"` to all links in statuses, to prevent SEO spam +- Extract RSS functionality from OStatus +- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities +
+ API Changes + +- **Breaking:** Admin API: Return link alongside with token on password reset +- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string. +- Admin API: Return `total` when querying for reports +- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) +- Admin API: Return link alongside with token on password reset +- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) +- Mastodon API: `pleroma.thread_muted` to the Status entity +- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message +
+ ### Added - Refreshing poll results for remote polls +- Authentication: Added rate limit for password-authorized actions / login existence checks +- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) +- - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). +
+ API Changes + - Job queue stats to the healthcheck page - Admin API: Add ability to require password reset - Mastodon API: Account entities now include `follow_requests_count` (planned Mastodon 3.x addition) @@ -14,37 +44,36 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance` - Mastodon API: Add `pleroma.unread_conversation_count` to the Account entity - OAuth: support for hierarchical permissions / [Mastodon 2.4.3 OAuth permissions](https://docs.joinmastodon.org/api/permissions/) -- Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed -- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) - Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints - Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array - Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body). - -### Removed -- **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media` - -### Changed -- **Breaking:** Elixir >=1.8 is now required (was >= 1.7) -- **Breaking:** Admin API: Return link alongside with token on password reset -- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) -- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler -- Admin API: Return `total` when querying for reports -- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) -- Admin API: Return link alongside with token on password reset -- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities -- OStatus: Extract RSS functionality -- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) -- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message +
### Fixed +- Report emails now include functional links to profiles of remote user accounts +
+ API Changes + - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) - Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname` -- Added `:instance, extended_nickname_format` setting to the default config -- Report emails now include functional links to profiles of remote user accounts +
-## [1.1.0] - 2019-??-?? -**Breaking:** The stable branch has been changed from `master` to `stable`, `master` now points to `release/1.0` +## [1.1.1] - 2019-10-18 +### Fixed +- One of the migrations between 1.0.0 and 1.1.0 wiping user info of the relay user because of unexpected behavior of postgresql's `jsonb_set`, resulting in inability to post in the default configuration. If you were affected, please run the following query in postgres console, the relay user will be recreated automatically: +``` +delete from users where ap_id = 'https://your.instance.hostname/relay'; +``` +- Bad user search matches + +## [1.1.0] - 2019-10-14 +**Breaking:** The stable branch has been changed from `master` to `stable`. If you want to keep using 1.0, the `release/1.0` branch will receive security updates for 6 months after 1.1 release. + +**OTP Note:** `pleroma_ctl` in 1.0 defaults to `master` and doesn't support specifying arbitrary branches, making `./pleroma_ctl update` fail. To fix this, fetch a version of `pleroma_ctl` from 1.1 using the command below and proceed with the update normally: +``` +curl -Lo ./bin/pleroma_ctl 'https://git.pleroma.social/pleroma/pleroma/raw/develop/rel/files/bin/pleroma_ctl' +``` ### Security - Mastodon API: respect post privacy in `/api/v1/statuses/:id/{favourited,reblogged}_by` @@ -52,16 +81,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** GNU Social API with Qvitter extensions support - Emoji: Remove longfox emojis. - Remove `Reply-To` header from report emails for admins. +- ActivityPub: The `/objects/:uuid/likes` endpoint. ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired - **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities. -- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string. - Configuration: added `config/description.exs`, from which `docs/config.md` is generated - Configuration: OpenGraph and TwitterCard providers enabled by default - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text -- Mastodon API: `pleroma.thread_muted` key in the Status entity - Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set - NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option - NodeInfo: Return `mailerEnabled` in `metadata` @@ -70,7 +98,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template – Pagination: (optional) return `total` alongside with `items` when paginating -- Add `rel="ugc"` to all links in statuses, to prevent SEO spam +- The `Pleroma.FlakeId` module has been replaced with the `flake_id` library. ### Fixed - Following from Osada @@ -81,21 +109,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Misskey's endless polls being unable to render - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity - Mastodon API: Notifications endpoint crashing if one notification failed to render +- Mastodon API: `exclude_replies` is correctly handled again. - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) - Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes -- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set -- Existing user id not being preserved on insert conflict +- Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) +- Mastodon API: Ensure the `account` field is not empty when rendering Notification entities. +- Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname` +- Mastodon API: Blocks are now treated consistently between the Streaming API and the Timeline APIs - Rich Media: Parser failing when no TTL can be found by image TTL setters - Rich Media: The crawled URL is now spliced into the rich media data. - ActivityPub S2S: sharedInbox usage has been mostly aligned with the rules in the AP specification. -- Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. -- Report email not being sent to admins when the reporter is a remote user -- Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances +- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set - ActivityPub: Deactivated user deletion - ActivityPub: Fix `/users/:nickname/inbox` crashing without an authenticated user - MRF: fix ability to follow a relay when AntiFollowbotPolicy was enabled -- Mastodon API: Blocks are now treated consistently between the Streaming API and the Timeline APIs -- Mastodon API: `exclude_replies` is correctly handled again. +- ActivityPub: Correct addressing of Undo. +- ActivityPub: Correct addressing of profile update activities. +- ActivityPub: Polls are now refreshed when necessary. +- Report emails now include functional links to profiles of remote user accounts +- Existing user id not being preserved on insert conflict +- Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. +- Report email not being sent to admins when the reporter is a remote user +- Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances ### Added - Expiring/ephemeral activites. All activities can have expires_at value set, which controls when they should be deleted automatically. @@ -109,6 +144,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header - Mastodon API, extension: Ability to reset avatar, profile banner, and background +- Mastodon API: Add support for `fields_attributes` API parameter (setting custom fields) - Mastodon API: Add support for categories for custom emojis by reusing the group feature. - Mastodon API: Add support for muting/unmuting notifications - Mastodon API: Add support for the `blocked_by` attribute in the relationship API (`GET /api/v1/accounts/relationships`). @@ -117,7 +153,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id - Mastodon API: Improve support for the user profile custom fields -- Mastodon API: follower/following counters are nullified when `hide_follows`/`hide_followers` and `hide_follows_count`/`hide_followers_count` are set +- Mastodon API: Add support for `fields_attributes` API parameter (setting custom fields) +- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`) - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID @@ -135,11 +172,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=` for resending account confirmation. - Pleroma API: Email change endpoint. - Admin API: Added moderation log -- Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). - Web response cache (currently, enabled for ActivityPub) -- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`) -- ActivityPub: Add ActivityPub actor's `discoverable` parameter. -- Admin API: Added moderation log filters (user/start date/end date/search/pagination) - Reverse Proxy: Do not retry failed requests to limit pressure on the peer ### Changed From c2f1cc4f165d4a5c23599c127300f325322fbfad Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 18 Oct 2019 15:56:14 +0300 Subject: [PATCH 49/84] Fix wrong list level in the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 002417f0e..0ebf358ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Refreshing poll results for remote polls - Authentication: Added rate limit for password-authorized actions / login existence checks - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) -- - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). +- Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
API Changes From 4e270964d77640ff0004c2d53a59369fa09bd0a0 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 22:29:28 +0000 Subject: [PATCH 50/84] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ebf358ed..cc6ed3f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Removed - **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media` +- **Breaking**: OStatus protocol support ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) From c6de0cbb4ae0cf2b04db5e1be79fab6138cbe71a Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 22:31:20 +0000 Subject: [PATCH 51/84] config: disable Websub and Salmon publisher modules --- config/config.exs | 8 +--- config/description.exs | 4 +- test/web/federator_test.exs | 87 ------------------------------------- 3 files changed, 2 insertions(+), 97 deletions(-) diff --git a/config/config.exs b/config/config.exs index f4d92102f..d0766a6e2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -59,10 +59,6 @@ scheduled_jobs = _ -> [] end -scheduled_jobs = - scheduled_jobs ++ - [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] - config :pleroma, Pleroma.Scheduler, global: true, overlap: true, @@ -243,9 +239,7 @@ config :pleroma, :instance, federation_incoming_replies_max_depth: 100, federation_reachability_timeout_days: 7, federation_publisher_modules: [ - Pleroma.Web.ActivityPub.Publisher, - Pleroma.Web.Websub, - Pleroma.Web.Salmon + Pleroma.Web.ActivityPub.Publisher ], allow_relay: true, rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy, diff --git a/config/description.exs b/config/description.exs index b007cf69c..571c64bc1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -581,9 +581,7 @@ config :pleroma, :config_description, [ type: [:list, :module], description: "List of modules for federation publishing", suggestions: [ - Pleroma.Web.ActivityPub.Publisher, - Pleroma.Web.Websub, - Pleroma.Web.Salmo + Pleroma.Web.ActivityPub.Publisher ] }, %{ diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 43a715706..bdaefdce1 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -111,93 +111,6 @@ defmodule Pleroma.Web.FederatorTest do all_enqueued(worker: PublisherWorker) ) end - - test "it federates only to reachable instances via Websub" do - user = insert(:user) - websub_topic = Pleroma.Web.OStatus.feed_path(user) - - sub1 = - insert(:websub_subscription, %{ - topic: websub_topic, - state: "active", - callback: "http://pleroma.soykaf.com/cb" - }) - - sub2 = - insert(:websub_subscription, %{ - topic: websub_topic, - state: "active", - callback: "https://pleroma2.soykaf.com/cb" - }) - - dt = NaiveDateTime.utc_now() - Instances.set_unreachable(sub2.callback, dt) - - Instances.set_consistently_unreachable(sub1.callback) - - {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"}) - - expected_callback = sub2.callback - expected_dt = NaiveDateTime.to_iso8601(dt) - - ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) - - assert ObanHelpers.member?( - %{ - "op" => "publish_one", - "params" => %{ - "callback" => expected_callback, - "unreachable_since" => expected_dt - } - }, - all_enqueued(worker: PublisherWorker) - ) - end - - test "it federates only to reachable instances via Salmon" do - user = insert(:user) - - _remote_user1 = - insert(:user, %{ - local: false, - nickname: "nick1@domain.com", - ap_id: "https://domain.com/users/nick1", - info: %{salmon: "https://domain.com/salmon"} - }) - - remote_user2 = - insert(:user, %{ - local: false, - nickname: "nick2@domain2.com", - ap_id: "https://domain2.com/users/nick2", - info: %{salmon: "https://domain2.com/salmon"} - }) - - remote_user2_id = remote_user2.id - - dt = NaiveDateTime.utc_now() - Instances.set_unreachable(remote_user2.ap_id, dt) - - Instances.set_consistently_unreachable("domain.com") - - {:ok, _activity} = - CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) - - expected_dt = NaiveDateTime.to_iso8601(dt) - - ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) - - assert ObanHelpers.member?( - %{ - "op" => "publish_one", - "params" => %{ - "recipient_id" => remote_user2_id, - "unreachable_since" => expected_dt - } - }, - all_enqueued(worker: PublisherWorker) - ) - end end describe "Receive an activity" do From 25b7ff56c371fa3b405fa339aaac6d4f0876ed85 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 22:52:46 +0000 Subject: [PATCH 52/84] application: don't start Federator.init/1 anymore --- lib/pleroma/application.ex | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 0bf218bc7..d681eecc8 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -161,11 +161,6 @@ defmodule Pleroma.Application do id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, restart: :temporary - }, - %{ - id: :federator_init, - start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, - restart: :temporary } ] end @@ -177,11 +172,6 @@ defmodule Pleroma.Application do start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, restart: :temporary }, - %{ - id: :federator_init, - start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, - restart: :temporary - }, %{ id: :internal_fetch_init, start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, From b16a460916e0384ec6ed34f80a390b9a4ed4d96d Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 22:53:45 +0000 Subject: [PATCH 53/84] federator: remove websub stuff --- lib/pleroma/web/federator/federator.ex | 42 ------------------------ lib/pleroma/workers/subscriber_worker.ex | 26 --------------- 2 files changed, 68 deletions(-) delete mode 100644 lib/pleroma/workers/subscriber_worker.ex diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 1a2da014a..8227d1a3a 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -11,18 +11,11 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.OStatus - alias Pleroma.Web.Websub alias Pleroma.Workers.PublisherWorker alias Pleroma.Workers.ReceiverWorker - alias Pleroma.Workers.SubscriberWorker require Logger - def init do - # To do: consider removing this call in favor of scheduled execution (`quantum`-based) - refresh_subscriptions(schedule_in: 60) - end - @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength def allowed_incoming_reply_depth?(depth) do @@ -53,18 +46,6 @@ defmodule Pleroma.Web.Federator do PublisherWorker.enqueue("publish", %{"activity_id" => activity.id}) end - def verify_websub(websub) do - SubscriberWorker.enqueue("verify_websub", %{"websub_id" => websub.id}) - end - - def request_subscription(websub) do - SubscriberWorker.enqueue("request_subscription", %{"websub_id" => websub.id}) - end - - def refresh_subscriptions(worker_args \\ []) do - SubscriberWorker.enqueue("refresh_subscriptions", %{}, worker_args ++ [max_attempts: 1]) - end - # Job Worker Callbacks @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} @@ -111,29 +92,6 @@ defmodule Pleroma.Web.Federator do end end - def perform(:request_subscription, websub) do - Logger.debug("Refreshing #{websub.topic}") - - with {:ok, websub} <- Websub.request_subscription(websub) do - Logger.debug("Successfully refreshed #{websub.topic}") - else - _e -> Logger.debug("Couldn't refresh #{websub.topic}") - end - end - - def perform(:verify_websub, websub) do - Logger.debug(fn -> - "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" - end) - - Websub.verify(websub) - end - - def perform(:refresh_subscriptions) do - Logger.debug("Federator running refresh subscriptions") - Websub.refresh_subscriptions() - end - def ap_enabled_actor(id) do user = User.get_cached_by_ap_id(id) diff --git a/lib/pleroma/workers/subscriber_worker.ex b/lib/pleroma/workers/subscriber_worker.ex deleted file mode 100644 index fc490e300..000000000 --- a/lib/pleroma/workers/subscriber_worker.ex +++ /dev/null @@ -1,26 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.SubscriberWorker do - alias Pleroma.Repo - alias Pleroma.Web.Federator - alias Pleroma.Web.Websub - - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" - - @impl Oban.Worker - def perform(%{"op" => "refresh_subscriptions"}, _job) do - Federator.perform(:refresh_subscriptions) - end - - def perform(%{"op" => "request_subscription", "websub_id" => websub_id}, _job) do - websub = Repo.get(Websub.WebsubClientSubscription, websub_id) - Federator.perform(:request_subscription, websub) - end - - def perform(%{"op" => "verify_websub", "websub_id" => websub_id}, _job) do - websub = Repo.get(Websub.WebsubServerSubscription, websub_id) - Federator.perform(:verify_websub, websub) - end -end From 4f82e42e4e581b32cd25fff862f880f7f5a87b81 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 22:57:37 +0000 Subject: [PATCH 54/84] websub: remove entirely --- lib/pleroma/user.ex | 7 - .../web/activity_pub/transmogrifier.ex | 15 - lib/pleroma/web/ostatus/ostatus.ex | 5 +- lib/pleroma/web/router.ex | 3 - .../web/templates/feed/feed/feed.xml.eex | 1 - lib/pleroma/web/websub/websub.ex | 332 ------------------ .../web/websub/websub_client_subscription.ex | 20 -- lib/pleroma/web/websub/websub_controller.ex | 99 ------ .../web/websub/websub_server_subscription.ex | 17 - test/support/factory.ex | 20 -- test/user_test.exs | 17 - test/web/activity_pub/transmogrifier_test.exs | 16 - test/web/ostatus/ostatus_test.exs | 54 --- test/web/websub/websub_controller_test.exs | 86 ----- test/web/websub/websub_test.exs | 236 ------------- 15 files changed, 1 insertion(+), 927 deletions(-) delete mode 100644 lib/pleroma/web/websub/websub.ex delete mode 100644 lib/pleroma/web/websub/websub_client_subscription.ex delete mode 100644 lib/pleroma/web/websub/websub_controller.ex delete mode 100644 lib/pleroma/web/websub/websub_server_subscription.ex delete mode 100644 test/web/websub/websub_controller_test.exs delete mode 100644 test/web/websub/websub_test.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d0d3e1f9a..cef011c68 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -28,7 +28,6 @@ defmodule Pleroma.User do alias Pleroma.Web.OAuth alias Pleroma.Web.OStatus alias Pleroma.Web.RelMe - alias Pleroma.Web.Websub alias Pleroma.Workers.BackgroundWorker require Logger @@ -437,12 +436,6 @@ defmodule Pleroma.User do {:error, "Could not follow user: #{followed.nickname} blocked you."} true -> - benchmark? = Pleroma.Config.get([:env]) == :benchmark - - if !followed.local && follower.local && !ap_enabled?(followed) && !benchmark? do - Websub.subscribe(follower, followed) - end - q = from(u in User, where: u.id == ^follower.id, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b56343beb..2c1ce9c55 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1073,8 +1073,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Repo.update_all(q, []) - maybe_retire_websub(user.ap_id) - q = from( a in Activity, @@ -1117,19 +1115,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> User.update_and_set_cache() end - def maybe_retire_websub(ap_id) do - # some sanity checks - if is_binary(ap_id) && String.length(ap_id) > 8 do - q = - from( - ws in Pleroma.Web.Websub.WebsubClientSubscription, - where: fragment("? like ?", ws.topic, ^"#{ap_id}%") - ) - - Repo.delete_all(q) - end - end - def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do Map.put(data, "url", url["href"]) end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 5de1ceef3..3dfc9b580 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -19,7 +19,6 @@ defmodule Pleroma.Web.OStatus do alias Pleroma.Web.OStatus.NoteHandler alias Pleroma.Web.OStatus.UnfollowHandler alias Pleroma.Web.WebFinger - alias Pleroma.Web.Websub def is_representable?(%Activity{} = activity) do object = Object.normalize(activity) @@ -314,11 +313,9 @@ defmodule Pleroma.Web.OStatus do @spec gather_user_info(String.t()) :: {:ok, map()} | {:error, any()} def gather_user_info(username) do - with {:ok, webfinger_data} <- WebFinger.finger(username), - {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do + with {:ok, webfinger_data} <- WebFinger.finger(username) do data = webfinger_data - |> Map.merge(feed_data) |> Map.put("fqn", username) {:ok, data} diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 80651f3ff..77fe938d5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -509,9 +509,6 @@ defmodule Pleroma.Web.Router do get("/users/:nickname", Feed.FeedController, :feed_redirect) post("/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming) - post("/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request) - get("/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation) - post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming) get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe) end diff --git a/lib/pleroma/web/templates/feed/feed/feed.xml.eex b/lib/pleroma/web/templates/feed/feed/feed.xml.eex index fbfdc46b5..692cfbd7f 100644 --- a/lib/pleroma/web/templates/feed/feed/feed.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/feed.xml.eex @@ -10,7 +10,6 @@ <%= @user.nickname <> "'s timeline" %> <%= most_recent_update(@activities, @user) %> <%= logo(@user) %> - diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex deleted file mode 100644 index b61f388b8..000000000 --- a/lib/pleroma/web/websub/websub.ex +++ /dev/null @@ -1,332 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Websub do - alias Ecto.Changeset - alias Pleroma.Activity - alias Pleroma.HTTP - alias Pleroma.Instances - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.Endpoint - alias Pleroma.Web.Federator - alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.OStatus - alias Pleroma.Web.OStatus.FeedRepresenter - alias Pleroma.Web.Router.Helpers - alias Pleroma.Web.Websub.WebsubClientSubscription - alias Pleroma.Web.Websub.WebsubServerSubscription - alias Pleroma.Web.XML - require Logger - - import Ecto.Query - - @behaviour Pleroma.Web.Federator.Publisher - - def verify(subscription, getter \\ &HTTP.get/3) do - challenge = Base.encode16(:crypto.strong_rand_bytes(8)) - lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) - lease_seconds = lease_seconds |> to_string - - params = %{ - "hub.challenge": challenge, - "hub.lease_seconds": lease_seconds, - "hub.topic": subscription.topic, - "hub.mode": "subscribe" - } - - url = hd(String.split(subscription.callback, "?")) - query = URI.parse(subscription.callback).query || "" - params = Map.merge(params, URI.decode_query(query)) - - with {:ok, response} <- getter.(url, [], params: params), - ^challenge <- response.body do - changeset = Changeset.change(subscription, %{state: "active"}) - Repo.update(changeset) - else - e -> - Logger.debug("Couldn't verify subscription") - Logger.debug(inspect(e)) - {:error, subscription} - end - end - - @supported_activities [ - "Create", - "Follow", - "Like", - "Announce", - "Undo", - "Delete" - ] - - def is_representable?(%Activity{data: %{"type" => type}} = activity) - when type in @supported_activities, - do: Visibility.is_public?(activity) - - def is_representable?(_), do: false - - def publish(topic, user, %{data: %{"type" => type}} = activity) - when type in @supported_activities do - response = - user - |> FeedRepresenter.to_simple_form([activity], [user]) - |> :xmerl.export_simple(:xmerl_xml) - |> to_string - - query = - from( - sub in WebsubServerSubscription, - where: sub.topic == ^topic and sub.state == "active", - where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until) - ) - - subscriptions = Repo.all(query) - - callbacks = Enum.map(subscriptions, & &1.callback) - reachable_callbacks_metadata = Instances.filter_reachable(callbacks) - reachable_callbacks = Map.keys(reachable_callbacks_metadata) - - subscriptions - |> Enum.filter(&(&1.callback in reachable_callbacks)) - |> Enum.each(fn sub -> - data = %{ - xml: response, - topic: topic, - callback: sub.callback, - secret: sub.secret, - unreachable_since: reachable_callbacks_metadata[sub.callback] - } - - Publisher.enqueue_one(__MODULE__, data) - end) - end - - def publish(_, _, _), do: "" - - def publish(actor, activity), do: publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) - - def sign(secret, doc) do - :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16() |> String.downcase() - end - - def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do - with {:ok, topic} <- valid_topic(params, user), - {:ok, lease_time} <- lease_time(params), - secret <- params["hub.secret"], - callback <- params["hub.callback"] do - subscription = get_subscription(topic, callback) - - data = %{ - state: subscription.state || "requested", - topic: topic, - secret: secret, - callback: callback - } - - change = Changeset.change(subscription, data) - websub = Repo.insert_or_update!(change) - - change = - Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)}) - - websub = Repo.update!(change) - - Federator.verify_websub(websub) - - {:ok, websub} - else - {:error, reason} -> - Logger.debug("Couldn't create subscription") - Logger.debug(inspect(reason)) - - {:error, reason} - end - end - - def incoming_subscription_request(user, params) do - Logger.info("Unhandled WebSub request for #{user.nickname}: #{inspect(params)}") - - {:error, "Invalid WebSub request"} - end - - defp get_subscription(topic, callback) do - Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) || - %WebsubServerSubscription{} - end - - # Temp hack for mastodon. - defp lease_time(%{"hub.lease_seconds" => ""}) do - # three days - {:ok, 60 * 60 * 24 * 3} - end - - defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do - {:ok, String.to_integer(lease_seconds)} - end - - defp lease_time(_) do - # three days - {:ok, 60 * 60 * 24 * 3} - end - - defp valid_topic(%{"hub.topic" => topic}, user) do - if topic == OStatus.feed_path(user) do - {:ok, OStatus.feed_path(user)} - else - {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} - end - end - - def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do - topic = subscribed.info.topic - # FIXME: Race condition, use transactions - {:ok, subscription} = - with subscription when not is_nil(subscription) <- - Repo.get_by(WebsubClientSubscription, topic: topic) do - subscribers = [subscriber.ap_id | subscription.subscribers] |> Enum.uniq() - change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) - Repo.update(change) - else - _e -> - subscription = %WebsubClientSubscription{ - topic: topic, - hub: subscribed.info.hub, - subscribers: [subscriber.ap_id], - state: "requested", - secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64(), - user: subscribed - } - - Repo.insert(subscription) - end - - requester.(subscription) - end - - def gather_feed_data(topic, getter \\ &HTTP.get/1) do - with {:ok, response} <- getter.(topic), - status when status in 200..299 <- response.status, - body <- response.body, - doc <- XML.parse_document(body), - uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc), - hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do - name = XML.string_from_xpath("/feed/author[1]/name", doc) - preferred_username = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc) - display_name = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc) - avatar = OStatus.make_avatar_object(doc) - bio = XML.string_from_xpath("/feed/author[1]/summary", doc) - - {:ok, - %{ - "uri" => uri, - "hub" => hub, - "nickname" => preferred_username || name, - "name" => display_name || name, - "host" => URI.parse(uri).host, - "avatar" => avatar, - "bio" => bio - }} - else - e -> - {:error, e} - end - end - - def request_subscription(websub, poster \\ &HTTP.post/3, timeout \\ 10_000) do - data = [ - "hub.mode": "subscribe", - "hub.topic": websub.topic, - "hub.secret": websub.secret, - "hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id) - ] - - # This checks once a second if we are confirmed yet - websub_checker = fn -> - helper = fn helper -> - :timer.sleep(1000) - websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted") - if websub, do: websub, else: helper.(helper) - end - - helper.(helper) - end - - task = Task.async(websub_checker) - - with {:ok, %{status: 202}} <- - poster.(websub.hub, {:form, data}, "Content-type": "application/x-www-form-urlencoded"), - {:ok, websub} <- Task.yield(task, timeout) do - {:ok, websub} - else - e -> - Task.shutdown(task) - - change = Ecto.Changeset.change(websub, %{state: "rejected"}) - {:ok, websub} = Repo.update(change) - - Logger.debug(fn -> "Couldn't confirm subscription: #{inspect(websub)}" end) - Logger.debug(fn -> "error: #{inspect(e)}" end) - - {:error, websub} - end - end - - def refresh_subscriptions(delta \\ 60 * 60 * 24) do - Logger.debug("Refreshing subscriptions") - - cut_off = NaiveDateTime.add(NaiveDateTime.utc_now(), delta) - - query = from(sub in WebsubClientSubscription, where: sub.valid_until < ^cut_off) - - subs = Repo.all(query) - - Enum.each(subs, fn sub -> - Federator.request_subscription(sub) - end) - end - - def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = params) do - signature = sign(secret || "", xml) - Logger.info(fn -> "Pushing #{topic} to #{callback}" end) - - with {:ok, %{status: code}} when code in 200..299 <- - HTTP.post( - callback, - xml, - [ - {"Content-Type", "application/atom+xml"}, - {"X-Hub-Signature", "sha1=#{signature}"} - ] - ) do - if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], - do: Instances.set_reachable(callback) - - Logger.info(fn -> "Pushed to #{callback}, code #{code}" end) - {:ok, code} - else - {_post_result, response} -> - unless params[:unreachable_since], do: Instances.set_reachable(callback) - Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end) - {:error, response} - end - end - - def gather_webfinger_links(%User{} = user) do - [ - %{ - "rel" => "http://schemas.google.com/g/2010#updates-from", - "type" => "application/atom+xml", - "href" => OStatus.feed_path(user) - }, - %{ - "rel" => "http://ostatus.org/schema/1.0/subscribe", - "template" => OStatus.remote_follow_path() - } - ] - end - - def gather_nodeinfo_protocol_names, do: ["ostatus"] -end diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex deleted file mode 100644 index 23a04b87d..000000000 --- a/lib/pleroma/web/websub/websub_client_subscription.ex +++ /dev/null @@ -1,20 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Websub.WebsubClientSubscription do - use Ecto.Schema - alias Pleroma.User - - schema "websub_client_subscriptions" do - field(:topic, :string) - field(:secret, :string) - field(:valid_until, :naive_datetime_usec) - field(:state, :string) - field(:subscribers, {:array, :string}, default: []) - field(:hub, :string) - belongs_to(:user, User, type: FlakeId.Ecto.CompatType) - - timestamps() - end -end diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex deleted file mode 100644 index 9e8b48b80..000000000 --- a/lib/pleroma/web/websub/websub_controller.ex +++ /dev/null @@ -1,99 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Websub.WebsubController do - use Pleroma.Web, :controller - - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.Federator - alias Pleroma.Web.Websub - alias Pleroma.Web.Websub.WebsubClientSubscription - - require Logger - - plug( - Pleroma.Web.FederatingPlug - when action in [ - :websub_subscription_request, - :websub_subscription_confirmation, - :websub_incoming - ] - ) - - def websub_subscription_request(conn, %{"nickname" => nickname} = params) do - user = User.get_cached_by_nickname(nickname) - - with {:ok, _websub} <- Websub.incoming_subscription_request(user, params) do - conn - |> send_resp(202, "Accepted") - else - {:error, reason} -> - conn - |> send_resp(500, reason) - end - end - - # TODO: Extract this into the Websub module - def websub_subscription_confirmation( - conn, - %{ - "id" => id, - "hub.mode" => "subscribe", - "hub.challenge" => challenge, - "hub.topic" => topic - } = params - ) do - Logger.debug("Got WebSub confirmation") - Logger.debug(inspect(params)) - - lease_seconds = - if params["hub.lease_seconds"] do - String.to_integer(params["hub.lease_seconds"]) - else - # Guess 3 days - 60 * 60 * 24 * 3 - end - - with %WebsubClientSubscription{} = websub <- - Repo.get_by(WebsubClientSubscription, id: id, topic: topic) do - valid_until = NaiveDateTime.add(NaiveDateTime.utc_now(), lease_seconds) - change = Ecto.Changeset.change(websub, %{state: "accepted", valid_until: valid_until}) - {:ok, _websub} = Repo.update(change) - - conn - |> send_resp(200, challenge) - else - _e -> - conn - |> send_resp(500, "Error") - end - end - - def websub_subscription_confirmation(conn, params) do - Logger.info("Invalid WebSub confirmation request: #{inspect(params)}") - - conn - |> send_resp(500, "Invalid parameters") - end - - def websub_incoming(conn, %{"id" => id}) do - with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), - signature <- String.downcase(signature), - %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), - {:ok, body, _conn} = read_body(conn), - ^signature <- Websub.sign(websub.secret, body) do - Federator.incoming_doc(body) - - conn - |> send_resp(200, "OK") - else - _e -> - Logger.debug("Can't handle incoming subscription post") - - conn - |> send_resp(500, "Error") - end - end -end diff --git a/lib/pleroma/web/websub/websub_server_subscription.ex b/lib/pleroma/web/websub/websub_server_subscription.ex deleted file mode 100644 index d0ef548da..000000000 --- a/lib/pleroma/web/websub/websub_server_subscription.ex +++ /dev/null @@ -1,17 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Websub.WebsubServerSubscription do - use Ecto.Schema - - schema "websub_server_subscriptions" do - field(:topic, :string) - field(:callback, :string) - field(:secret, :string) - field(:valid_until, :naive_datetime) - field(:state, :string) - - timestamps() - end -end diff --git a/test/support/factory.ex b/test/support/factory.ex index b180844cd..0fdb1e952 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -281,26 +281,6 @@ defmodule Pleroma.Factory do } end - def websub_subscription_factory do - %Pleroma.Web.Websub.WebsubServerSubscription{ - topic: "http://example.org", - callback: "http://example.org/callback", - secret: "here's a secret", - valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 100), - state: "requested" - } - end - - def websub_client_subscription_factory do - %Pleroma.Web.Websub.WebsubClientSubscription{ - topic: "http://example.org", - secret: "here's a secret", - valid_until: nil, - state: "requested", - subscribers: [] - } - end - def oauth_app_factory do %Pleroma.Web.OAuth.App{ client_name: "Some client", diff --git a/test/user_test.exs b/test/user_test.exs index 019e7b400..81d23e9a3 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -190,23 +190,6 @@ defmodule Pleroma.UserTest do refute User.following?(follower, followed) end - # This is a somewhat useless test. - # test "following a remote user will ensure a websub subscription is present" do - # user = insert(:user) - # {:ok, followed} = OStatus.make_user("shp@social.heldscal.la") - - # assert followed.local == false - - # {:ok, user} = User.follow(user, followed) - # assert User.ap_followers(followed) in user.following - - # query = from w in WebsubClientSubscription, - # where: w.topic == ^followed.info["topic"] - # websub = Repo.one(query) - - # assert websub - # end - describe "unfollow/2" do setup do setting = Pleroma.Config.get([:instance, :external_user_synchronization]) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6c35a6f4d..7823ff041 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -14,7 +14,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus - alias Pleroma.Web.Websub.WebsubClientSubscription import Mock import Pleroma.Factory @@ -1371,21 +1370,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end end - describe "maybe_retire_websub" do - test "it deletes all websub client subscripitions with the user as topic" do - subscription = %WebsubClientSubscription{topic: "https://niu.moe/users/rye.atom"} - {:ok, ws} = Repo.insert(subscription) - - subscription = %WebsubClientSubscription{topic: "https://niu.moe/users/pasty.atom"} - {:ok, ws2} = Repo.insert(subscription) - - Transmogrifier.maybe_retire_websub("https://niu.moe/users/rye") - - refute Repo.get(WebsubClientSubscription, ws.id) - assert Repo.get(WebsubClientSubscription, ws2.id) - end - end - describe "actor rewriting" do test "it fixes the actor URL property to be a proper URI" do data = %{ diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 70a0e4473..bbecba189 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -64,19 +64,6 @@ defmodule Pleroma.Web.OStatusTest do assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end - test "handle incoming notes with attachments - GS, subscription" do - incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - assert activity.data["type"] == "Create" - assert object.data["type"] == "Note" - assert object.data["actor"] == "https://social.heldscal.la/user/23211" - assert object.data["attachment"] |> length == 2 - assert object.data["external_url"] == "https://social.heldscal.la/notice/2020923" - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - end - test "handle incoming notes with tags" do incoming = File.read!("test/fixtures/ostatus_incoming_post_tag.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) @@ -221,29 +208,6 @@ defmodule Pleroma.Web.OStatusTest do refute String.contains?(retweeted_object.data["content"], "Test account") end - test "handle incoming favorites - GS, websub" do - capture_log(fn -> - incoming = File.read!("test/fixtures/favorite.xml") - {:ok, [[activity, favorited_activity]]} = OStatus.handle_incoming(incoming) - - assert activity.data["type"] == "Like" - assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == favorited_activity.data["object"] - - assert activity.data["id"] == - "tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00" - - refute activity.local - assert favorited_activity.data["type"] == "Create" - assert favorited_activity.data["actor"] == "https://shitposter.club/user/1" - - assert favorited_activity.data["object"] == - "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" - - refute favorited_activity.local - end) - end - test "handle conversation references" do incoming = File.read!("test/fixtures/mastodon_conversation.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) @@ -252,24 +216,6 @@ defmodule Pleroma.Web.OStatusTest do "tag:mastodon.social,2017-08-28:objectId=7876885:objectType=Conversation" end - test "handle incoming favorites with locally available object - GS, websub" do - note_activity = insert(:note_activity) - object = Object.normalize(note_activity) - - incoming = - File.read!("test/fixtures/favorite_with_local_note.xml") - |> String.replace("localid", object.data["id"]) - - {:ok, [[activity, favorited_activity]]} = OStatus.handle_incoming(incoming) - - assert activity.data["type"] == "Like" - assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == object.data["id"] - refute activity.local - assert note_activity.id == favorited_activity.id - assert favorited_activity.local - end - test_with_mock "handle incoming replies, fetching replied-to activities if we don't have them", OStatus, [:passthrough], diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs deleted file mode 100644 index f6d002b3b..000000000 --- a/test/web/websub/websub_controller_test.exs +++ /dev/null @@ -1,86 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Websub.WebsubControllerTest do - use Pleroma.Web.ConnCase - import Pleroma.Factory - alias Pleroma.Repo - alias Pleroma.Web.Websub - alias Pleroma.Web.Websub.WebsubClientSubscription - - clear_config_all([:instance, :federating]) do - Pleroma.Config.put([:instance, :federating], true) - end - - test "websub subscription request", %{conn: conn} do - user = insert(:user) - - path = Pleroma.Web.OStatus.pubsub_path(user) - - data = %{ - "hub.callback": "http://example.org/sub", - "hub.mode": "subscribe", - "hub.topic": Pleroma.Web.OStatus.feed_path(user), - "hub.secret": "a random secret", - "hub.lease_seconds": "100" - } - - conn = - conn - |> post(path, data) - - assert response(conn, 202) == "Accepted" - end - - test "websub subscription confirmation", %{conn: conn} do - websub = insert(:websub_client_subscription) - - params = %{ - "hub.mode" => "subscribe", - "hub.topic" => websub.topic, - "hub.challenge" => "some challenge", - "hub.lease_seconds" => "100" - } - - conn = - conn - |> get("/push/subscriptions/#{websub.id}", params) - - websub = Repo.get(WebsubClientSubscription, websub.id) - - assert response(conn, 200) == "some challenge" - assert websub.state == "accepted" - assert_in_delta NaiveDateTime.diff(websub.valid_until, NaiveDateTime.utc_now()), 100, 5 - end - - describe "websub_incoming" do - test "accepts incoming feed updates", %{conn: conn} do - websub = insert(:websub_client_subscription) - doc = "some stuff" - signature = Websub.sign(websub.secret, doc) - - conn = - conn - |> put_req_header("x-hub-signature", "sha1=" <> signature) - |> put_req_header("content-type", "application/atom+xml") - |> post("/push/subscriptions/#{websub.id}", doc) - - assert response(conn, 200) == "OK" - end - - test "rejects incoming feed updates with the wrong signature", %{conn: conn} do - websub = insert(:websub_client_subscription) - doc = "some stuff" - signature = Websub.sign("wrong secret", doc) - - conn = - conn - |> put_req_header("x-hub-signature", "sha1=" <> signature) - |> put_req_header("content-type", "application/atom+xml") - |> post("/push/subscriptions/#{websub.id}", doc) - - assert response(conn, 500) == "Error" - end - end -end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs deleted file mode 100644 index 46ca545de..000000000 --- a/test/web/websub/websub_test.exs +++ /dev/null @@ -1,236 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.WebsubTest do - use Pleroma.DataCase - use Oban.Testing, repo: Pleroma.Repo - - alias Pleroma.Tests.ObanHelpers - alias Pleroma.Web.Router.Helpers - alias Pleroma.Web.Websub - alias Pleroma.Web.Websub.WebsubClientSubscription - alias Pleroma.Web.Websub.WebsubServerSubscription - alias Pleroma.Workers.SubscriberWorker - - import Pleroma.Factory - import Tesla.Mock - - setup do - mock(fn env -> apply(HttpRequestMock, :request, [env]) end) - :ok - end - - test "a verification of a request that is accepted" do - sub = insert(:websub_subscription) - topic = sub.topic - - getter = fn _path, _headers, options -> - %{ - "hub.challenge": challenge, - "hub.lease_seconds": seconds, - "hub.topic": ^topic, - "hub.mode": "subscribe" - } = Keyword.get(options, :params) - - assert String.to_integer(seconds) > 0 - - {:ok, - %Tesla.Env{ - status: 200, - body: challenge - }} - end - - {:ok, sub} = Websub.verify(sub, getter) - assert sub.state == "active" - end - - test "a verification of a request that doesn't return 200" do - sub = insert(:websub_subscription) - - getter = fn _path, _headers, _options -> - {:ok, - %Tesla.Env{ - status: 500, - body: "" - }} - end - - {:error, sub} = Websub.verify(sub, getter) - # Keep the current state. - assert sub.state == "requested" - end - - test "an incoming subscription request" do - user = insert(:user) - - data = %{ - "hub.callback" => "http://example.org/sub", - "hub.mode" => "subscribe", - "hub.topic" => Pleroma.Web.OStatus.feed_path(user), - "hub.secret" => "a random secret", - "hub.lease_seconds" => "100" - } - - {:ok, subscription} = Websub.incoming_subscription_request(user, data) - assert subscription.topic == Pleroma.Web.OStatus.feed_path(user) - assert subscription.state == "requested" - assert subscription.secret == "a random secret" - assert subscription.callback == "http://example.org/sub" - end - - test "an incoming subscription request for an existing subscription" do - user = insert(:user) - - sub = - insert(:websub_subscription, state: "accepted", topic: Pleroma.Web.OStatus.feed_path(user)) - - data = %{ - "hub.callback" => sub.callback, - "hub.mode" => "subscribe", - "hub.topic" => Pleroma.Web.OStatus.feed_path(user), - "hub.secret" => "a random secret", - "hub.lease_seconds" => "100" - } - - {:ok, subscription} = Websub.incoming_subscription_request(user, data) - assert subscription.topic == Pleroma.Web.OStatus.feed_path(user) - assert subscription.state == sub.state - assert subscription.secret == "a random secret" - assert subscription.callback == sub.callback - assert length(Repo.all(WebsubServerSubscription)) == 1 - assert subscription.id == sub.id - end - - def accepting_verifier(subscription) do - {:ok, %{subscription | state: "accepted"}} - end - - test "initiate a subscription for a given user and topic" do - subscriber = insert(:user) - user = insert(:user, %{info: %Pleroma.User.Info{topic: "some_topic", hub: "some_hub"}}) - - {:ok, websub} = Websub.subscribe(subscriber, user, &accepting_verifier/1) - assert websub.subscribers == [subscriber.ap_id] - assert websub.topic == "some_topic" - assert websub.hub == "some_hub" - assert is_binary(websub.secret) - assert websub.user == user - assert websub.state == "accepted" - end - - test "discovers the hub and canonical url" do - topic = "https://mastodon.social/users/lambadalambda.atom" - - {:ok, discovered} = Websub.gather_feed_data(topic) - - expected = %{ - "hub" => "https://mastodon.social/api/push", - "uri" => "https://mastodon.social/users/lambadalambda", - "nickname" => "lambadalambda", - "name" => "Critical Value", - "host" => "mastodon.social", - "bio" => "a cool dude.", - "avatar" => %{ - "type" => "Image", - "url" => [ - %{ - "href" => - "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", - "mediaType" => "image/gif", - "type" => "Link" - } - ] - } - } - - assert expected == discovered - end - - test "calls the hub, requests topic" do - hub = "https://social.heldscal.la/main/push/hub" - topic = "https://social.heldscal.la/api/statuses/user_timeline/23211.atom" - websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) - - poster = fn ^hub, {:form, data}, _headers -> - assert Keyword.get(data, :"hub.mode") == "subscribe" - - assert Keyword.get(data, :"hub.callback") == - Helpers.websub_url( - Pleroma.Web.Endpoint, - :websub_subscription_confirmation, - websub.id - ) - - {:ok, %{status: 202}} - end - - task = Task.async(fn -> Websub.request_subscription(websub, poster) end) - - change = Ecto.Changeset.change(websub, %{state: "accepted"}) - {:ok, _} = Repo.update(change) - - {:ok, websub} = Task.await(task) - - assert websub.state == "accepted" - end - - test "rejects the subscription if it can't be accepted" do - hub = "https://social.heldscal.la/main/push/hub" - topic = "https://social.heldscal.la/api/statuses/user_timeline/23211.atom" - websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) - - poster = fn ^hub, {:form, _data}, _headers -> - {:ok, %{status: 202}} - end - - {:error, websub} = Websub.request_subscription(websub, poster, 1000) - assert websub.state == "rejected" - - websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) - - poster = fn ^hub, {:form, _data}, _headers -> - {:ok, %{status: 400}} - end - - {:error, websub} = Websub.request_subscription(websub, poster, 1000) - assert websub.state == "rejected" - end - - test "sign a text" do - signed = Websub.sign("secret", "text") - assert signed == "B8392C23690CCF871F37EC270BE1582DEC57A503" |> String.downcase() - - _signed = Websub.sign("secret", [["て"], ['す']]) - end - - describe "renewing subscriptions" do - test "it renews subscriptions that have less than a day of time left" do - day = 60 * 60 * 24 - now = NaiveDateTime.utc_now() - - still_good = - insert(:websub_client_subscription, %{ - valid_until: NaiveDateTime.add(now, 2 * day), - topic: "http://example.org/still_good", - hub: "http://example.org/still_good", - state: "accepted" - }) - - needs_refresh = - insert(:websub_client_subscription, %{ - valid_until: NaiveDateTime.add(now, day - 100), - topic: "http://example.org/needs_refresh", - hub: "http://example.org/needs_refresh", - state: "accepted" - }) - - _refresh = Websub.refresh_subscriptions() - ObanHelpers.perform(all_enqueued(worker: SubscriberWorker)) - - assert still_good == Repo.get(WebsubClientSubscription, still_good.id) - refute needs_refresh == Repo.get(WebsubClientSubscription, needs_refresh.id) - end - end -end From adb639db56fc40b07edaf9ed8cdf40d6aa2c573b Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 23:05:45 +0000 Subject: [PATCH 55/84] publisher: move remote_users() from Salmon module --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- lib/pleroma/web/federator/publisher.ex | 26 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 3866dacee..2aac4e8b9 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -129,7 +129,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [] end - Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers ++ fetchers + Pleroma.Web.Federator.Publisher.remote_users(actor, activity) ++ followers ++ fetchers end defp get_cc_ap_ids(ap_id, recipients) do diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 937064638..fb9b26649 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -80,4 +80,30 @@ defmodule Pleroma.Web.Federator.Publisher do links ++ module.gather_nodeinfo_protocol_names() end) end + + @doc """ + Gathers a set of remote users given an IR envelope. + """ + def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do + cc = Map.get(data, "cc", []) + + bcc = + data + |> Map.get("bcc", []) + |> Enum.reduce([], fn ap_id, bcc -> + case Pleroma.List.get_by_ap_id(ap_id) do + %Pleroma.List{user_id: ^user_id} = list -> + {:ok, following} = Pleroma.List.get_following(list) + bcc ++ Enum.map(following, & &1.ap_id) + + _ -> + bcc + end + end) + + [to, cc, bcc] + |> Enum.concat() + |> Enum.map(&User.get_cached_by_ap_id/1) + |> Enum.filter(fn user -> user && !user.local end) + end end From c00ae10af8187cb9d54a0bfbbf37a69a94298703 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 23:06:53 +0000 Subject: [PATCH 56/84] feed: don't advertise salmon endpoint --- lib/pleroma/web/templates/feed/feed/feed.xml.eex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/templates/feed/feed/feed.xml.eex b/lib/pleroma/web/templates/feed/feed/feed.xml.eex index 692cfbd7f..45df9dc09 100644 --- a/lib/pleroma/web/templates/feed/feed/feed.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/feed.xml.eex @@ -10,7 +10,6 @@ <%= @user.nickname <> "'s timeline" %> <%= most_recent_update(@activities, @user) %> <%= logo(@user) %> - <%= render @view_module, "_author.xml", assigns %> From a7b92bba68281f3861d1446b85743a0b65fe1115 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 23:07:20 +0000 Subject: [PATCH 57/84] webfinger: stop pulling Salmon data out of WebFinger --- lib/pleroma/web/web_finger/web_finger.ex | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index ecb39ee50..b4cc80179 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -108,7 +108,6 @@ defmodule Pleroma.Web.WebFinger do doc ), subject <- XML.string_from_xpath("//Subject", doc), - salmon <- XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc), subscribe_address <- XML.string_from_xpath( ~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template}, @@ -123,7 +122,6 @@ defmodule Pleroma.Web.WebFinger do "magic_key" => magic_key, "topic" => topic, "subject" => subject, - "salmon" => salmon, "subscribe_address" => subscribe_address, "ap_id" => ap_id } @@ -148,16 +146,6 @@ defmodule Pleroma.Web.WebFinger do {"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} -> Map.put(data, "ap_id", link["href"]) - {_, "magic-public-key"} -> - "data:application/magic-public-key," <> magic_key = link["href"] - Map.put(data, "magic_key", magic_key) - - {"application/atom+xml", "http://schemas.google.com/g/2010#updates-from"} -> - Map.put(data, "topic", link["href"]) - - {_, "salmon"} -> - Map.put(data, "salmon", link["href"]) - {_, "http://ostatus.org/schema/1.0/subscribe"} -> Map.put(data, "subscribe_address", link["template"]) From beb9861f9df080cd071c34f37c95e89d1b170138 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 23:07:54 +0000 Subject: [PATCH 58/84] router: disconnect Salmon --- lib/pleroma/web/router.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 77fe938d5..b3b5ada4e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -508,8 +508,6 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/feed", Feed.FeedController, :feed) get("/users/:nickname", Feed.FeedController, :feed_redirect) - post("/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming) - get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe) end From 835ad5237885cb1c95f678054733b904f20b0bbd Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 23:09:15 +0000 Subject: [PATCH 59/84] remove Salmon module --- lib/pleroma/user/info.ex | 6 - lib/pleroma/web/ostatus/feed_representer.ex | 2 - lib/pleroma/web/ostatus/ostatus.ex | 4 - lib/pleroma/web/ostatus/ostatus_controller.ex | 28 -- lib/pleroma/web/salmon/salmon.ex | 254 ------------------ test/web/salmon/salmon_test.exs | 101 ------- 6 files changed, 395 deletions(-) delete mode 100644 lib/pleroma/web/salmon/salmon.ex delete mode 100644 test/web/salmon/salmon_test.exs diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 4b5b43d7f..2d39abcb3 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -39,9 +39,6 @@ defmodule Pleroma.User.Info do field(:settings, :map, default: nil) field(:magic_key, :string, default: nil) field(:uri, :string, default: nil) - field(:topic, :string, default: nil) - field(:hub, :string, default: nil) - field(:salmon, :string, default: nil) field(:hide_followers_count, :boolean, default: false) field(:hide_follows_count, :boolean, default: false) field(:hide_followers, :boolean, default: false) @@ -262,9 +259,6 @@ defmodule Pleroma.User.Info do :locked, :magic_key, :uri, - :hub, - :topic, - :salmon, :hide_followers, :hide_follows, :hide_followers_count, diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index b7b97e505..fa7f7b564 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -40,8 +40,6 @@ defmodule Pleroma.Web.OStatus.FeedRepresenter do {:title, ['#{user.nickname}\'s timeline']}, {:updated, h.(most_recent_update)}, {:logo, [to_charlist(User.avatar_url(user) |> MediaProxy.url())]}, - {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []}, - {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []}, {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], []}, {:author, UserRepresenter.to_simple_form(user)} diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 3dfc9b580..a858759d3 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -37,10 +37,6 @@ defmodule Pleroma.Web.OStatus do def feed_path(user), do: "#{user.ap_id}/feed.atom" - def pubsub_path(user), do: "#{Web.base_url()}/push/hub/#{user.nickname}" - - def salmon_path(user), do: "#{user.ap_id}/salmon" - def remote_follow_path, do: "#{Web.base_url()}/ostatus_subscribe?acct={uri}" def handle_incoming(xml_string, options \\ []) do diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 20f2d9ddc..7466dd8ea 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -24,8 +24,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do {:ap_routes, params: ["uuid"]} when action in [:object, :activity] ) - plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming]) - plug( Pleroma.Plugs.SetFormatPlug when action in [:object, :activity, :notice] @@ -33,32 +31,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do action_fallback(:errors) - defp decode_or_retry(body) do - with {:ok, magic_key} <- Pleroma.Web.Salmon.fetch_magic_key(body), - {:ok, doc} <- Pleroma.Web.Salmon.decode_and_validate(magic_key, body) do - {:ok, doc} - else - _e -> - with [decoded | _] <- Pleroma.Web.Salmon.decode(body), - doc <- XML.parse_document(decoded), - uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc), - {:ok, _} <- Pleroma.Web.OStatus.make_user(uri, true), - {:ok, magic_key} <- Pleroma.Web.Salmon.fetch_magic_key(body), - {:ok, doc} <- Pleroma.Web.Salmon.decode_and_validate(magic_key, body) do - {:ok, doc} - end - end - end - - def salmon_incoming(conn, _) do - {:ok, body, _conn} = read_body(conn) - {:ok, doc} = decode_or_retry(body) - - Federator.incoming_doc(doc) - - send_resp(conn, 200, "") - end - def object(%{assigns: %{format: format}} = conn, %{"uuid" => _uuid}) when format in ["json", "activity+json"] do ActivityPubController.call(conn, :object) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex deleted file mode 100644 index 0ffe903cd..000000000 --- a/lib/pleroma/web/salmon/salmon.ex +++ /dev/null @@ -1,254 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Salmon do - @behaviour Pleroma.Web.Federator.Publisher - - use Bitwise - - alias Pleroma.Activity - alias Pleroma.HTTP - alias Pleroma.Instances - alias Pleroma.Keys - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.OStatus - alias Pleroma.Web.OStatus.ActivityRepresenter - alias Pleroma.Web.XML - - require Logger - - def decode(salmon) do - doc = XML.parse_document(salmon) - - {:xmlObj, :string, data} = :xmerl_xpath.string('string(//me:data[1])', doc) - {:xmlObj, :string, sig} = :xmerl_xpath.string('string(//me:sig[1])', doc) - {:xmlObj, :string, alg} = :xmerl_xpath.string('string(//me:alg[1])', doc) - {:xmlObj, :string, encoding} = :xmerl_xpath.string('string(//me:encoding[1])', doc) - {:xmlObj, :string, type} = :xmerl_xpath.string('string(//me:data[1]/@type)', doc) - - {:ok, data} = Base.url_decode64(to_string(data), ignore: :whitespace) - {:ok, sig} = Base.url_decode64(to_string(sig), ignore: :whitespace) - alg = to_string(alg) - encoding = to_string(encoding) - type = to_string(type) - - [data, type, encoding, alg, sig] - end - - def fetch_magic_key(salmon) do - with [data, _, _, _, _] <- decode(salmon), - doc <- XML.parse_document(data), - uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc), - {:ok, public_key} <- User.get_public_key_for_ap_id(uri), - magic_key <- encode_key(public_key) do - {:ok, magic_key} - end - end - - def decode_and_validate(magickey, salmon) do - [data, type, encoding, alg, sig] = decode(salmon) - - signed_text = - [data, type, encoding, alg] - |> Enum.map(&Base.url_encode64/1) - |> Enum.join(".") - - key = decode_key(magickey) - - verify = :public_key.verify(signed_text, :sha256, sig, key) - - if verify do - {:ok, data} - else - :error - end - end - - def decode_key("RSA." <> magickey) do - make_integer = fn bin -> - list = :erlang.binary_to_list(bin) - Enum.reduce(list, 0, fn el, acc -> acc <<< 8 ||| el end) - end - - [modulus, exponent] = - magickey - |> String.split(".") - |> Enum.map(fn n -> Base.url_decode64!(n, padding: false) end) - |> Enum.map(make_integer) - - {:RSAPublicKey, modulus, exponent} - end - - def encode_key({:RSAPublicKey, modulus, exponent}) do - modulus_enc = :binary.encode_unsigned(modulus) |> Base.url_encode64() - exponent_enc = :binary.encode_unsigned(exponent) |> Base.url_encode64() - - "RSA.#{modulus_enc}.#{exponent_enc}" - end - - def encode(private_key, doc) do - type = "application/atom+xml" - encoding = "base64url" - alg = "RSA-SHA256" - - signed_text = - [doc, type, encoding, alg] - |> Enum.map(&Base.url_encode64/1) - |> Enum.join(".") - - signature = - signed_text - |> :public_key.sign(:sha256, private_key) - |> to_string - |> Base.url_encode64() - - doc_base64 = - doc - |> Base.url_encode64() - - # Don't need proper xml building, these strings are safe to leave unescaped - salmon = """ - - - #{doc_base64} - #{encoding} - #{alg} - #{signature} - - """ - - {:ok, salmon} - end - - def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do - cc = Map.get(data, "cc", []) - - bcc = - data - |> Map.get("bcc", []) - |> Enum.reduce([], fn ap_id, bcc -> - case Pleroma.List.get_by_ap_id(ap_id) do - %Pleroma.List{user_id: ^user_id} = list -> - {:ok, following} = Pleroma.List.get_following(list) - bcc ++ Enum.map(following, & &1.ap_id) - - _ -> - bcc - end - end) - - [to, cc, bcc] - |> Enum.concat() - |> Enum.map(&User.get_cached_by_ap_id/1) - |> Enum.filter(fn user -> user && !user.local end) - end - - @doc "Pushes an activity to remote account." - def publish_one(%{recipient: %{info: %{salmon: salmon}}} = params), - do: publish_one(Map.put(params, :recipient, salmon)) - - def publish_one(%{recipient: url, feed: feed} = params) when is_binary(url) do - with {:ok, %{status: code}} when code in 200..299 <- - HTTP.post( - url, - feed, - [{"Content-Type", "application/magic-envelope+xml"}] - ) do - if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], - do: Instances.set_reachable(url) - - Logger.debug(fn -> "Pushed to #{url}, code #{code}" end) - {:ok, code} - else - e -> - unless params[:unreachable_since], do: Instances.set_reachable(url) - Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end) - {:error, "Unreachable instance"} - end - end - - def publish_one(%{recipient_id: recipient_id} = params) do - recipient = User.get_cached_by_id(recipient_id) - - params - |> Map.delete(:recipient_id) - |> Map.put(:recipient, recipient) - |> publish_one() - end - - def publish_one(_), do: :noop - - @supported_activities [ - "Create", - "Follow", - "Like", - "Announce", - "Undo", - "Delete" - ] - - def is_representable?(%Activity{data: %{"type" => type}} = activity) - when type in @supported_activities, - do: Visibility.is_public?(activity) - - def is_representable?(_), do: false - - @doc """ - Publishes an activity to remote accounts - """ - @spec publish(User.t(), Pleroma.Activity.t()) :: none - def publish(user, activity) - - def publish(%{keys: keys} = user, %{data: %{"type" => type}} = activity) - when type in @supported_activities do - feed = ActivityRepresenter.to_simple_form(activity, user, true) - - if feed do - feed = - ActivityRepresenter.wrap_with_entry(feed) - |> :xmerl.export_simple(:xmerl_xml) - |> to_string - - {:ok, private, _} = Keys.keys_from_pem(keys) - {:ok, feed} = encode(private, feed) - - remote_users = remote_users(user, activity) - - salmon_urls = Enum.map(remote_users, & &1.info.salmon) - reachable_urls_metadata = Instances.filter_reachable(salmon_urls) - reachable_urls = Map.keys(reachable_urls_metadata) - - remote_users - |> Enum.filter(&(&1.info.salmon in reachable_urls)) - |> Enum.each(fn remote_user -> - Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) - - Publisher.enqueue_one(__MODULE__, %{ - recipient_id: remote_user.id, - feed: feed, - unreachable_since: reachable_urls_metadata[remote_user.info.salmon] - }) - end) - end - end - - def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end) - - def gather_webfinger_links(%User{} = user) do - {:ok, _private, public} = Keys.keys_from_pem(user.keys) - magic_key = encode_key(public) - - [ - %{"rel" => "salmon", "href" => OStatus.salmon_path(user)}, - %{ - "rel" => "magic-public-key", - "href" => "data:application/magic-public-key,#{magic_key}" - } - ] - end - - def gather_nodeinfo_protocol_names, do: [] -end diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs deleted file mode 100644 index 153ec41ac..000000000 --- a/test/web/salmon/salmon_test.exs +++ /dev/null @@ -1,101 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Salmon.SalmonTest do - use Pleroma.DataCase - alias Pleroma.Activity - alias Pleroma.Keys - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.Salmon - import Mock - import Pleroma.Factory - - @magickey "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwQhh-1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" - - @wrong_magickey "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwQhh-1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAA" - - @magickey_friendica "RSA.AMwa8FUs2fWEjX0xN7yRQgegQffhBpuKNC6fa5VNSVorFjGZhRrlPMn7TQOeihlc9lBz2OsHlIedbYn2uJ7yCs0.AQAB" - - setup_all do - Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) - :ok - end - - test "decodes a salmon" do - {:ok, salmon} = File.read("test/fixtures/salmon.xml") - {:ok, doc} = Salmon.decode_and_validate(@magickey, salmon) - assert Regex.match?(~r/xml/, doc) - end - - test "errors on wrong magic key" do - {:ok, salmon} = File.read("test/fixtures/salmon.xml") - assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error - end - - test "it encodes a magic key from a public key" do - key = Salmon.decode_key(@magickey) - magic_key = Salmon.encode_key(key) - - assert @magickey == magic_key - end - - test "it decodes a friendica public key" do - _key = Salmon.decode_key(@magickey_friendica) - end - - test "encodes an xml payload with a private key" do - doc = File.read!("test/fixtures/incoming_note_activity.xml") - pem = File.read!("test/fixtures/private_key.pem") - {:ok, private, public} = Keys.keys_from_pem(pem) - - # Let's try a roundtrip. - {:ok, salmon} = Salmon.encode(private, doc) - {:ok, decoded_doc} = Salmon.decode_and_validate(Salmon.encode_key(public), salmon) - - assert doc == decoded_doc - end - - test "it gets a magic key" do - salmon = File.read!("test/fixtures/salmon2.xml") - {:ok, key} = Salmon.fetch_magic_key(salmon) - - assert key == - "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" - end - - test_with_mock "it pushes an activity to remote accounts it's addressed to", - Publisher, - [:passthrough], - [] do - user_data = %{ - info: %{ - salmon: "http://test-example.org/salmon" - }, - local: false - } - - mentioned_user = insert(:user, user_data) - note = insert(:note) - - activity_data = %{ - "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), - "type" => "Create", - "actor" => note.data["actor"], - "to" => note.data["to"] ++ [mentioned_user.ap_id], - "object" => note.data, - "published_at" => DateTime.utc_now() |> DateTime.to_iso8601(), - "context" => note.data["context"] - } - - {:ok, activity} = Repo.insert(%Activity{data: activity_data, recipients: activity_data["to"]}) - user = User.get_cached_by_ap_id(activity.data["actor"]) - {:ok, user} = User.ensure_keys_present(user) - - Salmon.publish(user, activity) - - assert called(Publisher.enqueue_one(Salmon, %{recipient_id: mentioned_user.id})) - end -end From 6a1f4c5145c05efdfa1b0d56ba25bf87e51c7f82 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 23:15:09 +0000 Subject: [PATCH 60/84] federator: remove OStatus incoming document support --- lib/pleroma/web/federator/federator.ex | 9 --------- lib/pleroma/workers/receiver_worker.ex | 4 ---- 2 files changed, 13 deletions(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 8227d1a3a..8c83f9aea 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -30,10 +30,6 @@ defmodule Pleroma.Web.Federator do # Client API - def incoming_doc(doc) do - ReceiverWorker.enqueue("incoming_doc", %{"body" => doc}) - end - def incoming_ap_doc(params) do ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) end @@ -62,11 +58,6 @@ defmodule Pleroma.Web.Federator do end end - def perform(:incoming_doc, doc) do - Logger.info("Got document, trying to parse") - OStatus.handle_incoming(doc) - end - def perform(:incoming_ap_doc, params) do Logger.info("Handling incoming AP activity") diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 83d528a66..8ad756b62 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -8,10 +8,6 @@ defmodule Pleroma.Workers.ReceiverWorker do use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker - def perform(%{"op" => "incoming_doc", "body" => doc}, _job) do - Federator.perform(:incoming_doc, doc) - end - def perform(%{"op" => "incoming_ap_doc", "params" => params}, _job) do Federator.perform(:incoming_ap_doc, params) end From d379b4876927701f0fa9e4886f9fd552fe71d9c9 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Thu, 17 Oct 2019 23:37:21 +0000 Subject: [PATCH 61/84] kill almost all of the OStatus module --- lib/pleroma/object/fetcher.ex | 12 +- lib/pleroma/user.ex | 26 +- lib/pleroma/web/federator/federator.ex | 1 - .../web/ostatus/activity_representer.ex | 313 ---------- lib/pleroma/web/ostatus/feed_representer.ex | 64 -- .../web/ostatus/handlers/delete_handler.ex | 18 - .../web/ostatus/handlers/follow_handler.ex | 26 - .../web/ostatus/handlers/note_handler.ex | 168 ----- .../web/ostatus/handlers/unfollow_handler.ex | 22 - lib/pleroma/web/ostatus/ostatus.ex | 388 ------------ lib/pleroma/web/ostatus/ostatus_controller.ex | 18 +- lib/pleroma/web/ostatus/user_representer.ex | 41 -- test/web/activity_pub/transmogrifier_test.exs | 27 - .../controllers/timeline_controller_test.exs | 7 +- .../mastodon_api/views/status_view_test.exs | 28 +- .../web/ostatus/activity_representer_test.exs | 300 --------- test/web/ostatus/feed_representer_test.exs | 59 -- .../delete_handling_test.exs | 48 -- test/web/ostatus/ostatus_controller_test.exs | 22 - test/web/ostatus/ostatus_test.exs | 591 ------------------ test/web/ostatus/user_representer_test.exs | 38 -- test/web/web_finger/web_finger_test.exs | 27 - 22 files changed, 20 insertions(+), 2224 deletions(-) delete mode 100644 lib/pleroma/web/ostatus/activity_representer.ex delete mode 100644 lib/pleroma/web/ostatus/feed_representer.ex delete mode 100644 lib/pleroma/web/ostatus/handlers/delete_handler.ex delete mode 100644 lib/pleroma/web/ostatus/handlers/follow_handler.ex delete mode 100644 lib/pleroma/web/ostatus/handlers/note_handler.ex delete mode 100644 lib/pleroma/web/ostatus/handlers/unfollow_handler.ex delete mode 100644 lib/pleroma/web/ostatus/ostatus.ex delete mode 100644 lib/pleroma/web/ostatus/user_representer.ex delete mode 100644 test/web/ostatus/activity_representer_test.exs delete mode 100644 test/web/ostatus/feed_representer_test.exs delete mode 100644 test/web/ostatus/incoming_documents/delete_handling_test.exs delete mode 100644 test/web/ostatus/ostatus_test.exs delete mode 100644 test/web/ostatus/user_representer_test.exs diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 5e064fd87..9436e2730 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.OStatus require Logger require Pleroma.Constants @@ -87,15 +86,8 @@ defmodule Pleroma.Object.Fetcher do {:fetch_object, %Object{} = object} -> {:ok, object} - _e -> - # Only fallback when receiving a fetch/normalization error with ActivityPub - Logger.info("Couldn't get object via AP, trying out OStatus fetching...") - - # FIXME: OStatus Object Containment? - case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity, false)} - e -> e - end + e -> + e end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index cef011c68..ec705b8f6 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -26,7 +26,6 @@ defmodule Pleroma.User do alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils alias Pleroma.Web.OAuth - alias Pleroma.Web.OStatus alias Pleroma.Web.RelMe alias Pleroma.Workers.BackgroundWorker @@ -609,12 +608,7 @@ defmodule Pleroma.User do Cachex.fetch!(:user_cache, key, fn -> user_info(user) end) end - def fetch_by_nickname(nickname) do - case ActivityPub.make_user_from_nickname(nickname) do - {:ok, user} -> {:ok, user} - _ -> OStatus.make_user(nickname) - end - end + def fetch_by_nickname(nickname), do: ActivityPub.make_user_from_nickname(nickname) def get_or_fetch_by_nickname(nickname) do with %User{} = user <- get_by_nickname(nickname) do @@ -1241,18 +1235,7 @@ defmodule Pleroma.User do def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy]) - def fetch_by_ap_id(ap_id) do - case ActivityPub.make_user_from_ap_id(ap_id) do - {:ok, user} -> - {:ok, user} - - _ -> - case OStatus.make_user(ap_id) do - {:ok, user} -> {:ok, user} - _ -> {:error, "Could not fetch by AP id"} - end - end - end + def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) def get_or_fetch_by_ap_id(ap_id) do user = get_cached_by_ap_id(ap_id) @@ -1307,11 +1290,6 @@ defmodule Pleroma.User do {:ok, key} end - # OStatus Magic Key - def public_key_from_info(%{magic_key: magic_key}) when not is_nil(magic_key) do - {:ok, Pleroma.Web.Salmon.decode_key(magic_key)} - end - def public_key_from_info(_), do: {:error, "not found key"} def get_public_key_for_ap_id(ap_id) do diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 8c83f9aea..e8a56ebd7 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.OStatus alias Pleroma.Workers.PublisherWorker alias Pleroma.Workers.ReceiverWorker diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex deleted file mode 100644 index 8e55b9f0b..000000000 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ /dev/null @@ -1,313 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.ActivityRepresenter do - alias Pleroma.Activity - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.OStatus.UserRepresenter - - require Logger - require Pleroma.Constants - - defp get_href(id) do - with %Object{data: %{"external_url" => external_url}} <- Object.get_cached_by_ap_id(id) do - external_url - else - _e -> id - end - end - - defp get_in_reply_to(activity) do - with %Object{data: %{"inReplyTo" => in_reply_to}} <- Object.normalize(activity) do - [ - {:"thr:in-reply-to", - [ref: to_charlist(in_reply_to), href: to_charlist(get_href(in_reply_to))], []} - ] - else - _ -> - [] - end - end - - defp get_mentions(to) do - Enum.map(to, fn id -> - cond do - # Special handling for the AP/Ostatus public collections - Pleroma.Constants.as_public() == id -> - {:link, - [ - rel: "mentioned", - "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", - href: "http://activityschema.org/collection/public" - ], []} - - # Ostatus doesn't handle follower collections, ignore these. - Regex.match?(~r/^#{Pleroma.Web.base_url()}.+followers$/, id) -> - [] - - true -> - {:link, - [ - rel: "mentioned", - "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", - href: id - ], []} - end - end) - end - - defp get_links(%{local: true}, %{"id" => object_id}) do - h = fn str -> [to_charlist(str)] end - - [ - {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []}, - {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []} - ] - end - - defp get_links(%{local: false}, %{"external_url" => external_url}) do - h = fn str -> [to_charlist(str)] end - - [ - {:link, [type: ['text/html'], href: h.(external_url), rel: 'alternate'], []} - ] - end - - defp get_links(_activity, _object_data), do: [] - - defp get_emoji_links(emojis) do - Enum.map(emojis, fn {emoji, file} -> - {:link, [name: to_charlist(emoji), rel: 'emoji', href: to_charlist(file)], []} - end) - end - - def to_simple_form(activity, user, with_author \\ false) - - def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do - h = fn str -> [to_charlist(str)] end - - object = Object.normalize(activity) - - updated_at = object.data["published"] - inserted_at = object.data["published"] - - attachments = - Enum.map(object.data["attachment"] || [], fn attachment -> - url = hd(attachment["url"]) - - {:link, - [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])], - []} - end) - - in_reply_to = get_in_reply_to(activity) - author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] - mentions = activity.recipients |> get_mentions - - categories = - (object.data["tag"] || []) - |> Enum.map(fn tag -> - if is_binary(tag) do - {:category, [term: to_charlist(tag)], []} - else - nil - end - end) - |> Enum.filter(& &1) - - emoji_links = get_emoji_links(object.data["emoji"] || %{}) - - summary = - if object.data["summary"] do - [{:summary, [], h.(object.data["summary"])}] - else - [] - end - - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, - {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, - # For notes, federate the object id. - {:id, h.(object.data["id"])}, - {:title, ['New note by #{user.nickname}']}, - {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))}, - {:published, h.(inserted_at)}, - {:updated, h.(updated_at)}, - {:"ostatus:conversation", [ref: h.(activity.data["context"])], - h.(activity.data["context"])}, - {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} - ] ++ - summary ++ - get_links(activity, object.data) ++ - categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links - end - - def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do - h = fn str -> [to_charlist(str)] end - - updated_at = activity.data["published"] - inserted_at = activity.data["published"] - - author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] - mentions = activity.recipients |> get_mentions - - [ - {:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']}, - {:id, h.(activity.data["id"])}, - {:title, ['New favorite by #{user.nickname}']}, - {:content, [type: 'html'], ['#{user.nickname} favorited something']}, - {:published, h.(inserted_at)}, - {:updated, h.(updated_at)}, - {:"activity:object", - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, - # For notes, federate the object id. - {:id, h.(activity.data["object"])} - ]}, - {:"ostatus:conversation", [ref: h.(activity.data["context"])], - h.(activity.data["context"])}, - {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, - {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, - {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []} - ] ++ author ++ mentions - end - - def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do - h = fn str -> [to_charlist(str)] end - - updated_at = activity.data["published"] - inserted_at = activity.data["published"] - - author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] - - retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) - retweeted_object = Object.normalize(retweeted_activity) - retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) - - retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true) - - mentions = - ([retweeted_user.ap_id] ++ activity.recipients) - |> Enum.uniq() - |> get_mentions() - - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, - {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, - {:id, h.(activity.data["id"])}, - {:title, ['#{user.nickname} repeated a notice']}, - {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']}, - {:published, h.(inserted_at)}, - {:updated, h.(updated_at)}, - {:"ostatus:conversation", [ref: h.(activity.data["context"])], - h.(activity.data["context"])}, - {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, - {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, - {:"activity:object", retweeted_xml} - ] ++ mentions ++ author - end - - def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) do - h = fn str -> [to_charlist(str)] end - - updated_at = activity.data["published"] - inserted_at = activity.data["published"] - - author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] - - mentions = (activity.recipients || []) |> get_mentions - - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, - {:"activity:verb", ['http://activitystrea.ms/schema/1.0/follow']}, - {:id, h.(activity.data["id"])}, - {:title, ['#{user.nickname} started following #{activity.data["object"]}']}, - {:content, [type: 'html'], - ['#{user.nickname} started following #{activity.data["object"]}']}, - {:published, h.(inserted_at)}, - {:updated, h.(updated_at)}, - {:"activity:object", - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/person']}, - {:id, h.(activity.data["object"])}, - {:uri, h.(activity.data["object"])} - ]}, - {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []} - ] ++ mentions ++ author - end - - # Only undos of follow for now. Will need to get redone once there are more - def to_simple_form( - %{data: %{"type" => "Undo", "object" => %{"type" => "Follow"} = follow_activity}} = - activity, - user, - with_author - ) do - h = fn str -> [to_charlist(str)] end - - updated_at = activity.data["published"] - inserted_at = activity.data["published"] - - author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] - - mentions = (activity.recipients || []) |> get_mentions - follow_activity = Activity.normalize(follow_activity) - - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, - {:"activity:verb", ['http://activitystrea.ms/schema/1.0/unfollow']}, - {:id, h.(activity.data["id"])}, - {:title, ['#{user.nickname} stopped following #{follow_activity.data["object"]}']}, - {:content, [type: 'html'], - ['#{user.nickname} stopped following #{follow_activity.data["object"]}']}, - {:published, h.(inserted_at)}, - {:updated, h.(updated_at)}, - {:"activity:object", - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/person']}, - {:id, h.(follow_activity.data["object"])}, - {:uri, h.(follow_activity.data["object"])} - ]}, - {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []} - ] ++ mentions ++ author - end - - def to_simple_form(%{data: %{"type" => "Delete"}} = activity, user, with_author) do - h = fn str -> [to_charlist(str)] end - - updated_at = activity.data["published"] - inserted_at = activity.data["published"] - - author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] - - [ - {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, - {:"activity:verb", ['http://activitystrea.ms/schema/1.0/delete']}, - {:id, h.(activity.data["object"])}, - {:title, ['An object was deleted']}, - {:content, [type: 'html'], ['An object was deleted']}, - {:published, h.(inserted_at)}, - {:updated, h.(updated_at)} - ] ++ author - end - - def to_simple_form(_, _, _), do: nil - - def wrap_with_entry(simple_form) do - [ - { - :entry, - [ - xmlns: 'http://www.w3.org/2005/Atom', - "xmlns:thr": 'http://purl.org/syndication/thread/1.0', - "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', - "xmlns:poco": 'http://portablecontacts.net/spec/1.0', - "xmlns:ostatus": 'http://ostatus.org/schema/1.0' - ], - simple_form - } - ] - end -end diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex deleted file mode 100644 index fa7f7b564..000000000 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ /dev/null @@ -1,64 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.FeedRepresenter do - alias Pleroma.User - alias Pleroma.Web.MediaProxy - alias Pleroma.Web.OStatus - alias Pleroma.Web.OStatus.ActivityRepresenter - alias Pleroma.Web.OStatus.UserRepresenter - - def to_simple_form(user, activities, _users) do - most_recent_update = - (List.first(activities) || user).updated_at - |> NaiveDateTime.to_iso8601() - - h = fn str -> [to_charlist(str)] end - - last_activity = List.last(activities) - - entries = - activities - |> Enum.map(fn activity -> - {:entry, ActivityRepresenter.to_simple_form(activity, user)} - end) - |> Enum.filter(fn {_, form} -> form end) - - [ - { - :feed, - [ - xmlns: 'http://www.w3.org/2005/Atom', - "xmlns:thr": 'http://purl.org/syndication/thread/1.0', - "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', - "xmlns:poco": 'http://portablecontacts.net/spec/1.0', - "xmlns:ostatus": 'http://ostatus.org/schema/1.0' - ], - [ - {:id, h.(OStatus.feed_path(user))}, - {:title, ['#{user.nickname}\'s timeline']}, - {:updated, h.(most_recent_update)}, - {:logo, [to_charlist(User.avatar_url(user) |> MediaProxy.url())]}, - {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], - []}, - {:author, UserRepresenter.to_simple_form(user)} - ] ++ - if last_activity do - [ - {:link, - [ - rel: 'next', - href: - to_charlist(OStatus.feed_path(user)) ++ - '?max_id=' ++ to_charlist(last_activity.id), - type: 'application/atom+xml' - ], []} - ] - else - [] - end ++ entries - } - ] - end -end diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex deleted file mode 100644 index ac2dc115c..000000000 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.DeleteHandler do - require Logger - alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.XML - - def handle_delete(entry, _doc \\ nil) do - with id <- XML.string_from_xpath("//id", entry), - %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, local: false) do - delete - end - end -end diff --git a/lib/pleroma/web/ostatus/handlers/follow_handler.ex b/lib/pleroma/web/ostatus/handlers/follow_handler.ex deleted file mode 100644 index 24513972e..000000000 --- a/lib/pleroma/web/ostatus/handlers/follow_handler.ex +++ /dev/null @@ -1,26 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.FollowHandler do - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML - - def handle(entry, doc) do - with {:ok, actor} <- OStatus.find_make_or_update_actor(doc), - id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry), - followed_uri when not is_nil(followed_uri) <- - XML.string_from_xpath("/entry/activity:object/id", entry), - {:ok, followed} <- OStatus.find_or_make_user(followed_uri), - {:locked, false} <- {:locked, followed.info.locked}, - {:ok, activity} <- ActivityPub.follow(actor, followed, id, false) do - User.follow(actor, followed) - {:ok, activity} - else - {:locked, true} -> - {:error, "It's not possible to follow locked accounts over OStatus"} - end - end -end diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex deleted file mode 100644 index 7fae14f7b..000000000 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ /dev/null @@ -1,168 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.NoteHandler do - require Logger - require Pleroma.Constants - - alias Pleroma.Activity - alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.CommonAPI - alias Pleroma.Web.Federator - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML - - @doc """ - Get the context for this note. Uses this: - 1. The context of the parent activity - 2. The conversation reference in the ostatus xml - 3. A newly generated context id. - """ - def get_context(entry, in_reply_to) do - context = - (XML.string_from_xpath("//ostatus:conversation[1]", entry) || - XML.string_from_xpath("//ostatus:conversation[1]/@ref", entry) || "") - |> String.trim() - - with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(in_reply_to) do - context - else - _e -> - if String.length(context) > 0 do - context - else - Utils.generate_context_id() - end - end - end - - def get_people_mentions(entry) do - :xmerl_xpath.string( - '//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', - entry - ) - |> Enum.map(fn person -> XML.string_from_xpath("@href", person) end) - end - - def get_collection_mentions(entry) do - transmogrify = fn - "http://activityschema.org/collection/public" -> - Pleroma.Constants.as_public() - - group -> - group - end - - :xmerl_xpath.string( - '//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/collection"]', - entry - ) - |> Enum.map(fn collection -> XML.string_from_xpath("@href", collection) |> transmogrify.() end) - end - - def get_mentions(entry) do - (get_people_mentions(entry) ++ get_collection_mentions(entry)) - |> Enum.filter(& &1) - end - - def get_emoji(entry) do - try do - :xmerl_xpath.string('//link[@rel="emoji"]', entry) - |> Enum.reduce(%{}, fn emoji, acc -> - Map.put(acc, XML.string_from_xpath("@name", emoji), XML.string_from_xpath("@href", emoji)) - end) - rescue - _e -> nil - end - end - - def make_to_list(actor, mentions) do - [ - actor.follower_address - ] ++ mentions - end - - def add_external_url(note, entry) do - url = XML.string_from_xpath("//link[@rel='alternate' and @type='text/html']/@href", entry) - Map.put(note, "external_url", url) - end - - def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do - with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do - activity - else - _e -> - with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), - in_reply_to_href when not is_nil(in_reply_to_href) <- - XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), - {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do - activity - else - _e -> nil - end - end - end - - # TODO: Clean this up a bit. - def handle_note(entry, doc \\ nil, options \\ []) do - with id <- XML.string_from_xpath("//id", entry), - activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), - [author] <- :xmerl_xpath.string('//author[1]', doc), - {:ok, actor} <- OStatus.find_make_or_update_actor(author), - content_html <- OStatus.get_content(entry), - cw <- OStatus.get_cw(entry), - in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), - options <- Keyword.put(options, :depth, (options[:depth] || 0) + 1), - in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to, options), - in_reply_to_object <- - (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil, - in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to, - attachments <- OStatus.get_attachments(entry), - context <- get_context(entry, in_reply_to), - tags <- OStatus.get_tags(entry), - mentions <- get_mentions(entry), - to <- make_to_list(actor, mentions), - date <- XML.string_from_xpath("//published", entry), - unlisted <- XML.string_from_xpath("//mastodon:scope", entry) == "unlisted", - cc <- if(unlisted, do: [Pleroma.Constants.as_public()], else: []), - note <- - CommonAPI.Utils.make_note_data( - actor.ap_id, - to, - context, - content_html, - attachments, - in_reply_to_activity, - [], - cw - ), - note <- note |> Map.put("id", id) |> Map.put("tag", tags), - note <- note |> Map.put("published", date), - note <- note |> Map.put("emoji", get_emoji(entry)), - note <- add_external_url(note, entry), - note <- note |> Map.put("cc", cc), - # TODO: Handle this case in make_note_data - note <- - if( - in_reply_to && !in_reply_to_activity, - do: note |> Map.put("inReplyTo", in_reply_to), - else: note - ) do - ActivityPub.create(%{ - to: to, - actor: actor, - context: context, - object: note, - published: date, - local: false, - additional: %{"cc" => cc} - }) - else - %Activity{} = activity -> {:ok, activity} - e -> {:error, e} - end - end -end diff --git a/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex b/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex deleted file mode 100644 index 2062432e3..000000000 --- a/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex +++ /dev/null @@ -1,22 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.UnfollowHandler do - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML - - def handle(entry, doc) do - with {:ok, actor} <- OStatus.find_make_or_update_actor(doc), - id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry), - followed_uri when not is_nil(followed_uri) <- - XML.string_from_xpath("/entry/activity:object/id", entry), - {:ok, followed} <- OStatus.find_or_make_user(followed_uri), - {:ok, activity} <- ActivityPub.unfollow(actor, followed, id, false) do - User.unfollow(actor, followed) - {:ok, activity} - end - end -end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex deleted file mode 100644 index a858759d3..000000000 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ /dev/null @@ -1,388 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus do - import Pleroma.Web.XML - require Logger - - alias Pleroma.Activity - alias Pleroma.HTTP - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.OStatus.DeleteHandler - alias Pleroma.Web.OStatus.FollowHandler - alias Pleroma.Web.OStatus.NoteHandler - alias Pleroma.Web.OStatus.UnfollowHandler - alias Pleroma.Web.WebFinger - - def is_representable?(%Activity{} = activity) do - object = Object.normalize(activity) - - cond do - is_nil(object) -> - false - - Visibility.is_public?(activity) && object.data["type"] == "Note" -> - true - - true -> - false - end - end - - def feed_path(user), do: "#{user.ap_id}/feed.atom" - - def remote_follow_path, do: "#{Web.base_url()}/ostatus_subscribe?acct={uri}" - - def handle_incoming(xml_string, options \\ []) do - with doc when doc != :error <- parse_document(xml_string) do - with {:ok, actor_user} <- find_make_or_update_actor(doc), - do: Pleroma.Instances.set_reachable(actor_user.ap_id) - - entries = :xmerl_xpath.string('//entry', doc) - - activities = - Enum.map(entries, fn entry -> - {:xmlObj, :string, object_type} = - :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) - - {:xmlObj, :string, verb} = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry) - Logger.debug("Handling #{verb}") - - try do - case verb do - 'http://activitystrea.ms/schema/1.0/delete' -> - with {:ok, activity} <- DeleteHandler.handle_delete(entry, doc), do: activity - - 'http://activitystrea.ms/schema/1.0/follow' -> - with {:ok, activity} <- FollowHandler.handle(entry, doc), do: activity - - 'http://activitystrea.ms/schema/1.0/unfollow' -> - with {:ok, activity} <- UnfollowHandler.handle(entry, doc), do: activity - - 'http://activitystrea.ms/schema/1.0/share' -> - with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), - do: [activity, retweeted_activity] - - 'http://activitystrea.ms/schema/1.0/favorite' -> - with {:ok, activity, favorited_activity} <- handle_favorite(entry, doc), - do: [activity, favorited_activity] - - _ -> - case object_type do - 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), - do: activity - - 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), - do: activity - - _ -> - Logger.error("Couldn't parse incoming document") - nil - end - end - rescue - e -> - Logger.error("Error occured while handling activity") - Logger.error(xml_string) - Logger.error(inspect(e)) - nil - end - end) - |> Enum.filter(& &1) - - {:ok, activities} - else - _e -> {:error, []} - end - end - - def make_share(entry, doc, retweeted_activity) do - with {:ok, actor} <- find_make_or_update_actor(doc), - %Object{} = object <- Object.normalize(retweeted_activity), - id when not is_nil(id) <- string_from_xpath("/entry/id", entry), - {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do - {:ok, activity} - end - end - - def handle_share(entry, doc) do - with {:ok, retweeted_activity} <- get_or_build_object(entry), - {:ok, activity} <- make_share(entry, doc, retweeted_activity) do - {:ok, activity, retweeted_activity} - else - e -> {:error, e} - end - end - - def make_favorite(entry, doc, favorited_activity) do - with {:ok, actor} <- find_make_or_update_actor(doc), - %Object{} = object <- Object.normalize(favorited_activity), - id when not is_nil(id) <- string_from_xpath("/entry/id", entry), - {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do - {:ok, activity} - end - end - - def get_or_build_object(entry) do - with {:ok, activity} <- get_or_try_fetching(entry) do - {:ok, activity} - else - _e -> - with [object] <- :xmerl_xpath.string('/entry/activity:object', entry) do - NoteHandler.handle_note(object, object) - end - end - end - - def get_or_try_fetching(entry) do - Logger.debug("Trying to get entry from db") - - with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry), - %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do - {:ok, activity} - else - _ -> - Logger.debug("Couldn't get, will try to fetch") - - with href when not is_nil(href) <- - string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry), - {:ok, [favorited_activity]} <- fetch_activity_from_url(href) do - {:ok, favorited_activity} - else - e -> Logger.debug("Couldn't find href: #{inspect(e)}") - end - end - end - - def handle_favorite(entry, doc) do - with {:ok, favorited_activity} <- get_or_try_fetching(entry), - {:ok, activity} <- make_favorite(entry, doc, favorited_activity) do - {:ok, activity, favorited_activity} - else - e -> {:error, e} - end - end - - def get_attachments(entry) do - :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry) - |> Enum.map(fn enclosure -> - with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure), - type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do - %{ - "type" => "Attachment", - "url" => [ - %{ - "type" => "Link", - "mediaType" => type, - "href" => href - } - ] - } - end - end) - |> Enum.filter(& &1) - end - - @doc """ - Gets the content from a an entry. - """ - def get_content(entry) do - string_from_xpath("//content", entry) - end - - @doc """ - Get the cw that mastodon uses. - """ - def get_cw(entry) do - case string_from_xpath("/*/summary", entry) do - cw when not is_nil(cw) -> cw - _ -> nil - end - end - - def get_tags(entry) do - :xmerl_xpath.string('//category', entry) - |> Enum.map(fn category -> string_from_xpath("/category/@term", category) end) - |> Enum.filter(& &1) - |> Enum.map(&String.downcase/1) - end - - def maybe_update(doc, user) do - case string_from_xpath("//author[1]/ap_enabled", doc) do - "true" -> - Transmogrifier.upgrade_user_from_ap_id(user.ap_id) - - _ -> - maybe_update_ostatus(doc, user) - end - end - - def maybe_update_ostatus(doc, user) do - old_data = Map.take(user, [:bio, :avatar, :name]) - - with false <- user.local, - avatar <- make_avatar_object(doc), - bio <- string_from_xpath("//author[1]/summary", doc), - name <- string_from_xpath("//author[1]/poco:displayName", doc), - new_data <- %{ - avatar: avatar || old_data.avatar, - name: name || old_data.name, - bio: bio || old_data.bio - }, - false <- new_data == old_data do - change = Ecto.Changeset.change(user, new_data) - User.update_and_set_cache(change) - else - _ -> - {:ok, user} - end - end - - def find_make_or_update_actor(doc) do - uri = string_from_xpath("//author/uri[1]", doc) - - with {:ok, %User{} = user} <- find_or_make_user(uri), - {:ap_enabled, false} <- {:ap_enabled, User.ap_enabled?(user)} do - maybe_update(doc, user) - else - {:ap_enabled, true} -> - {:error, :invalid_protocol} - - _ -> - {:error, :unknown_user} - end - end - - @spec find_or_make_user(String.t()) :: {:ok, User.t()} - def find_or_make_user(uri) do - case User.get_by_ap_id(uri) do - %User{} = user -> {:ok, user} - _ -> make_user(uri) - end - end - - @spec make_user(String.t(), boolean()) :: {:ok, User.t()} | {:error, any()} - def make_user(uri, update \\ false) do - with {:ok, info} <- gather_user_info(uri) do - with false <- update, - %User{} = user <- User.get_cached_by_ap_id(info["uri"]) do - {:ok, user} - else - _e -> User.insert_or_update_user(build_user_data(info)) - end - end - end - - defp build_user_data(info) do - %{ - name: info["name"], - nickname: info["nickname"] <> "@" <> info["host"], - ap_id: info["uri"], - info: info, - avatar: info["avatar"], - bio: info["bio"] - } - end - - # TODO: Just takes the first one for now. - def make_avatar_object(author_doc, rel \\ "avatar") do - href = string_from_xpath("//author[1]/link[@rel=\"#{rel}\"]/@href", author_doc) - type = string_from_xpath("//author[1]/link[@rel=\"#{rel}\"]/@type", author_doc) - - if href do - %{ - "type" => "Image", - "url" => [%{"type" => "Link", "mediaType" => type, "href" => href}] - } - else - nil - end - end - - @spec gather_user_info(String.t()) :: {:ok, map()} | {:error, any()} - def gather_user_info(username) do - with {:ok, webfinger_data} <- WebFinger.finger(username) do - data = - webfinger_data - |> Map.put("fqn", username) - - {:ok, data} - else - e -> - Logger.debug(fn -> "Couldn't gather info for #{username}" end) - {:error, e} - end - end - - # Regex-based 'parsing' so we don't have to pull in a full html parser - # It's a hack anyway. Maybe revisit this in the future - @mastodon_regex ~r// - @gs_regex ~r// - @gs_classic_regex ~r// - def get_atom_url(body) do - cond do - Regex.match?(@mastodon_regex, body) -> - [[_, match]] = Regex.scan(@mastodon_regex, body) - {:ok, match} - - Regex.match?(@gs_regex, body) -> - [[_, match]] = Regex.scan(@gs_regex, body) - {:ok, match} - - Regex.match?(@gs_classic_regex, body) -> - [[_, match]] = Regex.scan(@gs_classic_regex, body) - {:ok, match} - - true -> - Logger.debug(fn -> "Couldn't find Atom link in #{inspect(body)}" end) - {:error, "Couldn't find the Atom link"} - end - end - - def fetch_activity_from_atom_url(url, options \\ []) do - with true <- String.starts_with?(url, "http"), - {:ok, %{body: body, status: code}} when code in 200..299 <- - HTTP.get(url, [{:Accept, "application/atom+xml"}]) do - Logger.debug("Got document from #{url}, handling...") - handle_incoming(body, options) - else - e -> - Logger.debug("Couldn't get #{url}: #{inspect(e)}") - e - end - end - - def fetch_activity_from_html_url(url, options \\ []) do - Logger.debug("Trying to fetch #{url}") - - with true <- String.starts_with?(url, "http"), - {:ok, %{body: body}} <- HTTP.get(url, []), - {:ok, atom_url} <- get_atom_url(body) do - fetch_activity_from_atom_url(atom_url, options) - else - e -> - Logger.debug("Couldn't get #{url}: #{inspect(e)}") - e - end - end - - def fetch_activity_from_url(url, options \\ []) do - with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url, options) do - {:ok, activities} - else - _e -> fetch_activity_from_html_url(url, options) - end - rescue - e -> - Logger.debug("Couldn't get #{url}: #{inspect(e)}") - {:error, "Couldn't get #{url}: #{inspect(e)}"} - end -end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 7466dd8ea..6958519de 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -13,11 +13,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Endpoint - alias Pleroma.Web.Federator alias Pleroma.Web.Metadata.PlayerView - alias Pleroma.Web.OStatus.ActivityRepresenter alias Pleroma.Web.Router - alias Pleroma.Web.XML plug( Pleroma.Plugs.RateLimiter, @@ -151,23 +148,10 @@ defmodule Pleroma.Web.OStatus.OStatusController do |> render("object.json", %{object: object}) end - defp represent_activity(_conn, "activity+json", _, _) do + defp represent_activity(_conn, _, _, _) do {:error, :not_found} end - defp represent_activity(conn, _, activity, user) do - response = - activity - |> ActivityRepresenter.to_simple_form(user, true) - |> ActivityRepresenter.wrap_with_entry() - |> :xmerl.export_simple(:xmerl_xml) - |> to_string - - conn - |> put_resp_content_type("application/atom+xml") - |> send_resp(200, response) - end - def errors(conn, {:error, :not_found}) do render_error(conn, :not_found, "Not found") end diff --git a/lib/pleroma/web/ostatus/user_representer.ex b/lib/pleroma/web/ostatus/user_representer.ex deleted file mode 100644 index 852be6eb4..000000000 --- a/lib/pleroma/web/ostatus/user_representer.ex +++ /dev/null @@ -1,41 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.UserRepresenter do - alias Pleroma.User - - def to_simple_form(user) do - ap_id = to_charlist(user.ap_id) - nickname = to_charlist(user.nickname) - name = to_charlist(user.name) - bio = to_charlist(user.bio) - avatar_url = to_charlist(User.avatar_url(user)) - - banner = - if banner_url = User.banner_url(user) do - [{:link, [rel: 'header', href: banner_url], []}] - else - [] - end - - ap_enabled = - if user.local do - [{:ap_enabled, ['true']}] - else - [] - end - - [ - {:id, [ap_id]}, - {:"activity:object", ['http://activitystrea.ms/schema/1.0/person']}, - {:uri, [ap_id]}, - {:"poco:preferredUsername", [nickname]}, - {:"poco:displayName", [name]}, - {:"poco:note", [bio]}, - {:summary, [bio]}, - {:name, [nickname]}, - {:link, [rel: 'avatar', href: avatar_url], []} - ] ++ banner ++ ap_enabled - end -end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 7823ff041..252d98a7a 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -13,7 +13,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI - alias Pleroma.Web.OStatus import Mock import Pleroma.Factory @@ -1180,32 +1179,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert modified["object"]["actor"] == modified["object"]["attributedTo"] end - test "it translates ostatus IDs to external URLs" do - incoming = File.read!("test/fixtures/incoming_note_activity.xml") - {:ok, [referent_activity]} = OStatus.handle_incoming(incoming) - - user = insert(:user) - - {:ok, activity, _} = CommonAPI.favorite(referent_activity.id, user) - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - - assert modified["object"] == "http://gs.example.org:4040/index.php/notice/29" - end - - test "it translates ostatus reply_to IDs to external URLs" do - incoming = File.read!("test/fixtures/incoming_note_activity.xml") - {:ok, [referred_activity]} = OStatus.handle_incoming(incoming) - - user = insert(:user) - - {:ok, activity} = - CommonAPI.post(user, %{"status" => "HI!", "in_reply_to_status_id" => referred_activity.id}) - - {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - - assert modified["object"]["inReplyTo"] == "http://gs.example.org:4040/index.php/notice/29" - end - test "it strips internal hashtag data" do user = insert(:user) diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs index fc45c25de..61b6cea75 100644 --- a/test/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI - alias Pleroma.Web.OStatus clear_config([:instance, :public]) @@ -75,8 +74,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do {:ok, _activity} = CommonAPI.post(following, %{"status" => "test"}) - {:ok, [_activity]} = - OStatus.fetch_activity_from_url("https://shitposter.club/notice/2827873") + _activity = insert(:note_activity, local: false) conn = get(conn, "/api/v1/timelines/public", %{"local" => "False"}) @@ -271,9 +269,6 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do {:ok, activity} = CommonAPI.post(following, %{"status" => "test #2hu"}) - {:ok, [_activity]} = - OStatus.fetch_activity_from_url("https://shitposter.club/notice/2827873") - nconn = get(conn, "/api/v1/timelines/tag/2hu") assert [%{"id" => id}] = json_response(nconn, :ok) diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index 1d5a6e956..9375c5030 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -14,7 +14,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.OStatus import Pleroma.Factory import Tesla.Mock @@ -229,19 +228,20 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do assert status.in_reply_to_id == to_string(note.id) end - test "contains mentions" do - incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") - # a user with this ap id might be in the cache. - recipient = "https://pleroma.soykaf.com/users/lain" - user = insert(:user, %{ap_id: recipient}) - - {:ok, [activity]} = OStatus.handle_incoming(incoming) - - status = StatusView.render("show.json", %{activity: activity}) - - assert status.mentions == - Enum.map([user], fn u -> AccountView.render("mention.json", %{user: u}) end) - end + # XXX: fix this test + # test "contains mentions" do + # incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") + # # a user with this ap id might be in the cache. + # recipient = "https://pleroma.soykaf.com/users/lain" + # user = insert(:user, %{ap_id: recipient}) + # + # {:ok, [activity]} = OStatus.handle_incoming(incoming) + # + # status = StatusView.render("show.json", %{activity: activity}) + # + # assert status.mentions == + # Enum.map([user], fn u -> AccountView.render("mention.json", %{user: u}) end) + # end test "create mentions from the 'to' field" do %User{ap_id: recipient_ap_id} = insert(:user) diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs deleted file mode 100644 index a8d500890..000000000 --- a/test/web/ostatus/activity_representer_test.exs +++ /dev/null @@ -1,300 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do - use Pleroma.DataCase - - alias Pleroma.Activity - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.OStatus - alias Pleroma.Web.OStatus.ActivityRepresenter - - import Pleroma.Factory - import Tesla.Mock - - setup do - mock(fn env -> apply(HttpRequestMock, :request, [env]) end) - :ok - end - - test "an external note activity" do - incoming = File.read!("test/fixtures/mastodon-note-cw.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - - user = User.get_cached_by_ap_id(activity.data["actor"]) - - tuple = ActivityRepresenter.to_simple_form(activity, user) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary() - - assert String.contains?( - res, - ~s{} - ) - end - - test "a note activity" do - note_activity = insert(:note_activity) - object_data = Object.normalize(note_activity).data - - user = User.get_cached_by_ap_id(note_activity.data["actor"]) - - expected = """ - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - #{object_data["id"]} - New note by #{user.nickname} - #{object_data["content"]} - #{object_data["published"]} - #{object_data["published"]} - #{note_activity.data["context"]} - - #{object_data["summary"]} - - - - - - """ - - tuple = ActivityRepresenter.to_simple_form(note_activity, user) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary() - - assert clean(res) == clean(expected) - end - - test "a reply note" do - user = insert(:user) - note_object = insert(:note) - _note = insert(:note_activity, %{note: note_object}) - object = insert(:note, %{data: %{"inReplyTo" => note_object.data["id"]}}) - answer = insert(:note_activity, %{note: object}) - - Repo.update!( - Object.change(note_object, %{data: Map.put(note_object.data, "external_url", "someurl")}) - ) - - expected = """ - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - #{object.data["id"]} - New note by #{user.nickname} - #{object.data["content"]} - #{object.data["published"]} - #{object.data["published"]} - #{answer.data["context"]} - - 2hu - - - - - - - """ - - tuple = ActivityRepresenter.to_simple_form(answer, user) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary() - - assert clean(res) == clean(expected) - end - - test "an announce activity" do - note = insert(:note_activity) - user = insert(:user) - object = Object.normalize(note) - - {:ok, announce, _object} = ActivityPub.announce(user, object) - - announce = Activity.get_by_id(announce.id) - - note_user = User.get_cached_by_ap_id(note.data["actor"]) - note = Activity.get_by_id(note.id) - - note_xml = - ActivityRepresenter.to_simple_form(note, note_user, true) - |> :xmerl.export_simple_content(:xmerl_xml) - |> to_string - - expected = """ - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - #{announce.data["id"]} - #{user.nickname} repeated a notice - RT #{object.data["content"]} - #{announce.data["published"]} - #{announce.data["published"]} - #{announce.data["context"]} - - - - #{note_xml} - - - - """ - - announce_xml = - ActivityRepresenter.to_simple_form(announce, user) - |> :xmerl.export_simple_content(:xmerl_xml) - |> to_string - - assert clean(expected) == clean(announce_xml) - end - - test "a like activity" do - note = insert(:note) - user = insert(:user) - {:ok, like, _note} = ActivityPub.like(user, note) - - tuple = ActivityRepresenter.to_simple_form(like, user) - refute is_nil(tuple) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary() - - expected = """ - http://activitystrea.ms/schema/1.0/favorite - #{like.data["id"]} - New favorite by #{user.nickname} - #{user.nickname} favorited something - #{like.data["published"]} - #{like.data["published"]} - - http://activitystrea.ms/schema/1.0/note - #{note.data["id"]} - - #{like.data["context"]} - - - - - - """ - - assert clean(res) == clean(expected) - end - - test "a follow activity" do - follower = insert(:user) - followed = insert(:user) - - {:ok, activity} = - ActivityPub.insert(%{ - "type" => "Follow", - "actor" => follower.ap_id, - "object" => followed.ap_id, - "to" => [followed.ap_id] - }) - - tuple = ActivityRepresenter.to_simple_form(activity, follower) - - refute is_nil(tuple) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary() - - expected = """ - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/follow - #{activity.data["id"]} - #{follower.nickname} started following #{activity.data["object"]} - #{follower.nickname} started following #{activity.data["object"]} - #{activity.data["published"]} - #{activity.data["published"]} - - http://activitystrea.ms/schema/1.0/person - #{activity.data["object"]} - #{activity.data["object"]} - - - - """ - - assert clean(res) == clean(expected) - end - - test "an unfollow activity" do - follower = insert(:user) - followed = insert(:user) - {:ok, _activity} = ActivityPub.follow(follower, followed) - {:ok, activity} = ActivityPub.unfollow(follower, followed) - - tuple = ActivityRepresenter.to_simple_form(activity, follower) - - refute is_nil(tuple) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary() - - expected = """ - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/unfollow - #{activity.data["id"]} - #{follower.nickname} stopped following #{followed.ap_id} - #{follower.nickname} stopped following #{followed.ap_id} - #{activity.data["published"]} - #{activity.data["published"]} - - http://activitystrea.ms/schema/1.0/person - #{followed.ap_id} - #{followed.ap_id} - - - - """ - - assert clean(res) == clean(expected) - end - - test "a delete" do - user = insert(:user) - - activity = %Activity{ - data: %{ - "id" => "ap_id", - "type" => "Delete", - "actor" => user.ap_id, - "object" => "some_id", - "published" => "2017-06-18T12:00:18+00:00" - } - } - - tuple = ActivityRepresenter.to_simple_form(activity, nil) - - refute is_nil(tuple) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary() - - expected = """ - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/delete - #{activity.data["object"]} - An object was deleted - An object was deleted - #{activity.data["published"]} - #{activity.data["published"]} - """ - - assert clean(res) == clean(expected) - end - - test "an unknown activity" do - tuple = ActivityRepresenter.to_simple_form(%Activity{}, nil) - assert is_nil(tuple) - end - - defp clean(string) do - String.replace(string, ~r/\s/, "") - end -end diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs deleted file mode 100644 index d1cadf1e4..000000000 --- a/test/web/ostatus/feed_representer_test.exs +++ /dev/null @@ -1,59 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.FeedRepresenterTest do - use Pleroma.DataCase - import Pleroma.Factory - alias Pleroma.User - alias Pleroma.Web.OStatus - alias Pleroma.Web.OStatus.ActivityRepresenter - alias Pleroma.Web.OStatus.FeedRepresenter - alias Pleroma.Web.OStatus.UserRepresenter - - test "returns a feed of the last 20 items of the user" do - note_activity = insert(:note_activity) - user = User.get_cached_by_ap_id(note_activity.data["actor"]) - - tuple = FeedRepresenter.to_simple_form(user, [note_activity], [user]) - - most_recent_update = - note_activity.updated_at - |> NaiveDateTime.to_iso8601() - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> to_string - - user_xml = - UserRepresenter.to_simple_form(user) - |> :xmerl.export_simple_content(:xmerl_xml) - - entry_xml = - ActivityRepresenter.to_simple_form(note_activity, user) - |> :xmerl.export_simple_content(:xmerl_xml) - - expected = """ - - #{OStatus.feed_path(user)} - #{user.nickname}'s timeline - #{most_recent_update} - #{User.avatar_url(user)} - - - - - #{user_xml} - - - - #{entry_xml} - - - """ - - assert clean(res) == clean(expected) - end - - defp clean(string) do - String.replace(string, ~r/\s/, "") - end -end diff --git a/test/web/ostatus/incoming_documents/delete_handling_test.exs b/test/web/ostatus/incoming_documents/delete_handling_test.exs deleted file mode 100644 index cd0447af7..000000000 --- a/test/web/ostatus/incoming_documents/delete_handling_test.exs +++ /dev/null @@ -1,48 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.DeleteHandlingTest do - use Pleroma.DataCase - - import Pleroma.Factory - import Tesla.Mock - - alias Pleroma.Activity - alias Pleroma.Object - alias Pleroma.Web.OStatus - - setup do - mock(fn env -> apply(HttpRequestMock, :request, [env]) end) - :ok - end - - describe "deletions" do - test "it removes the mentioned activity" do - note = insert(:note_activity) - second_note = insert(:note_activity) - object = Object.normalize(note) - second_object = Object.normalize(second_note) - user = insert(:user) - - {:ok, like, _object} = Pleroma.Web.ActivityPub.ActivityPub.like(user, object) - - incoming = - File.read!("test/fixtures/delete.xml") - |> String.replace( - "tag:mastodon.sdf.org,2017-06-10:objectId=310513:objectType=Status", - object.data["id"] - ) - - {:ok, [delete]} = OStatus.handle_incoming(incoming) - - refute Activity.get_by_id(note.id) - refute Activity.get_by_id(like.id) - assert Object.get_by_ap_id(object.data["id"]).data["type"] == "Tombstone" - assert Activity.get_by_id(second_note.id) - assert Object.get_by_ap_id(second_object.data["id"]) - - assert delete.data["type"] == "Delete" - end - end -end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index b1af918d8..29804cfe1 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.CommonAPI - alias Pleroma.Web.OStatus.ActivityRepresenter setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -73,27 +72,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do end describe "GET object/2" do - test "gets an object", %{conn: conn} do - note_activity = insert(:note_activity) - object = Object.normalize(note_activity) - user = User.get_cached_by_ap_id(note_activity.data["actor"]) - [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, object.data["id"])) - url = "/objects/#{uuid}" - - conn = - conn - |> put_req_header("accept", "application/xml") - |> get(url) - - expected = - ActivityRepresenter.to_simple_form(note_activity, user, true) - |> ActivityRepresenter.wrap_with_entry() - |> :xmerl.export_simple(:xmerl_xml) - |> to_string - - assert response(conn, 200) == expected - end - test "redirects to /notice/id for html format", %{conn: conn} do note_activity = insert(:note_activity) object = Object.normalize(note_activity) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs deleted file mode 100644 index bbecba189..000000000 --- a/test/web/ostatus/ostatus_test.exs +++ /dev/null @@ -1,591 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatusTest do - use Pleroma.DataCase - alias Pleroma.Activity - alias Pleroma.Instances - alias Pleroma.Object - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML - - import ExUnit.CaptureLog - import Mock - import Pleroma.Factory - - setup_all do - Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) - :ok - end - - test "don't insert create notes twice" do - incoming = File.read!("test/fixtures/incoming_note_activity.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - assert {:ok, [activity]} == OStatus.handle_incoming(incoming) - end - - test "handle incoming note - GS, Salmon" do - incoming = File.read!("test/fixtures/incoming_note_activity.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - user = User.get_cached_by_ap_id(activity.data["actor"]) - assert user.info.note_count == 1 - assert activity.data["type"] == "Create" - assert object.data["type"] == "Note" - - assert object.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" - - assert activity.data["published"] == "2017-04-23T14:51:03+00:00" - assert object.data["published"] == "2017-04-23T14:51:03+00:00" - - assert activity.data["context"] == - "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" - - assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] - assert object.data["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"} - assert activity.local == false - end - - test "handle incoming notes - GS, subscription" do - incoming = File.read!("test/fixtures/ostatus_incoming_post.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - assert activity.data["type"] == "Create" - assert object.data["type"] == "Note" - assert object.data["actor"] == "https://social.heldscal.la/user/23211" - assert object.data["content"] == "Will it blend?" - user = User.get_cached_by_ap_id(activity.data["actor"]) - assert User.ap_followers(user) in activity.data["to"] - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - end - - test "handle incoming notes with tags" do - incoming = File.read!("test/fixtures/ostatus_incoming_post_tag.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - assert object.data["tag"] == ["nsfw"] - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - end - - test "handle incoming notes - Mastodon, salmon, reply" do - # It uses the context of the replied to object - Repo.insert!(%Object{ - data: %{ - "id" => "https://pleroma.soykaf.com/objects/c237d966-ac75-4fe3-a87a-d89d71a3a7a4", - "context" => "2hu" - } - }) - - incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - assert activity.data["type"] == "Create" - assert object.data["type"] == "Note" - assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["context"] == "2hu" - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - end - - test "handle incoming notes - Mastodon, with CW" do - incoming = File.read!("test/fixtures/mastodon-note-cw.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - assert activity.data["type"] == "Create" - assert object.data["type"] == "Note" - assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" - assert object.data["summary"] == "technologic" - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - end - - test "handle incoming unlisted messages, put public into cc" do - incoming = File.read!("test/fixtures/mastodon-note-unlisted.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["cc"] - refute "https://www.w3.org/ns/activitystreams#Public" in object.data["to"] - assert "https://www.w3.org/ns/activitystreams#Public" in object.data["cc"] - end - - test "handle incoming retweets - Mastodon, with CW" do - incoming = File.read!("test/fixtures/cw_retweet.xml") - {:ok, [[_activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) - retweeted_object = Object.normalize(retweeted_activity) - - assert retweeted_object.data["summary"] == "Hey." - end - - test "handle incoming notes - GS, subscription, reply" do - incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity) - - assert activity.data["type"] == "Create" - assert object.data["type"] == "Note" - assert object.data["actor"] == "https://social.heldscal.la/user/23211" - - assert object.data["content"] == - "@shpbot why not indeed." - - assert object.data["inReplyTo"] == - "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" - - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - end - - test "handle incoming retweets - GS, subscription" do - incoming = File.read!("test/fixtures/share-gs.xml") - {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) - - assert activity.data["type"] == "Announce" - assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == retweeted_activity.data["object"] - assert "https://pleroma.soykaf.com/users/lain" in activity.data["to"] - refute activity.local - - retweeted_activity = Activity.get_by_id(retweeted_activity.id) - retweeted_object = Object.normalize(retweeted_activity) - assert retweeted_activity.data["type"] == "Create" - assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" - refute retweeted_activity.local - assert retweeted_object.data["announcement_count"] == 1 - assert String.contains?(retweeted_object.data["content"], "mastodon") - refute String.contains?(retweeted_object.data["content"], "Test account") - end - - test "handle incoming retweets - GS, subscription - local message" do - incoming = File.read!("test/fixtures/share-gs-local.xml") - note_activity = insert(:note_activity) - object = Object.normalize(note_activity) - user = User.get_cached_by_ap_id(note_activity.data["actor"]) - - incoming = - incoming - |> String.replace("LOCAL_ID", object.data["id"]) - |> String.replace("LOCAL_USER", user.ap_id) - - {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) - - assert activity.data["type"] == "Announce" - assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == object.data["id"] - assert user.ap_id in activity.data["to"] - refute activity.local - - retweeted_activity = Activity.get_by_id(retweeted_activity.id) - assert note_activity.id == retweeted_activity.id - assert retweeted_activity.data["type"] == "Create" - assert retweeted_activity.data["actor"] == user.ap_id - assert retweeted_activity.local - assert Object.normalize(retweeted_activity).data["announcement_count"] == 1 - end - - test "handle incoming retweets - Mastodon, salmon" do - incoming = File.read!("test/fixtures/share.xml") - {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) - retweeted_object = Object.normalize(retweeted_activity) - - assert activity.data["type"] == "Announce" - assert activity.data["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["object"] == retweeted_activity.data["object"] - - assert activity.data["id"] == - "tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status" - - refute activity.local - assert retweeted_activity.data["type"] == "Create" - assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" - refute retweeted_activity.local - refute String.contains?(retweeted_object.data["content"], "Test account") - end - - test "handle conversation references" do - incoming = File.read!("test/fixtures/mastodon_conversation.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - - assert activity.data["context"] == - "tag:mastodon.social,2017-08-28:objectId=7876885:objectType=Conversation" - end - - test_with_mock "handle incoming replies, fetching replied-to activities if we don't have them", - OStatus, - [:passthrough], - [] do - incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity, false) - - assert activity.data["type"] == "Create" - assert object.data["type"] == "Note" - - assert object.data["inReplyTo"] == - "http://pleroma.example.org:4000/objects/55bce8fc-b423-46b1-af71-3759ab4670bc" - - assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] - - assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" - - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] - - assert called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) - end - - test_with_mock "handle incoming replies, not fetching replied-to activities beyond max_replies_depth", - OStatus, - [:passthrough], - [] do - incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") - - with_mock Pleroma.Web.Federator, - allowed_incoming_reply_depth?: fn _ -> false end do - {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity, false) - - refute called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) - end - end - - test "handle incoming follows" do - incoming = File.read!("test/fixtures/follow.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - assert activity.data["type"] == "Follow" - - assert activity.data["id"] == - "tag:social.heldscal.la,2017-05-07:subscription:23211:person:44803:2017-05-07T09:54:48+00:00" - - assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == "https://pawoo.net/users/pekorino" - refute activity.local - - follower = User.get_cached_by_ap_id(activity.data["actor"]) - followed = User.get_cached_by_ap_id(activity.data["object"]) - - assert User.following?(follower, followed) - end - - test "refuse following over OStatus if the followed's account is locked" do - incoming = File.read!("test/fixtures/follow.xml") - _user = insert(:user, info: %{locked: true}, ap_id: "https://pawoo.net/users/pekorino") - - {:ok, [{:error, "It's not possible to follow locked accounts over OStatus"}]} = - OStatus.handle_incoming(incoming) - end - - test "handle incoming unfollows with existing follow" do - incoming_follow = File.read!("test/fixtures/follow.xml") - {:ok, [_activity]} = OStatus.handle_incoming(incoming_follow) - - incoming = File.read!("test/fixtures/unfollow.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - - assert activity.data["type"] == "Undo" - - assert activity.data["id"] == - "undo:tag:social.heldscal.la,2017-05-07:subscription:23211:person:44803:2017-05-07T09:54:48+00:00" - - assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - embedded_object = activity.data["object"] - assert is_map(embedded_object) - assert embedded_object["type"] == "Follow" - assert embedded_object["object"] == "https://pawoo.net/users/pekorino" - refute activity.local - - follower = User.get_cached_by_ap_id(activity.data["actor"]) - followed = User.get_cached_by_ap_id(embedded_object["object"]) - - refute User.following?(follower, followed) - end - - test "it clears `unreachable` federation status of the sender" do - incoming_reaction_xml = File.read!("test/fixtures/share-gs.xml") - doc = XML.parse_document(incoming_reaction_xml) - actor_uri = XML.string_from_xpath("//author/uri[1]", doc) - reacted_to_author_uri = XML.string_from_xpath("//author/uri[2]", doc) - - Instances.set_consistently_unreachable(actor_uri) - Instances.set_consistently_unreachable(reacted_to_author_uri) - refute Instances.reachable?(actor_uri) - refute Instances.reachable?(reacted_to_author_uri) - - {:ok, _} = OStatus.handle_incoming(incoming_reaction_xml) - assert Instances.reachable?(actor_uri) - refute Instances.reachable?(reacted_to_author_uri) - end - - describe "new remote user creation" do - test "returns local users" do - local_user = insert(:user) - {:ok, user} = OStatus.find_or_make_user(local_user.ap_id) - - assert user == local_user - end - - test "tries to use the information in poco fields" do - uri = "https://social.heldscal.la/user/23211" - - {:ok, user} = OStatus.find_or_make_user(uri) - - user = User.get_cached_by_id(user.id) - assert user.name == "Constance Variable" - assert user.nickname == "lambadalambda@social.heldscal.la" - assert user.local == false - assert user.info.uri == uri - assert user.ap_id == uri - assert user.bio == "Call me Deacon Blues." - assert user.avatar["type"] == "Image" - - {:ok, user_again} = OStatus.find_or_make_user(uri) - - assert user == user_again - end - - test "find_or_make_user sets all the nessary input fields" do - uri = "https://social.heldscal.la/user/23211" - {:ok, user} = OStatus.find_or_make_user(uri) - - assert user.info == - %User.Info{ - id: user.info.id, - ap_enabled: false, - background: %{}, - banner: %{}, - blocks: [], - deactivated: false, - default_scope: "public", - domain_blocks: [], - follower_count: 0, - is_admin: false, - is_moderator: false, - keys: nil, - locked: false, - no_rich_text: false, - note_count: 0, - settings: nil, - source_data: %{}, - hub: "https://social.heldscal.la/main/push/hub", - magic_key: - "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB", - salmon: "https://social.heldscal.la/main/salmon/user/23211", - topic: "https://social.heldscal.la/api/statuses/user_timeline/23211.atom", - uri: "https://social.heldscal.la/user/23211" - } - end - - test "find_make_or_update_actor takes an author element and returns an updated user" do - uri = "https://social.heldscal.la/user/23211" - - {:ok, user} = OStatus.find_or_make_user(uri) - old_name = user.name - old_bio = user.bio - change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, name: nil}) - - {:ok, user} = Repo.update(change) - refute user.avatar - - doc = XML.parse_document(File.read!("test/fixtures/23211.atom")) - [author] = :xmerl_xpath.string('//author[1]', doc) - {:ok, user} = OStatus.find_make_or_update_actor(author) - assert user.avatar["type"] == "Image" - assert user.name == old_name - assert user.bio == old_bio - - {:ok, user_again} = OStatus.find_make_or_update_actor(author) - assert user_again == user - end - - test "find_or_make_user disallows protocol downgrade" do - user = insert(:user, %{local: true}) - {:ok, user} = OStatus.find_or_make_user(user.ap_id) - - assert User.ap_enabled?(user) - - user = - insert(:user, %{ - ap_id: "https://social.heldscal.la/user/23211", - info: %{ap_enabled: true}, - local: false - }) - - assert User.ap_enabled?(user) - - {:ok, user} = OStatus.find_or_make_user(user.ap_id) - assert User.ap_enabled?(user) - end - - test "find_make_or_update_actor disallows protocol downgrade" do - user = insert(:user, %{local: true}) - {:ok, user} = OStatus.find_or_make_user(user.ap_id) - - assert User.ap_enabled?(user) - - user = - insert(:user, %{ - ap_id: "https://social.heldscal.la/user/23211", - info: %{ap_enabled: true}, - local: false - }) - - assert User.ap_enabled?(user) - - {:ok, user} = OStatus.find_or_make_user(user.ap_id) - assert User.ap_enabled?(user) - - doc = XML.parse_document(File.read!("test/fixtures/23211.atom")) - [author] = :xmerl_xpath.string('//author[1]', doc) - {:error, :invalid_protocol} = OStatus.find_make_or_update_actor(author) - end - end - - describe "gathering user info from a user id" do - test "it returns user info in a hash" do - user = "shp@social.heldscal.la" - - # TODO: make test local - {:ok, data} = OStatus.gather_user_info(user) - - expected = %{ - "hub" => "https://social.heldscal.la/main/push/hub", - "magic_key" => - "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - "name" => "shp", - "nickname" => "shp", - "salmon" => "https://social.heldscal.la/main/salmon/user/29191", - "subject" => "acct:shp@social.heldscal.la", - "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - "uri" => "https://social.heldscal.la/user/29191", - "host" => "social.heldscal.la", - "fqn" => user, - "bio" => "cofe", - "avatar" => %{ - "type" => "Image", - "url" => [ - %{ - "href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", - "mediaType" => "image/jpeg", - "type" => "Link" - } - ] - }, - "subscribe_address" => "https://social.heldscal.la/main/ostatussub?profile={uri}", - "ap_id" => nil - } - - assert data == expected - end - - test "it works with the uri" do - user = "https://social.heldscal.la/user/29191" - - # TODO: make test local - {:ok, data} = OStatus.gather_user_info(user) - - expected = %{ - "hub" => "https://social.heldscal.la/main/push/hub", - "magic_key" => - "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - "name" => "shp", - "nickname" => "shp", - "salmon" => "https://social.heldscal.la/main/salmon/user/29191", - "subject" => "https://social.heldscal.la/user/29191", - "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - "uri" => "https://social.heldscal.la/user/29191", - "host" => "social.heldscal.la", - "fqn" => user, - "bio" => "cofe", - "avatar" => %{ - "type" => "Image", - "url" => [ - %{ - "href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", - "mediaType" => "image/jpeg", - "type" => "Link" - } - ] - }, - "subscribe_address" => "https://social.heldscal.la/main/ostatussub?profile={uri}", - "ap_id" => nil - } - - assert data == expected - end - end - - describe "fetching a status by it's HTML url" do - test "it builds a missing status from an html url" do - capture_log(fn -> - url = "https://shitposter.club/notice/2827873" - {:ok, [activity]} = OStatus.fetch_activity_from_url(url) - - assert activity.data["actor"] == "https://shitposter.club/user/1" - - assert activity.data["object"] == - "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" - end) - end - - test "it works for atom notes, too" do - url = "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056" - {:ok, [activity]} = OStatus.fetch_activity_from_url(url) - assert activity.data["actor"] == "https://social.sakamoto.gq/users/eal" - assert activity.data["object"] == url - end - end - - test "it doesn't add nil in the to field" do - incoming = File.read!("test/fixtures/nil_mention_entry.xml") - {:ok, [activity]} = OStatus.handle_incoming(incoming) - - assert activity.data["to"] == [ - "http://localhost:4001/users/atarifrosch@social.stopwatchingus-heidelberg.de/followers", - "https://www.w3.org/ns/activitystreams#Public" - ] - end - - describe "is_representable?" do - test "Note objects are representable" do - note_activity = insert(:note_activity) - - assert OStatus.is_representable?(note_activity) - end - - test "Article objects are not representable" do - note_activity = insert(:note_activity) - note_object = Object.normalize(note_activity) - - note_data = - note_object.data - |> Map.put("type", "Article") - - Cachex.clear(:object_cache) - - cs = Object.change(note_object, %{data: note_data}) - {:ok, _article_object} = Repo.update(cs) - - # the underlying object is now an Article instead of a note, so this should fail - refute OStatus.is_representable?(note_activity) - end - end - - describe "make_user/2" do - test "creates new user" do - {:ok, user} = OStatus.make_user("https://social.heldscal.la/user/23211") - - created_user = - User - |> Repo.get_by(ap_id: "https://social.heldscal.la/user/23211") - |> Map.put(:last_digest_emailed_at, nil) - - assert user.info - assert user == created_user - end - end -end diff --git a/test/web/ostatus/user_representer_test.exs b/test/web/ostatus/user_representer_test.exs deleted file mode 100644 index e3863d2e9..000000000 --- a/test/web/ostatus/user_representer_test.exs +++ /dev/null @@ -1,38 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatus.UserRepresenterTest do - use Pleroma.DataCase - alias Pleroma.Web.OStatus.UserRepresenter - - import Pleroma.Factory - alias Pleroma.User - - test "returns a user with id, uri, name and link" do - user = insert(:user, %{nickname: "レイン"}) - tuple = UserRepresenter.to_simple_form(user) - - res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> to_string - - expected = """ - #{user.ap_id} - http://activitystrea.ms/schema/1.0/person - #{user.ap_id} - #{user.nickname} - #{user.name} - #{user.bio} - #{user.bio} - #{user.nickname} - - - true - """ - - assert clean(res) == clean(expected) - end - - defp clean(string) do - String.replace(string, ~r/\s/, "") - end -end diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 696c1bd70..5aa8c73cf 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -45,19 +45,6 @@ defmodule Pleroma.Web.WebFingerTest do assert {:error, %Jason.DecodeError{}} = WebFinger.finger(user) end - test "returns the info for an OStatus user" do - user = "shp@social.heldscal.la" - - {:ok, data} = WebFinger.finger(user) - - assert data["magic_key"] == - "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" - - assert data["topic"] == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" - assert data["subject"] == "acct:shp@social.heldscal.la" - assert data["salmon"] == "https://social.heldscal.la/main/salmon/user/29191" - end - test "returns the ActivityPub actor URI for an ActivityPub user" do user = "framasoft@framatube.org" @@ -72,20 +59,6 @@ defmodule Pleroma.Web.WebFingerTest do assert data["ap_id"] == "https://gerzilla.de/channel/kaniini" end - test "returns the correctly for json ostatus users" do - user = "winterdienst@gnusocial.de" - - {:ok, data} = WebFinger.finger(user) - - assert data["magic_key"] == - "RSA.qfYaxztz7ZELrE4v5WpJrPM99SKI3iv9Y3Tw6nfLGk-4CRljNYqV8IYX2FXjeucC_DKhPNnlF6fXyASpcSmA_qupX9WC66eVhFhZ5OuyBOeLvJ1C4x7Hi7Di8MNBxY3VdQuQR0tTaS_YAZCwASKp7H6XEid3EJpGt0EQZoNzRd8=.AQAB" - - assert data["topic"] == "https://gnusocial.de/api/statuses/user_timeline/249296.atom" - assert data["subject"] == "acct:winterdienst@gnusocial.de" - assert data["salmon"] == "https://gnusocial.de/main/salmon/user/249296" - assert data["subscribe_address"] == "https://gnusocial.de/main/ostatussub?profile={uri}" - end - test "it work for AP-only user" do user = "kpherox@mstdn.jp" From 48059c03c91b2437779ac42581812c07530c1a34 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 00:30:01 +0000 Subject: [PATCH 62/84] fix up some tests --- lib/pleroma/object/fetcher.ex | 4 +- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +- test/user_test.exs | 5 -- test/web/ostatus/ostatus_controller_test.exs | 78 -------------------- 4 files changed, 5 insertions(+), 86 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9436e2730..8975fb47e 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -162,8 +162,8 @@ defmodule Pleroma.Object.Fetcher do {:ok, %{status: code}} when code in [404, 410] -> {:error, "Object has been deleted"} - e -> - {:error, e} + _ -> + {:error, "Could not fetch by AP id"} end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d391732a2..d631e43c6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1219,7 +1219,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do data <- maybe_update_follow_information(data) do {:ok, data} else - e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") + e -> + Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") + {:error, e} end end diff --git a/test/user_test.exs b/test/user_test.exs index 81d23e9a3..ad050b7da 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -457,11 +457,6 @@ defmodule Pleroma.UserTest do assert user == fetched_user end - test "fetches an external user via ostatus if no user exists" do - {:ok, fetched_user} = User.get_or_fetch_by_nickname("shp@social.heldscal.la") - assert fetched_user.nickname == "shp@social.heldscal.la" - end - test "returns nil if no user could be fetched" do {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la") assert fetched_user == "not found nonexistant@social.heldscal.la" diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 29804cfe1..58534396e 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -21,56 +21,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do Pleroma.Config.put([:instance, :federating], true) end - describe "salmon_incoming" do - test "decodes a salmon", %{conn: conn} do - user = insert(:user) - salmon = File.read!("test/fixtures/salmon.xml") - - assert capture_log(fn -> - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) - - assert response(conn, 200) - end) =~ "[error]" - end - - test "decodes a salmon with a changed magic key", %{conn: conn} do - user = insert(:user) - salmon = File.read!("test/fixtures/salmon.xml") - - assert capture_log(fn -> - conn = - conn - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) - - assert response(conn, 200) - end) =~ "[error]" - - # Wrong key - info = %{ - magic_key: - "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwrong1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" - } - - # Set a wrong magic-key for a user so it has to refetch - "http://gs.example.org:4040/index.php/user/1" - |> User.get_cached_by_ap_id() - |> User.update_info(&User.Info.remote_user_creation(&1, info)) - - assert capture_log(fn -> - conn = - build_conn() - |> put_req_header("content-type", "application/atom+xml") - |> post("/users/#{user.nickname}/salmon", salmon) - - assert response(conn, 200) - end) =~ "[error]" - end - end - describe "GET object/2" do test "redirects to /notice/id for html format", %{conn: conn} do note_activity = insert(:note_activity) @@ -121,16 +71,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do end describe "GET activity/2" do - test "gets an activity in xml format", %{conn: conn} do - note_activity = insert(:note_activity) - [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"])) - - conn - |> put_req_header("accept", "application/xml") - |> get("/activities/#{uuid}") - |> response(200) - end - test "redirects to /notice/id for html format", %{conn: conn} do note_activity = insert(:note_activity) [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"])) @@ -158,24 +98,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do assert response(conn, 500) == ~S({"error":"Something went wrong"}) end - test "404s on deleted objects", %{conn: conn} do - note_activity = insert(:note_activity) - object = Object.normalize(note_activity) - [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, object.data["id"])) - - conn - |> put_req_header("accept", "application/xml") - |> get("/objects/#{uuid}") - |> response(200) - - Object.delete(object) - - conn - |> put_req_header("accept", "application/xml") - |> get("/objects/#{uuid}") - |> response(404) - end - test "404s on private activities", %{conn: conn} do note_activity = insert(:direct_note_activity) [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"])) From 597cb8897b64ba8353ccc045addb4672b97ab18f Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 00:37:13 +0000 Subject: [PATCH 63/84] tests: remove some more ostatus tests --- test/object/fetcher_test.exs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 895a73d2c..851a503a7 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -71,24 +71,6 @@ defmodule Pleroma.Object.FetcherTest do assert object == object_again end - - test "it works with objects only available via Ostatus" do - {:ok, object} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") - - assert object == object_again - end - - test "it correctly stitches up conversations between ostatus and ap" do - last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - {:ok, object} = Fetcher.fetch_object_from_id(last) - - object = Object.get_by_ap_id(object.data["inReplyTo"]) - assert object - end end describe "implementation quirks" do From af9aa8e35809561a3edbd79ed1adb7de31056f38 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 00:51:53 +0000 Subject: [PATCH 64/84] tests: simplify object fetching and containment tests --- test/object/containment_test.exs | 2 +- test/object/fetcher_test.exs | 21 +++------------------ 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs index 61cd1b412..0dc2728b9 100644 --- a/test/object/containment_test.exs +++ b/test/object/containment_test.exs @@ -65,7 +65,7 @@ defmodule Pleroma.Object.ContainmentTest do assert capture_log(fn -> {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") end) =~ - "[error] Could not decode user at fetch https://n1u.moe/users/rye, {:error, :error}" + "[error] Could not decode user at fetch https://n1u.moe/users/rye" end end diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 851a503a7..9ae6b015d 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -27,31 +27,16 @@ defmodule Pleroma.Object.FetcherTest do end describe "actor origin containment" do - test_with_mock "it rejects objects with a bogus origin", - Pleroma.Web.OStatus, - [:passthrough], - [] do + test "it rejects objects with a bogus origin" do {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json") - - refute called(Pleroma.Web.OStatus.fetch_activity_from_url(:_)) end - test_with_mock "it rejects objects when attributedTo is wrong (variant 1)", - Pleroma.Web.OStatus, - [:passthrough], - [] do + test "it rejects objects when attributedTo is wrong (variant 1)" do {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity2.json") - - refute called(Pleroma.Web.OStatus.fetch_activity_from_url(:_)) end - test_with_mock "it rejects objects when attributedTo is wrong (variant 2)", - Pleroma.Web.OStatus, - [:passthrough], - [] do + test "it rejects objects when attributedTo is wrong (variant 2)" do {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity3.json") - - refute called(Pleroma.Web.OStatus.fetch_activity_from_url(:_)) end end From 6f110fc04ca31bf571fb80a2a852ab60ddd496ff Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 02:42:25 +0000 Subject: [PATCH 65/84] object fetcher: fix up error handling --- lib/pleroma/object/fetcher.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 8975fb47e..9436e2730 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -162,8 +162,8 @@ defmodule Pleroma.Object.Fetcher do {:ok, %{status: code}} when code in [404, 410] -> {:error, "Object has been deleted"} - _ -> - {:error, "Could not fetch by AP id"} + e -> + {:error, e} end end From 700c654208b74269fa5da453ffd8f5cc2724086d Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 02:52:08 +0000 Subject: [PATCH 66/84] tests: fix relay tests --- test/web/activity_pub/relay_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/activity_pub/relay_test.exs b/test/web/activity_pub/relay_test.exs index 0f7556538..4a0a03944 100644 --- a/test/web/activity_pub/relay_test.exs +++ b/test/web/activity_pub/relay_test.exs @@ -22,8 +22,8 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do describe "follow/1" do test "returns errors when user not found" do assert capture_log(fn -> - assert Relay.follow("test-ap-id") == {:error, "Could not fetch by AP id"} - end) =~ "Could not fetch by AP id" + {:error, _} = Relay.follow("test-ap-id") + end) =~ "Could not decode user at fetch" end test "returns activity" do @@ -41,8 +41,8 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do describe "unfollow/1" do test "returns errors when user not found" do assert capture_log(fn -> - assert Relay.unfollow("test-ap-id") == {:error, "Could not fetch by AP id"} - end) =~ "Could not fetch by AP id" + {:error, _} = Relay.unfollow("test-ap-id") + end) =~ "Could not decode user at fetch" end test "returns activity" do From 85ddcaf418bd24722ea222e0fe19d538675ee784 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 02:56:02 +0000 Subject: [PATCH 67/84] tests: fix up signature tests --- test/signature_test.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/signature_test.exs b/test/signature_test.exs index 96c8ba07a..6b168f2d9 100644 --- a/test/signature_test.exs +++ b/test/signature_test.exs @@ -69,8 +69,7 @@ defmodule Pleroma.SignatureTest do test "it returns error when not found user" do assert capture_log(fn -> - assert Signature.refetch_public_key(make_fake_conn("test-ap_id")) == - {:error, {:error, :ok}} + {:error, _} = Signature.refetch_public_key(make_fake_conn("test-ap_id")) end) =~ "[error] Could not decode user" end end From bf2107743ffcd3b7b9c32c49242f1dfd89dcfdb6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 03:26:50 +0000 Subject: [PATCH 68/84] object: containment: don't try to contain ostatus objects --- lib/pleroma/object/containment.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index f077a9f32..cd8623821 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -57,7 +57,9 @@ defmodule Pleroma.Object.Containment do id_uri = URI.parse(id) other_uri = URI.parse(other_id) - if id_uri.host == other_uri.host do + # We explicitly allow 'tag' URIs through, due to legacy OStatus objects + # being present in the ActivityPub network. + if id_uri.host == other_uri.host || other_uri.scheme == "tag" do :ok else :error From a177f22e0242f58db665688ed56dd5f1ccd1999c Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 03:41:38 +0000 Subject: [PATCH 69/84] object: fetcher: improve error reporting --- lib/pleroma/object/fetcher.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9436e2730..9af2e02ea 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Object.Fetcher do {:normalize, nil} <- {:normalize, Object.normalize(data, false)}, params <- prepare_activity_params(data), {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)}, - {:ok, activity} <- Transmogrifier.handle_incoming(params, options), + {:transmogrifier, {:ok, activity}} <- {:transmogrifier, Transmogrifier.handle_incoming(params, options)}, {:object, _data, %Object{} = object} <- {:object, data, Object.normalize(activity, false)} do {:ok, object} @@ -74,9 +74,12 @@ defmodule Pleroma.Object.Fetcher do {:containment, _} -> {:error, "Object containment failed."} - {:error, {:reject, nil}} -> + {:transmogrifier, {:error, {:reject, nil}}} -> {:reject, nil} + {:transmogrifier, _} -> + {:error, "Transmogrifier failure."} + {:object, data, nil} -> reinject_object(%Object{}, data) @@ -106,7 +109,8 @@ defmodule Pleroma.Object.Fetcher do with {:ok, object} <- fetch_object_from_id(id, options) do object else - _e -> + e -> + Logger.error("Error while fetching #{id}: #{inspect(e)}") nil end end @@ -153,7 +157,7 @@ defmodule Pleroma.Object.Fetcher do Logger.debug("Fetch headers: #{inspect(headers)}") - with true <- String.starts_with?(id, "http"), + with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")}, {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers), {:ok, data} <- Jason.decode(body), :ok <- Containment.contain_origin_from_id(id, data) do @@ -162,6 +166,9 @@ defmodule Pleroma.Object.Fetcher do {:ok, %{status: code}} when code in [404, 410] -> {:error, "Object has been deleted"} + {:scheme, _} -> + {:error, "Unsupported URI scheme"} + e -> {:error, e} end From 7295a05ceee441311bf56513f5fe889908f59bd5 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 03:56:31 +0000 Subject: [PATCH 70/84] object: containment: also allow OStatus object IDs through when comparing origins --- lib/pleroma/object/containment.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index cd8623821..6a621ac26 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Object.Containment do id_uri = URI.parse(id) actor_uri = URI.parse(get_actor(params)) - if id_uri.host == actor_uri.host do + if id_uri.host == actor_uri.host || id_uri.scheme == "tag" do :ok else :error From bae96de273250a0054d22c132bd847ab83928ca3 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 03:57:32 +0000 Subject: [PATCH 71/84] activitypub: tag containment checks for better error tracing --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d631e43c6..94c467b69 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -132,7 +132,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), {:fake, false, map, recipients} <- {:fake, fake, map, recipients}, - :ok <- Containment.contain_child(map), + {:containment, :ok} <- {:containment, Containment.contain_child(map)}, {:ok, map, object} <- insert_full_object(map) do {:ok, activity} = Repo.insert(%Activity{ From dbfdb1f6e3fe2bb626b6a81e2bad2753bea02a19 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 03:58:28 +0000 Subject: [PATCH 72/84] add some missing tesla fixtures --- .../https___shitposter.club_notice_2827873.json | 1 + test/fixtures/tesla_mock/moonman@shitposter.club.json | 1 + test/support/http_request_mock.ex | 10 +++++++++- test/web/activity_pub/transmogrifier_test.exs | 1 - 4 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json create mode 100644 test/fixtures/tesla_mock/moonman@shitposter.club.json diff --git a/test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json b/test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json new file mode 100644 index 000000000..4b7b4df44 --- /dev/null +++ b/test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://shitposter.club/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://shitposter.club/users/moonman","attachment":[],"attributedTo":"https://shitposter.club/users/moonman","cc":["https://shitposter.club/users/moonman/followers"],"content":"@neimzr4luzerz @dolus childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English","context":"tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26","conversation":"tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26","id":"tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment","inReplyTo":"tag:shitposter.club,2017-05-05:noticeId=2827849:objectType=comment","inReplyToStatusId":2827849,"published":"2017-05-05T08:51:48Z","sensitive":false,"summary":null,"tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Note"} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/moonman@shitposter.club.json b/test/fixtures/tesla_mock/moonman@shitposter.club.json new file mode 100644 index 000000000..8f9ced1dd --- /dev/null +++ b/test/fixtures/tesla_mock/moonman@shitposter.club.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://shitposter.club/schemas/litepub-0.1.jsonld",{"@language":"und"}],"attachment":[],"endpoints":{"oauthAuthorizationEndpoint":"https://shitposter.club/oauth/authorize","oauthRegistrationEndpoint":"https://shitposter.club/api/v1/apps","oauthTokenEndpoint":"https://shitposter.club/oauth/token","sharedInbox":"https://shitposter.club/inbox"},"followers":"https://shitposter.club/users/moonman/followers","following":"https://shitposter.club/users/moonman/following","icon":{"type":"Image","url":"https://shitposter.club/media/bda6e00074f6a02cbf32ddb0abec08151eb4c795e580927ff7ad638d00cde4c8.jpg?name=blob.jpg"},"id":"https://shitposter.club/users/moonman","image":{"type":"Image","url":"https://shitposter.club/media/4eefb90d-cdb2-2b4f-5f29-7612856a99d2/4eefb90d-cdb2-2b4f-5f29-7612856a99d2.jpeg"},"inbox":"https://shitposter.club/users/moonman/inbox","manuallyApprovesFollowers":false,"name":"Captain Howdy","outbox":"https://shitposter.club/users/moonman/outbox","preferredUsername":"moonman","publicKey":{"id":"https://shitposter.club/users/moonman#main-key","owner":"https://shitposter.club/users/moonman","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnOTitJ19ZqcOZHwSXQUM\nJq9ip4GNblp83LgwG1t5c2h2iaI3fXMsB4EaEBs8XHsoSFyDeDNRSPE3mtVgOnWv\n1eaXWMDerBT06th6DrElD9k5IoEPtZRY4HtZa1xGnte7+6RjuPOzZ1fR9C8WxGgi\nwb9iOUMhazpo85fC3iKCAL5XhiuA3Nas57MDJgueeI9BF+2oFelFZdMSWwG96uch\niDfp8nfpkmzYI6SWbylObjm8RsfZbGTosLHwWyJPEITeYI/5M0XwJe9dgVI1rVNU\n52kplWOGTo1rm6V0AMHaYAd9RpiXxe8xt5OeranrsE/5LvEQUl0fz7SE36YmsOaH\nTwIDAQAB\n-----END PUBLIC KEY-----\n\n"},"summary":"EMAIL:shitposterclub@gmail.com
XMPP: moon@talk.shitposter.club
PRONOUNS: none of your business

Purported leftist kike piece of shit","tag":[],"type":"Person","url":"https://shitposter.club/users/moonman"} \ No newline at end of file diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 4feb57f3a..7d65209fb 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -38,6 +38,14 @@ defmodule HttpRequestMock do }} end + def get("https://shitposter.club/users/moonman", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/moonman@shitposter.club.json") + }} + end + def get("https://mastodon.social/users/emelie/statuses/101849165031453009", _, _, _) do {:ok, %Tesla.Env{ @@ -620,7 +628,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.html") + body: File.read!("test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json") }} end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 252d98a7a..dbb6e59b0 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Object.Fetcher - alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub From 3c785b85a62267f07f5e6bdd4f12c96ca94639c1 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 04:08:25 +0000 Subject: [PATCH 73/84] object: fetcher: fix up formatting --- lib/pleroma/object/fetcher.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9af2e02ea..7758cb90b 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -66,7 +66,8 @@ defmodule Pleroma.Object.Fetcher do {:normalize, nil} <- {:normalize, Object.normalize(data, false)}, params <- prepare_activity_params(data), {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)}, - {:transmogrifier, {:ok, activity}} <- {:transmogrifier, Transmogrifier.handle_incoming(params, options)}, + {:transmogrifier, {:ok, activity}} <- + {:transmogrifier, Transmogrifier.handle_incoming(params, options)}, {:object, _data, %Object{} = object} <- {:object, data, Object.normalize(activity, false)} do {:ok, object} From a8eb1f09758fb863b0ad36ae8c4ecac6e1e872a6 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 04:14:26 +0000 Subject: [PATCH 74/84] tests: mastodon search: search for an account that is visible via activitypub, not ostatus --- .../web/mastodon_api/controllers/search_controller_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index ee413eef7..7953fad62 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -204,17 +204,17 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do conn = conn |> assign(:user, user) - |> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "true"}) + |> get("/api/v1/search", %{"q" => "mike@osada.macgirvin.com", "resolve" => "true"}) assert results = json_response(conn, 200) [account] = results["accounts"] - assert account["acct"] == "shp@social.heldscal.la" + assert account["acct"] == "mike@osada.macgirvin.com" end test "search doesn't fetch remote accounts if resolve is false", %{conn: conn} do conn = conn - |> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "false"}) + |> get("/api/v1/search", %{"q" => "mike@osada.macgirvin.com", "resolve" => "false"}) assert results = json_response(conn, 200) assert [] == results["accounts"] From e99fdfc32db231391184220eb28024d358821d27 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 15:34:36 +0000 Subject: [PATCH 75/84] object: containment: only allow OStatus references in test suite environment --- lib/pleroma/object/containment.ex | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 6a621ac26..1beb9c83d 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -32,6 +32,23 @@ defmodule Pleroma.Object.Containment do get_actor(%{"actor" => actor}) end + # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus + # objects being present in the test suite environment. Once these objects are + # removed, please also remove this. + if Mix.env() == :test do + defp compare_uris(_, %URI{scheme: "tag" <> _}), do: :ok + end + + defp compare_uris(%URI{} = id_uri, %URI{} = other_uri) do + if id_uri.host == other_uri.host do + :ok + else + :error + end + end + + defp compare_uris(_, _), do: :error + @doc """ Checks that an imported AP object's actor matches the domain it came from. """ @@ -41,11 +58,7 @@ defmodule Pleroma.Object.Containment do id_uri = URI.parse(id) actor_uri = URI.parse(get_actor(params)) - if id_uri.host == actor_uri.host || id_uri.scheme == "tag" do - :ok - else - :error - end + compare_uris(actor_uri, id_uri) end def contain_origin(id, %{"attributedTo" => actor} = params), @@ -57,13 +70,7 @@ defmodule Pleroma.Object.Containment do id_uri = URI.parse(id) other_uri = URI.parse(other_id) - # We explicitly allow 'tag' URIs through, due to legacy OStatus objects - # being present in the ActivityPub network. - if id_uri.host == other_uri.host || other_uri.scheme == "tag" do - :ok - else - :error - end + compare_uris(id_uri, other_uri) end def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}), From 44e64af5e76e1ace82aa66973da883e334ebfc93 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 15:39:15 +0000 Subject: [PATCH 76/84] object: containment: simplify the pattern match for OStatus testsuite hack --- lib/pleroma/object/containment.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 1beb9c83d..68535c09e 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -36,7 +36,7 @@ defmodule Pleroma.Object.Containment do # objects being present in the test suite environment. Once these objects are # removed, please also remove this. if Mix.env() == :test do - defp compare_uris(_, %URI{scheme: "tag" <> _}), do: :ok + defp compare_uris(_, %URI{scheme: "tag"}), do: :ok end defp compare_uris(%URI{} = id_uri, %URI{} = other_uri) do From c2ae6310dc09e65515c7b8774d2b85b5ef7da1a1 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 18 Oct 2019 15:46:46 +0000 Subject: [PATCH 77/84] tests: mastodon api: fix broken test that used OStatus --- .../mastodon_api/views/status_view_test.exs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index 9375c5030..c200ad8fe 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -228,20 +228,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do assert status.in_reply_to_id == to_string(note.id) end - # XXX: fix this test - # test "contains mentions" do - # incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") - # # a user with this ap id might be in the cache. - # recipient = "https://pleroma.soykaf.com/users/lain" - # user = insert(:user, %{ap_id: recipient}) - # - # {:ok, [activity]} = OStatus.handle_incoming(incoming) - # - # status = StatusView.render("show.json", %{activity: activity}) - # - # assert status.mentions == - # Enum.map([user], fn u -> AccountView.render("mention.json", %{user: u}) end) - # end + test "contains mentions" do + user = insert(:user) + mentioned = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "hi @#{mentioned.nickname}"}) + + status = StatusView.render("show.json", %{activity: activity}) + + assert status.mentions == + Enum.map([mentioned], fn u -> AccountView.render("mention.json", %{user: u}) end) + end test "create mentions from the 'to' field" do %User{ap_id: recipient_ap_id} = insert(:user) From 901bf0fb8c5407a3e74bc782e8eb28eb47dab6ef Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 19 Oct 2019 00:37:39 +0300 Subject: [PATCH 78/84] pleroma_ctl: Fix attempting to use RPC for config generation --- CHANGELOG.md | 4 ++++ rel/files/bin/pleroma_ctl | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ebf358ed..4f90f36a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname`
+## [1.1.2] - 2019-10-18 +### Fixed +- `pleroma_ctl` trying to connect to a running instance when generating the config, which of course doesn't exist. + ## [1.1.1] - 2019-10-18 ### Fixed - One of the migrations between 1.0.0 and 1.1.0 wiping user info of the relay user because of unexpected behavior of postgresql's `jsonb_set`, resulting in inability to post in the default configuration. If you were affected, please run the following query in postgres console, the relay user will be recreated automatically: diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index 90f87a990..9fc5b0bad 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -141,8 +141,8 @@ else ACTION="$1" shift - - if [ "$(echo \"$1\" | grep \"^-\" >/dev/null)" = false ]; then + echo "$1" | grep "^-" >/dev/null + if [ $? -eq 1 ]; then SUBACTION="$1" shift fi From 638457ba94931c35268cf5fb4d2a432a0def6bdd Mon Sep 17 00:00:00 2001 From: KokaKiwi Date: Fri, 11 Oct 2019 14:48:01 +0200 Subject: [PATCH 79/84] MastoFE: Add PWA manifest. --- config/config.exs | 10 +++++ config/description.exs | 39 +++++++++++++++++++ lib/pleroma/web/masto_fe_controller.ex | 6 +++ lib/pleroma/web/router.ex | 6 +++ .../web/templates/masto_fe/index.html.eex | 6 ++- lib/pleroma/web/views/masto_fe_view.ex | 19 +++++++++ 6 files changed, 85 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index d0766a6e2..a69d41d17 100644 --- a/config/config.exs +++ b/config/config.exs @@ -322,6 +322,16 @@ config :pleroma, :assets, ], default_mascot: :pleroma_fox_tan +config :pleroma, :manifest, + icons: [ + %{ + src: "/static/logo.png", + type: "image/png" + } + ], + theme_color: "#282c37", + background_color: "#191b22" + config :pleroma, :activitypub, unfollow_blocked: true, outgoing_blocks: true, diff --git a/config/description.exs b/config/description.exs index 571c64bc1..70e963399 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1098,6 +1098,45 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: :manifest, + type: :group, + description: + "This section describe PWA manifest instance-specific values. Currently this option relate only for MastoFE", + children: [ + %{ + key: :icons, + type: {:list, :map}, + description: "Describe the icons of the app", + suggestion: [ + %{ + src: "/static/logo.png" + }, + %{ + src: "/static/icon.png", + type: "image/png" + }, + %{ + src: "/static/icon.ico", + sizes: "72x72 96x96 128x128 256x256" + } + ] + }, + %{ + key: :theme_color, + type: :string, + description: "Describe the theme color of the app", + suggestions: ["#282c37", "mediumpurple"] + }, + %{ + key: :background_color, + type: :string, + description: "Describe the background color of the app", + suggestions: ["#191b22", "aliceblue"] + } + ] + }, %{ group: :pleroma, key: :mrf_simple, diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex index 87860f1d5..93b38e8f4 100644 --- a/lib/pleroma/web/masto_fe_controller.ex +++ b/lib/pleroma/web/masto_fe_controller.ex @@ -34,6 +34,12 @@ defmodule Pleroma.Web.MastoFEController do end end + @doc "GET /web/manifest.json" + def manifest(conn, _params) do + conn + |> render("manifest.json") + end + @doc "PUT /api/web/settings" def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do with {:ok, _} <- User.update_info(user, &User.Info.mastodon_settings_update(&1, settings)) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d68fb87da..e7d9ee238 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -591,6 +591,12 @@ defmodule Pleroma.Web.Router do get("/:version", Nodeinfo.NodeinfoController, :nodeinfo) end + scope "/", Pleroma.Web do + pipe_through(:api) + + get("/web/manifest.json", MastoFEController, :manifest) + end + scope "/", Pleroma.Web do pipe_through(:mastodon_html) diff --git a/lib/pleroma/web/templates/masto_fe/index.html.eex b/lib/pleroma/web/templates/masto_fe/index.html.eex index feff36fae..c330960fa 100644 --- a/lib/pleroma/web/templates/masto_fe/index.html.eex +++ b/lib/pleroma/web/templates/masto_fe/index.html.eex @@ -4,9 +4,13 @@ -<%= Pleroma.Config.get([:instance, :name]) %> +<%= Config.get([:instance, :name]) %> + + + + diff --git a/lib/pleroma/web/views/masto_fe_view.ex b/lib/pleroma/web/views/masto_fe_view.ex index 21b086d4c..85b164b59 100644 --- a/lib/pleroma/web/views/masto_fe_view.ex +++ b/lib/pleroma/web/views/masto_fe_view.ex @@ -99,4 +99,23 @@ defmodule Pleroma.Web.MastoFEView do defp present?(nil), do: false defp present?(false), do: false defp present?(_), do: true + + def render("manifest.json", _params) do + %{ + name: Config.get([:instance, :name]), + description: Config.get([:instance, :description]), + icons: Config.get([:manifest, :icons]), + theme_color: Config.get([:manifest, :theme_color]), + background_color: Config.get([:manifest, :background_color]), + display: "standalone", + scope: Pleroma.Web.base_url(), + start_url: masto_fe_path(Pleroma.Web.Endpoint, :index, ["getting-started"]), + categories: [ + "social" + ], + serviceworker: %{ + src: "/sw.js" + } + } + end end From df3c2dfd8bed82121f3dcfba18105ce8ee5daddf Mon Sep 17 00:00:00 2001 From: KokaKiwi Date: Sat, 19 Oct 2019 12:11:25 +0200 Subject: [PATCH 80/84] Add documentation for PWA manifest configuration. --- docs/configuration/cheatsheet.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8d85276ed..3427ae419 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -247,6 +247,35 @@ relates to mascots on the mastodon frontend * `default_mascot`: An element from `mascots` - This will be used as the default mascot on MastoFE (default: `:pleroma_fox_tan`) +## :manifest + +This section describe PWA manifest instance-specific values. Currently this option relate only for MastoFE. + +* `icons`: Describe the icons of the app, this a list of maps describing icons in the same way as the + [spec](https://www.w3.org/TR/appmanifest/#imageresource-and-its-members) describes it. + + Example: + + ```elixir + config :pleroma, :manifest, + icons: [ + %{ + src: "/static/logo.png" + }, + %{ + src: "/static/icon.png", + type: "image/png" + }, + %{ + src: "/static/icon.ico", + sizes: "72x72 96x96 128x128 256x256" + } + ] + ``` + +* `theme_color`: Describe the theme color of the app. (Example: `"#282c37"`, `"rebeccapurple"`) +* `background_color`: Describe the background color of the app. (Example: `"#191b22"`, `"aliceblue"`) + ## :mrf_simple * `media_removal`: List of instances to remove medias from * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from From 52ed2f8f2d962154ba61b31a3f5aab13dc7217fc Mon Sep 17 00:00:00 2001 From: eugenijm Date: Fri, 11 Oct 2019 06:40:58 +0300 Subject: [PATCH 81/84] Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all user's conversations as read --- CHANGELOG.md | 1 + docs/API/pleroma_api.md | 7 +++++ lib/pleroma/conversation/participation.ex | 13 +++++++++ .../controllers/pleroma_api_controller.ex | 9 +++++++ lib/pleroma/web/router.ex | 1 + test/conversation/participation_test.exs | 14 ++++++++++ .../pleroma_api_controller_test.exs | 27 +++++++++++++++++++ 7 files changed, 72 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d516080b4..19cd1e539 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array - Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body). - Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays +- Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 0517bbdd7..6c326dc9b 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -367,6 +367,13 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) +## `GET /api/v1/pleroma/conversations/read` +### Marks all user's conversations as read. +* Method `POST` +* Authentication: required +* Params: None +* Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy). + ## `GET /api/pleroma/emoji/packs` ### Lists the custom emoji packs on the server * Method `GET` diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index e17f49e58..41918fa78 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -69,6 +69,19 @@ defmodule Pleroma.Conversation.Participation do end end + def mark_all_as_read(user) do + {_, participations} = + __MODULE__ + |> where([p], p.user_id == ^user.id) + |> where([p], not p.read) + |> update([p], set: [read: true]) + |> select([p], p) + |> Repo.update_all([]) + + User.set_unread_conversation_count(user) + {:ok, participations} + end + def mark_as_unread(participation) do participation |> read_cng(%{read: false}) diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index 9d50a7ca9..fc39abf05 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -79,6 +79,15 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do end end + def read_conversations(%{assigns: %{user: user}} = conn, _params) do + with {:ok, participations} <- Participation.mark_all_as_read(user) do + conn + |> add_link_headers(participations) + |> put_view(ConversationView) + |> render("participations.json", participations: participations, for: user) + end + end + def read_notification(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do with {:ok, notification} <- Notification.read_one(user, notification_id) do conn diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d68fb87da..66e7a2263 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -266,6 +266,7 @@ defmodule Pleroma.Web.Router do get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses) get("/conversations/:id", PleromaAPIController, :conversation) + post("/conversations/read", PleromaAPIController, :read_conversations) end scope [] do diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index a5af0d1b2..64c350904 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -133,6 +133,20 @@ defmodule Pleroma.Conversation.ParticipationTest do refute participation.read end + test "it marks all the user's participations as read" do + user = insert(:user) + other_user = insert(:user) + participation1 = insert(:participation, %{read: false, user: user}) + participation2 = insert(:participation, %{read: false, user: user}) + participation3 = insert(:participation, %{read: false, user: other_user}) + + {:ok, [%{read: true}, %{read: true}]} = Participation.mark_all_as_read(user) + + assert Participation.get(participation1.id).read == true + assert Participation.get(participation2.id).read == true + assert Participation.get(participation3.id).read == false + end + test "gets all the participations for a user, ordered by updated at descending" do user = insert(:user) {:ok, activity_one} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"}) diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs index 8a6528cbb..9cccc8c8a 100644 --- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs @@ -95,6 +95,33 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do assert other_user in participation.recipients end + test "POST /api/v1/pleroma/conversations/read", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hi @#{other_user.nickname}", "visibility" => "direct"}) + + {:ok, _activity} = + CommonAPI.post(user, %{"status" => "Hi @#{other_user.nickname}", "visibility" => "direct"}) + + [participation2, participation1] = Participation.for_user(other_user) + assert Participation.get(participation2.id).read == false + assert Participation.get(participation1.id).read == false + assert User.get_cached_by_id(other_user.id).info.unread_conversation_count == 2 + + [%{"unread" => false}, %{"unread" => false}] = + conn + |> assign(:user, other_user) + |> post("/api/v1/pleroma/conversations/read", %{}) + |> json_response(200) + + [participation2, participation1] = Participation.for_user(other_user) + assert Participation.get(participation2.id).read == true + assert Participation.get(participation1.id).read == true + assert User.get_cached_by_id(other_user.id).info.unread_conversation_count == 0 + end + describe "POST /api/v1/pleroma/notifications/read" do test "it marks a single notification as read", %{conn: conn} do user1 = insert(:user) From f94e28e8b6540884e4098e47fa15fc2c3657f896 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 19 Oct 2019 21:59:04 +0300 Subject: [PATCH 82/84] mix.exs: Bump version for develop builds As per discussion on irc, we already used GNU-style versioning system for release candidates, so it makes sense to use it for develop builds as well. --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 120092f1b..dcb9d9ea8 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("1.0.0"), + version: version("1.1.50"), elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), From cd90735a4b0dad01cca84d93b4fd671289e992b7 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sat, 19 Oct 2019 23:16:15 +0000 Subject: [PATCH 83/84] tests: ostatus controller tests: fix sporadic compilation failure --- test/web/ostatus/ostatus_controller_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 58534396e..37b7b62f5 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do use Pleroma.Web.ConnCase - import ExUnit.CaptureLog import Pleroma.Factory alias Pleroma.Object From 34754938ba27a71212899f56f8e089a82facd111 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Sun, 20 Oct 2019 10:39:03 +0000 Subject: [PATCH 84/84] add missing migration to drop websub table --- .../repo/migrations/20191017225002_drop_websub_tables.exs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 priv/repo/migrations/20191017225002_drop_websub_tables.exs diff --git a/priv/repo/migrations/20191017225002_drop_websub_tables.exs b/priv/repo/migrations/20191017225002_drop_websub_tables.exs new file mode 100644 index 000000000..9e483b2a1 --- /dev/null +++ b/priv/repo/migrations/20191017225002_drop_websub_tables.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.DropWebsubTables do + use Ecto.Migration + + def change do + drop_if_exists(table(:websub_client_subscriptions)) + drop_if_exists(table(:websub_server_subscriptions)) + end +end