exclude replies on blocked domains

This commit is contained in:
Alexander Strizhakov 2020-05-20 17:45:06 +03:00
parent fa37bfff99
commit 7e6ec778d9
No known key found for this signature in database
GPG Key ID: 022896A53AEF1381
9 changed files with 469 additions and 175 deletions

View File

@ -22,8 +22,21 @@ defmodule Pleroma.LoadTesting.Activities do
@max_concurrency 10 @max_concurrency 10
@visibility ~w(public private direct unlisted) @visibility ~w(public private direct unlisted)
@types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote) @types [
@groups ~w(user friends non_friends) :simple,
:emoji,
:mentions,
:hell_thread,
:attachment,
:tag,
:like,
:reblog,
:simple_thread
]
@groups [:friends_local, :friends_remote, :non_friends_local, :non_friends_local]
@remote_groups [:friends_remote, :non_friends_remote]
@friends_groups [:friends_local, :friends_remote]
@non_friends_groups [:non_friends_local, :non_friends_remote]
@spec generate(User.t(), keyword()) :: :ok @spec generate(User.t(), keyword()) :: :ok
def generate(user, opts \\ []) do def generate(user, opts \\ []) do
@ -34,33 +47,24 @@ defmodule Pleroma.LoadTesting.Activities do
opts = Keyword.merge(@defaults, opts) opts = Keyword.merge(@defaults, opts)
friends = users = Users.prepare_users(user, opts)
user
|> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true)
|> Enum.shuffle()
non_friends = {:ok, _} = Agent.start_link(fn -> users[:non_friends_remote] end, name: :non_friends_remote)
user
|> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false)
|> Enum.shuffle()
task_data = task_data =
for visibility <- @visibility, for visibility <- @visibility,
type <- @types, type <- @types,
group <- @groups, group <- [:user | @groups],
do: {visibility, type, group} do: {visibility, type, group}
IO.puts("Starting generating #{opts[:iterations]} iterations of activities...") IO.puts("Starting generating #{opts[:iterations]} iterations of activities...")
friends_thread = Enum.take(friends, 5)
non_friends_thread = Enum.take(friends, 5)
public_long_thread = fn -> public_long_thread = fn ->
generate_long_thread("public", user, friends_thread, non_friends_thread, opts) generate_long_thread("public", users, opts)
end end
private_long_thread = fn -> private_long_thread = fn ->
generate_long_thread("private", user, friends_thread, non_friends_thread, opts) generate_long_thread("private", users, opts)
end end
iterations = opts[:iterations] iterations = opts[:iterations]
@ -73,10 +77,10 @@ defmodule Pleroma.LoadTesting.Activities do
i when i == iterations - 2 -> i when i == iterations - 2 ->
spawn(public_long_thread) spawn(public_long_thread)
spawn(private_long_thread) spawn(private_long_thread)
generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) generate_activities(users, Enum.shuffle(task_data), opts)
_ -> _ ->
generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) generate_activities(users, Enum.shuffle(task_data), opts)
end end
) )
end) end)
@ -127,16 +131,16 @@ defmodule Pleroma.LoadTesting.Activities do
end) end)
end end
defp generate_long_thread(visibility, user, friends, non_friends, _opts) do defp generate_long_thread(visibility, users, _opts) do
group = group =
if visibility == "public", if visibility == "public",
do: "friends", do: :friends_local,
else: "user" else: :user
tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50) tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50)
{:ok, activity} = {:ok, activity} =
CommonAPI.post(user, %{ CommonAPI.post(users[:user], %{
status: "Start of #{visibility} long thread", status: "Start of #{visibility} long thread",
visibility: visibility visibility: visibility
}) })
@ -150,31 +154,28 @@ defmodule Pleroma.LoadTesting.Activities do
Map.put(state, key, activity) Map.put(state, key, activity)
end) end)
acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]} acc = {activity.id, ["@" <> users[:user].nickname, "reply to long thread"]}
insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) insert_replies_for_long_thread(tasks, visibility, users, acc)
IO.puts("Generating #{visibility} long thread ended\n") IO.puts("Generating #{visibility} long thread ended\n")
end end
defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do defp insert_replies_for_long_thread(tasks, visibility, users, acc) do
Enum.reduce(tasks, acc, fn Enum.reduce(tasks, acc, fn
"friend", {id, data} -> :user, {id, data} ->
friend = Enum.random(friends) user = users[:user]
insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility)
"non_friend", {id, data} ->
non_friend = Enum.random(non_friends)
insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility)
"user", {id, data} ->
insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility) insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility)
group, {id, data} ->
replier = Enum.random(users[group])
insert_reply(replier, List.delete(data, "@" <> replier.nickname), id, visibility)
end) end)
end end
defp generate_activities(user, friends, non_friends, task_data, opts) do defp generate_activities(users, task_data, opts) do
Task.async_stream( Task.async_stream(
task_data, task_data,
fn {visibility, type, group} -> fn {visibility, type, group} ->
insert_activity(type, visibility, group, user, friends, non_friends, opts) insert_activity(type, visibility, group, users, opts)
end, end,
max_concurrency: @max_concurrency, max_concurrency: @max_concurrency,
timeout: 30_000 timeout: 30_000
@ -182,47 +183,46 @@ defmodule Pleroma.LoadTesting.Activities do
|> Stream.run() |> Stream.run()
end end
defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do defp insert_local_activity(visibility, group, users, status) do
{:ok, _activity} = {:ok, _} =
group group
|> get_actor(user, friends, non_friends) |> get_actor(users)
|> CommonAPI.post(%{status: "Simple status", visibility: visibility}) |> CommonAPI.post(%{status: status, visibility: visibility})
end end
defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do defp insert_remote_activity(visibility, group, users, status) do
{:ok, _activity} = actor = get_actor(group, users)
group {act_data, obj_data} = prepare_activity_data(actor, visibility, users[:user])
|> get_actor(user, friends, non_friends) {activity_data, object_data} = other_data(actor, status)
|> CommonAPI.post(%{
status: "Simple status with emoji :firefox:", activity_data
visibility: visibility |> Map.merge(act_data)
}) |> Map.put("object", Map.merge(object_data, obj_data))
|> Pleroma.Web.ActivityPub.ActivityPub.insert(false)
end end
defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do defp user_mentions(users) do
user_mentions = user_mentions =
get_random_mentions(friends, Enum.random(0..3)) ++ Enum.reduce(
get_random_mentions(non_friends, Enum.random(0..3)) @groups,
[],
fn group, acc ->
acc ++ get_random_mentions(users[group], Enum.random(0..2))
end
)
user_mentions =
if Enum.random([true, false]), if Enum.random([true, false]),
do: ["@" <> user.nickname | user_mentions], do: ["@" <> users[:user].nickname | user_mentions],
else: user_mentions else: user_mentions
{:ok, _activity} =
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{
status: Enum.join(user_mentions, ", ") <> " simple status with mentions",
visibility: visibility
})
end end
defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do defp hell_thread_mentions(users) do
mentions =
with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do
cached = cached =
([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10)) @groups
|> Enum.reduce([users[:user]], fn group, acc ->
acc ++ Enum.take(users[group], 5)
end)
|> Enum.map(&"@#{&1.nickname}") |> Enum.map(&"@#{&1.nickname}")
|> Enum.join(", ") |> Enum.join(", ")
@ -231,18 +231,56 @@ defmodule Pleroma.LoadTesting.Activities do
else else
{:ok, cached} -> cached {:ok, cached} -> cached
end end
{:ok, _activity} =
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{
status: mentions <> " hell thread status",
visibility: visibility
})
end end
defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do defp insert_activity(:simple, visibility, group, users, _opts)
actor = get_actor(group, user, friends, non_friends) when group in @remote_groups do
insert_remote_activity(visibility, group, users, "Remote status")
end
defp insert_activity(:simple, visibility, group, users, _opts) do
insert_local_activity(visibility, group, users, "Simple status")
end
defp insert_activity(:emoji, visibility, group, users, _opts)
when group in @remote_groups do
insert_remote_activity(visibility, group, users, "Remote status with emoji :firefox:")
end
defp insert_activity(:emoji, visibility, group, users, _opts) do
insert_local_activity(visibility, group, users, "Simple status with emoji :firefox:")
end
defp insert_activity(:mentions, visibility, group, users, _opts)
when group in @remote_groups do
mentions = user_mentions(users)
status = Enum.join(mentions, ", ") <> " remote status with mentions"
insert_remote_activity(visibility, group, users, status)
end
defp insert_activity(:mentions, visibility, group, users, _opts) do
mentions = user_mentions(users)
status = Enum.join(mentions, ", ") <> " simple status with mentions"
insert_remote_activity(visibility, group, users, status)
end
defp insert_activity(:hell_thread, visibility, group, users, _)
when group in @remote_groups do
mentions = hell_thread_mentions(users)
insert_remote_activity(visibility, group, users, mentions <> " remote hell thread status")
end
defp insert_activity(:hell_thread, visibility, group, users, _opts) do
mentions = hell_thread_mentions(users)
insert_local_activity(visibility, group, users, mentions <> " hell thread status")
end
defp insert_activity(:attachment, visibility, group, users, _opts) do
actor = get_actor(group, users)
obj_data = %{ obj_data = %{
"actor" => actor.ap_id, "actor" => actor.ap_id,
@ -268,67 +306,54 @@ defmodule Pleroma.LoadTesting.Activities do
}) })
end end
defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do defp insert_activity(:tag, visibility, group, users, _opts) do
{:ok, _activity} = insert_local_activity(visibility, group, users, "Status with #tag")
group
|> get_actor(user, friends, non_friends)
|> CommonAPI.post(%{status: "Status with #tag", visibility: visibility})
end end
defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do defp insert_activity(:like, visibility, group, users, opts) do
actor = get_actor(group, user, friends, non_friends) actor = get_actor(group, users)
with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
{:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do {:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do
:ok :ok
else else
{:error, _} -> {:error, _} ->
insert_activity("like", visibility, group, user, friends, non_friends, opts) insert_activity(:like, visibility, group, users, opts)
nil -> nil ->
Process.sleep(15) Process.sleep(15)
insert_activity("like", visibility, group, user, friends, non_friends, opts) insert_activity(:like, visibility, group, users, opts)
end end
end end
defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do defp insert_activity(:reblog, visibility, group, users, opts) do
actor = get_actor(group, user, friends, non_friends) actor = get_actor(group, users)
with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
{:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do {:ok, _activity} <- CommonAPI.repeat(activity_id, actor) do
:ok :ok
else else
{:error, _} -> {:error, _} ->
insert_activity("reblog", visibility, group, user, friends, non_friends, opts) insert_activity(:reblog, visibility, group, users, opts)
nil -> nil ->
Process.sleep(15) Process.sleep(15)
insert_activity("reblog", visibility, group, user, friends, non_friends, opts) insert_activity(:reblog, visibility, group, users, opts)
end end
end end
defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts) defp insert_activity(:simple_thread, "direct", group, users, _opts) do
when visibility in ["public", "unlisted", "private"] do actor = get_actor(group, users)
actor = get_actor(group, user, friends, non_friends)
tasks = get_reply_tasks(visibility, group)
{:ok, activity} = CommonAPI.post(user, %{status: "Simple status", visibility: visibility})
acc = {activity.id, ["@" <> actor.nickname, "reply to status"]}
insert_replies(tasks, visibility, user, friends, non_friends, acc)
end
defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do
actor = get_actor(group, user, friends, non_friends)
tasks = get_reply_tasks("direct", group) tasks = get_reply_tasks("direct", group)
list = list =
case group do case group do
"non_friends" -> :user ->
Enum.take(non_friends, 3) group = Enum.random(@friends_groups)
Enum.take(users[group], 3)
_ -> _ ->
Enum.take(friends, 3) Enum.take(users[group], 3)
end end
data = Enum.map(list, &("@" <> &1.nickname)) data = Enum.map(list, &("@" <> &1.nickname))
@ -339,40 +364,30 @@ defmodule Pleroma.LoadTesting.Activities do
visibility: "direct" visibility: "direct"
}) })
acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]} acc = {activity.id, ["@" <> users[:user].nickname | data] ++ ["reply to status"]}
insert_direct_replies(tasks, user, list, acc) insert_direct_replies(tasks, users[:user], list, acc)
end end
defp insert_activity("remote", _, "user", _, _, _, _), do: :ok defp insert_activity(:simple_thread, visibility, group, users, _opts) do
actor = get_actor(group, users)
tasks = get_reply_tasks(visibility, group)
defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do {:ok, activity} =
remote_friends = CommonAPI.post(users[:user], %{status: "Simple status", visibility: visibility})
Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true)
remote_non_friends = acc = {activity.id, ["@" <> actor.nickname, "reply to status"]}
Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false) insert_replies(tasks, visibility, users, acc)
actor = get_actor(group, user, remote_friends, remote_non_friends)
{act_data, obj_data} = prepare_activity_data(actor, visibility, user)
{activity_data, object_data} = other_data(actor)
activity_data
|> Map.merge(act_data)
|> Map.put("object", Map.merge(object_data, obj_data))
|> Pleroma.Web.ActivityPub.ActivityPub.insert(false)
end end
defp get_actor("user", user, _friends, _non_friends), do: user defp get_actor(:user, %{user: user}), do: user
defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends) defp get_actor(group, users), do: Enum.random(users[group])
defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends)
defp other_data(actor) do defp other_data(actor, content) do
%{host: host} = URI.parse(actor.ap_id) %{host: host} = URI.parse(actor.ap_id)
datetime = DateTime.utc_now() datetime = DateTime.utc_now()
context_id = "http://#{host}:4000/contexts/#{UUID.generate()}" context_id = "https://#{host}/contexts/#{UUID.generate()}"
activity_id = "http://#{host}:4000/activities/#{UUID.generate()}" activity_id = "https://#{host}/activities/#{UUID.generate()}"
object_id = "http://#{host}:4000/objects/#{UUID.generate()}" object_id = "https://#{host}/objects/#{UUID.generate()}"
activity_data = %{ activity_data = %{
"actor" => actor.ap_id, "actor" => actor.ap_id,
@ -389,7 +404,7 @@ defmodule Pleroma.LoadTesting.Activities do
"attributedTo" => actor.ap_id, "attributedTo" => actor.ap_id,
"bcc" => [], "bcc" => [],
"bto" => [], "bto" => [],
"content" => "Remote post", "content" => content,
"context" => context_id, "context" => context_id,
"conversation" => context_id, "conversation" => context_id,
"emoji" => %{}, "emoji" => %{},
@ -475,51 +490,65 @@ defmodule Pleroma.LoadTesting.Activities do
{act_data, obj_data} {act_data, obj_data}
end end
defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user) defp get_reply_tasks("public", :user) do
defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend) [:friends_local, :friends_remote, :non_friends_local, :non_friends_remote, :user]
defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend) end
defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"], defp get_reply_tasks("public", group) when group in @friends_groups do
do: ~w(friend user friend) [:non_friends_local, :non_friends_remote, :user, :friends_local, :friends_remote]
end
defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"], defp get_reply_tasks("public", group) when group in @non_friends_groups do
do: ~w(user friend user) [:user, :friends_local, :friends_remote, :non_friends_local, :non_friends_remote]
end
defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"], defp get_reply_tasks(visibility, :user) when visibility in ["unlisted", "private"] do
[:friends_local, :friends_remote, :user, :friends_local, :friends_remote]
end
defp get_reply_tasks(visibility, group)
when visibility in ["unlisted", "private"] and group in @friends_groups do
[:user, :friends_remote, :friends_local, :user]
end
defp get_reply_tasks(visibility, group)
when visibility in ["unlisted", "private"] and
group in @non_friends_groups,
do: [] do: []
defp get_reply_tasks("direct", "user"), do: ~w(friend user friend) defp get_reply_tasks("direct", :user), do: [:friends_local, :user, :friends_remote]
defp get_reply_tasks("direct", "friends"), do: ~w(user friend user)
defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user)
defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do defp get_reply_tasks("direct", group) when group in @friends_groups,
do: [:user, group, :user]
defp get_reply_tasks("direct", group) when group in @non_friends_groups do
[:user, :non_friends_remote, :user, :non_friends_local]
end
defp insert_replies(tasks, visibility, users, acc) do
Enum.reduce(tasks, acc, fn Enum.reduce(tasks, acc, fn
"friend", {id, data} -> :user, {id, data} ->
friend = Enum.random(friends) insert_reply(users[:user], data, id, visibility)
insert_reply(friend, data, id, visibility)
"non_friend", {id, data} -> group, {id, data} ->
non_friend = Enum.random(non_friends) replier = Enum.random(users[group])
insert_reply(non_friend, data, id, visibility) insert_reply(replier, data, id, visibility)
"user", {id, data} ->
insert_reply(user, data, id, visibility)
end) end)
end end
defp insert_direct_replies(tasks, user, list, acc) do defp insert_direct_replies(tasks, user, list, acc) do
Enum.reduce(tasks, acc, fn Enum.reduce(tasks, acc, fn
group, {id, data} when group in ["friend", "non_friend"] -> :user, {id, data} ->
{reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct")
{reply_id, data}
_, {id, data} ->
actor = Enum.random(list) actor = Enum.random(list)
{reply_id, _} = {reply_id, _} =
insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct") insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct")
{reply_id, data} {reply_id, data}
"user", {id, data} ->
{reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct")
{reply_id, data}
end) end)
end end

View File

@ -36,6 +36,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
fetch_home_timeline(user) fetch_home_timeline(user)
fetch_direct_timeline(user) fetch_direct_timeline(user)
fetch_public_timeline(user) fetch_public_timeline(user)
fetch_public_timeline(user, :with_blocks)
fetch_public_timeline(user, :local) fetch_public_timeline(user, :local)
fetch_public_timeline(user, :tag) fetch_public_timeline(user, :tag)
fetch_notifications(user) fetch_notifications(user)
@ -227,6 +228,76 @@ defmodule Pleroma.LoadTesting.Fetcher do
fetch_public_timeline(opts, "public timeline only media") fetch_public_timeline(opts, "public timeline only media")
end end
# TODO: remove using `:method` after benchmarks
defp fetch_public_timeline(user, :with_blocks) do
opts = opts_for_public_timeline(user)
remote_non_friends = Agent.get(:non_friends_remote, & &1)
Benchee.run(
%{
"public timeline without blocks" => fn opts ->
ActivityPub.fetch_public_activities(opts)
end
},
inputs: %{
"old filtering" => Map.delete(opts, :method),
"with psql fun" => Map.put(opts, :method, :fun),
"with unnest" => Map.put(opts, :method, :unnest)
}
)
Enum.each(remote_non_friends, fn non_friend ->
{:ok, _} = User.block(user, non_friend)
end)
user = User.get_by_id(user.id)
opts = Map.put(opts, "blocking_user", user)
Benchee.run(
%{
"public timeline with user block" => fn opts ->
ActivityPub.fetch_public_activities(opts)
end
},
inputs: %{
"old filtering" => Map.delete(opts, :method),
"with psql fun" => Map.put(opts, :method, :fun),
"with unnest" => Map.put(opts, :method, :unnest)
}
)
domains =
Enum.reduce(remote_non_friends, [], fn non_friend, domains ->
{:ok, _user} = User.unblock(user, non_friend)
%{host: host} = URI.parse(non_friend.ap_id)
[host | domains]
end)
domains = Enum.uniq(domains)
Enum.each(domains, fn domain ->
{:ok, _} = User.block_domain(user, domain)
end)
user = User.get_by_id(user.id)
opts = Map.put(opts, "blocking_user", user)
Benchee.run(
%{
"public timeline with domain block" => fn opts ->
ActivityPub.fetch_public_activities(opts)
end
},
inputs: %{
"old filtering" => Map.delete(opts, :method),
"with psql fun" => Map.put(opts, :method, :fun),
"with unnest" => Map.put(opts, :method, :unnest)
}
)
end
defp fetch_public_timeline(opts, title) when is_binary(title) do defp fetch_public_timeline(opts, title) when is_binary(title) do
first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last() first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last()

View File

@ -27,7 +27,7 @@ defmodule Pleroma.LoadTesting.Users do
make_friends(main_user, opts[:friends]) make_friends(main_user, opts[:friends])
Repo.get(User, main_user.id) User.get_by_id(main_user.id)
end end
def generate_users(max) do def generate_users(max) do
@ -166,4 +166,24 @@ defmodule Pleroma.LoadTesting.Users do
) )
|> Stream.run() |> Stream.run()
end end
@spec prepare_users(User.t(), keyword()) :: map()
def prepare_users(user, opts) do
friends_limit = opts[:friends_used]
non_friends_limit = opts[:non_friends_used]
%{
user: user,
friends_local: fetch_users(user, friends_limit, :local, true),
friends_remote: fetch_users(user, friends_limit, :external, true),
non_friends_local: fetch_users(user, non_friends_limit, :local, false),
non_friends_remote: fetch_users(user, non_friends_limit, :external, false)
}
end
defp fetch_users(user, limit, local, friends?) do
user
|> get_users(limit: limit, local: local, friends?: friends?)
|> Enum.shuffle()
end
end end

View File

@ -5,7 +5,6 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
import Ecto.Query import Ecto.Query
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.Web.MastodonAPI.TimelineController
def run(_args) do def run(_args) do
Mix.Pleroma.start_pleroma() Mix.Pleroma.start_pleroma()
@ -37,7 +36,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
Benchee.run( Benchee.run(
%{ %{
"Hashtag fetching, any" => fn tags -> "Hashtag fetching, any" => fn tags ->
TimelineController.hashtag_fetching( hashtag_fetching(
%{ %{
"any" => tags "any" => tags
}, },
@ -47,7 +46,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
end, end,
# Will always return zero results because no overlapping hashtags are generated. # Will always return zero results because no overlapping hashtags are generated.
"Hashtag fetching, all" => fn tags -> "Hashtag fetching, all" => fn tags ->
TimelineController.hashtag_fetching( hashtag_fetching(
%{ %{
"all" => tags "all" => tags
}, },
@ -67,7 +66,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
Benchee.run( Benchee.run(
%{ %{
"Hashtag fetching" => fn tag -> "Hashtag fetching" => fn tag ->
TimelineController.hashtag_fetching( hashtag_fetching(
%{ %{
"tag" => tag "tag" => tag
}, },
@ -80,4 +79,35 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
time: 5 time: 5
) )
end end
defp hashtag_fetching(params, user, local_only) do
tags =
[params["tag"], params["any"]]
|> List.flatten()
|> Enum.uniq()
|> Enum.filter(& &1)
|> Enum.map(&String.downcase(&1))
tag_all =
params
|> Map.get("all", [])
|> Enum.map(&String.downcase(&1))
tag_reject =
params
|> Map.get("none", [])
|> Enum.map(&String.downcase(&1))
_activities =
params
|> Map.put("type", "Create")
|> Map.put("local_only", local_only)
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("tag", tags)
|> Map.put("tag_all", tag_all)
|> Map.put("tag_reject", tag_reject)
|> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
end
end end

View File

@ -932,6 +932,33 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
query = query =
if has_named_binding?(query, :object), do: query, else: Activity.with_joined_object(query) if has_named_binding?(query, :object), do: query, else: Activity.with_joined_object(query)
# TODO: update after benchmarks
query =
case opts[:method] do
:fun ->
from(a in query,
where:
fragment(
"recipients_contain_blocked_domains(?, ?) = false",
a.recipients,
^domain_blocks
)
)
:unnest ->
from(a in query,
where:
fragment(
"NOT ? && (SELECT ARRAY(SELECT split_part(UNNEST(?), '/', 3)))",
^domain_blocks,
a.recipients
)
)
_ ->
query
end
from( from(
[activity, object: o] in query, [activity, object: o] in query,
where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids), where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids),

View File

@ -62,6 +62,13 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
only_media_param(), only_media_param(),
with_muted_param(), with_muted_param(),
exclude_visibilities_param(), exclude_visibilities_param(),
# TODO: remove after benchmarks
Operation.parameter(
:method,
:query,
%Schema{type: :string},
"Temp parameter"
),
reply_visibility_param() | pagination_params() reply_visibility_param() | pagination_params()
], ],
operationId: "TimelineController.public", operationId: "TimelineController.public",

View File

@ -109,14 +109,23 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
if restrict? and is_nil(user) do if restrict? and is_nil(user) do
render_error(conn, :unauthorized, "authorization required for timeline view") render_error(conn, :unauthorized, "authorization required for timeline view")
else else
activities = # TODO: return back after benchmarks
params =
params params
|> Map.put("type", ["Create", "Announce"]) |> Map.put("type", ["Create", "Announce"])
|> Map.put("local_only", local_only) |> Map.put("local_only", local_only)
|> Map.put("blocking_user", user) |> Map.put("blocking_user", user)
|> Map.put("muting_user", user) |> Map.put("muting_user", user)
|> Map.put("reply_filtering_user", user) |> Map.put("reply_filtering_user", user)
|> ActivityPub.fetch_public_activities()
params =
if params["method"] do
Map.put(params, :method, String.to_existing_atom(params["method"]))
else
params
end
activities = ActivityPub.fetch_public_activities(params)
conn conn
|> add_link_headers(activities, %{"local" => local_only}) |> add_link_headers(activities, %{"local" => local_only})

View File

@ -0,0 +1,33 @@
defmodule Pleroma.Repo.Migrations.AddRecipientsContainBlockedDomainsFunction do
use Ecto.Migration
@disable_ddl_transaction true
def up do
statement = """
CREATE OR REPLACE FUNCTION recipients_contain_blocked_domains(recipients varchar[], blocked_domains varchar[]) RETURNS boolean AS $$
DECLARE
recipient_domain varchar;
recipient varchar;
BEGIN
FOREACH recipient IN ARRAY recipients LOOP
recipient_domain = split_part(recipient, '/', 3)::varchar;
IF recipient_domain = ANY(blocked_domains) THEN
RETURN TRUE;
END IF;
END LOOP;
RETURN FALSE;
END;
$$ LANGUAGE plpgsql;
"""
execute(statement)
end
def down do
execute(
"drop function if exists recipients_contain_blocked_domains(recipients varchar[], blocked_domains varchar[])"
)
end
end

View File

@ -90,6 +90,74 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
res_conn = get(conn, "/api/v1/timelines/public") res_conn = get(conn, "/api/v1/timelines/public")
assert length(json_response_and_validate_schema(res_conn, 200)) == 1 assert length(json_response_and_validate_schema(res_conn, 200)) == 1
end end
test "doesn't return replies if follower is posting with blocked user" do
%{conn: conn, user: blocker} = oauth_access(["read:statuses"])
[blockee, friend] = insert_list(2, :user)
{:ok, blocker} = User.follow(blocker, friend)
{:ok, _} = User.block(blocker, blockee)
conn = assign(conn, :user, blocker)
{:ok, %{id: activity_id} = activity} = CommonAPI.post(friend, %{status: "hey!"})
{:ok, reply_from_blockee} =
CommonAPI.post(blockee, %{status: "heya", in_reply_to_status_id: activity})
{:ok, _reply_from_friend} =
CommonAPI.post(friend, %{status: "status", in_reply_to_status_id: reply_from_blockee})
res_conn = get(conn, "/api/v1/timelines/public")
[%{"id" => ^activity_id}] = json_response_and_validate_schema(res_conn, 200)
end
# TODO: update after benchmarks
test "doesn't return replies if follow is posting with users from blocked domain" do
%{conn: conn, user: blocker} = oauth_access(["read:statuses"])
friend = insert(:user)
blockee = insert(:user, ap_id: "https://example.com/users/blocked")
{:ok, blocker} = User.follow(blocker, friend)
{:ok, blocker} = User.block_domain(blocker, "example.com")
conn = assign(conn, :user, blocker)
{:ok, %{id: activity_id} = activity} = CommonAPI.post(friend, %{status: "hey!"})
{:ok, reply_from_blockee} =
CommonAPI.post(blockee, %{status: "heya", in_reply_to_status_id: activity})
{:ok, _reply_from_friend} =
CommonAPI.post(friend, %{status: "status", in_reply_to_status_id: reply_from_blockee})
res_conn = get(conn, "/api/v1/timelines/public?method=fun")
activities = json_response_and_validate_schema(res_conn, 200)
[%{"id" => ^activity_id}] = activities
end
# TODO: update after benchmarks
test "doesn't return replies if follow is posting with users from blocked domain with unnest param" do
%{conn: conn, user: blocker} = oauth_access(["read:statuses"])
friend = insert(:user)
blockee = insert(:user, ap_id: "https://example.com/users/blocked")
{:ok, blocker} = User.follow(blocker, friend)
{:ok, blocker} = User.block_domain(blocker, "example.com")
conn = assign(conn, :user, blocker)
{:ok, %{id: activity_id} = activity} = CommonAPI.post(friend, %{status: "hey!"})
{:ok, reply_from_blockee} =
CommonAPI.post(blockee, %{status: "heya", in_reply_to_status_id: activity})
{:ok, _reply_from_friend} =
CommonAPI.post(friend, %{status: "status", in_reply_to_status_id: reply_from_blockee})
res_conn = get(conn, "/api/v1/timelines/public?method=unnest")
activities = json_response_and_validate_schema(res_conn, 200)
[%{"id" => ^activity_id}] = activities
end
end end
defp local_and_remote_activities do defp local_and_remote_activities do