From c1b6952d2abe55668f240947e734127664a4cefa Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 13 Aug 2019 20:34:34 +0300 Subject: [PATCH] Mastodon API: Preloading and normalization optimizations - Try to normalize the activity instead of object wherever possible - Put the `user` key on non-home timelines as well so bookmarks and thread mutes are preloaded there as well - Skip trying to get the user when rendering mentions if the id == as:Public or user's follower collection - Preload the object when getting replied to activities and do not crash if it's not present This almost solves the problem of Pleroma hammering the db with a lot of queries when rendering timelines, the things left are 1. When rendering mentions and the user is not in cache, save it for later and request all uncached users in one go 2. Somehow get rid of needing to get the latest follow activity to detect the value of `requested` in a relationship. (create a database view for user relationship and cache it maybe?) --- lib/pleroma/activity.ex | 27 +++++++++++++++++-- .../mastodon_api/mastodon_api_controller.ex | 11 +++++--- .../web/mastodon_api/views/status_view.ex | 20 +++++++++----- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 46552c7be..baf1e7722 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -224,6 +224,29 @@ defmodule Pleroma.Activity do def get_create_by_object_ap_id(_), do: nil + def create_by_object_ap_id_with_object(ap_ids) when is_list(ap_ids) do + from( + activity in Activity, + where: + fragment( + "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)", + activity.data, + activity.data, + ^ap_ids + ), + where: fragment("(?)->>'type' = 'Create'", activity.data), + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", + o.data, + activity.data, + activity.data + ), + preload: [object: o] + ) + end + def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do from( activity in Activity, @@ -263,8 +286,8 @@ defmodule Pleroma.Activity do defp get_in_reply_to_activity_from_object(_), do: nil - def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do - get_in_reply_to_activity_from_object(Object.normalize(object)) + def get_in_reply_to_activity(%Activity{} = activity) do + get_in_reply_to_activity_from_object(Object.normalize(activity)) end def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"]) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 174e93468..96e0d82aa 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -435,6 +435,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Map.put("local_only", local_only) |> Map.put("blocking_user", user) |> Map.put("muting_user", user) + |> Map.put("user", user) |> ActivityPub.fetch_public_activities() |> Enum.reverse() @@ -885,8 +886,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{data: %{"object" => object}} <- Activity.get_by_id(id), - %Object{data: %{"likes" => likes}} <- Object.normalize(object) do + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{data: %{"likes" => likes}} <- Object.normalize(activity) do q = from(u in User, where: u.ap_id in ^likes) users = @@ -902,8 +903,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{data: %{"object" => object}} <- Activity.get_by_id(id), - %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{data: %{"announcements" => announces}} <- Object.normalize(activity) do q = from(u in User, where: u.ap_id in ^announces) users = @@ -944,6 +945,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Map.put("local_only", local_only) |> Map.put("blocking_user", user) |> Map.put("muting_user", user) + |> Map.put("user", user) |> Map.put("tag", tags) |> Map.put("tag_all", tag_all) |> Map.put("tag_reject", tag_reject) @@ -1350,6 +1352,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do params |> Map.put("type", "Create") |> Map.put("blocking_user", user) + |> Map.put("user", user) |> Map.put("muting_user", user) # we must filter the following list for the user to avoid leaking statuses the user diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 02819e116..492af1702 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do use Pleroma.Web, :view + require Pleroma.Constants + alias Pleroma.Activity alias Pleroma.HTML alias Pleroma.Object @@ -24,19 +26,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_replied_to_activities(activities) do activities |> Enum.map(fn - %{data: %{"type" => "Create", "object" => object}} -> - object = Object.normalize(object) - object.data["inReplyTo"] != "" && object.data["inReplyTo"] + %{data: %{"type" => "Create"}} = activity -> + object = Object.normalize(activity) + object && object.data["inReplyTo"] != "" && object.data["inReplyTo"] _ -> nil end) |> Enum.filter(& &1) - |> Activity.create_by_object_ap_id() + |> Activity.create_by_object_ap_id_with_object() |> Repo.all() |> Enum.reduce(%{}, fn activity, acc -> object = Object.normalize(activity) - Map.put(acc, object.data["id"], activity) + if object, do: Map.put(acc, object.data["id"], activity), else: acc end) end @@ -88,6 +90,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogged_activity = Activity.create_by_object_ap_id(activity_object.data["id"]) |> Activity.with_preloaded_bookmark(opts[:for]) + |> Activity.with_set_thread_muted_field(opts[:for]) |> Repo.one() reblogged = render("status.json", Map.put(opts, :activity, reblogged_activity)) @@ -142,6 +145,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do object = Object.normalize(activity) user = get_user(activity.data["actor"]) + user_follower_address = user.follower_address like_count = object.data["like_count"] || 0 announcement_count = object.data["announcement_count"] || 0 @@ -157,7 +161,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do mentions = (object.data["to"] ++ tag_mentions) |> Enum.uniq() - |> Enum.map(fn ap_id -> User.get_cached_by_ap_id(ap_id) end) + |> Enum.map(fn + Pleroma.Constants.as_public() -> nil + ^user_follower_address -> nil + ap_id -> User.get_cached_by_ap_id(ap_id) + end) |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end)