[#3213] Performance optimization of filtering by hashtags ("any" condition).

This commit is contained in:
Ivan Tashkinov 2021-03-07 11:33:21 +03:00
parent 5856f51717
commit 7f8785fd9b
3 changed files with 53 additions and 36 deletions

View File

@ -93,6 +93,7 @@ defmodule Pleroma.Pagination do
max_id: :string, max_id: :string,
offset: :integer, offset: :integer,
limit: :integer, limit: :integer,
skip_extra_order: :boolean,
skip_order: :boolean skip_order: :boolean
} }
@ -114,6 +115,8 @@ defmodule Pleroma.Pagination do
defp restrict(query, :order, %{skip_order: true}, _), do: query defp restrict(query, :order, %{skip_order: true}, _), do: query
defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query
defp restrict(query, :order, %{min_id: _}, table_binding) do defp restrict(query, :order, %{min_id: _}, table_binding) do
order_by( order_by(
query, query,

View File

@ -466,6 +466,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> Repo.one() |> Repo.one()
end end
defp fetch_paginated_optimized(query, opts, pagination) do
# Note: tag-filtering funcs may apply "ORDER BY objects.id DESC",
# and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan
opts = Map.put(opts, :skip_extra_order, true)
Pagination.fetch_paginated(query, opts, pagination)
end
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
list_memberships = Pleroma.List.memberships(opts[:user])
fetch_activities_query(recipients ++ list_memberships, opts)
|> fetch_paginated_optimized(opts, pagination)
|> Enum.reverse()
|> maybe_update_cc(list_memberships, opts[:user])
end
@spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()] @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
opts = Map.delete(opts, :user) opts = Map.delete(opts, :user)
@ -473,7 +490,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
[Constants.as_public()] [Constants.as_public()]
|> fetch_activities_query(opts) |> fetch_activities_query(opts)
|> restrict_unlisted(opts) |> restrict_unlisted(opts)
|> Pagination.fetch_paginated(opts, pagination) |> fetch_paginated_optimized(opts, pagination)
end end
@spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()] @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
@ -751,6 +768,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id) |> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id)
|> where([hto, ht], ht.name in ^tags) |> where([hto, ht], ht.name in ^tags)
|> select([hto], hto.object_id) |> select([hto], hto.object_id)
|> distinct([hto], true)
end end
defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
@ -789,9 +807,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end end
defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do
hashtag_ids =
from(ht in Hashtag, where: ht.name in ^tags, select: ht.id)
|> Repo.all()
# Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan
from( from(
[_activity, object] in query, [_activity, object] in query,
where: object.id in subquery(object_ids_query_for_tags(tags)) join: hto in "hashtags_objects",
on: hto.object_id == object.id,
where: hto.hashtag_id in ^hashtag_ids,
distinct: [desc: object.id],
order_by: [desc: object.id]
) )
end end
@ -1188,7 +1215,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
Map.put(opts, key, Hashtag.normalize_name(value)) Map.put(opts, key, Hashtag.normalize_name(value))
value when is_list(value) -> value when is_list(value) ->
Map.put(opts, key, Enum.map(value, &Hashtag.normalize_name/1)) normalized_value =
value
|> Enum.map(&Hashtag.normalize_name/1)
|> Enum.uniq()
Map.put(opts, key, normalized_value)
_ -> _ ->
opts opts
@ -1275,15 +1307,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end end
end end
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
list_memberships = Pleroma.List.memberships(opts[:user])
fetch_activities_query(recipients ++ list_memberships, opts)
|> Pagination.fetch_paginated(opts, pagination)
|> Enum.reverse()
|> maybe_update_cc(list_memberships, opts[:user])
end
@doc """ @doc """
Fetch favorites activities of user with order by sort adds to favorites Fetch favorites activities of user with order by sort adds to favorites
""" """

View File

@ -133,31 +133,22 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
end end
defp hashtag_fetching(params, user, local_only) do defp hashtag_fetching(params, user, local_only) do
tags = # Note: not sanitizing tag options at this stage (may be mix-cased, have duplicates etc.)
tags_any =
[params[:tag], params[:any]] [params[:tag], params[:any]]
|> List.flatten() |> List.flatten()
|> Enum.reject(&is_nil/1) |> Enum.filter(& &1)
|> Enum.map(&String.downcase/1)
|> Enum.uniq()
tag_all = tag_all = Map.get(params, :all, [])
params tag_reject = Map.get(params, :none, [])
|> Map.get(:all, [])
|> Enum.map(&String.downcase/1)
tag_reject =
params
|> Map.get(:none, [])
|> Enum.map(&String.downcase/1)
_activities =
params params
|> Map.put(:type, "Create") |> Map.put(:type, "Create")
|> Map.put(:local_only, local_only) |> Map.put(:local_only, local_only)
|> Map.put(:blocking_user, user) |> Map.put(:blocking_user, user)
|> Map.put(:muting_user, user) |> Map.put(:muting_user, user)
|> Map.put(:user, user) |> Map.put(:user, user)
|> Map.put(:tag, tags) |> Map.put(:tag, tags_any)
|> Map.put(:tag_all, tag_all) |> Map.put(:tag_all, tag_all)
|> Map.put(:tag_reject, tag_reject) |> Map.put(:tag_reject, tag_reject)
|> ActivityPub.fetch_public_activities() |> ActivityPub.fetch_public_activities()