Merge branch 'user-search-nickname-rank-boost' into 'develop'

Improved search results for localized nickname match

See merge request pleroma/pleroma!2733
This commit is contained in:
Haelwenn 2020-07-10 08:49:14 +00:00
commit 5ea6387572
4 changed files with 66 additions and 17 deletions

View File

@ -69,11 +69,15 @@ defmodule Pleroma.User.Search do
u in query, u in query,
where: where:
fragment( fragment(
# The fragment must _exactly_ match `users_fts_index`, otherwise the index won't work
""" """
(to_tsvector('simple', ?) || to_tsvector('simple', ?)) @@ to_tsquery('simple', ?) (
setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')
) @@ to_tsquery('simple', ?)
""", """,
u.name,
u.nickname, u.nickname,
u.name,
^query_string ^query_string
) )
) )
@ -88,15 +92,23 @@ defmodule Pleroma.User.Search do
|> Enum.join(" | ") |> Enum.join(" | ")
end end
# Considers nickname match, localized nickname match, name match; preferences nickname match
defp trigram_rank(query, query_string) do defp trigram_rank(query, query_string) do
from( from(
u in query, u in query,
select_merge: %{ select_merge: %{
search_rank: search_rank:
fragment( fragment(
"similarity(?, trim(? || ' ' || coalesce(?, '')))", """
similarity(?, ?) +
similarity(?, regexp_replace(?, '@.+', '')) +
similarity(?, trim(coalesce(?, '')))
""",
^query_string, ^query_string,
u.nickname, u.nickname,
^query_string,
u.nickname,
^query_string,
u.name u.name
) )
} }

View File

@ -0,0 +1,18 @@
defmodule Pleroma.Repo.Migrations.DropUserTrigramIndex do
@moduledoc "Drops unused trigram index on `users` (FTS index is being used instead)"
use Ecto.Migration
def up do
drop_if_exists(index(:users, [], name: :users_trigram_index))
end
def down do
create_if_not_exists(
index(:users, ["(trim(nickname || ' ' || coalesce(name, ''))) gist_trgm_ops"],
name: :users_trigram_index,
using: :gist
)
)
end
end

View File

@ -481,17 +481,17 @@ defmodule Mix.Tasks.Pleroma.UserTest do
moot = insert(:user, nickname: "moot") moot = insert(:user, nickname: "moot")
kawen = insert(:user, nickname: "kawen", name: "fediverse expert moon") kawen = insert(:user, nickname: "kawen", name: "fediverse expert moon")
{:ok, user} = User.follow(user, kawen) {:ok, user} = User.follow(user, moon)
assert [moon.id, kawen.id] == User.Search.search("moon") |> Enum.map(& &1.id) assert [moon.id, kawen.id] == User.Search.search("moon") |> Enum.map(& &1.id)
res = User.search("moo") |> Enum.map(& &1.id)
assert moon.id in res
assert moot.id in res
assert kawen.id in res
assert [moon.id, kawen.id] == User.Search.search("moon fediverse") |> Enum.map(& &1.id)
assert [kawen.id, moon.id] == res = User.search("moo") |> Enum.map(& &1.id)
User.Search.search("moon fediverse", for_user: user) |> Enum.map(& &1.id) assert Enum.sort([moon.id, moot.id, kawen.id]) == Enum.sort(res)
assert [kawen.id, moon.id] == User.Search.search("expert fediverse") |> Enum.map(& &1.id)
assert [moon.id, kawen.id] ==
User.Search.search("expert fediverse", for_user: user) |> Enum.map(& &1.id)
end end
end end

View File

@ -46,30 +46,49 @@ defmodule Pleroma.UserSearchTest do
assert length(User.search("john", limit: 3, offset: 3)) == 2 assert length(User.search("john", limit: 3, offset: 3)) == 2
end end
test "finds a user by full or partial nickname" do defp clear_virtual_fields(user) do
Map.merge(user, %{search_rank: nil, search_type: nil})
end
test "finds a user by full nickname or its leading fragment" do
user = insert(:user, %{nickname: "john"}) user = insert(:user, %{nickname: "john"})
Enum.each(["john", "jo", "j"], fn query -> Enum.each(["john", "jo", "j"], fn query ->
assert user == assert user ==
User.search(query) User.search(query)
|> List.first() |> List.first()
|> Map.put(:search_rank, nil) |> clear_virtual_fields()
|> Map.put(:search_type, nil)
end) end)
end end
test "finds a user by full or partial name" do test "finds a user by full name or leading fragment(s) of its words" do
user = insert(:user, %{name: "John Doe"}) user = insert(:user, %{name: "John Doe"})
Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query -> Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query ->
assert user == assert user ==
User.search(query) User.search(query)
|> List.first() |> List.first()
|> Map.put(:search_rank, nil) |> clear_virtual_fields()
|> Map.put(:search_type, nil)
end) end)
end end
test "matches by leading fragment of user domain" do
user = insert(:user, %{nickname: "arandom@dude.com"})
insert(:user, %{nickname: "iamthedude"})
assert [user.id] == User.search("dud") |> Enum.map(& &1.id)
end
test "ranks full nickname match higher than full name match" do
nicknamed_user = insert(:user, %{nickname: "hj@shigusegubu.club"})
named_user = insert(:user, %{nickname: "xyz@sample.com", name: "HJ"})
results = User.search("hj")
assert [nicknamed_user.id, named_user.id] == Enum.map(results, & &1.id)
assert Enum.at(results, 0).search_rank > Enum.at(results, 1).search_rank
end
test "finds users, considering density of matched tokens" do test "finds users, considering density of matched tokens" do
u1 = insert(:user, %{name: "Bar Bar plus Word Word"}) u1 = insert(:user, %{name: "Bar Bar plus Word Word"})
u2 = insert(:user, %{name: "Word Word Bar Bar Bar"}) u2 = insert(:user, %{name: "Word Word Bar Bar Bar"})