From 45e57dd187ecb9463f0114f75a05f03dbc9e206a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 10 Feb 2019 21:37:51 +0000 Subject: [PATCH 1/7] rich media: tighten fetching timeouts and size limits --- lib/pleroma/web/rich_media/parser.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 38f1cdeec..4341141df 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -9,6 +9,13 @@ defmodule Pleroma.Web.RichMedia.Parser do Pleroma.Web.RichMedia.Parsers.OEmbed ] + @hackney_options [ + pool: :media, + timeout: 2_000, + recv_timeout: 2_000, + max_body: 2_000_000 + ] + def parse(nil), do: {:error, "No URL provided"} if Mix.env() == :test do @@ -28,7 +35,7 @@ defmodule Pleroma.Web.RichMedia.Parser do defp parse_url(url) do try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media]) + {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data() rescue From 379d04692cdbf558c611c588c0e6a4262c02a58c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 11 Feb 2019 21:35:40 +0300 Subject: [PATCH 2/7] Filter summary in keywordpolicy --- .../web/activity_pub/mrf/keyword_policy.ex | 28 ++-- .../activity_pub/mrf/keyword_policy_test.exs | 132 ++++++++++++++++-- 2 files changed, 138 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index ce6d2e529..5fdc03414 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -12,9 +12,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do String.match?(string, pattern) end - defp check_reject(%{"object" => %{"content" => content}} = message) do + defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> - string_matches?(content, pattern) + string_matches?(content, pattern) or string_matches?(summary, pattern) end) do {:reject, nil} else @@ -22,10 +22,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end end - defp check_ftl_removal(%{"to" => to, "object" => %{"content" => content}} = message) do + defp check_ftl_removal( + %{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message + ) do if "https://www.w3.org/ns/activitystreams#Public" in to and Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> - string_matches?(content, pattern) + string_matches?(content, pattern) or string_matches?(summary, pattern) end) do to = List.delete(to, "https://www.w3.org/ns/activitystreams#Public") cc = ["https://www.w3.org/ns/activitystreams#Public" | message["cc"] || []] @@ -41,14 +43,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end end - defp check_replace(%{"object" => %{"content" => content}} = message) do - content = - Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), content, fn {pattern, replacement}, - acc -> - String.replace(acc, pattern, replacement) + defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do + {content, summary} = + Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), {content, summary}, fn {pattern, + replacement}, + {content_acc, + summary_acc} -> + {String.replace(content_acc, pattern, replacement), + String.replace(summary_acc, pattern, replacement)} end) - {:ok, put_in(message["object"]["content"], content)} + {:ok, + message + |> put_in(["object", "content"], content) + |> put_in(["object", "summary"], summary)} end @impl true diff --git a/test/web/activity_pub/mrf/keyword_policy_test.exs b/test/web/activity_pub/mrf/keyword_policy_test.exs index 67a5858d7..602892a37 100644 --- a/test/web/activity_pub/mrf/keyword_policy_test.exs +++ b/test/web/activity_pub/mrf/keyword_policy_test.exs @@ -12,18 +12,35 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do end describe "rejecting based on keywords" do - test "rejects if string matches" do + test "rejects if string matches in content" do Pleroma.Config.put([:mrf_keyword, :reject], ["pun"]) message = %{ "type" => "Create", - "object" => %{"content" => "just a daily reminder that compLAINer is a good pun"} + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } } assert {:reject, nil} == KeywordPolicy.filter(message) end - test "rejects if regex matches" do + test "rejects if string matches in summary" do + Pleroma.Config.put([:mrf_keyword, :reject], ["pun"]) + + message = %{ + "type" => "Create", + "object" => %{ + "summary" => "just a daily reminder that compLAINer is a good pun", + "content" => "" + } + } + + assert {:reject, nil} == KeywordPolicy.filter(message) + end + + test "rejects if regex matches in content" do Pleroma.Config.put([:mrf_keyword, :reject], [~r/comp[lL][aA][iI][nN]er/]) assert true == @@ -31,7 +48,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do message = %{ "type" => "Create", "object" => %{ - "content" => "just a daily reminder that #{content} is a good pun" + "content" => "just a daily reminder that #{content} is a good pun", + "summary" => "" + } + } + + {:reject, nil} == KeywordPolicy.filter(message) + end) + end + + test "rejects if regex matches in summary" do + Pleroma.Config.put([:mrf_keyword, :reject], [~r/comp[lL][aA][iI][nN]er/]) + + assert true == + Enum.all?(["complainer", "compLainer", "compLAiNer", "compLAINer"], fn content -> + message = %{ + "type" => "Create", + "object" => %{ + "summary" => "just a daily reminder that #{content} is a good pun", + "content" => "" } } @@ -41,13 +76,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do end describe "delisting from ftl based on keywords" do - test "delists if string matches" do + test "delists if string matches in content" do Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], ["pun"]) message = %{ "to" => ["https://www.w3.org/ns/activitystreams#Public"], "type" => "Create", - "object" => %{"content" => "just a daily reminder that compLAINer is a good pun"} + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } } {:ok, result} = KeywordPolicy.filter(message) @@ -55,7 +93,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"] end - test "delists if regex matches" do + test "delists if string matches in summary" do + Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], ["pun"]) + + message = %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create", + "object" => %{ + "summary" => "just a daily reminder that compLAINer is a good pun", + "content" => "" + } + } + + {:ok, result} = KeywordPolicy.filter(message) + assert ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"] + refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"] + end + + test "delists if regex matches in content" do Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], [~r/comp[lL][aA][iI][nN]er/]) assert true == @@ -64,7 +119,29 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do "type" => "Create", "to" => ["https://www.w3.org/ns/activitystreams#Public"], "object" => %{ - "content" => "just a daily reminder that #{content} is a good pun" + "content" => "just a daily reminder that #{content} is a good pun", + "summary" => "" + } + } + + {:ok, result} = KeywordPolicy.filter(message) + + ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"] and + not (["https://www.w3.org/ns/activitystreams#Public"] == result["to"]) + end) + end + + test "delists if regex matches in summary" do + Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], [~r/comp[lL][aA][iI][nN]er/]) + + assert true == + Enum.all?(["complainer", "compLainer", "compLAiNer", "compLAINer"], fn content -> + message = %{ + "type" => "Create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{ + "summary" => "just a daily reminder that #{content} is a good pun", + "content" => "" } } @@ -77,20 +154,33 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do end describe "replacing keywords" do - test "replaces keyword if string matches" do + test "replaces keyword if string matches in content" do Pleroma.Config.put([:mrf_keyword, :replace], [{"opensource", "free software"}]) message = %{ "type" => "Create", "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "object" => %{"content" => "ZFS is opensource"} + "object" => %{"content" => "ZFS is opensource", "summary" => ""} } {:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message) assert result == "ZFS is free software" end - test "replaces keyword if regex matches" do + test "replaces keyword if string matches in summary" do + Pleroma.Config.put([:mrf_keyword, :replace], [{"opensource", "free software"}]) + + message = %{ + "type" => "Create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{"summary" => "ZFS is opensource", "content" => ""} + } + + {:ok, %{"object" => %{"summary" => result}}} = KeywordPolicy.filter(message) + assert result == "ZFS is free software" + end + + test "replaces keyword if regex matches in content" do Pleroma.Config.put([:mrf_keyword, :replace], [ {~r/open(-|\s)?source\s?(software)?/, "free software"} ]) @@ -100,12 +190,30 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do message = %{ "type" => "Create", "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "object" => %{"content" => "ZFS is #{content}"} + "object" => %{"content" => "ZFS is #{content}", "summary" => ""} } {:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message) result == "ZFS is free software" end) end + + test "replaces keyword if regex matches in summary" do + Pleroma.Config.put([:mrf_keyword, :replace], [ + {~r/open(-|\s)?source\s?(software)?/, "free software"} + ]) + + assert true == + Enum.all?(["opensource", "open-source", "open source"], fn content -> + message = %{ + "type" => "Create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{"summary" => "ZFS is #{content}", "content" => ""} + } + + {:ok, %{"object" => %{"summary" => result}}} = KeywordPolicy.filter(message) + result == "ZFS is free software" + end) + end end end From ea1058929c4dd569c00864f5292ec0a689acd1c6 Mon Sep 17 00:00:00 2001 From: shibayashi Date: Tue, 12 Feb 2019 00:08:52 +0100 Subject: [PATCH 3/7] Use url[:scheme] instead of protocol to determine if https is enabled --- lib/pleroma/plugs/http_security_plug.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 2a266c407..3c8e6a18f 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -33,7 +33,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do end defp csp_string do - protocol = Config.get([Pleroma.Web.Endpoint, :protocol]) + scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] [ "default-src 'none'", @@ -46,7 +46,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "script-src 'self'", "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"), "manifest-src 'self'", - if protocol == "https" do + if scheme == "https" do "upgrade-insecure-requests" end ] From 84f22d1cb8cf953bf8f48c04d82ae05f780ec407 Mon Sep 17 00:00:00 2001 From: Hakaba Hitoyo Date: Tue, 12 Feb 2019 02:35:15 +0000 Subject: [PATCH 4/7] Mark streaming feature for mobile/web apps in Clients.md --- docs/Clients.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/Clients.md b/docs/Clients.md index 057f12392..043d04a0f 100644 --- a/docs/Clients.md +++ b/docs/Clients.md @@ -26,60 +26,71 @@ Feel free to contact us to be added to this list! - Source Code: - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy) - Platforms: iOS +- Features: No Streaming ### Nekonium - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/) - Source: - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin) - Platforms: Android +- Features: Streaming Ready ### Mastalab - Source Code: - Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79) - Platforms: Android +- Features: Streaming Ready ### Roma - Homepage: - Source Code: ??? - Platforms: iOS, Android +- Features: No Streaming ### Tootdon - Homepage: , - Source Code: ??? - Contact: [@tootdon@mstdn.jp](https://mstdn.jp/users/tootdon) - Platforms: Android, iOS +- Features: No Streaming ### Tusky - Homepage: - Source Code: - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck) - Platforms: Android +- Features: No Streaming ### Twidere - Homepage: - Source Code: , - Contact: - Platform: Android, iOS +- Features: No Streaming ## Alternative Web Interfaces ### Brutaldon - Homepage: - Source Code: - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc) +- Features: No Streaming ### Feather - Source Code: - Contact: [@kaniini@pleroma.site](https://pleroma.site/kaniini) +- Features: No Streaming ### Halcyon - Source Code: - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon) +- Features: Streaming Ready ### Pinafore - Homepage: - Source Code: - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore) - Note: Pleroma support is a secondary goal +- Features: No Streaming ### Sengi - Source Code: From 71ce564ecc442614995c021281a8f1e1a67fabc1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 22 Nov 2018 07:12:13 +0100 Subject: [PATCH 5/7] =?UTF-8?q?config/dev.exs:=20Don=E2=80=99t=20put=20sec?= =?UTF-8?q?ure=20cookies=20on=20dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/dev.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/dev.exs b/config/dev.exs index 8f89aa03c..f77bb9976 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -16,7 +16,8 @@ config :pleroma, Pleroma.Web.Endpoint, debug_errors: true, code_reloader: true, check_origin: false, - watchers: [] + watchers: [], + secure_cookie_flag: false config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Local From 00e8f0b07dd3dced84b0317e1c5c4156d249dec4 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 1 Feb 2019 13:10:50 +0100 Subject: [PATCH 6/7] Plugs.HTTPSecurityPlug: Add unsafe-eval to script-src when in dev mode This is needed to run dev mode mastofe at the same time --- lib/pleroma/plugs/http_security_plug.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 3c8e6a18f..05e935f2c 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -43,9 +43,11 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "media-src 'self' https:", "style-src 'self' 'unsafe-inline'", "font-src 'self'", - "script-src 'self'", "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"), "manifest-src 'self'", + if Mix.env() == :dev do + "script-src 'self' 'unsafe-eval'" + end, if scheme == "https" do "upgrade-insecure-requests" end From da4c662af31a2c45c767f2a9ed136272ee9fc2c8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 2 Feb 2019 19:06:26 +0100 Subject: [PATCH 7/7] Plugs.HTTPSecurityPlug: Add webpacker to connect-src --- lib/pleroma/plugs/http_security_plug.ex | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 05e935f2c..057553e24 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -34,6 +34,21 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do defp csp_string do scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] + websocket_url = String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws") + + connect_src = + if Mix.env() == :dev do + "connect-src 'self' http://localhost:3035/ " <> websocket_url + else + "connect-src 'self' " <> websocket_url + end + + script_src = + if Mix.env() == :dev do + "script-src 'self' 'unsafe-eval'" + else + "script-src 'self'" + end [ "default-src 'none'", @@ -43,11 +58,9 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "media-src 'self' https:", "style-src 'self' 'unsafe-inline'", "font-src 'self'", - "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"), "manifest-src 'self'", - if Mix.env() == :dev do - "script-src 'self' 'unsafe-eval'" - end, + connect_src, + script_src, if scheme == "https" do "upgrade-insecure-requests" end