a1batross
/
pleroma
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'tusooa/from/upstream-develop/tusooa/edits' into emr_develop

This commit is contained in:
a1batross 2022-06-24 16:26:33 +02:00
parent c15b43cce4 7726c399de
commit ab386eb6dc
16 changed files with 226 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
lib/pleroma/activity/html.ex
View File

@ -8,6 +8,40 @@ defmodule Pleroma.Activity.HTML do
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
# We store a list of cache keys related to an activity in a
# separate cache, scrubber_management_cache. It has the same
# size as scrubber_cache (see application.ex). Every time we add
# a cache to scrubber_cache, we update scrubber_management_cache.
#
# The most recent write of a certain key in the management cache
# is the same as the most recent write of any record related to that
# key in the main cache.
# Assuming LRW ( https://hexdocs.pm/cachex/Cachex.Policy.LRW.html ),
# this means when the management cache is evicted by cachex, all
# related records in the main cache will also have been evicted.
defp get_cache_keys_for(activity_id) do
with {:ok, list} when is_list(list) <- @cachex.get(:scrubber_management_cache, activity_id) do
list
else
_ -> []
end
end
defp add_cache_key_for(activity_id, additional_key) do
current = get_cache_keys_for(activity_id)
unless additional_key in current do
@cachex.put(:scrubber_management_cache, activity_id, [additional_key | current])
end
end
def invalidate_cache_for(activity_id) do
keys = get_cache_keys_for(activity_id)
Enum.map(keys, &@cachex.del(:scrubber_cache, &1))
@cachex.del(:scrubber_management_cache, activity_id)
end
def get_cached_scrubbed_html_for_activity(
content,
scrubbers,
@ -19,6 +53,8 @@ defmodule Pleroma.Activity.HTML do
@cachex.fetch!(:scrubber_cache, key, fn _key ->
object = Object.normalize(activity, fetch: false)
add_cache_key_for(activity.id, key)
HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
end)
end

1
lib/pleroma/application.ex
View File

@ -189,6 +189,7 @@ defmodule Pleroma.Application do
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500),
build_cachex("scrubber_management", limit: 2500),
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),

16
lib/pleroma/web/activity_pub/object_validator.ex
View File

@ -141,6 +141,22 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
end
end
def validate(
%{"type" => "Update", "object" => %{"type" => objtype} = object} = update_activity,
meta
)
when objtype in ~w[Question Answer Audio Video Event Article Note Page] do
with {:ok, object_data} <- cast_and_apply(object),
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
{:ok, update_activity} <-
update_activity
|> UpdateValidator.cast_and_validate()
|> Ecto.Changeset.apply_action(:insert) do
update_activity = stringify_keys(update_activity)
{:ok, update_activity, meta}
end
end
def validate(%{"type" => type} = object, meta)
when type in ~w[Accept Reject Follow Update Like EmojiReact Announce
ChatMessage Answer] do

4
lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
View File

@ -49,7 +49,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"])
defp fix_url(data), do: data
defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data
defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do
Map.put(data, "tag", Enum.filter(tag, &is_map/1))
end
defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag])
defp fix_tag(data), do: Map.drop(data, ["tag"])

25
lib/pleroma/web/activity_pub/side_effects.ex
View File

@ -455,7 +455,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
%{data: %{"type" => "Update", "object" => updated_object}} = object,
meta
) do
orig_object = Object.get_by_ap_id(updated_object["id"])
orig_object_ap_id = updated_object["id"]
orig_object = Object.get_by_ap_id(orig_object_ap_id)
orig_object_data = orig_object.data
if orig_object_data["type"] in @updatable_object_types do
@ -468,15 +469,21 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
|> Object.maybe_update_history(orig_object_data, updated)
|> maybe_update_poll(updated_object)
orig_object
|> Repo.preload(:hashtags)
|> Object.change(%{data: updated_object_data})
|> Object.update_and_set_cache()
changeset =
orig_object
|> Repo.preload(:hashtags)
|> Object.change(%{data: updated_object_data})
if updated do
object
|> Activity.normalize()
|> ActivityPub.notify_and_stream()
with {:ok, new_object} <- Repo.update(changeset),
{:ok, _} <- Object.invalid_object_cache(new_object),
{:ok, _} <- Object.set_cache(new_object),
# The metadata/utils.ex uses the object id for the cache.
{:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do
if updated do
object
|> Activity.normalize()
|> ActivityPub.notify_and_stream()
end
end
end

10
lib/pleroma/web/mastodon_api/controllers/status_controller.ex
View File

@ -195,8 +195,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
end
@doc "GET /api/v1/statuses/:id/history"
def show_history(%{assigns: %{user: user}} = conn, %{id: id} = params) do
with %Activity{} = activity <- Activity.get_by_id_with_object(id),
def show_history(%{assigns: assigns} = conn, %{id: id} = params) do
with user = assigns[:user],
%Activity{} = activity <- Activity.get_by_id_with_object(id),
true <- Visibility.visible_for_user?(activity, user) do
try_render(conn, "history.json",
activity: activity,
@ -210,8 +211,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
end
@doc "GET /api/v1/statuses/:id/source"
def show_source(%{assigns: %{user: user}} = conn, %{id: id} = _params) do
with %Activity{} = activity <- Activity.get_by_id_with_object(id),
def show_source(%{assigns: assigns} = conn, %{id: id} = _params) do
with user = assigns[:user],
%Activity{} = activity <- Activity.get_by_id_with_object(id),
true <- Visibility.visible_for_user?(activity, user) do
try_render(conn, "source.json",
activity: activity,

1
lib/pleroma/web/mastodon_api/views/instance_view.ex
View File

@ -68,6 +68,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
"shareable_emoji_packs",
"multifetch",
"pleroma:api/v1/notifications:include_types_filter",
"editing",
if Config.get([:activitypub, :blockers_visible]) do
"blockers_visible"
end,

40
lib/pleroma/web/mastodon_api/views/status_view.ex
View File

@ -272,6 +272,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"])
history_len =
1 +
(Object.history_for(object.data)
|> Map.get("orderedItems")
|> length())
# See render("history.json", ...) for more details
# Here the implicit index of the current content is 0
chrono_order = history_len - 1
content =
object
|> render_content()
@ -281,14 +291,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
|> Activity.HTML.get_cached_scrubbed_html_for_activity(
User.html_filter_policy(opts[:for]),
activity,
"mastoapi:content"
"mastoapi:content:#{chrono_order}"
)
content_plaintext =
content
|> Activity.HTML.get_cached_stripped_html_for_activity(
activity,
"mastoapi:content"
"mastoapi:content:#{chrono_order}"
)
summary = object.data["summary"] || ""
@ -408,11 +418,24 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
|> Enum.map(&Map.put(&1, "id", object.data["id"]))
|> Enum.map(&%Object{data: &1, id: object.id})
history = [object | past_history]
history =
[object | past_history]
# Mastodon expects the original to be at the first
|> Enum.reverse()
|> Enum.with_index()
|> Enum.map(fn {object, chrono_order} ->
%{
# The history is prepended every time there is a new edit.
# In chrono_order, the oldest item is always at 0, and so on.
# The chrono_order is an invariant kept between edits.
chrono_order: chrono_order,
object: object
}
end)
individual_opts =
opts
|> Map.put(:as, :object)
|> Map.put(:as, :item)
|> Map.put(:user, user)
|> Map.put(:hashtags, hashtags)
@ -421,7 +444,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
def render(
"history_item.json",
%{activity: activity, user: user, object: object, hashtags: hashtags} = opts
%{
activity: activity,
user: user,
item: %{object: object, chrono_order: chrono_order},
hashtags: hashtags
} = opts
) do
sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw")
@ -439,7 +467,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
|> Activity.HTML.get_cached_scrubbed_html_for_activity(
User.html_filter_policy(opts[:for]),
activity,
"mastoapi:content"
"mastoapi:content:#{chrono_order}"
)
summary = object.data["summary"] || ""

4
lib/pleroma/web/router.ex
View File

@ -552,8 +552,6 @@ defmodule Pleroma.Web.Router do
get("/bookmarks", StatusController, :bookmarks)
post("/statuses", StatusController, :create)
get("/statuses/:id/history", StatusController, :show_history)
get("/statuses/:id/source", StatusController, :show_source)
put("/statuses/:id", StatusController, :update)
delete("/statuses/:id", StatusController, :delete)
post("/statuses/:id/reblog", StatusController, :reblog)
@ -611,6 +609,8 @@ defmodule Pleroma.Web.Router do
get("/statuses/:id/card", StatusController, :card)
get("/statuses/:id/favourited_by", StatusController, :favourited_by)
get("/statuses/:id/reblogged_by", StatusController, :reblogged_by)
get("/statuses/:id/history", StatusController, :show_history)
get("/statuses/:id/source", StatusController, :show_source)
get("/custom_emojis", CustomEmojiController, :index)

8
lib/pleroma/web/streamer.ex
View File

@ -297,12 +297,16 @@ defmodule Pleroma.Web.Streamer do
defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop
defp push_to_socket(topic, %Activity{data: %{"type" => "Update"}} = item) do
anon_render = StreamerView.render("status_update.json", item)
create_activity =
Pleroma.Activity.get_create_by_object_ap_id(item.object.data["id"])
|> Map.put(:object, item.object)
anon_render = StreamerView.render("status_update.json", create_activity)
Registry.dispatch(@registry, topic, fn list ->
Enum.each(list, fn {pid, auth?} ->
if auth? do
send(pid, {:render_with_user, StreamerView, "status_update.json", item})
send(pid, {:render_with_user, StreamerView, "status_update.json", create_activity})
else
send(pid, {:text, anon_render})
end

4
lib/pleroma/web/views/streamer_view.ex
View File

@ -26,8 +26,6 @@ defmodule Pleroma.Web.StreamerView do
end
def render("status_update.json", %Activity{} = activity, %User{} = user) do
activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"])
%{
event: "status.update",
payload:
@ -68,8 +66,6 @@ defmodule Pleroma.Web.StreamerView do
end
def render("status_update.json", %Activity{} = activity) do
activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"])
%{
event: "status.update",
payload:

5
test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs
View File

@ -31,6 +31,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest
test "a basic note validates", %{note: note} do
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note)
end
test "a note from factory validates" do
note = insert(:note)
%{valid?: true} = ArticleNotePageValidator.cast_and_validate(note.data)
end
end
test "a Note from Roadhouse validates" do

35
test/pleroma/web/activity_pub/object_validators/update_handling_test.exs
View File

@ -60,7 +60,40 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateHandlingTest do
{:ok, update, _} = Builder.update(other_user, updated_note)
assert {:ok, _update, []} = ObjectValidator.validate(update, [])
assert {:ok, _update, _} = ObjectValidator.validate(update, [])
end
end
describe "update note" do
test "converts object into Pleroma's format" do
mastodon_tags = [
%{
"icon" => %{
"mediaType" => "image/png",
"type" => "Image",
"url" => "https://somewhere.org/emoji/url/1.png"
},
"id" => "https://somewhere.org/emoji/1",
"name" => ":some_emoji:",
"type" => "Emoji",
"updated" => "2021-04-07T11:00:00Z"
}
]
user = insert(:user)
note = insert(:note, user: user)
updated_note =
note.data
|> Map.put("content", "edited content")
|> Map.put("tag", mastodon_tags)
{:ok, update, _} = Builder.update(user, updated_note)
assert {:ok, _update, meta} = ObjectValidator.validate(update, [])
assert %{"emoji" => %{"some_emoji" => "https://somewhere.org/emoji/url/1.png"}} =
meta[:object_data]
end
end
end

22
test/pleroma/web/mastodon_api/controllers/status_controller_test.exs
View File

@ -1993,7 +1993,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
describe "get status history" do
setup do
oauth_access(["read:statuses"])
%{conn: build_conn()}
end
test "unedited post", %{conn: conn} do
@ -2032,14 +2032,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
conn = get(conn, "/api/v1/statuses/#{activity.id}/history")
assert [_, %{"spoiler_text" => "title 2"}, %{"spoiler_text" => "title 1"}] =
assert [%{"spoiler_text" => "title 1"}, %{"spoiler_text" => "title 2"}, _] =
json_response_and_validate_schema(conn, 200)
end
end
describe "get status source" do
setup do
oauth_access(["read:statuses"])
%{conn: build_conn()}
end
test "it returns the source", %{conn: conn} do
@ -2061,9 +2061,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
oauth_access(["write:statuses"])
end
test "it updates the status", %{conn: conn, user: user} do
test "it updates the status" do
%{conn: conn, user: user} = oauth_access(["write:statuses", "read:statuses"])
{:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"})
conn
|> get("/api/v1/statuses/#{activity.id}")
|> json_response_and_validate_schema(200)
response =
conn
|> put_req_header("content-type", "application/json")
@ -2075,6 +2081,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
assert response["content"] == "edited"
assert response["spoiler_text"] == "lol"
response =
conn
|> get("/api/v1/statuses/#{activity.id}")
|> json_response_and_validate_schema(200)
assert response["content"] == "edited"
assert response["spoiler_text"] == "lol"
end
test "it updates the attachments", %{conn: conn, user: user} do

16
test/pleroma/web/metadata/utils_test.exs
View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Metadata.UtilsTest do
use Pleroma.DataCase, async: true
use Pleroma.DataCase, async: false
import Pleroma.Factory
alias Pleroma.Web.Metadata.Utils
@ -22,6 +22,20 @@ defmodule Pleroma.Web.Metadata.UtilsTest do
assert Utils.scrub_html_and_truncate(note) == "Pleroma's really cool!"
end
test "it does not return old content after editing" do
user = insert(:user)
{:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew #def"})
object = Pleroma.Object.normalize(activity)
assert Utils.scrub_html_and_truncate(object) == "mew mew #def"
{:ok, update} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "mew mew #abc"})
update = Pleroma.Activity.normalize(update)
object = Pleroma.Object.normalize(update)
assert Utils.scrub_html_and_truncate(object) == "mew mew #abc"
end
end
describe "scrub_html_and_truncate/2" do

37
test/pleroma/web/streamer_test.exs
View File

@ -451,9 +451,9 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user, oauth_token)
{:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"})
edited = Pleroma.Activity.normalize(edited)
create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"])
assert_receive {:render_with_user, _, "status_update.json", ^edited}
assert_receive {:render_with_user, _, "status_update.json", ^create}
refute Streamer.filtered_by_user?(user, edited)
end
@ -462,9 +462,9 @@ defmodule Pleroma.Web.StreamerTest do
Streamer.get_topic_and_add_socket("user", user, oauth_token)
{:ok, edited} = CommonAPI.update(user, activity, %{status: "mew mew"})
edited = Pleroma.Activity.normalize(edited)
create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"])
assert_receive {:render_with_user, _, "status_update.json", ^edited}
assert_receive {:render_with_user, _, "status_update.json", ^create}
refute Streamer.filtered_by_user?(user, edited)
end
end
@ -528,6 +528,35 @@ defmodule Pleroma.Web.StreamerTest do
assert %{"id" => ^activity_id} = Jason.decode!(payload)
refute Streamer.filtered_by_user?(sender, edited)
end
test "it streams multiple edits in the 'public' stream correctly" do
sender = insert(:user)
Streamer.get_topic_and_add_socket("public", nil, nil)
{:ok, activity} = CommonAPI.post(sender, %{status: "hey"})
assert_receive {:text, _}
{:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"})
edited = Pleroma.Activity.normalize(edited)
%{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"])
assert_receive {:text, event}
assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event)
assert %{"id" => ^activity_id} = Jason.decode!(payload)
refute Streamer.filtered_by_user?(sender, edited)
{:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew 2"})
edited = Pleroma.Activity.normalize(edited)
%{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"])
assert_receive {:text, event}
assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event)
assert %{"id" => ^activity_id, "content" => "mew mew 2"} = Jason.decode!(payload)
refute Streamer.filtered_by_user?(sender, edited)
end
end
describe "thread_containment/2" do