From f84ed44cea1e5793dd899c74c38336a1721889e6 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 6 Jul 2022 01:19:53 -0400 Subject: [PATCH 1/4] Fix cannot get full history on object fetch --- .../web/activity_pub/object_validator.ex | 13 ++- test/pleroma/object/fetcher_test.exs | 80 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 3ccb4a3d6..9f446100d 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -103,8 +103,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do meta ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do - with {:ok, object_data} <- cast_and_apply(object), - meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + with {:ok, object_data} <- cast_and_apply_and_stringify_with_history(object), + meta = Keyword.put(meta, :object_data, object_data), {:ok, create_activity} <- create_activity |> CreateGenericValidator.cast_and_validate(meta) @@ -212,6 +212,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(o, m), do: {:error, {:validator_not_set, {o, m}}} + def cast_and_apply_and_stringify_with_history(object) do + do_separate_with_history(object, fn object -> + with {:ok, object_data} <- cast_and_apply(object), + object_data <- object_data |> stringify_keys() do + {:ok, object_data} + end + end) + end + def cast_and_apply(%{"type" => "ChatMessage"} = object) do ChatMessageValidator.cast_and_apply(object) end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 5a79e064f..1f97f36f7 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -456,4 +456,84 @@ defmodule Pleroma.Object.FetcherTest do } = refetched.data end end + + describe "fetch with history" do + setup do + object2 = %{ + "id" => "https://mastodon.social/2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 2", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + } + + mock(fn + %{ + method: :get, + url: "https://mastodon.social/2" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object2) + } + + %{ + method: :get, + url: "https://mastodon.social/users/emelie/collections/featured" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + Jason.encode!(%{ + "id" => "https://mastodon.social/users/emelie/collections/featured", + "type" => "OrderedCollection", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "orderedItems" => [], + "totalItems" => 0 + }) + } + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %{object2: object2} + end + + test "it gets history", %{object2: object2} do + {:ok, object} = Fetcher.fetch_object_from_id(object2["id"]) + + assert %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [%{}] + } + } = object.data + end + end end From 069554e9253a47f99225e12cc0ee99700fb89c6e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 7 Jul 2022 15:11:29 -0400 Subject: [PATCH 2/4] Guard against outdated Updates It is possible for an earlier Update to be received by us later. For this, we now (1) only allows Updates to poll counts if there is no updated field, or the updated field is the same as the last updated date or creation date; (2) does not allow updating anything if the updated field is older than the last updated date or creation date; (3) allows updating updatable fields otherwise (normal updates); (4) if only the updated field is changed, it does not create a new history item on its own. --- lib/pleroma/object/updater.ex | 65 ++++++++++--- .../web/activity_pub/side_effects_test.exs | 96 ++++++++++++++++++- test/pleroma/web/common_api_test.exs | 2 +- 3 files changed, 145 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 0b21f6c99..3d34c3f27 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -10,7 +10,10 @@ defmodule Pleroma.Object.Updater do |> Enum.reduce( %{data: orig_object_data, updated: false}, fn field, %{data: data, updated: updated} -> - updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) + updated = + updated or + (field != "updated" and + Map.get(updated_object, field) != Map.get(orig_object_data, field)) data = if Map.has_key?(updated_object, field) do @@ -136,21 +139,57 @@ defmodule Pleroma.Object.Updater do # This calculates the data of the new Object from an Update. # new_data's formerRepresentations is considered. def make_new_object_data_from_update_object(original_data, new_data) do - %{data: updated_data, updated: updated} = - original_data - |> update_content_fields(new_data) + update_is_reasonable = + with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]}, + {_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)}, + {_, last_updated} when not is_nil(last_updated) <- + {:last_updated, original_data["updated"] || original_data["published"]}, + {_, {:ok, last_updated_time, _}} <- + {:last_updated, DateTime.from_iso8601(last_updated)}, + :gt <- DateTime.compare(updated_time, last_updated_time) do + :update_everything + else + # only allow poll updates + {:cur_updated, _} -> :no_content_update + :eq -> :no_content_update + # allow all updates + {:last_updated, _} -> :update_everything + # allow no updates + _ -> false + end - %{updated_object: updated_data, used_history_in_new_object?: used_history_in_new_object?} = - updated_data - |> maybe_update_history(original_data, - updated: updated, - use_history_in_new_object?: true, - new_data: new_data - ) + %{ + updated_object: updated_data, + used_history_in_new_object?: used_history_in_new_object?, + updated: updated + } = + if update_is_reasonable == :update_everything do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + updated_data + |> maybe_update_history(original_data, + updated: updated, + use_history_in_new_object?: true, + new_data: new_data + ) + |> Map.put(:updated, updated) + else + %{ + updated_object: original_data, + used_history_in_new_object?: false, + updated: false + } + end updated_data = - updated_data - |> maybe_update_poll(new_data) + if update_is_reasonable != false do + updated_data + |> maybe_update_poll(new_data) + else + updated_data + end %{ updated_data: updated_data, diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 8e84db774..3b313b769 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -142,14 +142,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do describe "update notes" do setup do + make_time = fn -> + Pleroma.Web.ActivityPub.Utils.make_date() + end + user = insert(:user) - note = insert(:note, user: user) + note = insert(:note, user: user, data: %{"published" => make_time.()}) _note_activity = insert(:note_activity, note: note) updated_note = note.data |> Map.put("summary", "edited summary") |> Map.put("content", "edited content") + |> Map.put("updated", make_time.()) {:ok, update_data, []} = Builder.update(user, updated_note) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) @@ -170,8 +175,69 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do updated_note: updated_note } do {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + updated_time = updated_note["updated"] + new_note = Pleroma.Object.get_by_id(object_id) - assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data + + assert %{ + "summary" => "edited summary", + "content" => "edited content", + "updated" => ^updated_time + } = new_note.data + end + + test "it rejects updates with no updated attribute in object", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + updated_note = Map.drop(updated_note, ["updated"]) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than what we have in the original object", + %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, -10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than the last Update", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, +10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, update_time, _} = DateTime.from_iso8601(old_note.data["updated"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(update_time, -5))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data end test "it updates using object_data", %{ @@ -215,6 +281,14 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do note.data |> Map.put("summary", "edited summary 2") |> Map.put("content", "edited content 2") + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) {:ok, second_update_data, []} = Builder.update(user, second_updated_note) {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) @@ -238,7 +312,18 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do updated_note: updated_note } do {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) - %{data: _first_edit} = Pleroma.Object.get_by_id(object_id) + %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + + updated_note = + updated_note + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) @@ -270,7 +355,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do data["oneOf"] |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) - updated_question = data |> Map.put("oneOf", new_choices) + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", Pleroma.Web.ActivityPub.Utils.make_date()) {:ok, update_data, []} = Builder.update(user, updated_question) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 842c75e21..24f3a1a93 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1596,7 +1596,7 @@ defmodule Pleroma.Web.CommonAPITest do clear_config([:instance, :federating], true) with_mock Pleroma.Web.Federator, - publish: fn p -> nil end do + publish: fn _p -> nil end do {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) assert updated.data["object"]["content"] == "updated 2 :#{emoji2}:" From 11a6e8842079d8f29417e0de31fda85c7b7cb1be Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 7 Jul 2022 15:22:04 -0400 Subject: [PATCH 3/4] Test that Question updates are viable --- .../web/activity_pub/side_effects_test.exs | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 3b313b769..0db3a8ea6 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -341,7 +341,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do describe "update questions" do setup do user = insert(:user) - question = insert(:question, user: user) + + question = + insert(:question, + user: user, + data: %{"published" => Pleroma.Web.ActivityPub.Utils.make_date()} + ) %{user: user, data: question.data, id: question.id} end @@ -372,6 +377,59 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do refute Map.has_key?(new_question, "formerRepresentations") end + + test "allows updating choice count without updated field", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end + + test "allows updating choice count with updated field same as the creation date", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", data["published"]) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end end describe "EmojiReact objects" do From 04ded94a50fbabb194ab9e9c5cf8f08937f85d64 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 9 Jul 2022 18:00:42 -0400 Subject: [PATCH 4/4] Fix remote emoji in subject disappearing after edits --- lib/pleroma/web/common_api.ex | 9 +++++- test/pleroma/web/common_api_test.exs | 42 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index e5a78c102..89f5dd606 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -415,7 +415,14 @@ defmodule Pleroma.Web.CommonAPI do defp make_update_data(user, orig_object, changes) do kept_params = %{ - visibility: Visibility.get_visibility(orig_object) + visibility: Visibility.get_visibility(orig_object), + in_reply_to_id: + with replied_id when is_binary(replied_id) <- orig_object.data["inReplyTo"], + %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(replied_id) do + activity_id + else + _ -> nil + end } params = Map.merge(changes, kept_params) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 24f3a1a93..fcfcb9a2e 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1605,5 +1605,47 @@ defmodule Pleroma.Web.CommonAPITest do assert called(Pleroma.Web.Federator.publish(updated)) end end + + test "editing a post that copied a remote title with remote emoji should keep that emoji" do + remote_emoji_uri = "https://remote.org/emoji.png" + + note = + insert( + :note, + data: %{ + "summary" => ":remoteemoji:", + "emoji" => %{ + "remoteemoji" => remote_emoji_uri + }, + "tag" => [ + %{ + "type" => "Emoji", + "name" => "remoteemoji", + "icon" => %{"url" => remote_emoji_uri} + } + ] + } + ) + + note_activity = insert(:note_activity, note: note) + + user = insert(:user) + + {:ok, reply} = + CommonAPI.post(user, %{ + status: "reply", + spoiler_text: ":remoteemoji:", + in_reply_to_id: note_activity.id + }) + + assert reply.object.data["emoji"]["remoteemoji"] == remote_emoji_uri + + {:ok, edit} = + CommonAPI.update(user, reply, %{status: "reply mew mew", spoiler_text: ":remoteemoji:"}) + + edited_note = Pleroma.Object.normalize(edit) + + assert edited_note.data["emoji"]["remoteemoji"] == remote_emoji_uri + end end end