Merge branch 'features/validators-video' into 'develop'

Pipeline Ingestion: Video and Article

See merge request pleroma/pleroma!2908

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -84,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Event]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do

lib/pleroma/web/activity_pub/object_validator.ex

@@ -12,11 +12,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
@@ -149,10 +151,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
-  def validate(%{"type" => "Audio"} = object, meta) do
+  def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do
     with {:ok, object} <-
            object
-           |> AudioValidator.cast_and_validate()
+           |> AudioVideoValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Article"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> ArticleNoteValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
@@ -198,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio Event] do
+      when objtype in ~w[Question Answer Audio Video Event Article] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -232,14 +244,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     AnswerValidator.cast_and_apply(object)
   end
 
-  def cast_and_apply(%{"type" => "Audio"} = object) do
+  def cast_and_apply(%{"type" => type} = object) when type in ~w[Audio Video] do
-    AudioValidator.cast_and_apply(object)
+    AudioVideoValidator.cast_and_apply(object)
   end
 
   def cast_and_apply(%{"type" => "Event"} = object) do
     EventValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Article"} = object) do
+    ArticleNoteValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
   # is_struct/1 isn't present in Elixir 1.8.x
@@ -262,7 +278,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   def stringify_keys(object), do: object
 
   def fetch_actor(object) do
-    with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do
+    with actor <- Containment.get_actor(object),
+         {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do
       User.get_or_fetch_by_ap_id(actor)
     end
   end

lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
@@ -25,14 +25,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     # TODO: Write type
     field(:tag, {:array, :map}, default: [])
     field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
     field(:content, :string)
+
     field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
 
     # TODO: Remove actor on objects
     field(:actor, ObjectValidators.ObjectID)
 
     field(:attributedTo, ObjectValidators.ObjectID)
-    field(:summary, :string)
     field(:published, ObjectValidators.DateTime)
     field(:emoji, ObjectValidators.Emoji, default: %{})
     field(:sensitive, :boolean, default: false)
@@ -40,13 +45,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     field(:replies_count, :integer, default: 0)
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, :string)
+    field(:inReplyTo, ObjectValidators.ObjectID)
     field(:url, ObjectValidators.Uri)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
 
-    field(:likes, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
   end
 
   def cast_and_apply(data) do
@@ -62,19 +65,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
   end
 
   def cast_data(data) do
+    data = fix(data)
+
     %__MODULE__{}
     |> changeset(data)
   end
 
-  defp fix_url(%{"url" => url} = data) when is_list(url) do
+  defp fix_url(%{"url" => url} = data) when is_map(url) do
-    attachment =
+    Map.put(data, "url", url["href"])
-      Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end)
-
-    link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
-
-    data
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
   end
 
   defp fix_url(data), do: data
@@ -83,8 +81,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     data
     |> CommonFixes.fix_defaults()
     |> CommonFixes.fix_attribution()
-    |> Transmogrifier.fix_emoji()
+    |> CommonFixes.fix_actor()
     |> fix_url()
+    |> Transmogrifier.fix_emoji()
   end
 
   def changeset(struct, data) do
@@ -97,8 +96,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
 
   def validate_data(data_cng) do
     data_cng
-    |> validate_inclusion(:type, ["Audio"])
+    |> validate_inclusion(:type, ["Article", "Note"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
 
   import Ecto.Changeset
@@ -15,7 +16,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:mediaType, :string, default: "application/octet-stream")
     field(:name, :string)
 
-    embeds_many(:url, UrlObjectValidator)
+    embeds_many :url, UrlObjectValidator, primary_key: false do
+      field(:type, :string)
+      field(:href, ObjectValidators.Uri)
+      field(:mediaType, :string, default: "application/octet-stream")
+    end
   end
 
   def cast_and_validate(data) do
@@ -37,7 +42,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
 
     struct
     |> cast(data, [:type, :mediaType, :name])
-    |> cast_embed(:url, required: true)
+    |> cast_embed(:url, with: &url_changeset/2)
+    |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
+    |> validate_required([:type, :mediaType, :url])
+  end
+
+  def url_changeset(struct, data) do
+    data = fix_media_type(data)
+
+    struct
+    |> cast(data, [:type, :href, :mediaType])
+    |> validate_inclusion(:type, ["Link"])
+    |> validate_required([:type, :href, :mediaType])
   end
 
   def fix_media_type(data) do
@@ -75,6 +91,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
 
   def validate_data(cng) do
     cng
+    |> validate_inclusion(:type, ~w[Document Audio Image Video])
     |> validate_required([:mediaType, :url, :type])
   end
 end

lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex

@@ -0,0 +1,134 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EarmarkRenderer
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:bto, ObjectValidators.Recipients, default: [])
+    field(:bcc, ObjectValidators.Recipients, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
+    field(:content, :string)
+
+    field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:published, ObjectValidators.DateTime)
+    field(:emoji, ObjectValidators.Emoji, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, ObjectValidators.ObjectID)
+    field(:url, ObjectValidators.Uri)
+
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix_url(%{"url" => url} = data) when is_list(url) do
+    attachment =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and String.starts_with?(mime_type, ["video/", "audio/"])
+      end)
+
+    link_element =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and mime_type == "text/html"
+      end)
+
+    data
+    |> Map.put("attachment", [attachment])
+    |> Map.put("url", link_element["href"])
+  end
+
+  defp fix_url(data), do: data
+
+  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
+       when is_binary(content) do
+    content =
+      content
+      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
+      |> Pleroma.HTML.filter_tags()
+
+    Map.put(data, "content", content)
+  end
+
+  defp fix_content(data), do: data
+
+  defp fix(data) do
+    data
+    |> CommonFixes.fix_defaults()
+    |> CommonFixes.fix_attribution()
+    |> CommonFixes.fix_actor()
+    |> Transmogrifier.fix_emoji()
+    |> fix_url()
+    |> fix_content()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast_embed(:attachment)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Audio", "Video"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end

lib/pleroma/web/activity_pub/object_validators/common_fixes.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
+  alias Pleroma.Object.Containment
   alias Pleroma.Web.ActivityPub.Utils
 
   # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
@@ -19,4 +20,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
     data
     |> Map.put_new("actor", data["attributedTo"])
   end
+
+  def fix_actor(data) do
+    actor = Containment.get_actor(data)
+
+    data
+    |> Map.put("actor", actor)
+    |> Map.put("attributedTo", actor)
+  end
 end

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -10,9 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   import Ecto.Changeset
-  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   @primary_key false
 
@@ -75,14 +76,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     data
     |> fix_context(meta)
     |> fix_addressing(meta)
+    |> CommonFixes.fix_actor()
   end
 
   def validate_data(cng, meta \\ []) do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])
-    |> validate_actor_presence()
+    |> CommonValidations.validate_actor_presence()
-    |> validate_any_presence([:to, :cc])
+    |> CommonValidations.validate_any_presence([:to, :cc])
     |> validate_actors_match(meta)
     |> validate_context_match(meta)
     |> validate_object_nonexistence()

lib/pleroma/web/activity_pub/object_validators/note_validator.ex

@@ -1,73 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.Transmogrifier
-
-  import Ecto.Changeset
-
-  @primary_key false
-
-  embedded_schema do
-    field(:id, ObjectValidators.ObjectID, primary_key: true)
-    field(:to, ObjectValidators.Recipients, default: [])
-    field(:cc, ObjectValidators.Recipients, default: [])
-    field(:bto, ObjectValidators.Recipients, default: [])
-    field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
-    field(:tag, {:array, :map}, default: [])
-    field(:type, :string)
-
-    field(:name, :string)
-    field(:summary, :string)
-    field(:content, :string)
-
-    field(:context, :string)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
-
-    field(:actor, ObjectValidators.ObjectID)
-    field(:attributedTo, ObjectValidators.ObjectID)
-    field(:published, ObjectValidators.DateTime)
-    field(:emoji, ObjectValidators.Emoji, default: %{})
-    field(:sensitive, :boolean, default: false)
-    # TODO: Write type
-    field(:attachment, {:array, :map}, default: [])
-    field(:replies_count, :integer, default: 0)
-    field(:like_count, :integer, default: 0)
-    field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, ObjectValidators.ObjectID)
-    field(:url, ObjectValidators.Uri)
-
-    field(:likes, {:array, :string}, default: [])
-    field(:announcements, {:array, :string}, default: [])
-  end
-
-  def cast_and_validate(data) do
-    data
-    |> cast_data()
-    |> validate_data()
-  end
-
-  defp fix(data) do
-    data
-    |> Transmogrifier.fix_emoji()
-  end
-
-  def cast_data(data) do
-    data = fix(data)
-
-    %__MODULE__{}
-    |> cast(data, __schema__(:fields))
-  end
-
-  def validate_data(data_cng) do
-    data_cng
-    |> validate_inclusion(:type, ["Note"])
-    |> validate_required([:id, :actor, :to, :cc, :type, :content, :context])
-  end
-end

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -47,8 +47,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     # short identifier for PleromaFE to group statuses by context
     field(:context_id, :integer)
 
-    field(:likes, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
 
     field(:closed, ObjectValidators.DateTime)
     field(:voters, {:array, ObjectValidators.ObjectID}, default: [])

lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex

@@ -1,24 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-
-  import Ecto.Changeset
-  @primary_key false
-
-  embedded_schema do
-    field(:type, :string)
-    field(:href, ObjectValidators.Uri)
-    field(:mediaType, :string, default: "application/octet-stream")
-  end
-
-  def changeset(struct, data) do
-    struct
-    |> cast(data, __schema__(:fields))
-    |> validate_required([:type, :href, :mediaType])
-  end
-end

lib/pleroma/web/activity_pub/side_effects.ex

@@ -336,7 +336,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Question Event] do
+      when objtype in ~w[Audio Video Question Event Article] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   A module to handle coding from internal to wire ActivityPub and back.
   """
   alias Pleroma.Activity
-  alias Pleroma.EarmarkRenderer
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Maps
   alias Pleroma.Object
@@ -45,7 +44,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
-    |> fix_content
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -274,24 +272,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     Map.put(object, "url", url["href"])
   end
 
-  def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
+  def fix_url(%{"url" => url} = object) when is_list(url) do
-    attachment =
-      Enum.find(url, fn x ->
-        media_type = x["mediaType"] || x["mimeType"] || ""
-
-        is_map(x) and String.starts_with?(media_type, "video/")
-      end)
-
-    link_element =
-      Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end)
-
-    object
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
-  end
-
-  def fix_url(%{"type" => object_type, "url" => url} = object)
-      when object_type != "Video" and is_list(url) do
     first_element = Enum.at(url, 0)
 
     url_string =
@@ -371,18 +352,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_type(object, _), do: object
 
-  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object)
-       when is_binary(content) do
-    html_content =
-      content
-      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
-      |> Pleroma.HTML.filter_tags()
-
-    Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"})
-  end
-
-  defp fix_content(object), do: object
-
   # Reduce the object list to find the reported user.
   defp get_reported(objects) do
     Enum.reduce_while(objects, nil, fn ap_id, _ ->
@@ -455,7 +424,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ~w{Article Note Video Page} do
+      when objtype in ~w{Note Page} do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -549,7 +518,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype}} = data,
         _options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio Event} do
+      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
+    data = Map.put(data, "object", strip_internal_fields(data["object"]))
+
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}

test/fixtures/tesla_mock/wedistribute-create-article.json

@@ -0,0 +1 @@
+{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Create","actor":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","object":{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Article","name":"The end is near: Mastodon plans to drop OStatus support","content":"<!-- wp:paragraph {\"dropCap\":true} -->\n<p class=\"has-drop-cap\">The days of OStatus are numbered. The venerable protocol has served as a glue between many different types of servers since the early days of the Fediverse, connecting StatusNet (now GNU Social) to Friendica, Hubzilla, Mastodon, and Pleroma.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Now that many fediverse platforms support ActivityPub as a successor protocol, Mastodon appears to be drawing a line in the sand. In <a href=\"https:\/\/www.patreon.com\/posts\/mastodon-2-9-and-28121681\">a Patreon update<\/a>, Eugen Rochko writes:<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:quote -->\n<blockquote class=\"wp-block-quote\"><p>...OStatus...has overstayed its welcome in the code...and now that most of the network uses ActivityPub, it's time for it to go. <\/p><cite>Eugen Rochko, Mastodon creator<\/cite><\/blockquote>\n<!-- \/wp:quote -->\n\n<!-- wp:paragraph -->\n<p>The <a href=\"https:\/\/github.com\/tootsuite\/mastodon\/pull\/11205\">pull request<\/a> to remove Pubsubhubbub and Salmon, two of the main components of OStatus, has already been merged into Mastodon's master branch.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Some projects will be left in the dark as a side effect of this. GNU Social and PostActiv, for example, both only communicate using OStatus. While <a href=\"https:\/\/mastodon.social\/@dansup\/102076573310057902\">some discussion<\/a> exists regarding adopting ActivityPub for GNU Social, and <a href=\"https:\/\/notabug.org\/diogo\/gnu-social\/src\/activitypub\/plugins\/ActivityPub\">a plugin is in development<\/a>, it hasn't been formally adopted yet. We just hope that the <a href=\"https:\/\/status.fsf.org\/main\/public\">Free Software Foundation's instance<\/a> gets updated in time!<\/p>\n<!-- \/wp:paragraph -->","summary":"One of the largest platforms in the federated social web is dropping the protocol that it started with.","attributedTo":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","url":"https:\/\/wedistribute.org\/2019\/07\/mastodon-drops-ostatus\/","to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810","likes":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/likes","shares":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/shares"},"to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85809"}

test/web/activity_pub/object_validators/article_note_validator_test.exs

@@ -2,10 +2,10 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do
   use Pleroma.DataCase
 
-  alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
   alias Pleroma.Web.ActivityPub.Utils
 
   import Pleroma.Factory
@@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do
     end
 
     test "a basic note validates", %{note: note} do
-      %{valid?: true} = NoteValidator.cast_and_validate(note)
+      %{valid?: true} = ArticleNoteValidator.cast_and_validate(note)
     end
   end
 end

test/web/activity_pub/transmogrifier/article_handling_test.exs

@@ -0,0 +1,75 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.ArticleHandlingTest do
+  use Oban.Testing, repo: Pleroma.Repo
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  test "Pterotype (Wordpress Plugin) Article" do
+    Tesla.Mock.mock(fn %{url: "https://wedistribute.org/wp-json/pterotype/v1/actor/-blog"} ->
+      %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")}
+    end)
+
+    data =
+      File.read!("test/fixtures/tesla_mock/wedistribute-create-article.json") |> Jason.decode!()
+
+    {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+
+    object = Object.normalize(data["object"])
+
+    assert object.data["name"] == "The end is near: Mastodon plans to drop OStatus support"
+
+    assert object.data["summary"] ==
+             "One of the largest platforms in the federated social web is dropping the protocol that it started with."
+
+    assert object.data["url"] == "https://wedistribute.org/2019/07/mastodon-drops-ostatus/"
+  end
+
+  test "Plume Article" do
+    Tesla.Mock.mock(fn
+      %{url: "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"} ->
+        %Tesla.Env{
+          status: 200,
+          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json")
+        }
+
+      %{url: "https://baptiste.gelez.xyz/@/BaptisteGelez"} ->
+        %Tesla.Env{
+          status: 200,
+          body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json")
+        }
+    end)
+
+    {:ok, object} =
+      Fetcher.fetch_object_from_id(
+        "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
+      )
+
+    assert object.data["name"] == "This Month in Plume: June 2018"
+
+    assert object.data["url"] ==
+             "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
+  end
+
+  test "Prismo Article" do
+    Tesla.Mock.mock(fn %{url: "https://prismo.news/@mxb"} ->
+      %Tesla.Env{
+        status: 200,
+        body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json")
+      }
+    end)
+
+    data = File.read!("test/fixtures/prismo-url-map.json") |> Jason.decode!()
+
+    {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+    object = Object.normalize(data["object"])
+
+    assert object.data["url"] == "https://prismo.news/posts/83"
+  end
+end

test/web/activity_pub/transmogrifier/video_handling_test.exs

@@ -0,0 +1,93 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Transmogrifier.VideoHandlingTest do
+  use Oban.Testing, repo: Pleroma.Repo
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  test "skip converting the content when it is nil" do
+    data =
+      File.read!("test/fixtures/tesla_mock/framatube.org-video.json")
+      |> Jason.decode!()
+      |> Kernel.put_in(["object", "content"], nil)
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert object = Object.normalize(activity, false)
+
+    assert object.data["content"] == nil
+  end
+
+  test "it converts content of object to html" do
+    data = File.read!("test/fixtures/tesla_mock/framatube.org-video.json") |> Jason.decode!()
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert object = Object.normalize(activity, false)
+
+    assert object.data["content"] ==
+             "<p>Après avoir mené avec un certain succès la campagne « Dégooglisons Internet » en 2014, l’association Framasoft annonce fin 2019 arrêter progressivement un certain nombre de ses services alternatifs aux GAFAM. Pourquoi ?</p><p>Transcription par @aprilorg ici : <a href=\"https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft\">https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft</a></p>"
+  end
+
+  test "it remaps video URLs as attachments if necessary" do
+    {:ok, object} =
+      Fetcher.fetch_object_from_id(
+        "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
+      )
+
+    assert object.data["url"] ==
+             "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
+
+    assert object.data["attachment"] == [
+             %{
+               "type" => "Link",
+               "mediaType" => "video/mp4",
+               "name" => nil,
+               "url" => [
+                 %{
+                   "href" =>
+                     "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4",
+                   "mediaType" => "video/mp4",
+                   "type" => "Link"
+                 }
+               ]
+             }
+           ]
+
+    data = File.read!("test/fixtures/tesla_mock/framatube.org-video.json") |> Jason.decode!()
+
+    {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
+
+    assert object = Object.normalize(activity, false)
+
+    assert object.data["attachment"] == [
+             %{
+               "type" => "Link",
+               "mediaType" => "video/mp4",
+               "name" => nil,
+               "url" => [
+                 %{
+                   "href" =>
+                     "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4",
+                   "mediaType" => "video/mp4",
+                   "type" => "Link"
+                 }
+               ]
+             }
+           ]
+
+    assert object.data["url"] ==
+             "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"
+  end
+end

test/web/activity_pub/transmogrifier_test.exs

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
 
   alias Pleroma.Activity
   alias Pleroma.Object
-  alias Pleroma.Object.Fetcher
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -45,15 +44,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       assert "test" in object.data["tag"]
     end
 
-    test "it works for incoming notices with url not being a string (prismo)" do
-      data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!()
-
-      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      object = Object.normalize(data["object"])
-
-      assert object.data["url"] == "https://prismo.news/posts/83"
-    end
-
     test "it cleans up incoming notices which are not really DMs" do
       user = insert(:user)
       other_user = insert(:user)
@@ -355,83 +345,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       refute User.following?(User.get_cached_by_ap_id(data["actor"]), user)
     end
 
-    test "skip converting the content when it is nil" do
-      object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe"
-
-      {:ok, object} = Fetcher.fetch_and_contain_remote_object_from_id(object_id)
-
-      result =
-        Pleroma.Web.ActivityPub.Transmogrifier.fix_object(Map.merge(object, %{"content" => nil}))
-
-      assert result["content"] == nil
-    end
-
-    test "it converts content of object to html" do
-      object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe"
-
-      {:ok, %{"content" => content_markdown}} =
-        Fetcher.fetch_and_contain_remote_object_from_id(object_id)
-
-      {:ok, %Pleroma.Object{data: %{"content" => content}} = object} =
-        Fetcher.fetch_object_from_id(object_id)
-
-      assert content_markdown ==
-               "Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership\n\nTwenty Years in Jail: FreeBSD's Jails, Then and Now\n\nJails started as a limited virtualization system, but over the last two years they've..."
-
-      assert content ==
-               "<p>Support this and our other Michigan!/usr/group videos and meetings. Learn more at <a href=\"http://mug.org/membership\">http://mug.org/membership</a></p><p>Twenty Years in Jail: FreeBSD’s Jails, Then and Now</p><p>Jails started as a limited virtualization system, but over the last two years they’ve…</p>"
-
-      assert object.data["mediaType"] == "text/html"
-    end
-
-    test "it remaps video URLs as attachments if necessary" do
-      {:ok, object} =
-        Fetcher.fetch_object_from_id(
-          "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
-        )
-
-      assert object.data["url"] ==
-               "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
-
-      assert object.data["attachment"] == [
-               %{
-                 "type" => "Link",
-                 "mediaType" => "video/mp4",
-                 "url" => [
-                   %{
-                     "href" =>
-                       "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4",
-                     "mediaType" => "video/mp4",
-                     "type" => "Link"
-                   }
-                 ]
-               }
-             ]
-
-      {:ok, object} =
-        Fetcher.fetch_object_from_id(
-          "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"
-        )
-
-      assert object.data["attachment"] == [
-               %{
-                 "type" => "Link",
-                 "mediaType" => "video/mp4",
-                 "url" => [
-                   %{
-                     "href" =>
-                       "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4",
-                     "mediaType" => "video/mp4",
-                     "type" => "Link"
-                   }
-                 ]
-               }
-             ]
-
-      assert object.data["url"] ==
-               "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"
-    end
-
     test "it accepts Flag activities" do
       user = insert(:user)
       other_user = insert(:user)
@@ -1133,75 +1046,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
              }
     end
 
-    test "fixes data for video object" do
+    test "returns non-modified object" do
-      object = %{
-        "type" => "Video",
-        "url" => [
-          %{
-            "type" => "Link",
-            "mimeType" => "video/mp4",
-            "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "video/mp4",
-            "href" => "https://peertube46fb-ad81-2d4c2d1630e3-240.mp4"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d1630e3"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d16377-42"
-          }
-        ]
-      }
-
-      assert Transmogrifier.fix_url(object) == %{
-               "attachment" => [
-                 %{
-                   "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4",
-                   "mimeType" => "video/mp4",
-                   "type" => "Link"
-                 }
-               ],
-               "type" => "Video",
-               "url" => "https://peertube.-2d4c2d1630e3"
-             }
-    end
-
-    test "fixes url for not Video object" do
-      object = %{
-        "type" => "Text",
-        "url" => [
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d1630e3"
-          },
-          %{
-            "type" => "Link",
-            "mimeType" => "text/html",
-            "href" => "https://peertube.-2d4c2d16377-42"
-          }
-        ]
-      }
-
-      assert Transmogrifier.fix_url(object) == %{
-               "type" => "Text",
-               "url" => "https://peertube.-2d4c2d1630e3"
-             }
-
-      assert Transmogrifier.fix_url(%{"type" => "Text", "url" => []}) == %{
-               "type" => "Text",
-               "url" => ""
-             }
-    end
-
-    test "retunrs not modified object" do
       assert Transmogrifier.fix_url(%{"type" => "Text"}) == %{"type" => "Text"}
     end
   end