Merge remote-tracking branch 'upstream/develop' into neckbeard

neckbeard
Your New SJW Waifu 3 months ago
commit 11ea7d98bb
  1. 4
      .gitlab-ci.yml
  2. 3
      CHANGELOG.md
  3. 4
      Dockerfile
  4. 7
      ci/Dockerfile
  5. 12
      ci/README
  6. 4
      config/config.exs
  7. 24
      config/description.exs
  8. 2
      docs/configuration/howto_database_config.md
  9. 4
      docs/development/API/differences_in_mastoapi_responses.md
  10. 2
      docs/installation/generic_dependencies.include
  11. 2
      elixir_buildpack.config
  12. 9
      lib/mix/tasks/pleroma/config.ex
  13. 3
      lib/mix/tasks/pleroma/database.ex
  14. 5
      lib/pleroma/activity.ex
  15. 36
      lib/pleroma/activity/html.ex
  16. 31
      lib/pleroma/activity/ir/topics.ex
  17. 6
      lib/pleroma/activity/queries.ex
  18. 4
      lib/pleroma/application.ex
  19. 105
      lib/pleroma/bbs/handler.ex
  20. 15
      lib/pleroma/config/loader.ex
  21. 36
      lib/pleroma/constants.ex
  22. 1
      lib/pleroma/data_migration.ex
  23. 139
      lib/pleroma/migrators/context_objects_deletion_migrator.ex
  24. 2
      lib/pleroma/migrators/hashtags_table_migrator.ex
  25. 40
      lib/pleroma/notification.ex
  26. 7
      lib/pleroma/object.ex
  27. 34
      lib/pleroma/object/fetcher.ex
  28. 240
      lib/pleroma/object/updater.ex
  29. 28
      lib/pleroma/signature.ex
  30. 2
      lib/pleroma/upload.ex
  31. 19
      lib/pleroma/user.ex
  32. 16
      lib/pleroma/web/activity_pub/activity_pub.ex
  33. 52
      lib/pleroma/web/activity_pub/activity_pub_controller.ex
  34. 13
      lib/pleroma/web/activity_pub/builder.ex
  35. 45
      lib/pleroma/web/activity_pub/mrf.ex
  36. 3
      lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
  37. 6
      lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex
  38. 7
      lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex
  39. 47
      lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex
  40. 57
      lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
  41. 9
      lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
  42. 15
      lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex
  43. 7
      lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex
  44. 6
      lib/pleroma/web/activity_pub/mrf/normalize_markup.ex
  45. 3
      lib/pleroma/web/activity_pub/mrf/policy.ex
  46. 116
      lib/pleroma/web/activity_pub/object_validator.ex
  47. 12
      lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
  48. 3
      lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
  49. 3
      lib/pleroma/web/activity_pub/object_validators/common_fields.ex
  50. 7
      lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
  51. 2
      lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
  52. 2
      lib/pleroma/web/activity_pub/object_validators/event_validator.ex
  53. 2
      lib/pleroma/web/activity_pub/object_validators/question_validator.ex
  54. 4
      lib/pleroma/web/activity_pub/object_validators/update_validator.ex
  55. 97
      lib/pleroma/web/activity_pub/side_effects.ex
  56. 33
      lib/pleroma/web/activity_pub/transmogrifier.ex
  57. 23
      lib/pleroma/web/activity_pub/utils.ex
  58. 2
      lib/pleroma/web/activity_pub/views/user_view.ex
  59. 4
      lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
  60. 192
      lib/pleroma/web/api_spec/operations/status_operation.ex
  61. 10
      lib/pleroma/web/api_spec/operations/twitter_util_operation.ex
  62. 15
      lib/pleroma/web/api_spec/schemas/status.ex
  63. 35
      lib/pleroma/web/common_api.ex
  64. 5
      lib/pleroma/web/common_api/activity_draft.ex
  65. 39
      lib/pleroma/web/common_api/utils.ex
  66. 6
      lib/pleroma/web/federator.ex
  67. 1
      lib/pleroma/web/mastodon_api/controllers/notification_controller.ex
  68. 60
      lib/pleroma/web/mastodon_api/controllers/status_controller.ex
  69. 4
      lib/pleroma/web/mastodon_api/views/instance_view.ex
  70. 15
      lib/pleroma/web/mastodon_api/views/notification_view.ex
  71. 173
      lib/pleroma/web/mastodon_api/views/status_view.ex
  72. 15
      lib/pleroma/web/metadata/utils.ex
  73. 2
      lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
  74. 53
      lib/pleroma/web/plugs/http_signature_plug.ex
  75. 4
      lib/pleroma/web/router.ex
  76. 18
      lib/pleroma/web/streamer.ex
  77. 10
      lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex
  78. 95
      lib/pleroma/web/twitter_api/controllers/util_controller.ex
  79. 2
      lib/pleroma/web/twitter_api/views/util_view.ex
  80. 27
      lib/pleroma/web/views/streamer_view.ex
  81. 4
      lib/pleroma/web/web_finger.ex
  82. 5
      mix.exs
  83. 48
      priv/gettext/config_descriptions.pot
  84. 52
      priv/gettext/errors.pot
  85. 158
      priv/gettext/nl/LC_MESSAGES/errors.po
  86. 165
      priv/gettext/nl/LC_MESSAGES/posix_errors.po
  87. 567
      priv/gettext/nl/LC_MESSAGES/static_pages.po
  88. 49
      priv/gettext/static_pages.pot
  89. 51
      priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs
  90. 37
      priv/repo/migrations/20220711182322_add_associated_object_id_function.exs
  91. 37
      priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs
  92. 18
      priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs
  93. 156
      priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs
  94. 28
      priv/repo/migrations/20220905011454_generate_unset_user_keys.exs
  95. 3
      priv/static/schemas/litepub-0.1.jsonld
  96. 6
      priv/templates/sample_config.eex
  97. 2
      restarter/mix.exs
  98. 61
      test/fixtures/create-pleroma-reply-to-misskey-thread.json
  99. 50
      test/fixtures/tesla_mock/helene@p.helene.moe.json
  100. 65
      test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json
  101. Some files were not shown because too many files have changed in this diff Show More

@ -271,7 +271,7 @@ amd64:
MIX_ENV: prod
before_script: &before-release
- apt-get update && apt-get install -y cmake libmagic-dev
- echo "import Mix.Config" > config/prod.secret.exs
- echo "import Config" > config/prod.secret.exs
- mix local.hex --force
- mix local.rebar --force
script: &release
@ -290,7 +290,7 @@ amd64-musl:
variables: *release-variables
before_script: &before-release-musl
- apk add git build-base cmake file-dev openssl
- echo "import Mix.Config" > config/prod.secret.exs
- echo "import Config" > config/prod.secret.exs
- mix local.hex --force
- mix local.rebar --force
script: *release

@ -11,8 +11,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- MastoFE
### Changed
- **Breaking:** Elixir >=1.10 is now required (was >= 1.9)
- Allow users to remove their emails if instance does not need email to register
- Uploadfilter `Pleroma.Upload.Filter.Exiftool` has been renamed to `Pleroma.Upload.Filter.Exiftool.StripLocation`
- **Breaking**: `/api/v1/pleroma/backups` endpoints now requires `read:backups` scope instead of `read:accounts`
- Updated the recommended pleroma.vcl configuration for Varnish to target Varnish 7.0+
### Added
@ -34,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field.
- Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field
- Added move account API
- Enable remote users to interact with posts
### Fixed
- Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies

@ -1,11 +1,11 @@
FROM elixir:1.9-alpine as build
FROM elixir:1.10-alpine as build
COPY . .
ENV MIX_ENV=prod
RUN apk add git gcc g++ musl-dev make cmake file-dev &&\
echo "import Mix.Config" > config/prod.secret.exs &&\
echo "import Config" > config/prod.secret.exs &&\
mix local.hex --force &&\
mix local.rebar --force &&\
mix deps.get --only prod &&\

@ -1,7 +1,8 @@
FROM elixir:1.9.4
FROM elixir:1.10.4
# Single RUN statement, otherwise intermediate images are created
# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
RUN apt-get update &&\
apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
mix local.hex --force &&\
mix local.rebar --force

@ -0,0 +1,12 @@
## Dependencies
Assuming an AMD64 Alpine system, you're going to need the following packages
- `qemu qemu-openrc qemu-arm qemu-aarch64` for binfmt
- `docker-cli-buildx` for building the images
## Setting up
```
docker login git.pleroma.social:5050
doas rc-service qemu-binfmt start
```

@ -37,7 +37,7 @@
# FIGURATION! EDIT YOUR SECRET FILE (either prod.secret.exs, dev.secret.exs).
#
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
# and its dependencies with the aid of the Config module.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
@ -673,6 +673,8 @@ config :pleroma, :features, improved_hashtag_timeline: :auto
config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
config :pleroma, :delete_context_objects, fault_rate_allowance: 0.01
config :pleroma, :env, Mix.env()
config :http_signatures,

@ -495,6 +495,27 @@ config :pleroma, :config_description, [
}
]
},
%{
group: :pleroma,
key: :delete_context_objects,
type: :group,
description: "`delete_context_objects` background migration settings",
children: [
%{
key: :fault_rate_allowance,
type: :float,
description:
"Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if context object deletion failed for all records.",
suggestions: [0.01]
},
%{
key: :sleep_interval_ms,
type: :integer,
description:
"Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
}
]
},
%{
group: :pleroma,
key: :instance,
@ -984,7 +1005,8 @@ config :pleroma, :config_description, [
key: :birthday_min_age,
type: :integer,
description:
"Minimum required age for users to create account. Only used if birthday is required."
"Minimum required age (in days) for users to create account. Only used if birthday is required.",
suggestions: [6570]
}
]
},

@ -59,7 +59,7 @@ The configuration of Pleroma has traditionally been managed with a config file,
Here is an example of a server config stripped down after migration:
```
use Mix.Config
import Config
config :pleroma, Pleroma.Web.Endpoint,
url: [host: "cool.pleroma.site", scheme: "https", port: 443]

@ -40,6 +40,10 @@ Has these additional fields under the `pleroma` object:
- `parent_visible`: If the parent of this post is visible to the user or not.
- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.
The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes:
- `content_type`: The content type of the status source.
## Scheduled statuses
Has these additional fields in `params`:

@ -1,7 +1,7 @@
## Required dependencies
* PostgreSQL 9.6+
* Elixir 1.9+
* Elixir 1.10+
* Erlang OTP 22.2+
* git
* file / libmagic

@ -1,2 +1,2 @@
elixir_version=1.9.4
elixir_version=1.10.4
erlang_version=22.3.4.1

@ -304,13 +304,8 @@ defmodule Mix.Tasks.Pleroma.Config do
System.cmd("mix", ["format", path])
end
if Code.ensure_loaded?(Config.Reader) do
defp config_header, do: "import Config\r\n\r\n"
defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
else
defp config_header, do: "use Mix.Config\r\n\r\n"
defp read_file(config_file), do: Mix.Config.eval!(config_file)
end
defp config_header, do: "import Config\r\n\r\n"
defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
defp write_and_delete(config, file, delete?) do
config

@ -154,9 +154,8 @@ defmodule Mix.Tasks.Pleroma.Database do
|> join(:inner, [a], o in Object,
on:
fragment(
"(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
"(?->>'id') = associated_object_id((?))",
o.data,
a.data,
a.data
)
)

@ -53,7 +53,7 @@ defmodule Pleroma.Activity do
#
# ```
# |> join(:inner, [activity], o in Object,
# on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
# on: fragment("(?->>'id') = associated_object_id((?))",
# o.data, activity.data, activity.data))
# |> preload([activity, object], [object: object])
# ```
@ -69,9 +69,8 @@ defmodule Pleroma.Activity do
join(query, join_type, [activity], o in Object,
on:
fragment(
"(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
"(?->>'id') = associated_object_id(?)",
o.data,
activity.data,
activity.data
),
as: :object

@ -8,6 +8,40 @@ defmodule Pleroma.Activity.HTML do
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
# We store a list of cache keys related to an activity in a
# separate cache, scrubber_management_cache. It has the same
# size as scrubber_cache (see application.ex). Every time we add
# a cache to scrubber_cache, we update scrubber_management_cache.
#
# The most recent write of a certain key in the management cache
# is the same as the most recent write of any record related to that
# key in the main cache.
# Assuming LRW ( https://hexdocs.pm/cachex/Cachex.Policy.LRW.html ),
# this means when the management cache is evicted by cachex, all
# related records in the main cache will also have been evicted.
defp get_cache_keys_for(activity_id) do
with {:ok, list} when is_list(list) <- @cachex.get(:scrubber_management_cache, activity_id) do
list
else
_ -> []
end
end
defp add_cache_key_for(activity_id, additional_key) do
current = get_cache_keys_for(activity_id)
unless additional_key in current do
@cachex.put(:scrubber_management_cache, activity_id, [additional_key | current])
end
end
def invalidate_cache_for(activity_id) do
keys = get_cache_keys_for(activity_id)
Enum.map(keys, &@cachex.del(:scrubber_cache, &1))
@cachex.del(:scrubber_management_cache, activity_id)
end
def get_cached_scrubbed_html_for_activity(
content,
scrubbers,
@ -19,6 +53,8 @@ defmodule Pleroma.Activity.HTML do
@cachex.fetch!(:scrubber_cache, key, fn _key ->
object = Object.normalize(activity, fetch: false)
add_cache_key_for(activity.id, key)
HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
end)
end

@ -13,6 +13,14 @@ defmodule Pleroma.Activity.Ir.Topics do
|> List.flatten()
end
defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Delete"}}) do
["user", "user:pleroma_chat"]
end
defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Create"}}) do
[]
end
defp generate_topics(%{data: %{"type" => "Answer"}}, _) do
[]
end
@ -21,7 +29,7 @@ defmodule Pleroma.Activity.Ir.Topics do
["user", "list"] ++ visibility_tags(object, activity)
end
defp visibility_tags(object, activity) do
defp visibility_tags(object, %{data: %{"type" => type}} = activity) when type != "Announce" do
case Visibility.get_visibility(activity) do
"public" ->
if activity.local do
@ -31,6 +39,10 @@ defmodule Pleroma.Activity.Ir.Topics do
end
|> item_creation_tags(object, activity)
"local" ->
["public:local"]
|> item_creation_tags(object, activity)
"direct" ->
["direct"]
@ -39,6 +51,10 @@ defmodule Pleroma.Activity.Ir.Topics do
end
end
defp visibility_tags(_object, _activity) do
[]
end
defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
tags ++
remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
@ -63,7 +79,18 @@ defmodule Pleroma.Activity.Ir.Topics do
defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
defp attachment_topics(_object, %{local: true} = activity) do
case Visibility.get_visibility(activity) do
"public" ->
["public:media", "public:local:media"]
"local" ->
["public:local:media"]
_ ->
[]
end
end
defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]

@ -52,8 +52,7 @@ defmodule Pleroma.Activity.Queries do
activity in query,
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)",
activity.data,
"associated_object_id((?)) = ANY(?)",
activity.data,
^object_ids
)
@ -64,8 +63,7 @@ defmodule Pleroma.Activity.Queries do
from(activity in query,
where:
fragment(
"coalesce((?)->'object'->>'id', (?)->>'object') = ?",
activity.data,
"associated_object_id((?)) = ?",
activity.data,
^object_id
)

@ -199,6 +199,7 @@ defmodule Pleroma.Application do
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500),
build_cachex("scrubber_management", limit: 2500),
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
@ -248,7 +249,8 @@ defmodule Pleroma.Application do
defp background_migrators do
[
Pleroma.Migrators.HashtagsTableMigrator
Pleroma.Migrators.HashtagsTableMigrator,
Pleroma.Migrators.ContextObjectsDeletionMigrator
]
end

@ -42,8 +42,45 @@ defmodule Pleroma.BBS.Handler do
def puts_activity(activity) do
status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity})
IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
IO.puts(HTML.strip_tags(status.content))
status.content
|> String.split("<br/>")
|> Enum.map(&HTML.strip_tags/1)
|> Enum.map(&HtmlEntities.decode/1)
|> Enum.map(&IO.puts/1)
end
def puts_notification(activity, user) do
notification =
Pleroma.Web.MastodonAPI.NotificationView.render("show.json", %{
notification: activity,
for: user
})
IO.puts(
"== (#{notification.type}) #{notification.status.id} by #{notification.account.display_name} (#{notification.account.acct})"
)
notification.status.content
|> String.split("<br/>")
|> Enum.map(&HTML.strip_tags/1)
|> Enum.map(&HtmlEntities.decode/1)
|> (fn x ->
case x do
[content] ->
"> " <> content
[head | _tail] ->
# "> " <> hd <> "..."
head
|> String.slice(1, 80)
|> (fn x -> "> " <> x <> "..." end).()
end
end).()
|> IO.puts()
IO.puts("")
end
@ -53,6 +90,11 @@ defmodule Pleroma.BBS.Handler do
IO.puts("home - Show the home timeline")
IO.puts("p <text> - Post the given text")
IO.puts("r <id> <text> - Reply to the post with the given id")
IO.puts("t <id> - Show a thread from the given id")
IO.puts("n - Show notifications")
IO.puts("n read - Mark all notifactions as read")
IO.puts("f <id> - Favourites the post with the given id")
IO.puts("R <id> - Repeat the post with the given id")
IO.puts("quit - Quit")
state
@ -73,11 +115,53 @@ defmodule Pleroma.BBS.Handler do
state
end
def handle_command(%{user: user} = state, "t " <> activity_id) do
with %Activity{} = activity <- Activity.get_by_id(activity_id) do
activities =
ActivityPub.fetch_activities_for_context(activity.data["context"], %{
blocking_user: user,
user: user,
exclude_id: activity.id
})
case activities do
[] ->
activity_id
|> Activity.get_by_id()
|> puts_activity()
_ ->
activities
|> Enum.reverse()
|> Enum.each(&puts_activity/1)
end
else
_e -> IO.puts("Could not show this thread...")
end
state
end
def handle_command(%{user: user} = state, "n read") do
Pleroma.Notification.clear(user)
IO.puts("All notifications were marked as read")
state
end
def handle_command(%{user: user} = state, "n") do
user
|> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(%{})
|> Enum.each(&puts_notification(&1, user))
state
end
def handle_command(%{user: user} = state, "p " <> text) do
text = String.trim(text)
with {:ok, _activity} <- CommonAPI.post(user, %{status: text}) do
IO.puts("Posted!")
with {:ok, activity} <- CommonAPI.post(user, %{status: text}) do
IO.puts("Posted! ID: #{activity.id}")
else
_e -> IO.puts("Could not post...")
end
@ -85,6 +169,19 @@ defmodule Pleroma.BBS.Handler do
state
end
def handle_command(%{user: user} = state, "f " <> id) do
id = String.trim(id)
with %Activity{} = activity <- Activity.get_by_id(id),
{:ok, _activity} <- CommonAPI.favorite(user, activity) do
IO.puts("Favourited!")
else
_e -> IO.puts("Could not Favourite...")
end
state
end
def handle_command(state, "home") do
user = state.user
@ -123,7 +220,7 @@ defmodule Pleroma.BBS.Handler do
loop(%{state | counter: state.counter + 1})
{:error, :interrupted} ->
{:input, ^input, {:error, :interrupted}} ->
IO.puts("Caught Ctrl+C...")
loop(%{state | counter: state.counter + 1})

@ -19,21 +19,10 @@ defmodule Pleroma.Config.Loader do
:tesla
]
if Code.ensure_loaded?(Config.Reader) do
@reader Config.Reader
def read(path), do: @reader.read!(path)
else
# support for Elixir less than 1.9
@reader Mix.Config
def read(path) do
path
|> @reader.eval!()
|> elem(0)
end
end
@reader Config.Reader
@spec read(Path.t()) :: keyword()
def read(path), do: @reader.read!(path)
@spec merge(keyword(), keyword()) :: keyword()
def merge(c1, c2), do: @reader.merge(c1, c2)

@ -28,6 +28,42 @@ defmodule Pleroma.Constants do
~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
)
const(status_updatable_fields,
do: [
"source",
"tag",
"updated",
"emoji",
"content",
"summary",
"sensitive",
"attachment",
"generator"
]
)
const(updatable_object_types,
do: [
"Note",
"Question",
"Audio",
"Video",
"Event",
"Article",
"Page"
]
)
const(actor_types,
do: [
"Application",
"Group",
"Organization",
"Person",
"Service"
]
)
# basic regex, just there to weed out potential mistakes
# https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
const(mime_regex,

@ -42,4 +42,5 @@ defmodule Pleroma.DataMigration do
end
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
def delete_context_objects, do: get_by_name("delete_context_objects")
end

@ -0,0 +1,139 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Migrators.ContextObjectsDeletionMigrator do
defmodule State do
use Pleroma.Migrators.Support.BaseMigratorState
@impl Pleroma.Migrators.Support.BaseMigratorState
defdelegate data_migration(), to: Pleroma.DataMigration, as: :delete_context_objects
end
use Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Object
@doc "This migration removes objects created exclusively for contexts, containing only an `id` field."
@impl BaseMigrator
def feature_config_path, do: [:features, :delete_context_objects]
@impl BaseMigrator
def fault_rate_allowance, do: Config.get([:delete_context_objects, :fault_rate_allowance], 0)
@impl BaseMigrator
def perform do
data_migration_id = data_migration_id()
max_processed_id = get_stat(:max_processed_id, 0)
Logger.info("Deleting context objects from `objects` (from oid: #{max_processed_id})...")
query()
|> where([object], object.id > ^max_processed_id)
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|> Stream.each(fn objects ->
object_ids = Enum.map(objects, & &1.id)
results = Enum.map(object_ids, &delete_context_object(&1))
failed_ids =
results
|> Enum.filter(&(elem(&1, 0) == :error))
|> Enum.map(&elem(&1, 1))
chunk_affected_count =
results
|> Enum.filter(&(elem(&1, 0) == :ok))
|> length()
for failed_id <- failed_ids do
_ =
Repo.query(
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
[data_migration_id, failed_id]
)
end
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
[data_migration_id, object_ids -- failed_ids]
)
max_object_id = Enum.at(object_ids, -1)
put_stat(:max_processed_id, max_object_id)
increment_stat(:iteration_processed_count, length(object_ids))
increment_stat(:processed_count, length(object_ids))
increment_stat(:failed_count, length(failed_ids))
increment_stat(:affected_count, chunk_affected_count)
put_stat(:records_per_second, records_per_second())
persist_state()
# A quick and dirty approach to controlling the load this background migration imposes
sleep_interval = Config.get([:delete_context_objects, :sleep_interval_ms], 0)
Process.sleep(sleep_interval)
end)
|> Stream.run()
end
@impl BaseMigrator
def query do
# Context objects have no activity type, and only one field, `id`.
# Only those context objects are without types.
from(
object in Object,
where: fragment("(?)->'type' IS NULL", object.data),
select: %{
id: object.id
}
)
end
@spec delete_context_object(integer()) :: {:ok | :error, integer()}
defp delete_context_object(id) do
result =
%Object{id: id}
|> Repo.delete()
|> elem(0)
{result, id}
end
@impl BaseMigrator
def retry_failed do
data_migration_id = data_migration_id()
failed_objects_query()
|> Repo.chunk_stream(100, :one)
|> Stream.each(fn object ->
with {res, _} when res != :error <- delete_context_object(object.id) do
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = $2",
[data_migration_id, object.id]
)
end
end)
|> Stream.run()
put_stat(:failed_count, failures_count())
persist_state()
force_continue()
end
defp failed_objects_query do
from(o in Object)
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
on: dmf.record_id == o.id
)
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|> order_by([o], asc: o.id)
end
end

@ -183,7 +183,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
DELETE FROM hashtags_objects WHERE object_id IN
(SELECT DISTINCT objects.id FROM objects
JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
ON associated_object_id(activities) =
(objects.data->>'id')
AND activities.data->>'type' = 'Create'
WHERE activities.id IS NULL);

@ -117,9 +117,8 @@ defmodule Pleroma.Notification do
|> join(:left, [n, a], object in Object,
on:
fragment(
"(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
"(?->>'id') = associated_object_id(?)",
object.data,
a.data,
a.data
)
)
@ -193,13 +192,11 @@ defmodule Pleroma.Notification do
|> join(:left, [n, a], mutated_activity in Pleroma.Activity,
on:
fragment(
"COALESCE((?->'object')->>'id', ?->>'object')",
a.data,
"associated_object_id(?)",
a.data
) ==
fragment(
"COALESCE((?->'object')->>'id', ?->>'object')",
mutated_activity.data,
"associated_object_id(?)",
mutated_activity.data
) and
fragment("(?->>'type' = 'Like' or ?->>'type' = 'Announce')", a.data, a.data) and
@ -385,7 +382,7 @@ defmodule Pleroma.Notification do
end
def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do
when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do
do_create_notifications(activity, options)
end
@ -439,6 +436,9 @@ defmodule Pleroma.Notification do
activity
|> type_from_activity_object()
"Update" ->
"update"
t ->
raise "No notification type for activity type #{t}"
end
@ -513,7 +513,16 @@ defmodule Pleroma.Notification do
def get_notified_from_activity(activity, local_only \\ true)
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do
when type in [
"Create",
"Like",
"Announce",
"Follow",
"Move",
"EmojiReact",
"Flag",
"Update"
] do
potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
potential_receivers =
@ -553,6 +562,21 @@ defmodule Pleroma.Notification do
(User.all_superusers() |> Enum.map(fn user -> user.ap_id end)) -- [actor]
end
# Update activity: notify all who repeated this
def get_potential_receiver_ap_ids(%{data: %{"type" => "Update", "actor" => actor}} = activity) do
with %Object{data: %{"id" => object_id}} <- Object.normalize(activity, fetch: false) do
repeaters =
Activity.Queries.by_type("Announce")
|> Activity.Queries.by_object_id(object_id)
|> Activity.with_joined_user_actor()
|> where([a, u], u.local)
|> select([a, u], u.ap_id)
|> Repo.all()
repeaters -- [actor]
end
end
def get_potential_receiver_ap_ids(activity) do
[]
|> Utils.maybe_notify_to_recipients(activity)

@ -40,8 +40,7 @@ defmodule Pleroma.Object do
join(query, join_type, [{object, object_position}], a in Activity,
on:
fragment(
"COALESCE(?->'object'->>'id', ?->>'object') = (? ->> 'id') AND (?->>'type' = ?) ",
a.data,
"associated_object_id(?) = (? ->> 'id') AND (?->>'type' = ?) ",
a.data,
object.data,
a.data,
@ -208,10 +207,6 @@ defmodule Pleroma.Object do
end
end
def context_mapping(context) do
Object.change(%Object{}, %{data: %{"id" => context}})
end
def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
%ObjectTombstone{
id: id,

@ -26,8 +26,42 @@ defmodule Pleroma.Object.Fetcher do
end
defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
has_history? = fn
%{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true
_ -> false
end
internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
remote_history_exists? = has_history?.(new_data)
# If the remote history exists, we treat that as the only source of truth.
new_data =
if has_history?.(old_data) and not remote_history_exists? do
Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"])
else
new_data
end
# If the remote does not have history information, we need to manage it ourselves
new_data =
if not remote_history_exists? do
changed? =
Pleroma.Constants.status_updatable_fields()
|> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end)
%{updated_object: updated_object} =
new_data
|> Object.Updater.maybe_update_history(old_data,
updated: changed?,
use_history_in_new_object?: false
)
updated_object
else
new_data
end
Map.merge(new_data, internal_fields)
end

@ -0,0 +1,240 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Object.Updater do
require Pleroma.Constants
def update_content_fields(orig_object_data, updated_object) do
Pleroma.Constants.status_updatable_fields()
|> Enum.reduce(
%{data: orig_object_data, updated: false},
fn field, %{data: data, updated: updated} ->
updated =
updated or
(field != "updated" and
Map.get(updated_object, field) != Map.get(orig_object_data, field))
data =
if Map.has_key?(updated_object, field) do
Map.put(data, field, updated_object[field])
else
Map.drop(data, [field])
end
%{data: data, updated: updated}
end
)
end
def maybe_history(object) do
with history <- Map.get(object, "formerRepresentations"),
true <- is_map(history),
"OrderedCollection" <- Map.get(history, "type"),
true <- is_list(Map.get(history, "orderedItems")),
true <- is_integer(Map.get(history, "totalItems")) do
history
else
_ -> nil
end
end
def history_for(object) do
with history when not is_nil(history) <- maybe_history(object) do
history
else
_ -> history_skeleton()
end
end
defp history_skeleton do
%{
"type" => "OrderedCollection",
"totalItems" => 0,
"orderedItems" => []
}
end
def maybe_update_history(
updated_object,
orig_object_data,
opts
) do
updated = opts[:updated]
use_history_in_new_object? = opts[:use_history_in_new_object?]
if not updated do
%{updated_object: updated_object, used_history_in_new_object?: false}
else
# Put edit history
# Note that we may have got the edit history by first fetching the object
{new_history, used_history_in_new_object?} =
with true <- use_history_in_new_object?,
updated_history when not is_nil(updated_history) <- maybe_history(opts[:new_data]) do
{updated_history, true}
else
_ ->
history = history_for(orig_object_data)
latest_history_item =
orig_object_data
|> Map.drop(["id", "formerRepresentations"])
updated_history =
history
|> Map.put("orderedItems", [latest_history_item | history["orderedItems"]])
|> Map.put("totalItems", history["totalItems"] + 1)
{updated_history, false}
end
updated_object =
updated_object
|> Map.put("formerRepresentations", new_history)
%{updated_object: updated_object, used_history_in_new_object?: used_history_in_new_object?}
end
end
defp maybe_update_poll(to_be_updated, updated_object) do
choice_key = fn data ->
if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf"
end
with true <- to_be_updated["type"] == "Question",
key <- choice_key.(updated_object),
true <- key == choice_key.(to_be_updated),
orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])),
new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])),
true <- orig_choices == new_choices do
# Choices are the same, but counts are different
to_be_updated
|> Map.put(key, updated_object[key])
else
# Choices (or vote type) have changed, do not allow this
_ -> to_be_updated
end
end
# This calculates the data to be sent as the object of an Update.
# new_data's formerRepresentations is not considered.
# formerRepresentations is added to the returned data.
def make_update_object_data(original_data, new_data, date) do
%{data: updated_data, updated: updated} =
original_data
|> update_content_fields(new_data)
if not updated do
updated_data
else
%{updated_object: updated_data} =
updated_data
|> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false)
updated_data
|> Map.put("updated", date)
end
end
# This calculates the data of the new Object from an Update.
# new_data's formerRepresentations is considered.
def make_new_object_data_from_update_object(original_data, new_data) do
update_is_reasonable =
with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]},
{_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)},
{_, last_updated} when not is_nil(last_updated) <-
{:last_updated, original_data["updated"] || original_data["published"]},
{_, {:ok, last_updated_time, _}} <-
{:last_updated, DateTime.from_iso8601(last_updated)},
:gt <- DateTime.compare(updated_time, last_updated_time) do
:update_everything
else
# only allow poll updates
{:cur_updated, _} -> :no_content_update
:eq -> :no_content_update
# allow all updates
{:last_updated, _} -> :update_everything
# allow no updates
_ -> false
end
%{
updated_object: updated_data,
used_history_in_new_object?: used_history_in_new_object?,
updated: updated
} =
if update_is_reasonable == :update_everything do
%{data: updated_data, updated: updated} =
original_data
|> update_content_fields(new_data)
updated_data
|> maybe_update_history(original_data,
updated: updated,
use_history_in_new_object?: true,
new_data: new_data
)
|> Map.put(:updated, updated)
else
%{
updated_object: original_data,
used_history_in_new_object?: false,
updated: false
}
end
updated_data =
if update_is_reasonable != false do
updated_data
|> maybe_update_poll(new_data)
else
updated_data
end
%{
updated_data: updated_data,
updated: updated,
used_history_in_new_object?: used_history_in_new_object?
}
end
def for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do
new_items =
Enum.map(items, fun)
|> Enum.reduce_while(
{:ok, []},
fn
{:ok, item}, {:ok, acc} -> {:cont, {:ok, acc ++ [item]}}
e, _acc -> {:halt, e}
end
)
case new_items do
{:ok, items} -> {:ok, Map.put(history, "orderedItems", items)}
e -> e
end
end
def for_each_history_item(history, _, _) do
{:ok, history}
end
def do_with_history(object, fun) do
with history <- object["formerRepresentations"],
object <- Map.drop(object, ["formerRepresentations"]),
{_, {:ok, object}} <- {:main_body, fun.(object)},
{_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do
object =
if history do
Map.put(object, "formerRepresentations", history)
else
object
end
{:ok, object}
else
{:main_body, e} -> e
{:history_items, e} -> e
end
end
end

@ -10,17 +10,14 @@ defmodule Pleroma.Signature do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
@known_suffixes ["/publickey", "/main-key"]
def key_id_to_actor_id(key_id) do
uri =
URI.parse(key_id)
key_id
|> URI.parse()
|> Map.put(:fragment, nil)
uri =
if not is_nil(uri.path) and String.ends_with?(uri.path, "/publickey") do
Map.put(uri, :path, String.replace(uri.path, "/publickey", ""))
else
uri
end
|> remove_suffix(@known_suffixes)
maybe_ap_id = URI.to_string(uri)
@ -36,6 +33,16 @@ defmodule Pleroma.Signature do
end
end
defp remove_suffix(uri, [test | rest]) do
if not is_nil(uri.path) and String.ends_with?(uri.path, test) do
Map.put(uri, :path, String.replace(uri.path, test, ""))
else
remove_suffix(uri, rest)
end
end
defp remove_suffix(uri, []), do: uri
def fetch_public_key(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
@ -59,9 +66,8 @@ defmodule Pleroma.Signature do
end
end
def sign(%User{} = user, headers) do
with {:ok, %{keys: keys}} <- User.ensure_keys_present(user),
{:ok, private_key, _} <- Keys.keys_from_pem(keys) do
def sign(%User{keys: keys} = user, headers) do
with {