Merge branch 'feature/object-pruning' into 'develop'

Object pruning

See merge request pleroma/pleroma!1181
This commit is contained in:
lambda 2019-05-22 11:04:30 +00:00
commit 714d8d4ef9
7 changed files with 97 additions and 3 deletions

View File

@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Metadata: RelMe provider - Metadata: RelMe provider
- OAuth: added support for refresh tokens - OAuth: added support for refresh tokens
- Emoji packs and emoji pack manager - Emoji packs and emoji pack manager
- Object pruning (`mix pleroma.database prune_objects`)
### Changed ### Changed
- **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer

View File

@ -239,7 +239,8 @@ config :pleroma, :instance,
welcome_message: nil, welcome_message: nil,
max_report_comment_size: 1000, max_report_comment_size: 1000,
safe_dm_mentions: false, safe_dm_mentions: false,
healthcheck: false healthcheck: false,
remote_post_retention_days: 90
config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800

View File

@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer,
* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
* `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``. * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database
## :app_account_creation ## :app_account_creation
REST API for creating an account settings REST API for creating an account settings

View File

@ -5,6 +5,7 @@
defmodule Mix.Tasks.Pleroma.Database do defmodule Mix.Tasks.Pleroma.Database do
alias Mix.Tasks.Pleroma.Common alias Mix.Tasks.Pleroma.Common
alias Pleroma.Conversation alias Pleroma.Conversation
alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
require Logger require Logger
@ -23,6 +24,10 @@ defmodule Mix.Tasks.Pleroma.Database do
Options: Options:
- `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
## Prune old objects from the database
mix pleroma.database prune_objects
## Create a conversation for all existing DMs. Can be safely re-run. ## Create a conversation for all existing DMs. Can be safely re-run.
mix pleroma.database bump_all_conversations mix pleroma.database bump_all_conversations
@ -72,4 +77,46 @@ defmodule Mix.Tasks.Pleroma.Database do
Enum.each(users, &User.remove_duplicated_following/1) Enum.each(users, &User.remove_duplicated_following/1)
Enum.each(users, &User.update_follower_count/1) Enum.each(users, &User.update_follower_count/1)
end end
def run(["prune_objects" | args]) do
import Ecto.Query
{options, [], []} =
OptionParser.parse(
args,
strict: [
vacuum: :boolean
]
)
Common.start_pleroma()
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
Logger.info("Pruning objects older than #{deadline} days")
time_deadline =
NaiveDateTime.utc_now()
|> NaiveDateTime.add(-(deadline * 86_400))
public = "https://www.w3.org/ns/activitystreams#Public"
from(o in Object,
where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public),
where: o.inserted_at < ^time_deadline,
where:
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
)
|> Repo.delete_all()
if Keyword.get(options, :vacuum) do
Logger.info("Runnning VACUUM FULL")
Repo.query!(
"vacuum full;",
[],
timeout: :infinity
)
end
end
end end

View File

@ -130,6 +130,13 @@ defmodule Pleroma.Object do
end end
end end
def prune(%Object{data: %{"id" => id}} = object) do
with {:ok, object} <- Repo.delete(object),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
{:ok, object}
end
end
def set_cache(%Object{data: %{"id" => ap_id}} = object) do def set_cache(%Object{data: %{"id" => ap_id}} = object) do
Cachex.put(:object_cache, "object:#{ap_id}", object) Cachex.put(:object_cache, "object:#{ap_id}", object)
{:ok, object} {:ok, object}

View File

@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do
@httpoison Application.get_env(:pleroma, :httpoison) @httpoison Application.get_env(:pleroma, :httpoison)
defp reinject_object(data) do
Logger.debug("Reinjecting object #{data["id"]}")
with data <- Transmogrifier.fix_object(data),
{:ok, object} <- Object.create(data) do
{:ok, object}
else
e ->
Logger.error("Error while processing object: #{inspect(e)}")
{:error, e}
end
end
# TODO: # TODO:
# This will create a Create activity, which we need internally at the moment. # This will create a Create activity, which we need internally at the moment.
def fetch_object_from_id(id) do def fetch_object_from_id(id) do
@ -26,12 +39,17 @@ defmodule Pleroma.Object.Fetcher do
"object" => data "object" => data
}, },
:ok <- Containment.contain_origin(id, params), :ok <- Containment.contain_origin(id, params),
{:ok, activity} <- Transmogrifier.handle_incoming(params) do {:ok, activity} <- Transmogrifier.handle_incoming(params),
{:ok, Object.normalize(activity, false)} {:object, _data, %Object{} = object} <-
{:object, data, Object.normalize(activity, false)} do
{:ok, object}
else else
{:error, {:reject, nil}} -> {:error, {:reject, nil}} ->
{:reject, nil} {:reject, nil}
{:object, data, nil} ->
reinject_object(data)
object = %Object{} -> object = %Object{} ->
{:ok, object} {:ok, object}

View File

@ -87,4 +87,23 @@ defmodule Pleroma.Object.FetcherTest do
) )
end end
end end
describe "pruning" do
test "it can refetch pruned objects" do
object_id = "http://mastodon.example.org/@admin/99541947525187367"
{:ok, object} = Fetcher.fetch_object_from_id(object_id)
assert object
{:ok, _object} = Object.prune(object)
refute Object.get_by_ap_id(object_id)
{:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id)
assert object.data["id"] == object_two.data["id"]
assert object.id != object_two.id
end
end
end end