Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into feature/local-only-scope

This commit is contained in:
lain 2020-11-04 11:47:41 +01:00
commit c7bcbfbc1d
50 changed files with 1118 additions and 32 deletions

View File

@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Support for local-only statuses - Support for local-only statuses
- Support pagination of blocks and mutes - Support pagination of blocks and mutes
- App metrics: ability to restrict access to specified IP whitelist. - App metrics: ability to restrict access to specified IP whitelist.
- Account backup
- Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance. - Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
@ -40,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Pleroma API: Pagination for remote/local packs and emoji. - Pleroma API: Pagination for remote/local packs and emoji.
- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type` - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
</details> </details>
@ -58,6 +60,8 @@ switched to a new configuration mechanism, however it was not officially removed
- Allow sending chat messages to yourself. - Allow sending chat messages to yourself.
- Fix remote users with a whitespace name. - Fix remote users with a whitespace name.
- OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting. - OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
- Mastodon API: Current user is now included in conversation if it's the only participant
- Mastodon API: Fixed last_status.account being not filled with account data
## Unreleased (Patch) ## Unreleased (Patch)

View File

@ -551,6 +551,7 @@ config :pleroma, Oban,
queues: [ queues: [
activity_expiration: 10, activity_expiration: 10,
token_expiration: 5, token_expiration: 5,
backup: 1,
federator_incoming: 50, federator_incoming: 50,
federator_outgoing: 50, federator_outgoing: 50,
ingestion_queue: 50, ingestion_queue: 50,
@ -835,6 +836,11 @@ config :floki, :html_parser, Floki.HTMLParser.FastHtml
config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
config :pleroma, Pleroma.User.Backup,
purge_after_days: 30,
limit_days: 7,
dir: nil
# Import environment specific config. This must remain at the bottom # Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above. # of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs" import_config "#{Mix.env()}.exs"

View File

@ -2297,6 +2297,12 @@ config :pleroma, :config_description, [
description: "Activity expiration queue", description: "Activity expiration queue",
suggestions: [10] suggestions: [10]
}, },
%{
key: :backup,
type: :integer,
description: "Backup queue",
suggestions: [1]
},
%{ %{
key: :attachments_cleanup, key: :attachments_cleanup,
type: :integer, type: :integer,
@ -3732,6 +3738,26 @@ config :pleroma, :config_description, [
} }
] ]
}, },
%{
group: :pleroma,
key: Pleroma.User.Backup,
type: :group,
description: "Account Backup",
children: [
%{
key: :purge_after_days,
type: :integer,
description: "Remove backup achives after N days",
suggestions: [30]
},
%{
key: :limit_days,
type: :integer,
description: "Limit user to export not more often than once per N days",
suggestions: [7]
}
]
},
%{ %{
group: :prometheus, group: :prometheus,
key: Pleroma.Web.Endpoint.MetricsExporter, key: Pleroma.Web.Endpoint.MetricsExporter,

View File

@ -173,11 +173,14 @@ Returned data:
"created_at": "2020-04-21T15:06:45.000Z", "created_at": "2020-04-21T15:06:45.000Z",
"emojis": [], "emojis": [],
"id": "12", "id": "12",
"unread": false "unread": false,
"idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f"
} }
] ]
``` ```
- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation.
### Posting a chat message ### Posting a chat message
Posting a chat message for given Chat id works like this: Posting a chat message for given Chat id works like this:

View File

@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
{"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]} {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
] ]
``` ```
## `POST /api/v1/pleroma/backups`
### Create a user backup archive
* Method: `POST`
* Authentication: required
* Params: none
* Response: JSON
* Example response:
```json
[{
"content_type": "application/zip",
"file_size": 0,
"inserted_at": "2020-09-10T16:18:03.000Z",
"processed": false,
"url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
}]
```
## `GET /api/v1/pleroma/backups`
### Lists user backups
* Method: `GET`
* Authentication: not required
* Params: none
* Response: JSON
* Example response:
```json
[{
"content_type": "application/zip",
"file_size": 55457,
"inserted_at": "2020-09-10T16:18:03.000Z",
"processed": true,
"url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
}]
```

View File

@ -1078,6 +1078,20 @@ Control favicons for instances.
* `enabled`: Allow/disallow displaying and getting instances favicons * `enabled`: Allow/disallow displaying and getting instances favicons
## Pleroma.User.Backup
!!! note
Requires enabled email
* `:purge_after_days` an integer, remove backup achives after N days.
* `:limit_days` an integer, limit user to export not more often than once per N days.
* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
1. the directory named by the TMPDIR environment variable
2. the directory named by the TEMP environment variable
3. the directory named by the TMP environment variable
4. C:\TMP on Windows or /tmp on Unix-like operating systems
5. as a last resort, the current working directory
## Frontend management ## Frontend management
Frontends in Pleroma are swappable - you can specify which one to use here. Frontends in Pleroma are swappable - you can specify which one to use here.

View File

@ -14,6 +14,7 @@ defmodule Pleroma.Activity do
alias Pleroma.ReportNote alias Pleroma.ReportNote
alias Pleroma.ThreadMute alias Pleroma.ThreadMute
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
import Ecto.Changeset import Ecto.Changeset
import Ecto.Query import Ecto.Query
@ -155,6 +156,18 @@ defmodule Pleroma.Activity do
def get_bookmark(_, _), do: nil def get_bookmark(_, _), do: nil
def get_report(activity_id) do
opts = %{
type: "Flag",
skip_preload: true,
preload_report_notes: true
}
ActivityPub.fetch_activities_query([], opts)
|> where(id: ^activity_id)
|> Repo.one()
end
def change(struct, params \\ %{}) do def change(struct, params \\ %{}) do
struct struct
|> cast(params, [:data, :recipients]) |> cast(params, [:data, :recipients])

View File

@ -168,7 +168,11 @@ defmodule Pleroma.Application do
build_cachex("web_resp", limit: 2500), build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500), build_cachex("failed_proxy_url", limit: 2500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
build_cachex("chat_message_id_idempotency_key",
expiration: chat_message_id_idempotency_key_expiration(),
limit: 500_000
)
] ]
end end
@ -178,6 +182,9 @@ defmodule Pleroma.Application do
defp idempotency_expiration, defp idempotency_expiration,
do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
defp chat_message_id_idempotency_key_expiration,
do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
defp seconds_valid_interval, defp seconds_valid_interval,
do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))

View File

@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
def new do def new do
endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
case Tesla.get(endpoint <> "/new") do case Pleroma.HTTP.get(endpoint <> "/new") do
{:error, _} -> {:error, _} ->
%{error: :kocaptcha_service_unavailable} %{error: :kocaptcha_service_unavailable}

View File

@ -189,4 +189,30 @@ defmodule Pleroma.Emails.UserEmail do
Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
end end
def backup_is_ready_email(backup, admin_user_id \\ nil) do
%{user: user} = Pleroma.Repo.preload(backup, :user)
download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
html_body =
if is_nil(admin_user_id) do
"""
<p>You requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="#{download_url}">#{download_url}</a></p>
"""
else
admin = Pleroma.Repo.get(User, admin_user_id)
"""
<p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
<p><a href="#{download_url}">#{download_url}</a></p>
"""
end
new()
|> to(recipient(user))
|> from(sender())
|> subject("Your account archive is ready")
|> html_body(html_body)
end
end end

View File

@ -594,7 +594,7 @@ defmodule Pleroma.Emoji.Pack do
end end
defp download_archive(url, sha) do defp download_archive(url, sha) do
with {:ok, %{body: archive}} <- Tesla.get(url) do with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do
if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
{:ok, archive} {:ok, archive}
else else
@ -617,7 +617,7 @@ defmodule Pleroma.Emoji.Pack do
end end
defp update_sha_and_save_metadata(pack, data) do defp update_sha_and_save_metadata(pack, data) do
with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]), with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]),
:ok <- validate_has_all_files(pack, zip) do :ok <- validate_has_all_files(pack, zip) do
fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16() fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()

View File

@ -655,6 +655,16 @@ defmodule Pleroma.ModerationLog do
"@#{actor_nickname} deleted chat message ##{subject_id}" "@#{actor_nickname} deleted chat message ##{subject_id}"
end end
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "create_backup",
"subject" => %{"nickname" => user_nickname}
}
}) do
"@#{actor_nickname} requested account backup for @#{user_nickname}"
end
defp nicknames_to_string(nicknames) do defp nicknames_to_string(nicknames) do
nicknames nicknames
|> Enum.map(&"@#{&1}") |> Enum.map(&"@#{&1}")

258
lib/pleroma/user/backup.ex Normal file
View File

@ -0,0 +1,258 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.Backup do
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
import Pleroma.Web.Gettext
require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.Bookmark
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
alias Pleroma.Workers.BackupWorker
schema "backups" do
field(:content_type, :string)
field(:file_name, :string)
field(:file_size, :integer, default: 0)
field(:processed, :boolean, default: false)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
timestamps()
end
def create(user, admin_id \\ nil) do
with :ok <- validate_email_enabled(),
:ok <- validate_user_email(user),
:ok <- validate_limit(user, admin_id),
{:ok, backup} <- user |> new() |> Repo.insert() do
BackupWorker.process(backup, admin_id)
end
end
def new(user) do
rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
%__MODULE__{
user_id: user.id,
content_type: "application/zip",
file_name: name
}
end
def delete(backup) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
Repo.delete(backup)
end
end
defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
defp validate_limit(user, nil) do
case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} ->
days = Pleroma.Config.get([__MODULE__, :limit_days])
diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
if diff > days do
:ok
else
{:error,
dngettext(
"errors",
"Last export was less than a day ago",
"Last export was less than %{days} days ago",
days,
days: days
)}
end
nil ->
:ok
end
end
defp validate_email_enabled do
if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
:ok
else
{:error, dgettext("errors", "Backups require enabled email")}
end
end
defp validate_user_email(%User{email: nil}) do
{:error, dgettext("errors", "Email is required")}
end
defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
def get_last(user_id) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> limit(1)
|> Repo.one()
end
def list(%User{id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> Repo.all()
end
def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> where([b], b.id != ^latest_id)
|> Repo.all()
|> Enum.each(&BackupWorker.delete/1)
end
def get(id), do: Repo.get(__MODULE__, id)
def process(%__MODULE__{} = backup) do
with {:ok, zip_file} <- export(backup),
{:ok, %{size: size}} <- File.stat(zip_file),
{:ok, _upload} <- upload(backup, zip_file) do
backup
|> cast(%{file_size: size, processed: true}, [:file_size, :processed])
|> Repo.update()
end
end
@files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
def export(%__MODULE__{} = backup) do
backup = Repo.preload(backup, :user)
name = String.trim_trailing(backup.file_name, ".zip")
dir = dir(name)
with :ok <- File.mkdir(dir),
:ok <- actor(dir, backup.user),
:ok <- statuses(dir, backup.user),
:ok <- likes(dir, backup.user),
:ok <- bookmarks(dir, backup.user),
{:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
{:ok, _} <- File.rm_rf(dir) do
{:ok, to_string(zip_path)}
end
end
def dir(name) do
dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
Path.join(dir, name)
end
def upload(%__MODULE__{} = backup, zip_path) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
upload = %Pleroma.Upload{
name: backup.file_name,
tempfile: zip_path,
content_type: backup.content_type,
path: Path.join("backups", backup.file_name)
}
with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
:ok <- File.rm(zip_path) do
{:ok, upload}
end
end
defp actor(dir, user) do
with {:ok, json} <-
UserView.render("user.json", %{user: user})
|> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
|> Jason.encode() do
File.write(Path.join(dir, "actor.json"), json)
end
end
defp write_header(file, name) do
IO.write(
file,
"""
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": "#{name}.json",
"type": "OrderedCollection",
"orderedItems": [
"""
)
end
defp write(query, dir, name, fun) do
path = Path.join(dir, "#{name}.json")
with {:ok, file} <- File.open(path, [:write, :utf8]),
:ok <- write_header(file, name) do
total =
query
|> Pleroma.Repo.chunk_stream(100)
|> Enum.reduce(0, fn i, acc ->
with {:ok, data} <- fun.(i),
{:ok, str} <- Jason.encode(data),
:ok <- IO.write(file, str <> ",\n") do
acc + 1
else
_ -> acc
end
end)
with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do
File.close(file)
end
end
end
defp bookmarks(dir, %{id: user_id} = _user) do
Bookmark
|> where(user_id: ^user_id)
|> join(:inner, [b], activity in assoc(b, :activity))
|> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
|> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
end
defp likes(dir, user) do
user.ap_id
|> Activity.Queries.by_actor()
|> Activity.Queries.by_type("Like")
|> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
|> write(dir, "likes", fn a -> {:ok, a.object} end)
end
defp statuses(dir, user) do
opts =
%{}
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:actor_id, user.ap_id)
[
[Pleroma.Constants.as_public(), user.ap_id],
User.following(user),
Pleroma.List.memberships(user)
]
|> Enum.concat()
|> ActivityPub.fetch_activities_query(opts)
|> write(dir, "outbox", fn a ->
with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
{:ok, Map.delete(activity, "@context")}
end
end)
end
end

View File

@ -187,7 +187,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
{:ok, notifications} = Notification.create_notifications(activity, do_send: false) {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
{:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
if in_reply_to = object.data["inReplyTo"] do if in_reply_to = object.data["inReplyTo"] && object.data["type"] != "Answer" do
Object.increase_replies_count(in_reply_to) Object.increase_replies_count(in_reply_to)
end end
@ -312,6 +312,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
{:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id) {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
{:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id) {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
Cachex.put(
:chat_message_id_idempotency_key_cache,
cm_ref.id,
meta[:idempotency_key]
)
{ {
["user", "user:pleroma_chat"], ["user", "user:pleroma_chat"],
{user, %{cm_ref | chat: chat, object: object}} {user, %{cm_ref | chat: chat, object: object}}

View File

@ -26,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
plug( plug(
OAuthScopesPlug, OAuthScopesPlug,
%{scopes: ["read:accounts"], admin: true} %{scopes: ["read:accounts"], admin: true}
when action in [:right_get, :show_user_credentials] when action in [:right_get, :show_user_credentials, :create_backup]
) )
plug( plug(
@ -441,6 +441,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
json(conn, %{"status_visibility" => counters}) json(conn, %{"status_visibility" => counters})
end end
def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
with %User{} = user <- User.get_by_nickname(nickname),
{:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do
ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
json(conn, "")
end
end
defp page_params(params) do defp page_params(params) do
{ {
fetch_integer_param(params, "page", 1), fetch_integer_param(params, "page", 1),

View File

@ -38,7 +38,7 @@ defmodule Pleroma.Web.AdminAPI.ReportController do
end end
def show(conn, %{id: id}) do def show(conn, %{id: id}) do
with %Activity{} = report <- Activity.get_by_id(id) do with %Activity{} = report <- Activity.get_report(id) do
render(conn, "show.json", Report.extract_report_info(report)) render(conn, "show.json", Report.extract_report_info(report))
else else
_ -> {:error, :not_found} _ -> {:error, :not_found}

View File

@ -0,0 +1,79 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
alias OpenApiSpex.Operation
alias OpenApiSpex.Schema
alias Pleroma.Web.ApiSpec.Schemas.ApiError
def open_api_operation(action) do
operation = String.to_existing_atom("#{action}_operation")
apply(__MODULE__, operation, [])
end
def index_operation do
%Operation{
tags: ["Backups"],
summary: "List backups",
security: [%{"oAuth" => ["read:account"]}],
operationId: "PleromaAPI.BackupController.index",
responses: %{
200 =>
Operation.response(
"An array of backups",
"application/json",
%Schema{
type: :array,
items: backup()
}
),
400 => Operation.response("Bad Request", "application/json", ApiError)
}
}
end
def create_operation do
%Operation{
tags: ["Backups"],
summary: "Create a backup",
security: [%{"oAuth" => ["read:account"]}],
operationId: "PleromaAPI.BackupController.create",
responses: %{
200 =>
Operation.response(
"An array of backups",
"application/json",
%Schema{
type: :array,
items: backup()
}
),
400 => Operation.response("Bad Request", "application/json", ApiError)
}
}
end
defp backup do
%Schema{
title: "Backup",
description: "Response schema for a backup",
type: :object,
properties: %{
inserted_at: %Schema{type: :string, format: :"date-time"},
content_type: %Schema{type: :string},
file_name: %Schema{type: :string},
file_size: %Schema{type: :integer},
processed: %Schema{type: :boolean}
},
example: %{
"content_type" => "application/zip",
"file_name" =>
"https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip",
"file_size" => 4105,
"inserted_at" => "2020-09-08T16:42:07.000Z",
"processed" => true
}
}
end
end

View File

@ -46,7 +46,8 @@ defmodule Pleroma.Web.CommonAPI do
{_, {:ok, %Activity{} = activity, _meta}} <- {_, {:ok, %Activity{} = activity, _meta}} <-
{:common_pipeline, {:common_pipeline,
Pipeline.common_pipeline(create_activity_data, Pipeline.common_pipeline(create_activity_data,
local: true local: true,
idempotency_key: opts[:idempotency_key]
)} do )} do
{:ok, activity} {:ok, activity}
else else

View File

@ -33,8 +33,15 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do
end end
activity = Activity.get_by_id_with_object(last_activity_id) activity = Activity.get_by_id_with_object(last_activity_id)
# Conversations return all users except the current user.
users = Enum.reject(participation.recipients, &(&1.id == user.id)) # Conversations return all users except the current user,
# except when the current user is the only participant
users =
if length(participation.recipients) > 1 do
Enum.reject(participation.recipients, &(&1.id == user.id))
else
participation.recipients
end
%{ %{
id: participation.id |> to_string(), id: participation.id |> to_string(),
@ -43,7 +50,8 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do
last_status: last_status:
render(StatusView, "show.json", render(StatusView, "show.json",
activity: activity, activity: activity,
direct_conversation_id: participation.id direct_conversation_id: participation.id,
for: user
) )
} }
end end

View File

@ -0,0 +1,28 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.PleromaAPI.BackupController do
use Pleroma.Web, :controller
alias Pleroma.User.Backup
alias Pleroma.Web.Plugs.OAuthScopesPlug
action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation
def index(%{assigns: %{user: user}} = conn, _params) do
backups = Backup.list(user)
render(conn, "index.json", backups: backups)
end
def create(%{assigns: %{user: user}} = conn, _params) do
with {:ok, _} <- Backup.create(user) do
backups = Backup.list(user)
render(conn, "index.json", backups: backups)
end
end
end

View File

@ -80,7 +80,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
%User{} = recipient <- User.get_cached_by_ap_id(chat.recipient), %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
{:ok, activity} <- {:ok, activity} <-
CommonAPI.post_chat_message(user, recipient, params[:content], CommonAPI.post_chat_message(user, recipient, params[:content],
media_id: params[:media_id] media_id: params[:media_id],
idempotency_key: idempotency_key(conn)
), ),
message <- Object.normalize(activity, false), message <- Object.normalize(activity, false),
cm_ref <- MessageReference.for_chat_and_object(chat, message) do cm_ref <- MessageReference.for_chat_and_object(chat, message) do
@ -169,4 +170,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
|> render("show.json", chat: chat) |> render("show.json", chat: chat)
end end
end end
defp idempotency_key(conn) do
case get_req_header(conn, "idempotency-key") do
[key] -> key
_ -> nil
end
end
end end

View File

@ -0,0 +1,28 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.PleromaAPI.BackupView do
use Pleroma.Web, :view
alias Pleroma.User.Backup
alias Pleroma.Web.CommonAPI.Utils
def render("show.json", %{backup: %Backup{} = backup}) do
%{
content_type: backup.content_type,
url: download_url(backup),
file_size: backup.file_size,
processed: backup.processed,
inserted_at: Utils.to_masto_date(backup.inserted_at)
}
end
def render("index.json", %{backups: backups}) do
render_many(backups, __MODULE__, "show.json")
end
def download_url(%Backup{file_name: file_name}) do
Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name
end
end

View File

@ -5,6 +5,7 @@
defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
use Pleroma.Web, :view use Pleroma.Web, :view
alias Pleroma.Maps
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MastodonAPI.StatusView
@ -37,6 +38,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object) Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object)
) )
} }
|> put_idempotency_key()
end end
def render("index.json", opts) do def render("index.json", opts) do
@ -47,4 +49,13 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
Map.put(opts, :as, :chat_message_reference) Map.put(opts, :as, :chat_message_reference)
) )
end end
defp put_idempotency_key(data) do
with {:ok, idempotency_key} <- Cachex.get(:chat_message_id_idempotency_key_cache, data.id) do
data
|> Maps.put_if_present(:idempotency_key, idempotency_key)
else
_ -> data
end
end
end end

View File

@ -243,6 +243,8 @@ defmodule Pleroma.Web.Router do
get("/chats/:id", ChatController, :show) get("/chats/:id", ChatController, :show)
get("/chats/:id/messages", ChatController, :messages) get("/chats/:id/messages", ChatController, :messages)
delete("/chats/:id/messages/:message_id", ChatController, :delete_message) delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
post("/backups", AdminAPIController, :create_backup)
end end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
@ -373,6 +375,9 @@ defmodule Pleroma.Web.Router do
put("/mascot", MascotController, :update) put("/mascot", MascotController, :update)
post("/scrobble", ScrobbleController, :create) post("/scrobble", ScrobbleController, :create)
get("/backups", BackupController, :index)
post("/backups", BackupController, :create)
end end
scope [] do scope [] do

View File

@ -0,0 +1,54 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :backup, max_attempts: 1
alias Oban.Job
alias Pleroma.User.Backup
def process(backup, admin_user_id \\ nil) do
%{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id}
|> new()
|> Oban.insert()
end
def schedule_deletion(backup) do
days = Pleroma.Config.get([Backup, :purge_after_days])
time = 60 * 60 * 24 * days
scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
%{"op" => "delete", "backup_id" => backup.id}
|> new(scheduled_at: scheduled_at)
|> Oban.insert()
end
def delete(backup) do
%{"op" => "delete", "backup_id" => backup.id}
|> new()
|> Oban.insert()
end
def perform(%Job{
args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id}
}) do
with {:ok, %Backup{} = backup} <-
backup_id |> Backup.get() |> Backup.process(),
{:ok, _job} <- schedule_deletion(backup),
:ok <- Backup.remove_outdated(backup),
{:ok, _} <-
backup
|> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id)
|> Pleroma.Emails.Mailer.deliver() do
{:ok, backup}
end
end
def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do
case Backup.get(backup_id) do
%Backup{} = backup -> Backup.delete(backup)
nil -> :ok
end
end
end

View File

@ -134,7 +134,7 @@ defmodule Pleroma.Mixfile do
{:cachex, "~> 3.2"}, {:cachex, "~> 3.2"},
{:poison, "~> 3.0", override: true}, {:poison, "~> 3.0", override: true},
{:tesla, {:tesla,
git: "https://github.com/teamon/tesla/", git: "https://github.com/teamon/tesla.git",
ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30",
override: true}, override: true},
{:castore, "~> 0.1"}, {:castore, "~> 0.1"},
@ -196,7 +196,7 @@ defmodule Pleroma.Mixfile do
ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
{:restarter, path: "./restarter"}, {:restarter, path: "./restarter"},
{:majic, {:majic,
git: "https://git.pleroma.social/pleroma/elixir-libraries/majic", branch: "develop"}, git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", branch: "develop"},
{:open_api_spex, {:open_api_spex,
git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"}, ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"},

View File

@ -66,7 +66,7 @@
"jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
"libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
"linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"}, "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"},
"majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]}, "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]},
"makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"}, "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"},
"makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"},
"meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"},
@ -115,7 +115,7 @@
"swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"}, "swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"},
"syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"},
"telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"},
"tesla": {:git, "https://github.com/teamon/tesla/", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]}, "tesla": {:git, "https://github.com/teamon/tesla.git", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]},
"timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"},
"trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
"tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"}, "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"},

View File

@ -0,0 +1,17 @@
defmodule Pleroma.Repo.Migrations.CreateBackups do
use Ecto.Migration
def change do
create_if_not_exists table(:backups) do
add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
add(:file_name, :string, null: false)
add(:content_type, :string, null: false)
add(:processed, :boolean, null: false, default: false)
add(:file_size, :bigint)
timestamps()
end
create_if_not_exists(index(:backups, [:user_id]))
end
end

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.6 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

View File

@ -0,0 +1,244 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.BackupTest do
use Oban.Testing, repo: Pleroma.Repo
use Pleroma.DataCase
import Mock
import Pleroma.Factory
import Swoosh.TestAssertions
alias Pleroma.Bookmark
alias Pleroma.Tests.ObanHelpers
alias Pleroma.User.Backup
alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.BackupWorker
setup do
clear_config([Pleroma.Upload, :uploader])
clear_config([Backup, :limit_days])
clear_config([Pleroma.Emails.Mailer, :enabled], true)
end
test "it requries enabled email" do
Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false)
user = insert(:user)
assert {:error, "Backups require enabled email"} == Backup.create(user)
end
test "it requries user's email" do
user = insert(:user, %{email: nil})
assert {:error, "Email is required"} == Backup.create(user)
end
test "it creates a backup record and an Oban job" do
%{id: user_id} = user = insert(:user)
assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
assert_enqueued(worker: BackupWorker, args: args)
backup = Backup.get(args["backup_id"])
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
end
test "it return an error if the export limit is over" do
%{id: user_id} = user = insert(:user)
limit_days = Pleroma.Config.get([Backup, :limit_days])
assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
backup = Backup.get(args["backup_id"])
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
end
test "it process a backup record" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
%{id: user_id} = user = insert(:user)
assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
assert {:ok, backup} = perform_job(BackupWorker, args)
assert backup.file_size > 0
assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
assert_enqueued(worker: BackupWorker, args: delete_job_args)
assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
refute Backup.get(backup_id)
email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup)
assert_email_sent(
to: {user.name, user.email},
html_body: email.html_body
)
end
test "it removes outdated backups after creating a fresh one" do
Pleroma.Config.put([Backup, :limit_days], -1)
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
user = insert(:user)
assert {:ok, job1} = Backup.create(user)
assert {:ok, %Backup{}} = ObanHelpers.perform(job1)
assert {:ok, job2} = Backup.create(user)
assert Pleroma.Repo.aggregate(Backup, :count) == 2
assert {:ok, backup2} = ObanHelpers.perform(job2)
ObanHelpers.perform_all()
assert [^backup2] = Pleroma.Repo.all(Backup)
end
test "it creates a zip archive with user data" do
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, %{object: %{data: %{"id" => id1}}} = status1} =
CommonAPI.post(user, %{status: "status1"})
{:ok, %{object: %{data: %{"id" => id2}}} = status2} =
CommonAPI.post(user, %{status: "status2"})
{:ok, %{object: %{data: %{"id" => id3}}} = status3} =
CommonAPI.post(user, %{status: "status3"})
CommonAPI.favorite(user, status1.id)
CommonAPI.favorite(user, status2.id)
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.export(backup)
assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
assert %{
"@context" => [
"https://www.w3.org/ns/activitystreams",
"http://localhost:4001/schemas/litepub-0.1.jsonld",
%{"@language" => "und"}
],
"bookmarks" => "bookmarks.json",
"followers" => "http://cofe.io/users/cofe/followers",
"following" => "http://cofe.io/users/cofe/following",
"id" => "http://cofe.io/users/cofe",
"inbox" => "http://cofe.io/users/cofe/inbox",
"likes" => "likes.json",
"name" => "Cofe",
"outbox" => "http://cofe.io/users/cofe/outbox",
"preferredUsername" => "cofe",
"publicKey" => %{
"id" => "http://cofe.io/users/cofe#main-key",
"owner" => "http://cofe.io/users/cofe"
},
"type" => "Person",
"url" => "http://cofe.io/users/cofe"
} = Jason.decode!(json)
assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "outbox.json",
"orderedItems" => [
%{
"object" => %{
"actor" => "http://cofe.io/users/cofe",
"content" => "status1",
"type" => "Note"
},
"type" => "Create"
},
%{
"object" => %{
"actor" => "http://cofe.io/users/cofe",
"content" => "status2"
}
},
%{
"actor" => "http://cofe.io/users/cofe",
"object" => %{
"content" => "status3"
}
}
],
"totalItems" => 3,
"type" => "OrderedCollection"
} = Jason.decode!(json)
assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "likes.json",
"orderedItems" => [^id1, ^id2],
"totalItems" => 2,
"type" => "OrderedCollection"
} = Jason.decode!(json)
assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "bookmarks.json",
"orderedItems" => [^id2, ^id3],
"totalItems" => 2,
"type" => "OrderedCollection"
} = Jason.decode!(json)
:zip.zip_close(zipfile)
File.rm!(path)
end
describe "it uploads and deletes a backup archive" do
setup do
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com"
)
clear_config([Pleroma.Upload, :uploader])
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, status1} = CommonAPI.post(user, %{status: "status1"})
{:ok, status2} = CommonAPI.post(user, %{status: "status2"})
{:ok, status3} = CommonAPI.post(user, %{status: "status3"})
CommonAPI.favorite(user, status1.id)
CommonAPI.favorite(user, status2.id)
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.export(backup)
[path: path, backup: backup]
end
test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
with_mock ExAws,
request: fn
%{http_method: :put} -> {:ok, :ok}
%{http_method: :delete} -> {:ok, %{status_code: 204}}
end do
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
end
end
test "Local", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
end
end

View File

@ -27,6 +27,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AnswerHandlingTest do
}) })
object = Object.normalize(activity) object = Object.normalize(activity)
assert object.data["repliesCount"] == nil
data = data =
File.read!("test/fixtures/mastodon-vote.json") File.read!("test/fixtures/mastodon-vote.json")
@ -41,7 +42,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AnswerHandlingTest do
assert answer_object.data["inReplyTo"] == object.data["id"] assert answer_object.data["inReplyTo"] == object.data["id"]
new_object = Object.get_by_ap_id(object.data["id"]) new_object = Object.get_by_ap_id(object.data["id"])
assert new_object.data["replies_count"] == object.data["replies_count"] assert new_object.data["repliesCount"] == nil
assert Enum.any?( assert Enum.any?(
new_object.data["oneOf"], new_object.data["oneOf"],

View File

@ -11,7 +11,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
import Swoosh.TestAssertions import Swoosh.TestAssertions
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.MFA alias Pleroma.MFA
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Repo alias Pleroma.Repo
@ -978,6 +977,73 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
response["status_visibility"] response["status_visibility"]
end end
end end
describe "/api/pleroma/backups" do
test "it creates a backup", %{conn: conn} do
admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true)
token = insert(:oauth_admin_token, user: admin)
user = %{id: user_id, nickname: user_nickname} = insert(:user)
assert "" ==
conn
|> assign(:user, admin)
|> assign(:token, token)
|> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
|> json_response(200)
assert [backup] = Repo.all(Pleroma.User.Backup)
ObanHelpers.perform_all()
email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)
assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup")
assert_email_sent(to: {user.name, user.email}, html_body: email.html_body)
log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}"
assert [
%{
data: %{
"action" => "create_backup",
"actor" => %{
"id" => ^admin_id,
"nickname" => ^admin_nickname
},
"message" => ^log_message,
"subject" => %{
"id" => ^user_id,
"nickname" => ^user_nickname
}
}
}
] = Pleroma.ModerationLog |> Repo.all()
end
test "it doesn't limit admins", %{conn: conn} do
admin = insert(:user, is_admin: true)
token = insert(:oauth_admin_token, user: admin)
user = insert(:user)
assert "" ==
conn
|> assign(:user, admin)
|> assign(:token, token)
|> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
|> json_response(200)
assert [_backup] = Repo.all(Pleroma.User.Backup)
assert "" ==
conn
|> assign(:user, admin)
|> assign(:token, token)
|> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
|> json_response(200)
assert Repo.aggregate(Pleroma.User.Backup, :count) == 2
end
end
end end
# Needed for testing # Needed for testing

View File

@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do
alias Pleroma.Chat alias Pleroma.Chat
alias Pleroma.Chat.MessageReference alias Pleroma.Chat.MessageReference
alias Pleroma.Config
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo

View File

@ -5,7 +5,6 @@
defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
use Pleroma.Web.ConnCase, async: true use Pleroma.Web.ConnCase, async: true
import Pleroma.Factory import Pleroma.Factory
alias Pleroma.Config
@dir "test/tmp/instance_static" @dir "test/tmp/instance_static"
@default_instance_panel ~s(<p>Welcome to <a href="https://pleroma.social" target="_blank">Pleroma!</a></p>) @default_instance_panel ~s(<p>Welcome to <a href="https://pleroma.social" target="_blank">Pleroma!</a></p>)

View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.OAuthAppControllerTest do
import Pleroma.Factory import Pleroma.Factory
alias Pleroma.Config
alias Pleroma.Web alias Pleroma.Web
setup do setup do

View File

@ -7,7 +7,6 @@ defmodule Pleroma.Web.AdminAPI.RelayControllerTest do
import Pleroma.Factory import Pleroma.Factory
alias Pleroma.Config
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User

View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
import Pleroma.Factory import Pleroma.Factory
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.ReportNote alias Pleroma.ReportNote
@ -38,12 +37,21 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
status_ids: [activity.id] status_ids: [activity.id]
}) })
conn
|> put_req_header("content-type", "application/json")
|> post("/api/pleroma/admin/reports/#{report_id}/notes", %{
content: "this is an admin note"
})
response = response =
conn conn
|> get("/api/pleroma/admin/reports/#{report_id}") |> get("/api/pleroma/admin/reports/#{report_id}")
|> json_response_and_validate_schema(:ok) |> json_response_and_validate_schema(:ok)
assert response["id"] == report_id assert response["id"] == report_id
[notes] = response["notes"]
assert notes["content"] == "this is an admin note"
end end
test "returns 404 when report id is invalid", %{conn: conn} do test "returns 404 when report id is invalid", %{conn: conn} do

View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.StatusControllerTest do
import Pleroma.Factory import Pleroma.Factory
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User

View File

@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do
import Mock import Mock
import Pleroma.Factory import Pleroma.Factory
alias Pleroma.Config
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Repo alias Pleroma.Repo

View File

@ -55,14 +55,35 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do
account_ids = Enum.map(res_accounts, & &1["id"]) account_ids = Enum.map(res_accounts, & &1["id"])
assert length(res_accounts) == 2 assert length(res_accounts) == 2
assert user_one.id not in account_ids
assert user_two.id in account_ids assert user_two.id in account_ids
assert user_three.id in account_ids assert user_three.id in account_ids
assert is_binary(res_id) assert is_binary(res_id)
assert unread == false assert unread == false
assert res_last_status["id"] == direct.id assert res_last_status["id"] == direct.id
assert res_last_status["account"]["id"] == user_one.id
assert Participation.unread_count(user_one) == 0 assert Participation.unread_count(user_one) == 0
end end
test "includes the user if the user is the only participant", %{
user: user_one,
conn: conn
} do
{:ok, _direct} = create_direct_message(user_one, [])
res_conn = get(conn, "/api/v1/conversations")
assert response = json_response_and_validate_schema(res_conn, 200)
assert [
%{
"accounts" => [account]
}
] = response
assert user_one.id == account["id"]
end
test "observes limit params", %{ test "observes limit params", %{
user: user_one, user: user_one,
user_two: user_two, user_two: user_two,

View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
import Pleroma.Factory import Pleroma.Factory
import Tesla.Mock import Tesla.Mock
alias Pleroma.Config
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI

View File

@ -36,9 +36,11 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do
assert conversation.id == participation.id |> to_string() assert conversation.id == participation.id |> to_string()
assert conversation.last_status.id == activity.id assert conversation.last_status.id == activity.id
assert conversation.last_status.account.id == user.id
assert [account] = conversation.accounts assert [account] = conversation.accounts
assert account.id == other_user.id assert account.id == other_user.id
assert conversation.last_status.pleroma.direct_conversation_id == participation.id assert conversation.last_status.pleroma.direct_conversation_id == participation.id
end end
end end

View File

@ -0,0 +1,85 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
use Pleroma.Web.ConnCase
alias Pleroma.User.Backup
alias Pleroma.Web.PleromaAPI.BackupView
setup do
clear_config([Pleroma.Upload, :uploader])
clear_config([Backup, :limit_days])
oauth_access(["read:accounts"])
end
test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do
assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
backup = Backup.get(backup_id)
response =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
assert [
%{
"content_type" => "application/zip",
"url" => url,
"file_size" => 0,
"processed" => false,
"inserted_at" => _
}
] = response
assert url == BackupView.download_url(backup)
Pleroma.Tests.ObanHelpers.perform_all()
assert [
%{
"url" => ^url,
"processed" => true
}
] =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
end
test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do
assert [
%{
"content_type" => "application/zip",
"url" => url,
"file_size" => 0,
"processed" => false,
"inserted_at" => _
}
] =
conn
|> post("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
Pleroma.Tests.ObanHelpers.perform_all()
assert [
%{
"url" => ^url,
"processed" => true
}
] =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
days = Pleroma.Config.get([Backup, :limit_days])
assert %{"error" => "Last export was less than #{days} days ago"} ==
conn
|> post("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(400)
end
end

View File

@ -82,11 +82,13 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do
result = result =
conn conn
|> put_req_header("content-type", "application/json") |> put_req_header("content-type", "application/json")
|> put_req_header("idempotency-key", "123")
|> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"}) |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"})
|> json_response_and_validate_schema(200) |> json_response_and_validate_schema(200)
assert result["content"] == "Hallo!!" assert result["content"] == "Hallo!!"
assert result["chat_id"] == chat.id |> to_string() assert result["chat_id"] == chat.id |> to_string()
assert result["idempotency_key"] == "123"
end end
test "it fails if there is no content", %{conn: conn, user: user} do test "it fails if there is no content", %{conn: conn, user: user} do

View File

@ -6,7 +6,6 @@ defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do
use Pleroma.Web.ConnCase use Pleroma.Web.ConnCase
use Oban.Testing, repo: Pleroma.Repo use Oban.Testing, repo: Pleroma.Repo
alias Pleroma.Config
alias Pleroma.Tests.ObanHelpers alias Pleroma.Tests.ObanHelpers
import Pleroma.Factory import Pleroma.Factory

View File

@ -25,7 +25,9 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
} }
{:ok, upload} = ActivityPub.upload(file, actor: user.ap_id) {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
{:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis :firefox:")
{:ok, activity} =
CommonAPI.post_chat_message(user, recipient, "kippis :firefox:", idempotency_key: "123")
chat = Chat.get(user.id, recipient.ap_id) chat = Chat.get(user.id, recipient.ap_id)
@ -42,6 +44,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
assert chat_message[:created_at] assert chat_message[:created_at]
assert chat_message[:unread] == false assert chat_message[:unread] == false
assert match?([%{shortcode: "firefox"}], chat_message[:emojis]) assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
assert chat_message[:idempotency_key] == "123"
clear_config([:rich_media, :enabled], true) clear_config([:rich_media, :enabled], true)

View File

@ -5,7 +5,6 @@
defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
use Pleroma.Web.ConnCase use Pleroma.Web.ConnCase
alias Pleroma.Config
alias Plug.Conn alias Plug.Conn
describe "http security enabled" do describe "http security enabled" do

View File

@ -255,7 +255,9 @@ defmodule Pleroma.Web.StreamerTest do
} do } do
other_user = insert(:user) other_user = insert(:user)
{:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno") {:ok, create_activity} =
CommonAPI.post_chat_message(other_user, user, "hey cirno", idempotency_key: "123")
object = Object.normalize(create_activity, false) object = Object.normalize(create_activity, false)
chat = Chat.get(user.id, other_user.ap_id) chat = Chat.get(user.id, other_user.ap_id)
cm_ref = MessageReference.for_chat_and_object(chat, object) cm_ref = MessageReference.for_chat_and_object(chat, object)

View File

@ -5,7 +5,6 @@
defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do
use Pleroma.Web.ConnCase use Pleroma.Web.ConnCase
alias Pleroma.Config
alias Pleroma.MFA alias Pleroma.MFA
alias Pleroma.MFA.TOTP alias Pleroma.MFA.TOTP
alias Pleroma.User alias Pleroma.User

View File

@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do
Oban test helpers. Oban test helpers.
""" """
require Ecto.Query
alias Pleroma.Repo alias Pleroma.Repo
def wipe_all do def wipe_all do
@ -15,6 +17,7 @@ defmodule Pleroma.Tests.ObanHelpers do
def perform_all do def perform_all do
Oban.Job Oban.Job
|> Ecto.Query.where(state: "available")
|> Repo.all() |> Repo.all()
|> perform() |> perform()
end end