sjw
/
pleroma
mirror of https://git.pleroma.social/sjw/pleroma.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'href/pleroma-classic-flakeids' into flake-test

This commit is contained in:
lain 2019-01-16 12:29:31 +01:00
parent 608cc65d43 ceaf2cfb76
commit fc3a3c9403
19 changed files with 311 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/pleroma/PasswordResetToken.ex
View File

@ -10,7 +10,7 @@ defmodule Pleroma.PasswordResetToken do
alias Pleroma.{User, PasswordResetToken, Repo}
schema "password_reset_tokens" do
belongs_to(:user, User)
belongs_to(:user, User, type: Pleroma.FlakeId)
field(:token, :string)
field(:used, :boolean, default: false)

1
lib/pleroma/activity.ex
View File

@ -8,6 +8,7 @@ defmodule Pleroma.Activity do
import Ecto.Query
@type t :: %__MODULE__{}
@primary_key {:id, Pleroma.FlakeId, autogenerate: true}
# https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
@mastodon_notification_types %{

1
lib/pleroma/application.ex
View File

@ -99,6 +99,7 @@ defmodule Pleroma.Application do
],
id: :cachex_idem
),
worker(Pleroma.FlakeId, []),
worker(Pleroma.Web.Federator.RetryQueue, []),
worker(Pleroma.Web.Federator, []),
worker(Pleroma.Stats, []),

2
lib/pleroma/filter.ex
View File

@ -8,7 +8,7 @@ defmodule Pleroma.Filter do
alias Pleroma.{User, Repo}
schema "filters" do
belongs_to(:user, User)
belongs_to(:user, User, type: Pleroma.FlakeId)
field(:filter_id, :integer)
field(:hide, :boolean, default: false)
field(:whole_word, :boolean, default: true)

183
lib/pleroma/flake_id.ex Normal file
View File

@ -0,0 +1,183 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.FlakeId do
@moduledoc """
Flake is a decentralized, k-ordered id generation service.
Adapted from:
* [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License,
* [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0
"""
@type t :: binary
@behaviour Ecto.Type
use GenServer
require Logger
alias __MODULE__
import Kernel, except: [to_string: 1]
defstruct node: nil, time: 0, sq: 0
@doc "Converts a binary Flake to a String"
def to_string(<<0::integer-size(64), id::integer-size(64)>>) do
Kernel.to_string(id)
end
def to_string(flake = <<_::integer-size(64), _::integer-size(48), _::integer-size(16)>>) do
encode_base62(flake)
end
def to_string(s), do: s
for i <- [-1, 0] do
def from_string(unquote(i)), do: <<0::integer-size(128)>>
def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>>
end
def from_string(flake = <<_::integer-size(128)>>), do: flake
def from_string(string) when is_binary(string) and byte_size(string) < 18 do
case Integer.parse(string) do
{id, _} -> <<0::integer-size(64), id::integer-size(64)>>
_ -> nil
end
end
def from_string(string) do
string |> decode_base62 |> from_integer
end
def to_integer(<<integer::integer-size(128)>>), do: integer
def from_integer(integer) do
<<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
<<integer::integer-size(128)>>
end
@doc "Generates a Flake"
@spec get :: binary
def get, do: to_string(:gen_server.call(:flake, :get))
# -- Ecto.Type API
@impl Ecto.Type
def type, do: :uuid
@impl Ecto.Type
def cast(value) do
{:ok, FlakeId.to_string(value)}
end
@impl Ecto.Type
def load(value) do
{:ok, FlakeId.to_string(value)}
end
@impl Ecto.Type
def dump(value) do
{:ok, FlakeId.from_string(value)}
end
def autogenerate(), do: get()
# -- GenServer API
def start_link do
:gen_server.start_link({:local, :flake}, __MODULE__, [], [])
end
@impl GenServer
def init([]) do
{:ok, %FlakeId{node: mac(), time: time()}}
end
@impl GenServer
def handle_call(:get, _from, state) do
{flake, new_state} = get(time(), state)
{:reply, flake, new_state}
end
# Matches when the calling time is the same as the state time. Incr. sq
defp get(time, %FlakeId{time: time, node: node, sq: seq}) do
new_state = %FlakeId{time: time, node: node, sq: seq + 1}
{gen_flake(new_state), new_state}
end
# Matches when the times are different, reset sq
defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do
new_state = %FlakeId{time: newtime, node: node, sq: 0}
{gen_flake(new_state), new_state}
end
# Error when clock is running backwards
defp get(newtime, %FlakeId{time: time}) when newtime < time do
{:error, :clock_running_backwards}
end
defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do
<<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
end
defp nthchar_base62(n) when n <= 9, do: ?0 + n
defp nthchar_base62(n) when n <= 35, do: ?A + n - 10
defp nthchar_base62(n), do: ?a + n - 36
defp encode_base62(<<integer::integer-size(128)>>) do
integer
|> encode_base62([])
|> List.to_string()
end
defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc)
defp encode_base62(int, []) when int == 0, do: '0'
defp encode_base62(int, acc) when int == 0, do: acc
defp encode_base62(int, acc) do
r = rem(int, 62)
id = div(int, 62)
acc = [nthchar_base62(r) | acc]
encode_base62(id, acc)
end
defp decode_base62(s) do
decode_base62(String.to_charlist(s), 0)
end
defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9,
do: decode_base62(cs, 62 * acc + (c - ?0))
defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z,
do: decode_base62(cs, 62 * acc + (c - ?A + 10))
defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z,
do: decode_base62(cs, 62 * acc + (c - ?a + 36))
defp decode_base62([], acc), do: acc
defp time do
{mega_seconds, seconds, micro_seconds} = :erlang.timestamp()
1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
end
defp mac do
{:ok, addresses} = :inet.getifaddrs()
ifaces_with_mac =
Enum.reduce(addresses, [], fn {iface, attrs}, acc ->
if attrs[:hwaddr], do: [iface | acc], else: acc
end)
iface = Enum.at(ifaces_with_mac, :rand.uniform(length(ifaces_with_mac)) - 1)
mac(iface)
end
defp mac(name) do
{:ok, addresses} = :inet.getifaddrs()
proplist = :proplists.get_value(name, addresses)
hwaddr = Enum.take(:proplists.get_value(:hwaddr, proplist), 6)
<<worker::integer-size(48)>> = :binary.list_to_bin(hwaddr)
worker
end
end

2
lib/pleroma/list.ex
View File

@ -8,7 +8,7 @@ defmodule Pleroma.List do
alias Pleroma.{User, Repo, Activity}
schema "lists" do
belongs_to(:user, Pleroma.User)
belongs_to(:user, User, type: Pleroma.FlakeId)
field(:title, :string)
field(:following, {:array, :string}, default: [])

6
lib/pleroma/notification.ex
View File

@ -9,8 +9,8 @@ defmodule Pleroma.Notification do
schema "notifications" do
field(:seen, :boolean, default: false)
belongs_to(:user, Pleroma.User)
belongs_to(:activity, Pleroma.Activity)
belongs_to(:user, User, type: Pleroma.FlakeId)
belongs_to(:activity, Activity, type: Pleroma.FlakeId)
timestamps()
end
@ -96,7 +96,7 @@ defmodule Pleroma.Notification do
end
end
def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity)
def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
when type in ["Create", "Like", "Announce", "Follow"] do
users = get_notified_from_activity(activity)

2
lib/pleroma/user.ex
View File

@ -17,6 +17,8 @@ defmodule Pleroma.User do
@type t :: %__MODULE__{}
@primary_key {:id, Pleroma.FlakeId, autogenerate: true}
@email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
@strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/

4
lib/pleroma/web/activity_pub/activity_pub.ex
View File

@ -405,6 +405,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> Enum.reverse()
end
defp restrict_since(query, %{"since_id" => ""}), do: query
defp restrict_since(query, %{"since_id" => since_id}) do
from(activity in query, where: activity.id > ^since_id)
end
@ -460,6 +462,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_local(query, _), do: query
defp restrict_max(query, %{"max_id" => ""}), do: query
defp restrict_max(query, %{"max_id" => max_id}) do
from(activity in query, where: activity.id < ^max_id)
end

5
lib/pleroma/web/activity_pub/transmogrifier.ex
View File

@ -863,15 +863,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
maybe_retire_websub(user.ap_id)
# Only do this for recent activties, don't go through the whole db.
# Only look at the last 1000 activities.
since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000
q =
from(
a in Activity,
where: ^old_follower_address in a.recipients,
where: a.id > ^since,
update: [
set: [
recipients:

4
lib/pleroma/web/activity_pub/views/user_view.ex
View File

@ -160,7 +160,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
"partOf" => iri,
"totalItems" => info.note_count,
"orderedItems" => collection,
"next" => "#{iri}?max_id=#{min_id - 1}"
"next" => "#{iri}?max_id=#{min_id}"
}
if max_qid == nil do
@ -207,7 +207,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
"partOf" => iri,
"totalItems" => -1,
"orderedItems" => collection,
"next" => "#{iri}?max_id=#{min_id - 1}"
"next" => "#{iri}?max_id=#{min_id}"
}
if max_qid == nil do

2
lib/pleroma/web/oauth/authorization.ex
View File

@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
field(:token, :string)
field(:valid_until, :naive_datetime)
field(:used, :boolean, default: false)
belongs_to(:user, Pleroma.User)
belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
belongs_to(:app, App)
timestamps()

2
lib/pleroma/web/oauth/token.ex
View File

@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Token do
field(:token, :string)
field(:refresh_token, :string)
field(:valid_until, :naive_datetime)
belongs_to(:user, Pleroma.User)
belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
belongs_to(:app, App)
timestamps()

2
lib/pleroma/web/push/subscription.ex
View File

@ -10,7 +10,7 @@ defmodule Pleroma.Web.Push.Subscription do
alias Pleroma.Web.Push.Subscription
schema "push_subscriptions" do
belongs_to(:user, User)
belongs_to(:user, User, type: Pleroma.FlakeId)
belongs_to(:token, Token)
field(:endpoint, :string)
field(:key_p256dh, :string)

24
lib/pleroma/web/twitter_api/twitter_api_controller.ex
View File

@ -265,8 +265,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
end
def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
id = String.to_integer(id)
with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id),
activities <-
ActivityPub.fetch_activities_for_context(context, %{
@ -340,38 +338,42 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
end
def favorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
{:ok, activity} <- TwitterAPI.fav(user, id) do
with {:ok, activity} <- TwitterAPI.fav(user, id) do
conn
|> put_view(ActivityView)
|> render("activity.json", %{activity: activity, for: user})
else
_ -> json_reply(conn, 400, Jason.encode!(%{}))
end
end
def unfavorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
{:ok, activity} <- TwitterAPI.unfav(user, id) do
with {:ok, activity} <- TwitterAPI.unfav(user, id) do
conn
|> put_view(ActivityView)
|> render("activity.json", %{activity: activity, for: user})
else
_ -> json_reply(conn, 400, Jason.encode!(%{}))
end
end
def retweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
{:ok, activity} <- TwitterAPI.repeat(user, id) do
with {:ok, activity} <- TwitterAPI.repeat(user, id) do
conn
|> put_view(ActivityView)
|> render("activity.json", %{activity: activity, for: user})
else
_ -> json_reply(conn, 400, Jason.encode!(%{}))
end
end
def unretweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
{:ok, activity} <- TwitterAPI.unrepeat(user, id) do
with {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
conn
|> put_view(ActivityView)
|> render("activity.json", %{activity: activity, for: user})
else
_ -> json_reply(conn, 400, Jason.encode!(%{}))
end
end
@ -556,7 +558,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
def approve_friend_request(conn, %{"user_id" => uid} = _params) do
with followed <- conn.assigns[:user],
uid when is_number(uid) <- String.to_integer(uid),
%User{} = follower <- Repo.get(User, uid),
{:ok, follower} <- User.maybe_follow(follower, followed),
%Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
@ -578,7 +579,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
def deny_friend_request(conn, %{"user_id" => uid} = _params) do
with followed <- conn.assigns[:user],
uid when is_number(uid) <- String.to_integer(uid),
%User{} = follower <- Repo.get(User, uid),
%Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
{:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),

2
lib/pleroma/web/websub/websub_client_subscription.ex
View File

@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do
field(:state, :string)
field(:subscribers, {:array, :string}, default: [])
field(:hub, :string)
belongs_to(:user, User)
belongs_to(:user, User, type: Pleroma.FlakeId)
timestamps()
end

52
priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs Normal file
View File

@ -0,0 +1,52 @@
defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do
use Ecto.Migration
# This migrates from int serial IDs to custom Flake:
# 1- create a temporary uuid column
# 2- fill this column with compatibility ids (see below)
# 3- remove pkeys constraints
# 4- update relation pkeys with the new ids
# 5- rename the temporary column to id
# 6- re-create the constraints
def change do
# Old serial int ids are transformed to 128bits with extra padding.
# The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility
# with previously issued ids.
#execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
#execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
execute "ALTER TABLE activities DROP CONSTRAINT activities_pkey CASCADE;"
execute "ALTER TABLE users DROP CONSTRAINT users_pkey CASCADE;"
execute "ALTER TABLE activities ALTER COLUMN id DROP default;"
execute "ALTER TABLE users ALTER COLUMN id DROP default;"
execute "ALTER TABLE activities ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
execute "ALTER TABLE users ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
execute "ALTER TABLE activities ADD PRIMARY KEY (id);"
execute "ALTER TABLE users ADD PRIMARY KEY (id);"
# Fkeys:
# Activities - Referenced by:
# TABLE "notifications" CONSTRAINT "notifications_activity_id_fkey" FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE
# Users - Referenced by:
# TABLE "filters" CONSTRAINT "filters_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
# TABLE "lists" CONSTRAINT "lists_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
# TABLE "notifications" CONSTRAINT "notifications_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
# TABLE "oauth_authorizations" CONSTRAINT "oauth_authorizations_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
# TABLE "oauth_tokens" CONSTRAINT "oauth_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
# TABLE "password_reset_tokens" CONSTRAINT "password_reset_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
# TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
# TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
execute "ALTER TABLE notifications ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid);"
execute "ALTER TABLE notifications ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE;"
for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do
execute "ALTER TABLE #{table} ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid);"
execute "ALTER TABLE #{table} ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;"
end
end
end

41
test/flake_id_test.exs Normal file
View File

@ -0,0 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.FlakeIdTest do
use Pleroma.DataCase
import Kernel, except: [to_string: 1]
import Pleroma.FlakeId
describe "fake flakes (compatibility with older serial integers)" do
test "from_string/1" do
fake_flake = <<0::integer-size(64), 42::integer-size(64)>>
assert from_string("42") == fake_flake
end
test "zero or -1 is a null flake" do
fake_flake = <<0::integer-size(128)>>
assert from_string("0") == fake_flake
assert from_string("-1") == fake_flake
end
test "to_string/1" do
fake_flake = <<0::integer-size(64), 42::integer-size(64)>>
assert to_string(fake_flake) == "42"
end
end
test "ecto type behaviour" do
flake = <<0, 0, 1, 104, 80, 229, 2, 235, 140, 22, 69, 201, 53, 210, 0, 0>>
flake_s = "9eoozpwTul5mjSEDRI"
assert cast(flake) == {:ok, flake_s}
assert cast(flake_s) == {:ok, flake_s}
assert load(flake) == {:ok, flake_s}
assert load(flake_s) == {:ok, flake_s}
assert dump(flake_s) == {:ok, flake}
assert dump(flake) == {:ok, flake}
end
end

6
test/web/twitter_api/twitter_api_controller_test.exs
View File

@ -797,7 +797,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
|> with_credentials(current_user.nickname, "test")
|> post("/api/favorites/create/1.json")
assert json_response(conn, 500)
assert json_response(conn, 400)
end
end
@ -1621,7 +1621,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
conn =
build_conn()
|> assign(:user, user)
|> post("/api/pleroma/friendships/approve", %{"user_id" => to_string(other_user.id)})
|> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id})
assert relationship = json_response(conn, 200)
assert other_user.id == relationship["id"]
@ -1644,7 +1644,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
conn =
build_conn()
|> assign(:user, user)
|> post("/api/pleroma/friendships/deny", %{"user_id" => to_string(other_user.id)})
|> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id})
assert relationship = json_response(conn, 200)
assert other_user.id == relationship["id"]