Add Pleroma.JobQueueMonitor

This commit is contained in:
Egor Kislitsyn 2019-09-26 18:49:57 +07:00
parent 6abe12dced
commit 75290cbfbd
5 changed files with 133 additions and 1 deletions

View File

@ -42,6 +42,7 @@ defmodule Pleroma.Application do
hackney_pool_children() ++ hackney_pool_children() ++
[ [
Pleroma.Stats, Pleroma.Stats,
Pleroma.JobQueueMonitor,
{Oban, Pleroma.Config.get(Oban)} {Oban, Pleroma.Config.get(Oban)}
] ++ ] ++
task_children(@env) ++ task_children(@env) ++

View File

@ -14,6 +14,7 @@ defmodule Pleroma.Healthcheck do
active: 0, active: 0,
idle: 0, idle: 0,
memory_used: 0, memory_used: 0,
job_queue_stats: nil,
healthy: true healthy: true
@type t :: %__MODULE__{ @type t :: %__MODULE__{
@ -21,6 +22,7 @@ defmodule Pleroma.Healthcheck do
active: non_neg_integer(), active: non_neg_integer(),
idle: non_neg_integer(), idle: non_neg_integer(),
memory_used: number(), memory_used: number(),
job_queue_stats: map(),
healthy: boolean() healthy: boolean()
} }
@ -30,6 +32,7 @@ defmodule Pleroma.Healthcheck do
memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2) memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2)
} }
|> assign_db_info() |> assign_db_info()
|> assign_job_queue_stats()
|> check_health() |> check_health()
end end
@ -55,6 +58,11 @@ defmodule Pleroma.Healthcheck do
Map.merge(healthcheck, db_info) Map.merge(healthcheck, db_info)
end end
defp assign_job_queue_stats(healthcheck) do
stats = Pleroma.JobQueueMonitor.stats()
Map.put(healthcheck, :job_queue_stats, stats)
end
@spec check_health(Healthcheck.t()) :: Healthcheck.t() @spec check_health(Healthcheck.t()) :: Healthcheck.t()
def check_health(%{pool_size: pool_size, active: active} = check) def check_health(%{pool_size: pool_size, active: active} = check)
when active >= pool_size do when active >= pool_size do

View File

@ -0,0 +1,115 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.JobQueueMonitor do
use GenServer
@initial_state %{workers: %{}, queues: %{}, processed_jobs: 0, enqueued: 0}
@queue %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0}
@operation %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0}
def start_link(_) do
GenServer.start_link(__MODULE__, @initial_state, name: __MODULE__)
end
@impl true
def init(state) do
:telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil)
:telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil)
{:ok, state}
end
def stats do
GenServer.call(__MODULE__, :stats)
end
def enqueue({:ok, job}) do
meta = Map.take(job, [:args, :queue, :worker])
GenServer.cast(__MODULE__, {:process_enqueue, meta})
{:ok, job}
end
def enqueue(result), do: result
def handle_event([:oban, status], %{duration: duration}, meta, _) do
GenServer.cast(__MODULE__, {:process_event, status, duration, meta})
end
@impl true
def handle_call(:stats, _from, state) do
{:reply, state, state}
end
def handle_cast({:process_enqueue, meta}, state) do
state =
state
|> Map.update!(:workers, fn workers ->
workers
|> Map.put_new(meta.worker, %{})
|> Map.update!(meta.worker, &update_worker(&1, :enqueue, meta))
end)
|> Map.update!(:queues, fn workers ->
workers
|> Map.put_new(meta.queue, @queue)
|> Map.update!(meta.queue, fn queue -> Map.update!(queue, :enqueued, &(&1 + 1)) end)
end)
|> Map.update!(:enqueued, &(&1 + 1))
{:noreply, state}
end
@impl true
def handle_cast({:process_event, status, duration, meta}, state) do
state =
state
|> Map.update!(:workers, fn workers ->
workers
|> Map.put_new(meta.worker, %{})
|> Map.update!(meta.worker, &update_worker(&1, status, meta, duration))
end)
|> Map.update!(:queues, fn workers ->
workers
|> Map.put_new(meta.queue, @queue)
|> Map.update!(meta.queue, &update_queue(&1, status, meta, duration))
end)
|> Map.update!(:processed_jobs, &(&1 + 1))
|> decr_enqueued()
{:noreply, state}
end
defp update_worker(worker, status, meta, duration \\ 0) do
worker
|> Map.put_new(meta.args["op"], @operation)
|> Map.update!(meta.args["op"], &update_op(&1, status, meta, duration))
end
defp update_op(op, :enqueue, _meta, _duration) do
op
|> Map.update!(:enqueued, &(&1 + 1))
end
defp update_op(op, status, _meta, _duration) do
op
|> Map.update!(:processed_jobs, &(&1 + 1))
|> Map.update!(status, &(&1 + 1))
|> decr_enqueued()
end
defp update_queue(queue, status, _meta, _duration) do
queue
|> Map.update!(:processed_jobs, &(&1 + 1))
|> Map.update!(status, &(&1 + 1))
|> decr_enqueued()
end
defp decr_enqueued(map) do
Map.update!(map, :enqueued, fn
0 -> 0
enqueued -> enqueued - 1
end)
end
end

View File

@ -40,6 +40,7 @@ defmodule Pleroma.Workers.WorkerHelper do
unquote(caller_module) unquote(caller_module)
|> apply(:new, [params, worker_args]) |> apply(:new, [params, worker_args])
|> Pleroma.Repo.insert() |> Pleroma.Repo.insert()
|> Pleroma.JobQueueMonitor.enqueue()
end end
end end
end end

View File

@ -9,7 +9,14 @@ defmodule Pleroma.HealthcheckTest do
test "system_info/0" do test "system_info/0" do
result = Healthcheck.system_info() |> Map.from_struct() result = Healthcheck.system_info() |> Map.from_struct()
assert Map.keys(result) == [:active, :healthy, :idle, :memory_used, :pool_size] assert Map.keys(result) == [
:active,
:healthy,
:idle,
:job_queue_stats,
:memory_used,
:pool_size
]
end end
describe "check_health/1" do describe "check_health/1" do