From c077dc7af5e2a378223a8d2862df1d52877ea245 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Oct 2019 11:52:21 -0500 Subject: [PATCH 1/6] Initial doc about storing remote media --- docs/administration/storing_remote_media.md | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/administration/storing_remote_media.md diff --git a/docs/administration/storing_remote_media.md b/docs/administration/storing_remote_media.md new file mode 100644 index 000000000..7edda2753 --- /dev/null +++ b/docs/administration/storing_remote_media.md @@ -0,0 +1,36 @@ +# Storing Remote Media + +Pleroma does not store remote/federated media by default. The best way to achieve this is to change Nginx to keep its reverse proxy cache +forever and to activate the `MediaProxyWarmingPolicy` MRF policy in Pleroma which will automatically fetch all media through the proxy +as soon as the post is received by your instance. + +## Nginx + +We should be using `proxy_store` here I think??? + +``` + location ~ ^/(media|proxy) { + proxy_cache pleroma_media_cache; + slice 1m; + proxy_cache_key $host$uri$is_args$args$slice_range; + proxy_set_header Range $slice_range; + proxy_http_version 1.1; + proxy_cache_valid 200 206 301 304 1h; + proxy_cache_lock on; + proxy_ignore_client_abort on; + proxy_buffering on; + chunked_transfer_encoding on; + proxy_ignore_headers Cache-Control; + proxy_hide_header Cache-Control; + proxy_pass http://127.0.0.1:4000; + } +``` + +## Pleroma + +Add to your `prod.secret.exs`: + +``` +config :pleroma, :instance, + rewrite_policy: [Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] +``` From a1ad8dc34993445033595c8f52e0ee1815e5567d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Oct 2019 14:07:59 -0500 Subject: [PATCH 2/6] Leverage nginx proxy cache to store items with a 1 year TTL with no size limit. It does not purge items when they expire, but will only update them if the origin's copy has changed for some reason. If origin is offline/unavailable or gone forever it will still serve the cached copies. --- docs/administration/storing_remote_media.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/administration/storing_remote_media.md b/docs/administration/storing_remote_media.md index 7edda2753..0abb85a77 100644 --- a/docs/administration/storing_remote_media.md +++ b/docs/administration/storing_remote_media.md @@ -6,22 +6,25 @@ as soon as the post is received by your instance. ## Nginx -We should be using `proxy_store` here I think??? - ``` +proxy_cache_path /long/term/storage/path/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m + inactive=1y use_temp_path=off; + location ~ ^/(media|proxy) { proxy_cache pleroma_media_cache; slice 1m; proxy_cache_key $host$uri$is_args$args$slice_range; proxy_set_header Range $slice_range; proxy_http_version 1.1; - proxy_cache_valid 200 206 301 304 1h; + proxy_cache_valid 206 301 302 304 1h; + proxy_cache_valid 200 1y; proxy_cache_lock on; + proxy_cache_use_stale error timeout invalid_header updating; proxy_ignore_client_abort on; proxy_buffering on; chunked_transfer_encoding on; - proxy_ignore_headers Cache-Control; - proxy_hide_header Cache-Control; + proxy_ignore_headers Cache-Control Expires; + proxy_hide_header Cache-Control Expires; proxy_pass http://127.0.0.1:4000; } ``` From b9d164fb89af65c2aef83c2867c937ea39a9e995 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Oct 2019 14:12:01 -0500 Subject: [PATCH 3/6] Formatting --- docs/administration/storing_remote_media.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/administration/storing_remote_media.md b/docs/administration/storing_remote_media.md index 0abb85a77..74d333342 100644 --- a/docs/administration/storing_remote_media.md +++ b/docs/administration/storing_remote_media.md @@ -7,8 +7,8 @@ as soon as the post is received by your instance. ## Nginx ``` -proxy_cache_path /long/term/storage/path/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m - inactive=1y use_temp_path=off; + proxy_cache_path /long/term/storage/path/pleroma-media-cache levels=1:2 + keys_zone=pleroma_media_cache:10m inactive=1y use_temp_path=off; location ~ ^/(media|proxy) { proxy_cache pleroma_media_cache; From 47a551837ade9b5c5b7291c83bc3e787e9c6a17d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Oct 2019 15:13:42 -0500 Subject: [PATCH 4/6] Remove proxy_cache_lock suggestion --- docs/administration/storing_remote_media.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/administration/storing_remote_media.md b/docs/administration/storing_remote_media.md index 74d333342..619300e7e 100644 --- a/docs/administration/storing_remote_media.md +++ b/docs/administration/storing_remote_media.md @@ -18,7 +18,6 @@ as soon as the post is received by your instance. proxy_http_version 1.1; proxy_cache_valid 206 301 302 304 1h; proxy_cache_valid 200 1y; - proxy_cache_lock on; proxy_cache_use_stale error timeout invalid_header updating; proxy_ignore_client_abort on; proxy_buffering on; From 752d0c683357277f5926b7b7011b3f945a7610d1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 22 Oct 2019 15:14:04 -0500 Subject: [PATCH 5/6] Relocate to configuration subdir --- docs/{administration => configuration}/storing_remote_media.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{administration => configuration}/storing_remote_media.md (100%) diff --git a/docs/administration/storing_remote_media.md b/docs/configuration/storing_remote_media.md similarity index 100% rename from docs/administration/storing_remote_media.md rename to docs/configuration/storing_remote_media.md From 41db52729eee0158c90d69a8dfc0d87d2a866de0 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 14 May 2020 09:14:59 +0000 Subject: [PATCH 6/6] Apply suggestion to docs/configuration/storing_remote_media.md --- docs/configuration/storing_remote_media.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/storing_remote_media.md b/docs/configuration/storing_remote_media.md index 619300e7e..7e91fe7d9 100644 --- a/docs/configuration/storing_remote_media.md +++ b/docs/configuration/storing_remote_media.md @@ -1,7 +1,7 @@ # Storing Remote Media Pleroma does not store remote/federated media by default. The best way to achieve this is to change Nginx to keep its reverse proxy cache -forever and to activate the `MediaProxyWarmingPolicy` MRF policy in Pleroma which will automatically fetch all media through the proxy +for a year and to activate the `MediaProxyWarmingPolicy` MRF policy in Pleroma which will automatically fetch all media through the proxy as soon as the post is received by your instance. ## Nginx