From cf4c5e1fe815b677dbc1a8c4606efcc8e95216c9 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Thu, 14 Jan 2021 10:57:50 -0800 Subject: [PATCH 01/15] Implement #53 --- src/main.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index d9d6a4a..87d302f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,7 +58,15 @@ async fn main() -> std::io::Result<()> { // Proxy media through Libreddit .route("/proxy/{url:.*}/", web::get().to(proxy::handler)) // Browse user profile - .route("/{scope:u|user}/{username}/", web::get().to(user::profile)) + .service( + web::scope("/{scope:user|u}").service( + web::scope("/{username}").route("/", web::get().to(user::profile)).service( + web::scope("/comments/{id}/{title}") + .route("/", web::get().to(post::item)) + .route("/{comment_id}/", web::get().to(post::item)), + ), + ), + ) // Configure settings .service(web::resource("/settings/").route(web::get().to(settings::get)).route(web::post().to(settings::set))) // Subreddit services From 18684c934b719972b909fc28f464515348c34cb9 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Thu, 14 Jan 2021 11:45:04 -0800 Subject: [PATCH 02/15] Refactor subreddit searching --- src/search.rs | 65 +++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/src/search.rs b/src/search.rs index 5f43b83..35330c5 100644 --- a/src/search.rs +++ b/src/search.rs @@ -1,5 +1,5 @@ // CRATES -use crate::utils::{error, fetch_posts, param, prefs, Post, Preferences, request, val}; +use crate::utils::{error, fetch_posts, param, prefs, request, val, Post, Preferences}; use actix_web::{HttpRequest, HttpResponse}; use askama::Template; @@ -34,42 +34,19 @@ struct SearchTemplate { // SERVICES pub async fn find(req: HttpRequest) -> HttpResponse { let path = format!("{}.json?{}", req.path(), req.query_string()); + let sub = req.match_info().get("sub").unwrap_or("").to_string(); + let sort = if param(&path, "sort").is_empty() { "relevance".to_string() } else { param(&path, "sort") }; - let sub = req.match_info().get("sub").unwrap_or("").to_string(); - let mut subreddits: Vec = Vec::new(); - - if param(&path, "restrict_sr") == "" { - let subreddit_search_path = format!("/subreddits/search.json?q={}&limit=3", param(&path, "q")); - let res; - let subreddit_list; - // Send a request to the url - match request(&subreddit_search_path).await { - // If success, receive JSON in response - Ok(response) => { - res = response; - subreddit_list = res["data"]["children"].as_array(); - } - // If the Reddit API returns an error, exit this function - Err(_msg) => {subreddit_list = None;} - } - - // For each subreddit from subreddit list - if !subreddit_list.is_none() { - for subreddit in subreddit_list.unwrap() { - subreddits.push(Subreddit { - name: val(subreddit, "display_name_prefixed"), - url: val(subreddit, "url"), - description: val(subreddit, "public_description"), - subscribers: subreddit["data"]["subscribers"].as_u64().unwrap_or_default() as i64, - }); - } - } - } + let subreddits = if param(&path, "restrict_sr").is_empty() { + search_subreddits(param(&path, "q")).await + } else { + Vec::new() + }; match fetch_posts(&path, String::new()).await { Ok((posts, after)) => HttpResponse::Ok().content_type("text/html").body( @@ -93,3 +70,29 @@ pub async fn find(req: HttpRequest) -> HttpResponse { Err(msg) => error(msg).await, } } + +async fn search_subreddits(q: String) -> Vec { + let subreddit_search_path = format!("/subreddits/search.json?q={}&limit=3", q.replace(' ', "+")); + + // Send a request to the url + match request(&subreddit_search_path).await { + // If success, receive JSON in response + Ok(response) => { + match response["data"]["children"].as_array() { + // For each subreddit from subreddit list + Some(list) => list + .iter() + .map(|subreddit| Subreddit { + name: val(subreddit, "display_name_prefixed"), + url: val(subreddit, "url"), + description: val(subreddit, "public_description"), + subscribers: subreddit["data"]["subscribers"].as_u64().unwrap_or_default() as i64, + }) + .collect::>(), + _ => Vec::new(), + } + } + // If the Reddit API returns an error, exit this function + _ => Vec::new(), + } +} From 32b8637c7e96745147c1b504c55996cd5fb4ea42 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Thu, 14 Jan 2021 14:56:28 -0800 Subject: [PATCH 03/15] Handle failed redirects --- src/utils.rs | 14 +++++++++----- templates/utils.html | 5 ++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index d6acf27..cfab28e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -133,9 +133,10 @@ pub fn prefs(req: HttpRequest) -> Preferences { // Grab a query param from a url pub fn param(path: &str, value: &str) -> String { - let url = Url::parse(format!("https://libredd.it/{}", path).as_str()).unwrap(); - let pairs: HashMap<_, _> = url.query_pairs().into_owned().collect(); - pairs.get(value).unwrap_or(&String::new()).to_owned() + match Url::parse(format!("https://libredd.it/{}", path).as_str()) { + Ok(url) => url.query_pairs().into_owned().collect::>().get(value).unwrap_or(&String::new()).to_owned(), + _ => String::new(), + } } // Parse Cookie value from request @@ -360,9 +361,12 @@ pub async fn request(path: &str) -> Result { // Get first number of response HTTP status code match payload.status().to_string().chars().next() { // If success - Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap().to_vec()).unwrap()), + Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), // If redirection - Some('3') => Err((true, payload.headers().get("location").unwrap().to_str().unwrap().to_string())), + Some('3') => match payload.headers().get("location") { + Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), + None => Err((false, "Page not found".to_string())) + } // Otherwise _ => Err((false, "Page not found".to_string())), } diff --git a/templates/utils.html b/templates/utils.html index 968f198..cb3c65a 100644 --- a/templates/utils.html +++ b/templates/utils.html @@ -28,5 +28,8 @@ {%- endmacro %} {% macro render_flair(flair) -%} -{% for flair_part in flair %}{% if flair_part.flair_part_type == "emoji" %}{% else if flair_part.flair_part_type == "text" %}{{ flair_part.value }}{% endif %}{% endfor %} +{% for flair_part in flair %} + {% if flair_part.flair_part_type == "emoji" %} + {% else if flair_part.flair_part_type == "text" %}{{ flair_part.value }}{% endif %} +{% endfor %} {%- endmacro %} From 0c014ad41b484b5d3de23170b76b2e1822ccf67e Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Thu, 14 Jan 2021 15:13:52 -0800 Subject: [PATCH 04/15] Comment utils.rs --- src/utils.rs | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index cfab28e..1f4206a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -20,6 +20,7 @@ pub struct Flair { pub foreground_color: String, } +// Part of flair, either emoji or text pub struct FlairPart { pub flair_part_type: String, pub value: String, @@ -172,19 +173,23 @@ pub fn format_num(num: i64) -> String { pub async fn media(data: &Value) -> (String, String) { let post_type: &str; - let url = if !data["preview"]["reddit_video_preview"]["fallback_url"].is_null() { + // If post is a video, return the video + let url = if data["preview"]["reddit_video_preview"]["fallback_url"].is_string() { post_type = "video"; format_url(data["preview"]["reddit_video_preview"]["fallback_url"].as_str().unwrap_or_default()) - } else if !data["secure_media"]["reddit_video"]["fallback_url"].is_null() { + } else if data["secure_media"]["reddit_video"]["fallback_url"].is_string() { post_type = "video"; format_url(data["secure_media"]["reddit_video"]["fallback_url"].as_str().unwrap_or_default()) + // Handle images, whether GIFs or pics } else if data["post_hint"].as_str().unwrap_or("") == "image" { let preview = data["preview"]["images"][0].clone(); match preview["variants"]["mp4"].as_object() { + // Return the mp4 if the media is a gif Some(gif) => { post_type = "gif"; format_url(gif["source"]["url"].as_str().unwrap_or_default()) } + // Return the picture if the media is an image None => { post_type = "image"; format_url(preview["source"]["url"].as_str().unwrap_or_default()) @@ -202,10 +207,13 @@ pub async fn media(data: &Value) -> (String, String) { } pub fn parse_rich_flair(flair_type: String, rich_flair: Option<&Vec>, text_flair: Option<&str>) -> Vec { + // Parse type of flair match flair_type.as_str() { + // If flair contains emojis and text "richtext" => match rich_flair { Some(rich) => rich .iter() + // For each part of the flair, extract text and emojis .map(|part| { let value = |name: &str| part[name].as_str().unwrap_or_default(); FlairPart { @@ -220,6 +228,7 @@ pub fn parse_rich_flair(flair_type: String, rich_flair: Option<&Vec>, tex .collect::>(), None => Vec::new(), }, + // If flair contains only text "text" => match text_flair { Some(text) => vec![FlairPart { flair_part_type: "text".to_string(), @@ -234,8 +243,10 @@ pub fn parse_rich_flair(flair_type: String, rich_flair: Option<&Vec>, tex pub fn time(unix_time: i64) -> String { let time = OffsetDateTime::from_unix_timestamp(unix_time); let time_delta = OffsetDateTime::now_utc() - time; + // If the time difference is more than a month, show full date if time_delta > Duration::days(30) { - time.format("%b %d '%y") // %b %e '%y + time.format("%b %d '%y") + // Otherwise, show relative date/time } else if time_delta.whole_days() > 0 { format!("{}d ago", time_delta.whole_days()) } else if time_delta.whole_hours() > 0 { @@ -365,8 +376,8 @@ pub async fn request(path: &str) -> Result { // If redirection Some('3') => match payload.headers().get("location") { Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), - None => Err((false, "Page not found".to_string())) - } + None => Err((false, "Page not found".to_string())), + }, // Otherwise _ => Err((false, "Page not found".to_string())), } @@ -375,12 +386,14 @@ pub async fn request(path: &str) -> Result { } } - fn err(u: String, m: String) -> Result { + // Print error if debugging then return error based on error message + fn err(url: String, msg: String) -> Result { #[cfg(debug_assertions)] - dbg!(format!("{} - {}", u, m)); - Err(m) + dbg!(format!("{} - {}", url, msg)); + Err(msg) }; + // Parse JSON from body. If parsing fails, return error fn json(url: String, body: String) -> Result { match from_str(body.as_str()) { Ok(json) => Ok(json), @@ -388,13 +401,20 @@ pub async fn request(path: &str) -> Result { } } + // Make request to Reddit using send function match send(&url).await { + // If success, parse and return body Ok(body) => json(url, body), + // Follow any redirects Err((true, location)) => match send(location.as_str()).await { + // If success, parse and return body Ok(body) => json(url, body), + // Follow any redirects again Err((true, location)) => err(url, location), + // Return errors if request fails Err((_, msg)) => err(url, msg), }, + // Return errors if request fails Err((_, msg)) => err(url, msg), } } From bca2a7e54044204d04aa2ecbc64f2bc3998260ea Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 10:58:53 -0800 Subject: [PATCH 05/15] Error logging --- README.md | 2 +- src/utils.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9128667..c10c1c8 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Results from Google Lighthouse ([Libreddit Report](https://lighthouse-dot-webdot For transparency, I hope to describe all the ways Libreddit handles user privacy. -**Logging:** In production (when running the binary, hosting with docker, or using the official instances), Libreddit logs nothing. When debugging (running from source without `--release`), Libreddit logs post IDs and URL paths fetched to aid troubleshooting but nothing else. +**Logging:** In production (when running the binary, hosting with docker, or using the official instances), Libreddit logs errors. When debugging (running from source without `--release`), Libreddit logs post IDs and URL paths fetched to aid with troubleshooting. **DNS:** Both official domains (`libredd.it` and `libreddit.spike.codes`) use Cloudflare as the DNS resolver. Though, the sites are not proxied through Cloudflare meaning Cloudflare doesn't have access to user traffic. diff --git a/src/utils.rs b/src/utils.rs index 1f4206a..1af0d73 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -388,7 +388,7 @@ pub async fn request(path: &str) -> Result { // Print error if debugging then return error based on error message fn err(url: String, msg: String) -> Result { - #[cfg(debug_assertions)] + // #[cfg(debug_assertions)] dbg!(format!("{} - {}", url, msg)); Err(msg) }; From 21ff8d7b6f070aa48125b7e09e1f7e7007025740 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 11:21:59 -0800 Subject: [PATCH 06/15] Fix #56 --- templates/subreddit.html | 4 ++-- templates/user.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/subreddit.html b/templates/subreddit.html index 0a61ff0..2593465 100644 --- a/templates/subreddit.html +++ b/templates/subreddit.html @@ -76,11 +76,11 @@
{% if ends.0 != "" %} - PREV + PREV {% endif %} {% if ends.1 != "" %} - NEXT + NEXT {% endif %}
diff --git a/templates/user.html b/templates/user.html index 841d295..f12f8e4 100644 --- a/templates/user.html +++ b/templates/user.html @@ -86,11 +86,11 @@
{% if ends.0 != "" %} - PREV + PREV {% endif %} {% if ends.1 != "" %} - NEXT + NEXT {% endif %}
From aef0442e9d768150ba07909969952b2a7f8034bb Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 11:24:12 -0800 Subject: [PATCH 07/15] Add rate-limit warning --- src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 1af0d73..b17cec1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -382,7 +382,7 @@ pub async fn request(path: &str) -> Result { _ => Err((false, "Page not found".to_string())), } } - Err(_) => Err((false, "Couldn't send request to Reddit".to_string())), + Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), } } From 373ce55203ce56e3979ba19efb7d1e55bf1aed54 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 11:27:06 -0800 Subject: [PATCH 08/15] Recommend secondary instance --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c10c1c8..d6075ab 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,14 @@ > An alternative private front-end to Reddit -Libre + Reddit = [Libreddit](https://libredd.it) +Libre + Reddit = [Libreddit](https://libreddit.spike.codes) - 🚀 Fast: written in Rust for blazing fast speeds and safety - ☁️ Light: no JavaScript, no ads, no tracking - 🕵 Private: all requests are proxied through the server, including media - 🔒 Secure: strong [Content Security Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP) prevents browser requests to Reddit -Like [Invidious](https://github.com/iv-org/invidious) but for Reddit. Browse the coldest takes of [r/unpopularopinion](https://libredd.it/r/unpopularopinion) without being [tracked](#reddit). +Like [Invidious](https://github.com/iv-org/invidious) but for Reddit. Browse the coldest takes of [r/unpopularopinion](https://libreddit.spike.codes/r/unpopularopinion) without being [tracked](#reddit). ## Contents - [Screenshot](#screenshot) @@ -127,7 +127,7 @@ For transparency, I hope to describe all the ways Libreddit handles user privacy **DNS:** Both official domains (`libredd.it` and `libreddit.spike.codes`) use Cloudflare as the DNS resolver. Though, the sites are not proxied through Cloudflare meaning Cloudflare doesn't have access to user traffic. -**Cookies:** Libreddit uses optional cookies to store any configured settings in [the settings menu](https://libredd.it/settings). This is not a cross-site cookie and the cookie holds no personal data, only a value of the possible layout. +**Cookies:** Libreddit uses optional cookies to store any configured settings in [the settings menu](https://libreddit.spike.codes/settings). This is not a cross-site cookie and the cookie holds no personal data, only a value of the possible layout. **Hosting:** The official instances (`libredd.it` and `libreddit.spike.codes`) are hosted on [Repl.it](https://repl.it/) which monitors usage to prevent abuse. I can understand if this invalidates certain users' threat models and therefore, selfhosting and browsing through Tor are welcomed. From 078d6fe25bcd42abc6373d2a3ff7dd71acb9e5eb Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 15:05:55 -0800 Subject: [PATCH 09/15] Request about pages before posts --- src/subreddit.rs | 23 ++++++++++++----------- src/user.rs | 6 ++++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/subreddit.rs b/src/subreddit.rs index 7e68f46..35b4fde 100644 --- a/src/subreddit.rs +++ b/src/subreddit.rs @@ -34,19 +34,20 @@ pub async fn page(req: HttpRequest) -> HttpResponse { .to_string(); let sort = req.match_info().get("sort").unwrap_or("hot").to_string(); - let sub = if !sub_name.contains('+') && sub_name != "popular" && sub_name != "all" { - subreddit(&sub_name).await.unwrap_or_default() - } else if sub_name.contains('+') { - Subreddit { - name: sub_name, - ..Subreddit::default() - } - } else { - Subreddit::default() - }; - match fetch_posts(&path, String::new()).await { Ok((posts, after)) => { + // If you can get subreddit posts, also request subreddit metadata + let sub = if !sub_name.contains('+') && sub_name != "popular" && sub_name != "all" { + subreddit(&sub_name).await.unwrap_or_default() + } else if sub_name.contains('+') { + Subreddit { + name: sub_name, + ..Subreddit::default() + } + } else { + Subreddit::default() + }; + let s = SubredditTemplate { sub, posts, diff --git a/src/user.rs b/src/user.rs index 9a7502f..be11069 100644 --- a/src/user.rs +++ b/src/user.rs @@ -24,12 +24,14 @@ pub async fn profile(req: HttpRequest) -> HttpResponse { let sort = param(&path, "sort"); let username = req.match_info().get("username").unwrap_or("").to_string(); - // Request user profile data and user posts/comments from Reddit - let user = user(&username).await.unwrap_or_default(); + // Request user posts/comments from Reddit let posts = fetch_posts(&path, "Comment".to_string()).await; match posts { Ok((posts, after)) => { + // If you can get user posts, also request user data + let user = user(&username).await.unwrap_or_default(); + let s = UserTemplate { user, posts, From 59021b9331ab74eaa9f0be5ad1245b3110c42652 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 15:28:51 -0800 Subject: [PATCH 10/15] Switch back to ureq temporarily --- Cargo.lock | 89 +++++++++++++++++++++++++++--------- Cargo.toml | 1 + src/utils.rs | 124 +++++++++++++++++++++++++++++++-------------------- 3 files changed, 144 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d041577..38a5fe5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ "futures-util", "http", "log", - "rustls", + "rustls 0.18.1", "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", @@ -193,10 +193,10 @@ dependencies = [ "actix-service", "actix-utils", "futures-util", - "rustls", + "rustls 0.18.1", "tokio-rustls", "webpki", - "webpki-roots", + "webpki-roots 0.20.0", ] [[package]] @@ -249,7 +249,7 @@ dependencies = [ "mime", "pin-project 1.0.4", "regex", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", @@ -393,7 +393,7 @@ dependencies = [ "mime", "percent-encoding", "rand", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", @@ -529,6 +529,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chunked_transfer" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" + [[package]] name = "const_fn" version = "0.4.5" @@ -672,9 +678,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "futures" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309f13e3f4be6d5917178c84db67c0b9a09177ac16d4f9a7313a767a68adaa77" +checksum = "da9052a1a50244d8d5aa9bf55cbc2fb6f357c86cc52e46c62ed390a7180cf150" dependencies = [ "futures-channel", "futures-core", @@ -686,9 +692,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3b03bd32f6ec7885edeb99acd1e47e20e34fd4dfd3c6deed6fcac8a9d28f6a" +checksum = "f2d31b7ec7efab6eefc7c57233bb10b847986139d88cc2f5a02a1ae6871a1846" dependencies = [ "futures-core", "futures-sink", @@ -696,21 +702,21 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed8aeae2b6ab243ebabe6f54cd4cf53054d98883d5d326128af7d57a9ca5cd3d" +checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65" [[package]] name = "futures-io" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41234e71d5e8ca73d01563974ef6f50e516d71e18f1a2f1184742e31f5d469f" +checksum = "28be053525281ad8259d47e4de5de657b25e7bac113458555bb4b70bc6870500" [[package]] name = "futures-macro" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3520e0eb4e704e88d771b92d51273ee212997f0d8282f17f5d8ff1cb39104e42" +checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -720,24 +726,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c72d188479368953c6c8c7140e40d7a4401674ab3b98a41e60e515d6cbdbe5de" +checksum = "caf5c69029bda2e743fddd0582d1083951d65cc9539aebf8812f36c3491342d6" [[package]] name = "futures-task" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08944cea9021170d383287169859c0ca8147d9ec285978393109954448f33cc7" +checksum = "13de07eb8ea81ae445aca7b69f5f7bf15d7bf4912d8ca37d6645c77ae8a58d86" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd206efbe2ca683b2ce138ccdf61e1b0a63f5816dcedc9d8654c500ba0cea6" +checksum = "632a8cd0f2a4b3fdea1657f08bde063848c3bd00f9bbf6e256b8be78802e624b" dependencies = [ "futures-channel", "futures-core", @@ -986,6 +992,7 @@ dependencies = [ "serde", "serde_json", "time", + "ureq", "url", ] @@ -1413,6 +1420,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1749,7 +1769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls", + "rustls 0.18.1", "tokio", "webpki", ] @@ -1890,6 +1910,22 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96014ded8c85822677daee4f909d18acccca744810fd4f8ffc492c284f2324bc" +dependencies = [ + "base64 0.13.0", + "chunked_transfer", + "log", + "once_cell", + "rustls 0.19.0", + "url", + "webpki", + "webpki-roots 0.21.0", +] + [[package]] name = "url" version = "2.2.0" @@ -1997,6 +2033,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index e35d9e5..dce6390 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" base64 = "0.13.0" actix-web = { version = "3.3.2", features = ["rustls"] } askama = "0.10.5" +ureq = "2.0.1" serde = { version = "1.0.118", default_features = false, features = ["derive"] } serde_json = "1.0" async-recursion = "0.3.1" diff --git a/src/utils.rs b/src/utils.rs index b17cec1..421dd69 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -363,58 +363,86 @@ pub async fn request(path: &str) -> Result { let url = format!("https://www.reddit.com{}", path); // Send request using awc - async fn send(url: &str) -> Result { - let client = actix_web::client::Client::default(); - let response = client.get(url).send().await; + // async fn send(url: &str) -> Result { + // let client = actix_web::client::Client::default(); + // let response = client.get(url).send().await; - match response { - Ok(mut payload) => { - // Get first number of response HTTP status code - match payload.status().to_string().chars().next() { - // If success - Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), - // If redirection - Some('3') => match payload.headers().get("location") { - Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), - None => Err((false, "Page not found".to_string())), - }, - // Otherwise - _ => Err((false, "Page not found".to_string())), + // match response { + // Ok(mut payload) => { + // // Get first number of response HTTP status code + // match payload.status().to_string().chars().next() { + // // If success + // Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), + // // If redirection + // Some('3') => match payload.headers().get("location") { + // Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), + // None => Err((false, "Page not found".to_string())), + // }, + // // Otherwise + // _ => Err((false, "Page not found".to_string())), + // } + // } + // Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), + // } + // } + + // // Print error if debugging then return error based on error message + // fn err(url: String, msg: String) -> Result { + // // #[cfg(debug_assertions)] + // dbg!(format!("{} - {}", url, msg)); + // Err(msg) + // }; + + // // Parse JSON from body. If parsing fails, return error + // fn json(url: String, body: String) -> Result { + // match from_str(body.as_str()) { + // Ok(json) => Ok(json), + // Err(_) => err(url, "Failed to parse page JSON data".to_string()), + // } + // } + + // // Make request to Reddit using send function + // match send(&url).await { + // // If success, parse and return body + // Ok(body) => json(url, body), + // // Follow any redirects + // Err((true, location)) => match send(location.as_str()).await { + // // If success, parse and return body + // Ok(body) => json(url, body), + // // Follow any redirects again + // Err((true, location)) => err(url, location), + // // Return errors if request fails + // Err((_, msg)) => err(url, msg), + // }, + // // Return errors if request fails + // Err((_, msg)) => err(url, msg), + // } + + // Send request using ureq + match ureq::get(&url).call() { + // If response is success + Ok(response) => { + // Parse the response from Reddit as JSON + match from_str(&response.into_string().unwrap()) { + Ok(json) => Ok(json), + Err(_) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - Failed to parse page JSON data", url)); + Err("Failed to parse page JSON data".to_string()) } } - Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), } - } - - // Print error if debugging then return error based on error message - fn err(url: String, msg: String) -> Result { - // #[cfg(debug_assertions)] - dbg!(format!("{} - {}", url, msg)); - Err(msg) - }; - - // Parse JSON from body. If parsing fails, return error - fn json(url: String, body: String) -> Result { - match from_str(body.as_str()) { - Ok(json) => Ok(json), - Err(_) => err(url, "Failed to parse page JSON data".to_string()), + // If response is error + Err(ureq::Error::Status(_, _)) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - Page not found", url)); + Err("Page not found".to_string()) + } + // If failed to send request + Err(e) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - {}", url, e)); + Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string()) } - } - - // Make request to Reddit using send function - match send(&url).await { - // If success, parse and return body - Ok(body) => json(url, body), - // Follow any redirects - Err((true, location)) => match send(location.as_str()).await { - // If success, parse and return body - Ok(body) => json(url, body), - // Follow any redirects again - Err((true, location)) => err(url, location), - // Return errors if request fails - Err((_, msg)) => err(url, msg), - }, - // Return errors if request fails - Err((_, msg)) => err(url, msg), } } From 3a9b2dba320a60dd88850dfe94b58c46ffdb9d54 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 15:35:09 -0800 Subject: [PATCH 11/15] Fix error log --- src/utils.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 421dd69..f526f88 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -440,7 +440,6 @@ pub async fn request(path: &str) -> Result { } // If failed to send request Err(e) => { - #[cfg(debug_assertions)] dbg!(format!("{} - {}", url, e)); Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string()) } From 009902147841786f9b7928c0c938232aad613d97 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 15:55:10 -0800 Subject: [PATCH 12/15] Refactor flair spacing --- static/style.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/style.css b/static/style.css index 33d1964..28c7391 100644 --- a/static/style.css +++ b/static/style.css @@ -543,7 +543,8 @@ a.search_subreddit:hover { .post_flair { background: var(--accent); color: var(--background); - padding: 5px; + padding: 4px; + margin-right: 5px; border-radius: 5px; font-size: 12px; font-weight: bold; From 65543a43b218f1e3887fd5bca812d08c4cd9f04d Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 20:29:34 -0800 Subject: [PATCH 13/15] Make User-Agent Reddit-compliant --- Cargo.lock | 57 ++----------------- Cargo.toml | 1 - src/utils.rs | 152 +++++++++++++++++++++++++-------------------------- 3 files changed, 82 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38a5fe5..a42651b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ "futures-util", "http", "log", - "rustls 0.18.1", + "rustls", "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", @@ -193,10 +193,10 @@ dependencies = [ "actix-service", "actix-utils", "futures-util", - "rustls 0.18.1", + "rustls", "tokio-rustls", "webpki", - "webpki-roots 0.20.0", + "webpki-roots", ] [[package]] @@ -249,7 +249,7 @@ dependencies = [ "mime", "pin-project 1.0.4", "regex", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -393,7 +393,7 @@ dependencies = [ "mime", "percent-encoding", "rand", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -529,12 +529,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chunked_transfer" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" - [[package]] name = "const_fn" version = "0.4.5" @@ -992,7 +986,6 @@ dependencies = [ "serde", "serde_json", "time", - "ureq", "url", ] @@ -1420,19 +1413,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" -dependencies = [ - "base64 0.13.0", - "log", - "ring", - "sct", - "webpki", -] - [[package]] name = "ryu" version = "1.0.5" @@ -1769,7 +1749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls 0.18.1", + "rustls", "tokio", "webpki", ] @@ -1910,22 +1890,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "ureq" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96014ded8c85822677daee4f909d18acccca744810fd4f8ffc492c284f2324bc" -dependencies = [ - "base64 0.13.0", - "chunked_transfer", - "log", - "once_cell", - "rustls 0.19.0", - "url", - "webpki", - "webpki-roots 0.21.0", -] - [[package]] name = "url" version = "2.2.0" @@ -2033,15 +1997,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "webpki-roots" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" -dependencies = [ - "webpki", -] - [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index dce6390..e35d9e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ edition = "2018" base64 = "0.13.0" actix-web = { version = "3.3.2", features = ["rustls"] } askama = "0.10.5" -ureq = "2.0.1" serde = { version = "1.0.118", default_features = false, features = ["derive"] } serde_json = "1.0" async-recursion = "0.3.1" diff --git a/src/utils.rs b/src/utils.rs index f526f88..96dafaa 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -363,85 +363,85 @@ pub async fn request(path: &str) -> Result { let url = format!("https://www.reddit.com{}", path); // Send request using awc - // async fn send(url: &str) -> Result { - // let client = actix_web::client::Client::default(); - // let response = client.get(url).send().await; + async fn send(url: &str) -> Result { + let client = actix_web::client::Client::default(); + let response = client.get(url).header("User-Agent", format!("web:libreddit:{}", env!("CARGO_PKG_VERSION"))).send().await; - // match response { - // Ok(mut payload) => { - // // Get first number of response HTTP status code - // match payload.status().to_string().chars().next() { - // // If success - // Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), - // // If redirection - // Some('3') => match payload.headers().get("location") { - // Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), - // None => Err((false, "Page not found".to_string())), - // }, - // // Otherwise - // _ => Err((false, "Page not found".to_string())), - // } - // } - // Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), - // } - // } - - // // Print error if debugging then return error based on error message - // fn err(url: String, msg: String) -> Result { - // // #[cfg(debug_assertions)] - // dbg!(format!("{} - {}", url, msg)); - // Err(msg) - // }; - - // // Parse JSON from body. If parsing fails, return error - // fn json(url: String, body: String) -> Result { - // match from_str(body.as_str()) { - // Ok(json) => Ok(json), - // Err(_) => err(url, "Failed to parse page JSON data".to_string()), - // } - // } - - // // Make request to Reddit using send function - // match send(&url).await { - // // If success, parse and return body - // Ok(body) => json(url, body), - // // Follow any redirects - // Err((true, location)) => match send(location.as_str()).await { - // // If success, parse and return body - // Ok(body) => json(url, body), - // // Follow any redirects again - // Err((true, location)) => err(url, location), - // // Return errors if request fails - // Err((_, msg)) => err(url, msg), - // }, - // // Return errors if request fails - // Err((_, msg)) => err(url, msg), - // } - - // Send request using ureq - match ureq::get(&url).call() { - // If response is success - Ok(response) => { - // Parse the response from Reddit as JSON - match from_str(&response.into_string().unwrap()) { - Ok(json) => Ok(json), - Err(_) => { - #[cfg(debug_assertions)] - dbg!(format!("{} - Failed to parse page JSON data", url)); - Err("Failed to parse page JSON data".to_string()) + match response { + Ok(mut payload) => { + // Get first number of response HTTP status code + match payload.status().to_string().chars().next() { + // If success + Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), + // If redirection + Some('3') => match payload.headers().get("location") { + Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), + None => Err((false, "Page not found".to_string())), + }, + // Otherwise + _ => Err((false, "Page not found".to_string())), } } - } - // If response is error - Err(ureq::Error::Status(_, _)) => { - #[cfg(debug_assertions)] - dbg!(format!("{} - Page not found", url)); - Err("Page not found".to_string()) - } - // If failed to send request - Err(e) => { - dbg!(format!("{} - {}", url, e)); - Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string()) + Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), } } + + // Print error if debugging then return error based on error message + fn err(url: String, msg: String) -> Result { + // #[cfg(debug_assertions)] + dbg!(format!("{} - {}", url, msg)); + Err(msg) + }; + + // Parse JSON from body. If parsing fails, return error + fn json(url: String, body: String) -> Result { + match from_str(body.as_str()) { + Ok(json) => Ok(json), + Err(_) => err(url, "Failed to parse page JSON data".to_string()), + } + } + + // Make request to Reddit using send function + match send(&url).await { + // If success, parse and return body + Ok(body) => json(url, body), + // Follow any redirects + Err((true, location)) => match send(location.as_str()).await { + // If success, parse and return body + Ok(body) => json(url, body), + // Follow any redirects again + Err((true, location)) => err(url, location), + // Return errors if request fails + Err((_, msg)) => err(url, msg), + }, + // Return errors if request fails + Err((_, msg)) => err(url, msg), + } + + // Send request using ureq + // match ureq::get(&url).call() { + // // If response is success + // Ok(response) => { + // // Parse the response from Reddit as JSON + // match from_str(&response.into_string().unwrap()) { + // Ok(json) => Ok(json), + // Err(_) => { + // #[cfg(debug_assertions)] + // dbg!(format!("{} - Failed to parse page JSON data", url)); + // Err("Failed to parse page JSON data".to_string()) + // } + // } + // } + // // If response is error + // Err(ureq::Error::Status(_, _)) => { + // #[cfg(debug_assertions)] + // dbg!(format!("{} - Page not found", url)); + // Err("Page not found".to_string()) + // } + // // If failed to send request + // Err(e) => { + // dbg!(format!("{} - {}", url, e)); + // Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string()) + // } + // } } From 1fd688eeed78ba47567920cd6e7414dca7931957 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 20:57:51 -0800 Subject: [PATCH 14/15] Improve awc error log --- src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index 96dafaa..c6e957b 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -382,7 +382,7 @@ pub async fn request(path: &str) -> Result { _ => Err((false, "Page not found".to_string())), } } - Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), + Err(e) => { dbg!(e); Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())) }, } } From 2385fa33ec8052cd391629b55bceb5a48489eb42 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 21:26:51 -0800 Subject: [PATCH 15/15] Use ureq until AWC IO error is fixed --- Cargo.lock | 57 +++++++++++++++++-- Cargo.toml | 1 + src/utils.rs | 152 +++++++++++++++++++++++++-------------------------- 3 files changed, 128 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a42651b..38a5fe5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ "futures-util", "http", "log", - "rustls", + "rustls 0.18.1", "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", @@ -193,10 +193,10 @@ dependencies = [ "actix-service", "actix-utils", "futures-util", - "rustls", + "rustls 0.18.1", "tokio-rustls", "webpki", - "webpki-roots", + "webpki-roots 0.20.0", ] [[package]] @@ -249,7 +249,7 @@ dependencies = [ "mime", "pin-project 1.0.4", "regex", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", @@ -393,7 +393,7 @@ dependencies = [ "mime", "percent-encoding", "rand", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", @@ -529,6 +529,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chunked_transfer" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" + [[package]] name = "const_fn" version = "0.4.5" @@ -986,6 +992,7 @@ dependencies = [ "serde", "serde_json", "time", + "ureq", "url", ] @@ -1413,6 +1420,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1749,7 +1769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls", + "rustls 0.18.1", "tokio", "webpki", ] @@ -1890,6 +1910,22 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96014ded8c85822677daee4f909d18acccca744810fd4f8ffc492c284f2324bc" +dependencies = [ + "base64 0.13.0", + "chunked_transfer", + "log", + "once_cell", + "rustls 0.19.0", + "url", + "webpki", + "webpki-roots 0.21.0", +] + [[package]] name = "url" version = "2.2.0" @@ -1997,6 +2033,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index e35d9e5..dce6390 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" base64 = "0.13.0" actix-web = { version = "3.3.2", features = ["rustls"] } askama = "0.10.5" +ureq = "2.0.1" serde = { version = "1.0.118", default_features = false, features = ["derive"] } serde_json = "1.0" async-recursion = "0.3.1" diff --git a/src/utils.rs b/src/utils.rs index c6e957b..7154224 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -363,85 +363,85 @@ pub async fn request(path: &str) -> Result { let url = format!("https://www.reddit.com{}", path); // Send request using awc - async fn send(url: &str) -> Result { - let client = actix_web::client::Client::default(); - let response = client.get(url).header("User-Agent", format!("web:libreddit:{}", env!("CARGO_PKG_VERSION"))).send().await; + // async fn send(url: &str) -> Result { + // let client = actix_web::client::Client::default(); + // let response = client.get(url).header("User-Agent", format!("web:libreddit:{}", env!("CARGO_PKG_VERSION"))).send().await; - match response { - Ok(mut payload) => { - // Get first number of response HTTP status code - match payload.status().to_string().chars().next() { - // If success - Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), - // If redirection - Some('3') => match payload.headers().get("location") { - Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), - None => Err((false, "Page not found".to_string())), - }, - // Otherwise - _ => Err((false, "Page not found".to_string())), - } - } - Err(e) => { dbg!(e); Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())) }, - } - } - - // Print error if debugging then return error based on error message - fn err(url: String, msg: String) -> Result { - // #[cfg(debug_assertions)] - dbg!(format!("{} - {}", url, msg)); - Err(msg) - }; - - // Parse JSON from body. If parsing fails, return error - fn json(url: String, body: String) -> Result { - match from_str(body.as_str()) { - Ok(json) => Ok(json), - Err(_) => err(url, "Failed to parse page JSON data".to_string()), - } - } - - // Make request to Reddit using send function - match send(&url).await { - // If success, parse and return body - Ok(body) => json(url, body), - // Follow any redirects - Err((true, location)) => match send(location.as_str()).await { - // If success, parse and return body - Ok(body) => json(url, body), - // Follow any redirects again - Err((true, location)) => err(url, location), - // Return errors if request fails - Err((_, msg)) => err(url, msg), - }, - // Return errors if request fails - Err((_, msg)) => err(url, msg), - } - - // Send request using ureq - // match ureq::get(&url).call() { - // // If response is success - // Ok(response) => { - // // Parse the response from Reddit as JSON - // match from_str(&response.into_string().unwrap()) { - // Ok(json) => Ok(json), - // Err(_) => { - // #[cfg(debug_assertions)] - // dbg!(format!("{} - Failed to parse page JSON data", url)); - // Err("Failed to parse page JSON data".to_string()) + // match response { + // Ok(mut payload) => { + // // Get first number of response HTTP status code + // match payload.status().to_string().chars().next() { + // // If success + // Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), + // // If redirection + // Some('3') => match payload.headers().get("location") { + // Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), + // None => Err((false, "Page not found".to_string())), + // }, + // // Otherwise + // _ => Err((false, "Page not found".to_string())), // } // } - // } - // // If response is error - // Err(ureq::Error::Status(_, _)) => { - // #[cfg(debug_assertions)] - // dbg!(format!("{} - Page not found", url)); - // Err("Page not found".to_string()) - // } - // // If failed to send request - // Err(e) => { - // dbg!(format!("{} - {}", url, e)); - // Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string()) + // Err(e) => { dbg!(e); Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())) }, // } // } + + // // Print error if debugging then return error based on error message + // fn err(url: String, msg: String) -> Result { + // // #[cfg(debug_assertions)] + // dbg!(format!("{} - {}", url, msg)); + // Err(msg) + // }; + + // // Parse JSON from body. If parsing fails, return error + // fn json(url: String, body: String) -> Result { + // match from_str(body.as_str()) { + // Ok(json) => Ok(json), + // Err(_) => err(url, "Failed to parse page JSON data".to_string()), + // } + // } + + // // Make request to Reddit using send function + // match send(&url).await { + // // If success, parse and return body + // Ok(body) => json(url, body), + // // Follow any redirects + // Err((true, location)) => match send(location.as_str()).await { + // // If success, parse and return body + // Ok(body) => json(url, body), + // // Follow any redirects again + // Err((true, location)) => err(url, location), + // // Return errors if request fails + // Err((_, msg)) => err(url, msg), + // }, + // // Return errors if request fails + // Err((_, msg)) => err(url, msg), + // } + + // Send request using ureq + match ureq::get(&url).call() { + // If response is success + Ok(response) => { + // Parse the response from Reddit as JSON + match from_str(&response.into_string().unwrap()) { + Ok(json) => Ok(json), + Err(_) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - Failed to parse page JSON data", url)); + Err("Failed to parse page JSON data".to_string()) + } + } + } + // If response is error + Err(ureq::Error::Status(_, _)) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - Page not found", url)); + Err("Page not found".to_string()) + } + // If failed to send request + Err(e) => { + dbg!(format!("{} - {}", url, e)); + Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string()) + } + } }