From 59021b9331ab74eaa9f0be5ad1245b3110c42652 Mon Sep 17 00:00:00 2001 From: spikecodes <19519553+spikecodes@users.noreply.github.com> Date: Fri, 15 Jan 2021 15:28:51 -0800 Subject: [PATCH] Switch back to ureq temporarily --- Cargo.lock | 89 +++++++++++++++++++++++++++--------- Cargo.toml | 1 + src/utils.rs | 124 +++++++++++++++++++++++++++++++-------------------- 3 files changed, 144 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d041577..38a5fe5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ "futures-util", "http", "log", - "rustls", + "rustls 0.18.1", "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", @@ -193,10 +193,10 @@ dependencies = [ "actix-service", "actix-utils", "futures-util", - "rustls", + "rustls 0.18.1", "tokio-rustls", "webpki", - "webpki-roots", + "webpki-roots 0.20.0", ] [[package]] @@ -249,7 +249,7 @@ dependencies = [ "mime", "pin-project 1.0.4", "regex", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", @@ -393,7 +393,7 @@ dependencies = [ "mime", "percent-encoding", "rand", - "rustls", + "rustls 0.18.1", "serde", "serde_json", "serde_urlencoded", @@ -529,6 +529,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chunked_transfer" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" + [[package]] name = "const_fn" version = "0.4.5" @@ -672,9 +678,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "futures" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309f13e3f4be6d5917178c84db67c0b9a09177ac16d4f9a7313a767a68adaa77" +checksum = "da9052a1a50244d8d5aa9bf55cbc2fb6f357c86cc52e46c62ed390a7180cf150" dependencies = [ "futures-channel", "futures-core", @@ -686,9 +692,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3b03bd32f6ec7885edeb99acd1e47e20e34fd4dfd3c6deed6fcac8a9d28f6a" +checksum = "f2d31b7ec7efab6eefc7c57233bb10b847986139d88cc2f5a02a1ae6871a1846" dependencies = [ "futures-core", "futures-sink", @@ -696,21 +702,21 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed8aeae2b6ab243ebabe6f54cd4cf53054d98883d5d326128af7d57a9ca5cd3d" +checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65" [[package]] name = "futures-io" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41234e71d5e8ca73d01563974ef6f50e516d71e18f1a2f1184742e31f5d469f" +checksum = "28be053525281ad8259d47e4de5de657b25e7bac113458555bb4b70bc6870500" [[package]] name = "futures-macro" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3520e0eb4e704e88d771b92d51273ee212997f0d8282f17f5d8ff1cb39104e42" +checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -720,24 +726,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c72d188479368953c6c8c7140e40d7a4401674ab3b98a41e60e515d6cbdbe5de" +checksum = "caf5c69029bda2e743fddd0582d1083951d65cc9539aebf8812f36c3491342d6" [[package]] name = "futures-task" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08944cea9021170d383287169859c0ca8147d9ec285978393109954448f33cc7" +checksum = "13de07eb8ea81ae445aca7b69f5f7bf15d7bf4912d8ca37d6645c77ae8a58d86" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd206efbe2ca683b2ce138ccdf61e1b0a63f5816dcedc9d8654c500ba0cea6" +checksum = "632a8cd0f2a4b3fdea1657f08bde063848c3bd00f9bbf6e256b8be78802e624b" dependencies = [ "futures-channel", "futures-core", @@ -986,6 +992,7 @@ dependencies = [ "serde", "serde_json", "time", + "ureq", "url", ] @@ -1413,6 +1420,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1749,7 +1769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls", + "rustls 0.18.1", "tokio", "webpki", ] @@ -1890,6 +1910,22 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96014ded8c85822677daee4f909d18acccca744810fd4f8ffc492c284f2324bc" +dependencies = [ + "base64 0.13.0", + "chunked_transfer", + "log", + "once_cell", + "rustls 0.19.0", + "url", + "webpki", + "webpki-roots 0.21.0", +] + [[package]] name = "url" version = "2.2.0" @@ -1997,6 +2033,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" +dependencies = [ + "webpki", +] + [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index e35d9e5..dce6390 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" base64 = "0.13.0" actix-web = { version = "3.3.2", features = ["rustls"] } askama = "0.10.5" +ureq = "2.0.1" serde = { version = "1.0.118", default_features = false, features = ["derive"] } serde_json = "1.0" async-recursion = "0.3.1" diff --git a/src/utils.rs b/src/utils.rs index b17cec1..421dd69 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -363,58 +363,86 @@ pub async fn request(path: &str) -> Result { let url = format!("https://www.reddit.com{}", path); // Send request using awc - async fn send(url: &str) -> Result { - let client = actix_web::client::Client::default(); - let response = client.get(url).send().await; + // async fn send(url: &str) -> Result { + // let client = actix_web::client::Client::default(); + // let response = client.get(url).send().await; - match response { - Ok(mut payload) => { - // Get first number of response HTTP status code - match payload.status().to_string().chars().next() { - // If success - Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), - // If redirection - Some('3') => match payload.headers().get("location") { - Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), - None => Err((false, "Page not found".to_string())), - }, - // Otherwise - _ => Err((false, "Page not found".to_string())), + // match response { + // Ok(mut payload) => { + // // Get first number of response HTTP status code + // match payload.status().to_string().chars().next() { + // // If success + // Some('2') => Ok(String::from_utf8(payload.body().limit(20_000_000).await.unwrap_or_default().to_vec()).unwrap_or_default()), + // // If redirection + // Some('3') => match payload.headers().get("location") { + // Some(location) => Err((true, location.to_str().unwrap_or_default().to_string())), + // None => Err((false, "Page not found".to_string())), + // }, + // // Otherwise + // _ => Err((false, "Page not found".to_string())), + // } + // } + // Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), + // } + // } + + // // Print error if debugging then return error based on error message + // fn err(url: String, msg: String) -> Result { + // // #[cfg(debug_assertions)] + // dbg!(format!("{} - {}", url, msg)); + // Err(msg) + // }; + + // // Parse JSON from body. If parsing fails, return error + // fn json(url: String, body: String) -> Result { + // match from_str(body.as_str()) { + // Ok(json) => Ok(json), + // Err(_) => err(url, "Failed to parse page JSON data".to_string()), + // } + // } + + // // Make request to Reddit using send function + // match send(&url).await { + // // If success, parse and return body + // Ok(body) => json(url, body), + // // Follow any redirects + // Err((true, location)) => match send(location.as_str()).await { + // // If success, parse and return body + // Ok(body) => json(url, body), + // // Follow any redirects again + // Err((true, location)) => err(url, location), + // // Return errors if request fails + // Err((_, msg)) => err(url, msg), + // }, + // // Return errors if request fails + // Err((_, msg)) => err(url, msg), + // } + + // Send request using ureq + match ureq::get(&url).call() { + // If response is success + Ok(response) => { + // Parse the response from Reddit as JSON + match from_str(&response.into_string().unwrap()) { + Ok(json) => Ok(json), + Err(_) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - Failed to parse page JSON data", url)); + Err("Failed to parse page JSON data".to_string()) } } - Err(_) => Err((false, "Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string())), } - } - - // Print error if debugging then return error based on error message - fn err(url: String, msg: String) -> Result { - // #[cfg(debug_assertions)] - dbg!(format!("{} - {}", url, msg)); - Err(msg) - }; - - // Parse JSON from body. If parsing fails, return error - fn json(url: String, body: String) -> Result { - match from_str(body.as_str()) { - Ok(json) => Ok(json), - Err(_) => err(url, "Failed to parse page JSON data".to_string()), + // If response is error + Err(ureq::Error::Status(_, _)) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - Page not found", url)); + Err("Page not found".to_string()) + } + // If failed to send request + Err(e) => { + #[cfg(debug_assertions)] + dbg!(format!("{} - {}", url, e)); + Err("Couldn't send request to Reddit, this instance may be being rate-limited. Try another.".to_string()) } - } - - // Make request to Reddit using send function - match send(&url).await { - // If success, parse and return body - Ok(body) => json(url, body), - // Follow any redirects - Err((true, location)) => match send(location.as_str()).await { - // If success, parse and return body - Ok(body) => json(url, body), - // Follow any redirects again - Err((true, location)) => err(url, location), - // Return errors if request fails - Err((_, msg)) => err(url, msg), - }, - // Return errors if request fails - Err((_, msg)) => err(url, msg), } }