309 lines
9.4 KiB
Rust
309 lines
9.4 KiB
Rust
use serde::Serialize;
|
|
use wreq::multipart::Form;
|
|
use std::env;
|
|
use wreq::Client;
|
|
use wreq::Proxy;
|
|
use wreq::Response;
|
|
use wreq::Version;
|
|
use wreq::header::HeaderValue;
|
|
use wreq::redirect::Policy;
|
|
use wreq_util::Emulation;
|
|
use rand::seq::SliceRandom;
|
|
|
|
use crate::util::proxy;
|
|
use crate::util::flaresolverr::FlareSolverrRequest;
|
|
use crate::util::flaresolverr::Flaresolverr;
|
|
|
|
// A Send + Sync error type for all async paths
|
|
type AnyErr = Box<dyn std::error::Error + Send + Sync + 'static>;
|
|
|
|
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
|
|
pub struct Requester {
|
|
#[serde(skip)]
|
|
client: Client,
|
|
proxy: bool,
|
|
flaresolverr_session: Option<String>,
|
|
use_random_proxy: bool,
|
|
}
|
|
|
|
impl Requester {
|
|
pub fn new() -> Self {
|
|
let client = Client::builder()
|
|
.cert_verification(false)
|
|
.emulation(Emulation::Firefox136)
|
|
.cookie_store(true)
|
|
.redirect(Policy::default())
|
|
.build()
|
|
.expect("Failed to create HTTP client");
|
|
|
|
let requester = Requester {
|
|
client,
|
|
proxy: false,
|
|
flaresolverr_session: None,
|
|
use_random_proxy: false,
|
|
};
|
|
|
|
proxy::init_all_proxies_background(requester.clone());
|
|
|
|
requester
|
|
}
|
|
|
|
pub fn set_proxy(&mut self, proxy: bool) {
|
|
if proxy {
|
|
println!("Proxy enabled");
|
|
}
|
|
self.proxy = proxy;
|
|
}
|
|
|
|
pub fn set_random_proxy(&mut self, random: bool) {
|
|
self.use_random_proxy = random;
|
|
}
|
|
|
|
pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
|
|
let client = Client::builder()
|
|
.cert_verification(false)
|
|
.emulation(Emulation::Firefox136)
|
|
.cookie_store(true)
|
|
.build()
|
|
.expect("Failed to create HTTP client");
|
|
|
|
let mut request = client.get(url).version(Version::HTTP_11);
|
|
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
let proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy);
|
|
}
|
|
}
|
|
|
|
request.send().await
|
|
}
|
|
|
|
pub async fn get_raw_with_proxy(
|
|
&mut self,
|
|
url: &str,
|
|
proxy: Proxy,
|
|
) -> Result<Response, wreq::Error> {
|
|
let client = Client::builder()
|
|
.cert_verification(false)
|
|
.emulation(Emulation::Firefox136)
|
|
.cookie_store(true)
|
|
.build()
|
|
.expect("Failed to create HTTP client");
|
|
|
|
client
|
|
.get(url)
|
|
.version(Version::HTTP_11)
|
|
.proxy(proxy)
|
|
.send()
|
|
.await
|
|
}
|
|
|
|
pub async fn get_raw_with_headers(
|
|
&mut self,
|
|
url: &str,
|
|
headers: Vec<(String, String)>,
|
|
) -> Result<Response, wreq::Error> {
|
|
let client = Client::builder()
|
|
.cert_verification(false)
|
|
.emulation(Emulation::Firefox136)
|
|
.cookie_store(true)
|
|
.build()
|
|
.expect("Failed to create HTTP client");
|
|
|
|
let mut request = client.get(url).version(Version::HTTP_11);
|
|
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
let proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy);
|
|
}
|
|
}
|
|
// Set custom headers
|
|
for (key, value) in headers.iter() {
|
|
request = request.header(key, value);
|
|
}
|
|
request.send().await
|
|
}
|
|
|
|
pub async fn post_json<S>(
|
|
&mut self,
|
|
url: &str,
|
|
data: &S,
|
|
headers: Vec<(String, String)>,
|
|
) -> Result<Response, wreq::Error>
|
|
where
|
|
S: Serialize + ?Sized,
|
|
{
|
|
let mut request = self.client.post(url).version(Version::HTTP_11).json(data);
|
|
|
|
// Set custom headers
|
|
for (key, value) in headers.iter() {
|
|
request = request.header(key, value);
|
|
}
|
|
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
let proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy);
|
|
}
|
|
}
|
|
|
|
request.send().await
|
|
}
|
|
|
|
pub async fn post(
|
|
&mut self,
|
|
url: &str,
|
|
data: &str,
|
|
headers: Vec<(&str, &str)>,
|
|
) -> Result<Response, wreq::Error> {
|
|
let mut request = self.client.post(url).version(Version::HTTP_11).body(data.to_string());
|
|
|
|
// Set custom headers
|
|
for (key, value) in headers.iter() {
|
|
request = request.header(key.to_string(), value.to_string());
|
|
}
|
|
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
let proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy);
|
|
}
|
|
}
|
|
|
|
request.send().await
|
|
}
|
|
|
|
pub async fn post_multipart(
|
|
&mut self,
|
|
url: &str,
|
|
form: Form,
|
|
headers: Vec<(String, String)>,
|
|
_http_version: Option<Version>,
|
|
) -> Result<Response, wreq::Error>
|
|
{
|
|
let http_version = match _http_version {
|
|
Some(v) => v,
|
|
None => Version::HTTP_11,
|
|
};
|
|
|
|
let mut request = self.client.post(url).multipart(form).version(http_version);
|
|
|
|
// Set custom headers
|
|
for (key, value) in headers.iter() {
|
|
request = request.header(key, value);
|
|
}
|
|
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
let proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy);
|
|
}
|
|
}
|
|
|
|
request.send().await
|
|
}
|
|
|
|
pub async fn get(&mut self, url: &str, _http_version: Option<Version>) -> Result<String, AnyErr> {
|
|
let http_version = match _http_version {
|
|
Some(v) => v,
|
|
None => Version::HTTP_11,
|
|
};
|
|
loop {
|
|
let mut request = self.client.get(url).version(http_version);
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
let proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy);
|
|
}
|
|
}
|
|
// else if self.use_random_proxy {
|
|
// let proxies = proxy::all_proxies_snapshot().await.unwrap_or_default();
|
|
// if !proxies.is_empty() {
|
|
// let mut random_proxy = proxies.choose_mut(&mut rand::thread_rng()).unwrap().clone();
|
|
// request = request.proxy(random_proxy);
|
|
// }
|
|
// }
|
|
let response = request.send().await?;
|
|
if response.status().is_success() || response.status().as_u16() == 404 {
|
|
return Ok(response.text().await?);
|
|
}
|
|
if response.status().as_u16() == 429 {
|
|
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
|
continue;
|
|
} else {
|
|
println!(
|
|
"Direct request to {} failed with status: {}",
|
|
url,
|
|
response.status()
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
// If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately,
|
|
// so no non-Send error value lives across later `.await`s.
|
|
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
|
|
let mut flare = Flaresolverr::new(flare_url);
|
|
if self.proxy && env::var("BURP_URL").is_ok() {
|
|
flare.set_proxy(true);
|
|
}
|
|
|
|
let res = flare
|
|
.solve(FlareSolverrRequest {
|
|
cmd: "request.get".to_string(),
|
|
url: url.to_string(),
|
|
maxTimeout: 60000,
|
|
})
|
|
.await
|
|
.map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?;
|
|
|
|
// Rebuild client and apply UA/cookies from FlareSolverr
|
|
let cookie_origin = url.split('/').take(3).collect::<Vec<&str>>().join("/");
|
|
|
|
self.client = Client::builder()
|
|
.cert_verification(false)
|
|
.emulation(Emulation::Firefox136)
|
|
.cookie_store(true)
|
|
.redirect(Policy::default())
|
|
.build()
|
|
.expect("Failed to create HTTP client");
|
|
|
|
let useragent = res.solution.userAgent;
|
|
self.client
|
|
.update()
|
|
.headers(|headers| {
|
|
headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
|
|
})
|
|
.apply()
|
|
.unwrap();
|
|
|
|
if let Ok(origin) = url::Url::parse(&cookie_origin) {
|
|
for cookie in res.solution.cookies {
|
|
let header =
|
|
HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
|
|
self.client.set_cookie(&origin, header);
|
|
}
|
|
}
|
|
|
|
// Retry the original URL with the updated client & (optional) proxy
|
|
let mut request = self.client.get(url).version(Version::HTTP_11);
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
let proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy);
|
|
}
|
|
}
|
|
|
|
let response = request.send().await?;
|
|
if response.status().is_success() {
|
|
return Ok(response.text().await?);
|
|
}
|
|
|
|
// Fall back to FlareSolverr-provided body
|
|
Ok(res.solution.response)
|
|
}
|
|
}
|