122 lines
4.3 KiB
Rust
122 lines
4.3 KiB
Rust
use wreq::header::HeaderValue;
|
|
use wreq::redirect::Policy;
|
|
use wreq::Client;
|
|
use wreq::Proxy;
|
|
use wreq::Version;
|
|
use wreq_util::Emulation;
|
|
use std::env;
|
|
|
|
use crate::util::flaresolverr::FlareSolverrRequest;
|
|
use crate::util::flaresolverr::Flaresolverr;
|
|
|
|
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
|
|
pub struct Requester {
|
|
#[serde(skip)]
|
|
client: Client,
|
|
proxy: bool,
|
|
flaresolverr_session: Option<String>,
|
|
}
|
|
|
|
impl Requester {
|
|
pub fn new() -> Self {
|
|
let client = Client::builder()
|
|
.cert_verification(false)
|
|
.emulation(Emulation::Firefox136)
|
|
.cookie_store(true)
|
|
.redirect(Policy::default())
|
|
.build()
|
|
.expect("Failed to create HTTP client");
|
|
|
|
Requester {
|
|
client,
|
|
proxy: false,
|
|
flaresolverr_session: None,
|
|
}
|
|
}
|
|
pub fn set_proxy(&mut self, proxy: bool) {
|
|
self.proxy = proxy;
|
|
}
|
|
|
|
pub fn set_flaresolverr_session(&mut self, session: String) {
|
|
self.flaresolverr_session = Some(session);
|
|
}
|
|
|
|
fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String {
|
|
if location.starts_with("http://") || location.starts_with("https://") {
|
|
location.to_string()
|
|
} else if location.starts_with("//") {
|
|
format!("{}{}", "https:", location) // Replace with your base URL
|
|
} else if location.starts_with("/") {
|
|
let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/");
|
|
format!("{}{}", base_url, location)
|
|
} else {
|
|
format!("{}/{}", prev_url, location)
|
|
}
|
|
}
|
|
|
|
pub async fn get(&self, url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
|
let mut request = self.client.get(url).version(Version::HTTP_11);
|
|
let mut proxy;
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy.clone());
|
|
}
|
|
}
|
|
|
|
let mut response = request.send().await?;
|
|
|
|
if response.status().is_success() {
|
|
return Ok(response.text().await?);
|
|
} else {
|
|
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
|
|
let mut flare = Flaresolverr::new(flare_url);
|
|
if self.proxy && env::var("BURP_URL").is_ok() {
|
|
flare.set_proxy(true);
|
|
}
|
|
let result = flare
|
|
.solve(FlareSolverrRequest {
|
|
cmd: "request.get".to_string(),
|
|
url: url.to_string(),
|
|
maxTimeout: 60000,
|
|
})
|
|
.await;
|
|
match result {
|
|
Ok(res) => {
|
|
let useragent = res.solution.userAgent;
|
|
self.client.update()
|
|
.headers(|headers| {
|
|
headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
|
|
})
|
|
.apply()
|
|
.unwrap();
|
|
for cookie in res.solution.cookies {
|
|
let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
|
|
// Parse the domain string into a Url
|
|
let cookie_url = url.split("/").collect::<Vec<&str>>()[..3].join("/");
|
|
if let Ok(url) = url::Url::parse(cookie_url.as_str()) {
|
|
self.client.set_cookie(&url, header);
|
|
}
|
|
}
|
|
request = self.client.get(url).version(Version::HTTP_11);
|
|
if self.proxy {
|
|
if let Ok(proxy_url) = env::var("BURP_URL") {
|
|
proxy = Proxy::all(&proxy_url).unwrap();
|
|
request = request.proxy(proxy.clone());
|
|
}
|
|
}
|
|
|
|
response = request.send().await?;
|
|
if response.status().is_success() {
|
|
return Ok(response.text().await?);
|
|
}
|
|
|
|
Ok(res.solution.response)
|
|
}
|
|
Err(e) => {
|
|
return Err(format!("Failed to solve FlareSolverr: {e}").into());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |