use wreq::header::HeaderValue; use wreq::redirect::Policy; use wreq::Client; use wreq::Proxy; use wreq::Version; use wreq_util::Emulation; use std::env; use crate::util::flaresolverr::FlareSolverrRequest; use crate::util::flaresolverr::Flaresolverr; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub struct Requester { #[serde(skip)] client: Client, proxy: bool, flaresolverr_session: Option, } impl Requester { pub fn new() -> Self { let client = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) .cookie_store(true) .redirect(Policy::default()) .build() .expect("Failed to create HTTP client"); Requester { client, proxy: false, flaresolverr_session: None, } } pub fn set_proxy(&mut self, proxy: bool) { self.proxy = proxy; } pub fn set_flaresolverr_session(&mut self, session: String) { self.flaresolverr_session = Some(session); } fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String { if location.starts_with("http://") || location.starts_with("https://") { location.to_string() } else if location.starts_with("//") { format!("{}{}", "https:", location) // Replace with your base URL } else if location.starts_with("/") { let base_url = prev_url.split('/').take(3).collect::>().join("/"); format!("{}{}", base_url, location) } else { format!("{}/{}", prev_url, location) } } pub async fn get(&self, url: &str) -> Result> { let mut request = self.client.get(url).version(Version::HTTP_11); let mut proxy; if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy.clone()); } } let mut response = request.send().await?; if response.status().is_success() { return Ok(response.text().await?); } else { let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); let mut flare = Flaresolverr::new(flare_url); if self.proxy && env::var("BURP_URL").is_ok() { flare.set_proxy(true); } let result = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), url: url.to_string(), maxTimeout: 60000, }) .await; match result { Ok(res) => { let useragent = res.solution.userAgent; self.client.update() .headers(|headers| { headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap()); }) .apply() .unwrap(); for cookie in res.solution.cookies { let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap(); // Parse the domain string into a Url let cookie_url = url.split("/").collect::>()[..3].join("/"); if let Ok(url) = url::Url::parse(cookie_url.as_str()) { self.client.set_cookie(&url, header); } } request = self.client.get(url).version(Version::HTTP_11); if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy.clone()); } } response = request.send().await?; if response.status().is_success() { return Ok(response.text().await?); } Ok(res.solution.response) } Err(e) => { return Err(format!("Failed to solve FlareSolverr: {e}").into()); } } } } }