fixes for perfectgirls and missav

src/util/requester.rs

@@ -0,0 +1,123 @@
+use wreq::header::HeaderValue;
+use wreq::redirect::Policy;
+use wreq::Client;
+use wreq::Proxy;
+use wreq::Version;
+use wreq_util::Emulation;
+use std::env;
+
+use crate::util::flaresolverr::FlareSolverrRequest;
+use crate::util::flaresolverr::Flaresolverr;
+
+#[derive(Debug, Clone)]
+pub struct Requester {
+    client: Client,
+    proxy: bool,
+    flaresolverr_session: Option<String>,
+}
+
+impl Requester {
+    pub fn new() -> Self {
+        let client = Client::builder()
+        .cert_verification(false)
+        .emulation(Emulation::Firefox136)
+        .cookie_store(true)
+        .redirect(Policy::default())
+        .build()
+            .expect("Failed to create HTTP client");
+
+        Requester {
+            client,
+            proxy: false,
+            flaresolverr_session: None,
+        }
+    }
+    pub fn set_proxy(&mut self, proxy: bool) {
+        self.proxy = proxy;
+    }
+
+    pub fn set_flaresolverr_session(&mut self, session: String) {
+        self.flaresolverr_session = Some(session);
+    }
+
+    fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String {
+        if location.starts_with("http://") || location.starts_with("https://") {
+            location.to_string()
+        } else if location.starts_with("//") {
+            format!("{}{}", "https:", location) // Replace with your base URL
+        } else if location.starts_with("/") {
+            let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/");
+            format!("{}{}", base_url, location)
+        } else {
+            format!("{}/{}", prev_url, location)
+        }
+    }
+
+    pub async fn get(&self, url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+        let mut request = self.client.get(url).version(Version::HTTP_11);
+        let mut proxy;
+        if self.proxy {
+            if let Ok(proxy_url) = env::var("BURP_URL") {
+                proxy = Proxy::all(&proxy_url).unwrap();
+                request = request.proxy(proxy.clone());
+            }
+        }
+
+        let mut response = request.send().await?;
+    
+        if response.status().is_success() {
+            return Ok(response.text().await?);
+        } else {
+            let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
+            let mut flare = Flaresolverr::new(flare_url);
+            if self.proxy && env::var("BURP_URL").is_ok() {
+                flare.set_proxy(true);
+            }
+            let result = flare
+                .solve(FlareSolverrRequest {
+                    cmd: "request.get".to_string(),
+                    url: url.to_string(),
+                    maxTimeout: 60000,
+                })
+                .await;
+            match result {
+                Ok(res) => {
+                    let useragent = res.solution.userAgent;
+                    self.client.update()
+                        .headers(|headers| {
+                            headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
+                        })
+                        .apply()
+                        .unwrap();
+                    for cookie in res.solution.cookies {
+                        let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
+                        // Parse the domain string into a Url
+                        let cookie_url = url.split("/").collect::<Vec<&str>>()[..3].join("/");
+                        println!("{}", cookie_url);
+                        if let Ok(url) = url::Url::parse(cookie_url.as_str()) {
+                            println!("{:?}, {:?}", cookie_url, header);
+                            self.client.set_cookie(&url, header);
+                        }
+                    }
+                    request = self.client.get(url).version(Version::HTTP_11);
+                    if self.proxy {
+                        if let Ok(proxy_url) = env::var("BURP_URL") {
+                            proxy = Proxy::all(&proxy_url).unwrap();
+                            request = request.proxy(proxy.clone());
+                        }
+                    }
+
+                    response = request.send().await?;
+                    if response.status().is_success() {
+                        return Ok(response.text().await?);
+                    }
+
+                    Ok(res.solution.response)
+                }
+                Err(e) => {
+                    return Err(format!("Failed to solve FlareSolverr: {e}").into());
+                }
+            }
+        }
+    }
+}