fixes for perfectgirls and missav

src/util/flaresolverr.rs

@@ -1,7 +1,7 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, env};
 
 use serde_json::json;
-use wreq::Client;
+use wreq::{Client, Proxy};
 use wreq_util::Emulation;
 
 #[derive(serde::Serialize, serde::Deserialize, Debug)]
@@ -13,31 +13,31 @@ pub struct FlareSolverrRequest {
 
 #[derive(serde::Serialize, serde::Deserialize, Debug)]
 pub struct FlaresolverrCookie {
-    name: String,         //"cf_clearance",
-    value: String, //"lnKoXclrIp_mDrWJFfktPGm8GDyxjSpzy9dx0qDTiRg-1748689259-1.2.1.1-AIFERAPCdCSvvdu1mposNdUpKV9wHZXBpSI2L9k9TaKkPcqmomON_XEb6ZtRBtrmQu_DC8AzKllRg2vNzVKOUsvv9ndjQ.vv8Z7cNkgzpIbGFy96kXyAYH2mUk3Q7enZovDlEbK5kpV3Sbmd2M3_bUCBE1WjAMMdXlyNElH1LOpUm149O9hrluXjAffo4SwHI4HO0UckBPWBlBqhznKPgXxU0g8VHLDeYnQKViY8rP2ud4tyzKnJUxuYXzr4aWBNMp6TESp49vesRiel_Y5m.rlTY4zSb517S9iPbEQiYHRI.uH5mMHVI3jvJl0Mx94tPrpFnkhDdmzL3DRSllJe9k786Lf21I9WBoH2cCR3yHw",
-    domain: String, //".discord.com",
-    path: String,  //"/",
-    expires: f64,  //1780225259.237105,
-    size: u64,     //438,
-    httpOnly: bool, //true,
-    secure: bool,  //true,
-    session: bool, //false,
-    sameSite: Option<String>, //"None",
-    priority: String, //"Medium",
-    sameParty: bool, //false,
-    sourceScheme: String, //"Secure",
-    sourcePort: u32, //443,
-    partitionKey: Option<String>, //"https://perverzija.com"
+    pub name: String,         //"cf_clearance",
+    pub value: String, //"lnKoXclrIp_mDrWJFfktPGm8GDyxjSpzy9dx0qDTiRg-1748689259-1.2.1.1-AIFERAPCdCSvvdu1mposNdUpKV9wHZXBpSI2L9k9TaKkPcqmomON_XEb6ZtRBtrmQu_DC8AzKllRg2vNzVKOUsvv9ndjQ.vv8Z7cNkgzpIbGFy96kXyAYH2mUk3Q7enZovDlEbK5kpV3Sbmd2M3_bUCBE1WjAMMdXlyNElH1LOpUm149O9hrluXjAffo4SwHI4HO0UckBPWBlBqhznKPgXxU0g8VHLDeYnQKViY8rP2ud4tyzKnJUxuYXzr4aWBNMp6TESp49vesRiel_Y5m.rlTY4zSb517S9iPbEQiYHRI.uH5mMHVI3jvJl0Mx94tPrpFnkhDdmzL3DRSllJe9k786Lf21I9WBoH2cCR3yHw",
+    pub domain: String, //".discord.com",
+    pub path: String,  //"/",
+    pub expires: f64,  //1780225259.237105,
+    pub size: u64,     //438,
+    pub httpOnly: bool, //true,
+    pub secure: bool,  //true,
+    pub session: bool, //false,
+    pub sameSite: Option<String>, //"None",
+    pub priority: String, //"Medium",
+    pub sameParty: bool, //false,
+    pub sourceScheme: String, //"Secure",
+    pub sourcePort: u32, //443,
+    pub partitionKey: Option<String>, 
 }
 
 #[derive(serde::Serialize, serde::Deserialize, Debug)]  
 pub struct FlareSolverrSolution {
-    url: String, 
-    status: u32,
+    pub url: String, 
+    pub status: u32,
     pub response: String, 
-    headers: HashMap<String, String>,
-    cookies: Vec<FlaresolverrCookie>, 
-    userAgent: String,
+    pub headers: HashMap<String, String>,
+    pub cookies: Vec<FlaresolverrCookie>, 
+    pub userAgent: String,
 }
 // impl FlareSolverrSolution {
 //     fn to_client(&self,){
@@ -65,15 +65,21 @@ pub struct FlareSolverrResponse {
     version: String,
 }
 pub struct Flaresolverr {
-    url: String
+    url: String,
+    proxy: bool,
 }
 impl Flaresolverr {
     pub fn new(url: String) -> Self {
         Flaresolverr { 
-            url: url
+            url: url,
+            proxy: false,
         }
     }
 
+    pub fn set_proxy(&mut self, proxy: bool) {
+        self.proxy = proxy;
+    }
+
     pub async fn solve(
         &self,
         request: FlareSolverrRequest,
@@ -82,15 +88,23 @@ impl Flaresolverr {
         .emulation(Emulation::Firefox136)
         .build()?;
 
-    let response = client
+        let mut request = client
         .post(&self.url)
         .header("Content-Type", "application/json")
         .json(&json!({
             "cmd": request.cmd,
             "url": request.url,
             "maxTimeout": request.maxTimeout,
-        }))
-        .send().await?;
+        }));
+
+        if self.proxy {
+            if let Ok(proxy_url) = env::var("BURP_URL") {
+                let proxy = Proxy::all(&proxy_url).unwrap();
+                request = request.proxy(proxy.clone());
+            }
+        }
+
+    let response = request.send().await?;
 
     let body: FlareSolverrResponse = response.json::<FlareSolverrResponse>().await?;
     Ok(body)

src/util/mod.rs

@@ -1,6 +1,9 @@
+use crate::providers::AnyProvider;
+
 pub mod time;
 pub mod flaresolverr;
 pub mod cache;
+pub mod requester;
 
 pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
     let s = s.trim();
@@ -40,4 +43,11 @@ pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {
     }
 
     result
+}
+
+pub fn get_all_providers() -> Vec<AnyProvider>{
+
+    
+
+    return vec![];
 }

src/util/requester.rs

@@ -0,0 +1,123 @@
+use wreq::header::HeaderValue;
+use wreq::redirect::Policy;
+use wreq::Client;
+use wreq::Proxy;
+use wreq::Version;
+use wreq_util::Emulation;
+use std::env;
+
+use crate::util::flaresolverr::FlareSolverrRequest;
+use crate::util::flaresolverr::Flaresolverr;
+
+#[derive(Debug, Clone)]
+pub struct Requester {
+    client: Client,
+    proxy: bool,
+    flaresolverr_session: Option<String>,
+}
+
+impl Requester {
+    pub fn new() -> Self {
+        let client = Client::builder()
+        .cert_verification(false)
+        .emulation(Emulation::Firefox136)
+        .cookie_store(true)
+        .redirect(Policy::default())
+        .build()
+            .expect("Failed to create HTTP client");
+
+        Requester {
+            client,
+            proxy: false,
+            flaresolverr_session: None,
+        }
+    }
+    pub fn set_proxy(&mut self, proxy: bool) {
+        self.proxy = proxy;
+    }
+
+    pub fn set_flaresolverr_session(&mut self, session: String) {
+        self.flaresolverr_session = Some(session);
+    }
+
+    fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String {
+        if location.starts_with("http://") || location.starts_with("https://") {
+            location.to_string()
+        } else if location.starts_with("//") {
+            format!("{}{}", "https:", location) // Replace with your base URL
+        } else if location.starts_with("/") {
+            let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/");
+            format!("{}{}", base_url, location)
+        } else {
+            format!("{}/{}", prev_url, location)
+        }
+    }
+
+    pub async fn get(&self, url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+        let mut request = self.client.get(url).version(Version::HTTP_11);
+        let mut proxy;
+        if self.proxy {
+            if let Ok(proxy_url) = env::var("BURP_URL") {
+                proxy = Proxy::all(&proxy_url).unwrap();
+                request = request.proxy(proxy.clone());
+            }
+        }
+
+        let mut response = request.send().await?;
+    
+        if response.status().is_success() {
+            return Ok(response.text().await?);
+        } else {
+            let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
+            let mut flare = Flaresolverr::new(flare_url);
+            if self.proxy && env::var("BURP_URL").is_ok() {
+                flare.set_proxy(true);
+            }
+            let result = flare
+                .solve(FlareSolverrRequest {
+                    cmd: "request.get".to_string(),
+                    url: url.to_string(),
+                    maxTimeout: 60000,
+                })
+                .await;
+            match result {
+                Ok(res) => {
+                    let useragent = res.solution.userAgent;
+                    self.client.update()
+                        .headers(|headers| {
+                            headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
+                        })
+                        .apply()
+                        .unwrap();
+                    for cookie in res.solution.cookies {
+                        let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
+                        // Parse the domain string into a Url
+                        let cookie_url = url.split("/").collect::<Vec<&str>>()[..3].join("/");
+                        println!("{}", cookie_url);
+                        if let Ok(url) = url::Url::parse(cookie_url.as_str()) {
+                            println!("{:?}, {:?}", cookie_url, header);
+                            self.client.set_cookie(&url, header);
+                        }
+                    }
+                    request = self.client.get(url).version(Version::HTTP_11);
+                    if self.proxy {
+                        if let Ok(proxy_url) = env::var("BURP_URL") {
+                            proxy = Proxy::all(&proxy_url).unwrap();
+                            request = request.proxy(proxy.clone());
+                        }
+                    }
+
+                    response = request.send().await?;
+                    if response.status().is_success() {
+                        return Ok(response.text().await?);
+                    }
+
+                    Ok(res.solution.response)
+                }
+                Err(e) => {
+                    return Err(format!("Failed to solve FlareSolverr: {e}").into());
+                }
+            }
+        }
+    }
+}