fixes for perfectgirls and missav

Cargo.toml

@@ -11,12 +11,13 @@ env_logger = "0.11.8"
 error-chain = "0.12.4"
 futures = "0.3.31"
 htmlentity = "1.3.2"
-ntex = { version = "2.0", features = ["tokio"] }
+ntex = { version = "2.15.1", features = ["tokio"] }
 ntex-files = "2.0.0"
 serde = "1.0.219"
-serde_json = "1.0.140"
+serde_json = "1.0.143"
-tokio = { version = "1", features = ["full"] }
+tokio = { version = "1.47.1", features = ["full"] }
-wreq =  { version = "5", features = ["full"] }
+wreq =  { version = "5.3.0", features = ["full", "cookies"] }
 wreq-util = "2"
 percent-encoding = "2.1"
 capitalize = "0.3.4"
+url = "2.5.4"

src/main.rs

@@ -7,10 +7,6 @@ use dotenvy::dotenv;
 use ntex_files as fs;
 use ntex::web;
 
-
-#[macro_use]
-extern crate cute;
-
 mod api;
 mod db;
 mod models;

src/providers/missav.rs

@@ -13,6 +13,7 @@ use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
 use crate::videos::ServerOptions;
 use crate::videos::{VideoItem};
 use crate::DbPool;
+use crate::util::requester::Requester;
 
 
 error_chain! {
@@ -26,11 +27,15 @@ error_chain! {
 #[derive(Debug, Clone)]
 pub struct MissavProvider {
     url: String,
+    requester: Requester,
 }
 impl MissavProvider {
     pub fn new() -> Self {
+        let mut requester   = Requester::new();
+        requester.set_proxy(true);
         MissavProvider {
             url: "https://missav.ws".to_string(),
+            requester
         }
     }
     async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
@@ -52,20 +57,7 @@ impl MissavProvider {
         };
 
         
-        let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
+        let text = self.requester.get(&url_str).await.unwrap();
-
-        let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
-        
-        let mut response = client.get(url_str.clone())
-        // .proxy(proxy.clone())
-        .send().await?;
-        if response.status().is_redirection(){
-            response = client.get(response.headers()["Location"].to_str().unwrap())
-            // .proxy(proxy.clone())
-            .send().await?;
-        }
-        if response.status().is_success() {
-            let text = response.text().await?;
         let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool).await;
         if !video_items.is_empty() {
             cache.remove(&url_str);
@@ -74,35 +66,8 @@ impl MissavProvider {
             return Ok(old_items);
         }
         Ok(video_items)
-        } else {
-            let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
-            let flare = Flaresolverr::new(flare_url);
-            let result = flare
-                .solve(FlareSolverrRequest {
-                    cmd: "request.get".to_string(),
-                    url: url_str.clone(),
-                    maxTimeout: 60000,
-                })
-                .await;
-            let video_items = match result {
-                Ok(res) => {
-                    // println!("FlareSolverr response: {}", res);
-                    self.get_video_items_from_html(res.solution.response, pool).await
-                }
-                Err(e) => {
-                    println!("Error solving FlareSolverr: {}", e);
-                    return Err("Failed to solve FlareSolverr".into());
-                }
-            };
-            if !video_items.is_empty() {
-                cache.remove(&url_str);
-                cache.insert(url_str.clone(), video_items.clone());
-            } else {
-                return Ok(old_items);
-            }
-            Ok(video_items)
-        }
     }
+
     async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str, sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
         let search_string = query.replace(" ", "%20");
         let url_str = format!(
@@ -124,20 +89,7 @@ impl MissavProvider {
                 vec![]
             }
         };    
-        let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
+        let text = self.requester.get(&url_str).await.unwrap();
-
-        let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
-        
-        let mut response = client.get(url_str.clone())
-        // .proxy(proxy.clone())
-        .send().await?;
-        if response.status().is_redirection(){
-            response = client.get(response.headers()["Location"].to_str().unwrap())
-            // .proxy(proxy.clone())
-            .send().await?;
-        }
-        if response.status().is_success() {
-            let text = response.text().await?;
         let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool).await;
         if !video_items.is_empty() {
             cache.remove(&url_str);
@@ -146,33 +98,6 @@ impl MissavProvider {
             return Ok(old_items);
         }
         Ok(video_items)
-        } else {
-            let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
-            let flare = Flaresolverr::new(flare_url);
-            let result = flare
-                .solve(FlareSolverrRequest {
-                    cmd: "request.get".to_string(),
-                    url: url_str.clone(),
-                    maxTimeout: 60000,
-                })
-                .await;
-            let video_items = match result {
-                Ok(res) => {
-                    self.get_video_items_from_html(res.solution.response, pool).await
-                }
-                Err(e) => {
-                    println!("Error solving FlareSolverr: {}", e);
-                    return Err("Failed to solve FlareSolverr".into());
-                }
-            };
-            if !video_items.is_empty() {
-                cache.remove(&url_str);
-                cache.insert(url_str.clone(), video_items.clone());
-            } else{
-                return Ok(old_items);
-            }
-            Ok(video_items)
-        }
     }
 
     async fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
@@ -223,26 +148,7 @@ impl MissavProvider {
             }
         }
         drop(conn);
-        
+        let vid = self.requester.get(&url_str).await.unwrap();
-
-        let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
-
-        let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
-        
-        let mut response = client.get(url_str.clone())
-        // .proxy(proxy.clone())
-        .send().await?;
-        if response.status().is_redirection(){
-            response = client.get(response.headers()["Location"].to_str().unwrap())
-            // .proxy(proxy.clone())
-            .send().await?;
-        }
-        let vid;
-        if response.status().is_success() {
-            vid = response.text().await?;
-        } else {
-            return Err("Failed to fetch video details".into());
-        }
         let mut title = vid.split("<meta property=\"og:title\" content=\"").collect::<Vec<&str>>()[1]
             .split("\"")
             .collect::<Vec<&str>>()[0].trim()

src/providers/perfectgirls.rs

@@ -221,11 +221,14 @@ impl PerfectgirlsProvider {
                 .to_string();
             let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
 
-            let thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>()[1]
+            let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>()[1]
                 .split("data-original=\"").collect::<Vec<&str>>()[1]
                 .split("\"")
                 .collect::<Vec<&str>>()[0]
                 .to_string();
+            if thumb.starts_with("//"){
+                thumb = format!("https:{}",thumb);
+            }
         
             let mut tags = vec![];
             if video_segment.contains("href=\"/channels/"){

src/util/flaresolverr.rs

@@ -1,7 +1,7 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, env};
 
 use serde_json::json;
-use wreq::Client;
+use wreq::{Client, Proxy};
 use wreq_util::Emulation;
 
 #[derive(serde::Serialize, serde::Deserialize, Debug)]
@@ -13,31 +13,31 @@ pub struct FlareSolverrRequest {
 
 #[derive(serde::Serialize, serde::Deserialize, Debug)]
 pub struct FlaresolverrCookie {
-    name: String,         //"cf_clearance",
+    pub name: String,         //"cf_clearance",
-    value: String, //"lnKoXclrIp_mDrWJFfktPGm8GDyxjSpzy9dx0qDTiRg-1748689259-1.2.1.1-AIFERAPCdCSvvdu1mposNdUpKV9wHZXBpSI2L9k9TaKkPcqmomON_XEb6ZtRBtrmQu_DC8AzKllRg2vNzVKOUsvv9ndjQ.vv8Z7cNkgzpIbGFy96kXyAYH2mUk3Q7enZovDlEbK5kpV3Sbmd2M3_bUCBE1WjAMMdXlyNElH1LOpUm149O9hrluXjAffo4SwHI4HO0UckBPWBlBqhznKPgXxU0g8VHLDeYnQKViY8rP2ud4tyzKnJUxuYXzr4aWBNMp6TESp49vesRiel_Y5m.rlTY4zSb517S9iPbEQiYHRI.uH5mMHVI3jvJl0Mx94tPrpFnkhDdmzL3DRSllJe9k786Lf21I9WBoH2cCR3yHw",
+    pub value: String, //"lnKoXclrIp_mDrWJFfktPGm8GDyxjSpzy9dx0qDTiRg-1748689259-1.2.1.1-AIFERAPCdCSvvdu1mposNdUpKV9wHZXBpSI2L9k9TaKkPcqmomON_XEb6ZtRBtrmQu_DC8AzKllRg2vNzVKOUsvv9ndjQ.vv8Z7cNkgzpIbGFy96kXyAYH2mUk3Q7enZovDlEbK5kpV3Sbmd2M3_bUCBE1WjAMMdXlyNElH1LOpUm149O9hrluXjAffo4SwHI4HO0UckBPWBlBqhznKPgXxU0g8VHLDeYnQKViY8rP2ud4tyzKnJUxuYXzr4aWBNMp6TESp49vesRiel_Y5m.rlTY4zSb517S9iPbEQiYHRI.uH5mMHVI3jvJl0Mx94tPrpFnkhDdmzL3DRSllJe9k786Lf21I9WBoH2cCR3yHw",
-    domain: String, //".discord.com",
+    pub domain: String, //".discord.com",
-    path: String,  //"/",
+    pub path: String,  //"/",
-    expires: f64,  //1780225259.237105,
+    pub expires: f64,  //1780225259.237105,
-    size: u64,     //438,
+    pub size: u64,     //438,
-    httpOnly: bool, //true,
+    pub httpOnly: bool, //true,
-    secure: bool,  //true,
+    pub secure: bool,  //true,
-    session: bool, //false,
+    pub session: bool, //false,
-    sameSite: Option<String>, //"None",
+    pub sameSite: Option<String>, //"None",
-    priority: String, //"Medium",
+    pub priority: String, //"Medium",
-    sameParty: bool, //false,
+    pub sameParty: bool, //false,
-    sourceScheme: String, //"Secure",
+    pub sourceScheme: String, //"Secure",
-    sourcePort: u32, //443,
+    pub sourcePort: u32, //443,
-    partitionKey: Option<String>, //"https://perverzija.com"
+    pub partitionKey: Option<String>, 
 }
 
 #[derive(serde::Serialize, serde::Deserialize, Debug)]  
 pub struct FlareSolverrSolution {
-    url: String, 
+    pub url: String, 
-    status: u32,
+    pub status: u32,
     pub response: String, 
-    headers: HashMap<String, String>,
+    pub headers: HashMap<String, String>,
-    cookies: Vec<FlaresolverrCookie>, 
+    pub cookies: Vec<FlaresolverrCookie>, 
-    userAgent: String,
+    pub userAgent: String,
 }
 // impl FlareSolverrSolution {
 //     fn to_client(&self,){
@@ -65,15 +65,21 @@ pub struct FlareSolverrResponse {
     version: String,
 }
 pub struct Flaresolverr {
-    url: String
+    url: String,
+    proxy: bool,
 }
 impl Flaresolverr {
     pub fn new(url: String) -> Self {
         Flaresolverr { 
-            url: url
+            url: url,
+            proxy: false,
         }
     }
 
+    pub fn set_proxy(&mut self, proxy: bool) {
+        self.proxy = proxy;
+    }
+
     pub async fn solve(
         &self,
         request: FlareSolverrRequest,
@@ -82,15 +88,23 @@ impl Flaresolverr {
         .emulation(Emulation::Firefox136)
         .build()?;
 
-    let response = client
+        let mut request = client
         .post(&self.url)
         .header("Content-Type", "application/json")
         .json(&json!({
             "cmd": request.cmd,
             "url": request.url,
             "maxTimeout": request.maxTimeout,
-        }))
+        }));
-        .send().await?;
+
+        if self.proxy {
+            if let Ok(proxy_url) = env::var("BURP_URL") {
+                let proxy = Proxy::all(&proxy_url).unwrap();
+                request = request.proxy(proxy.clone());
+            }
+        }
+
+    let response = request.send().await?;
 
     let body: FlareSolverrResponse = response.json::<FlareSolverrResponse>().await?;
     Ok(body)

src/util/mod.rs

@@ -1,6 +1,9 @@
+use crate::providers::AnyProvider;
+
 pub mod time;
 pub mod flaresolverr;
 pub mod cache;
+pub mod requester;
 
 pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
     let s = s.trim();
@@ -41,3 +44,10 @@ pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {
 
     result
 }
+
+pub fn get_all_providers() -> Vec<AnyProvider>{
+
+    
+
+    return vec![];
+}

src/util/requester.rs

@@ -0,0 +1,123 @@
+use wreq::header::HeaderValue;
+use wreq::redirect::Policy;
+use wreq::Client;
+use wreq::Proxy;
+use wreq::Version;
+use wreq_util::Emulation;
+use std::env;
+
+use crate::util::flaresolverr::FlareSolverrRequest;
+use crate::util::flaresolverr::Flaresolverr;
+
+#[derive(Debug, Clone)]
+pub struct Requester {
+    client: Client,
+    proxy: bool,
+    flaresolverr_session: Option<String>,
+}
+
+impl Requester {
+    pub fn new() -> Self {
+        let client = Client::builder()
+        .cert_verification(false)
+        .emulation(Emulation::Firefox136)
+        .cookie_store(true)
+        .redirect(Policy::default())
+        .build()
+            .expect("Failed to create HTTP client");
+
+        Requester {
+            client,
+            proxy: false,
+            flaresolverr_session: None,
+        }
+    }
+    pub fn set_proxy(&mut self, proxy: bool) {
+        self.proxy = proxy;
+    }
+
+    pub fn set_flaresolverr_session(&mut self, session: String) {
+        self.flaresolverr_session = Some(session);
+    }
+
+    fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String {
+        if location.starts_with("http://") || location.starts_with("https://") {
+            location.to_string()
+        } else if location.starts_with("//") {
+            format!("{}{}", "https:", location) // Replace with your base URL
+        } else if location.starts_with("/") {
+            let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/");
+            format!("{}{}", base_url, location)
+        } else {
+            format!("{}/{}", prev_url, location)
+        }
+    }
+
+    pub async fn get(&self, url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+        let mut request = self.client.get(url).version(Version::HTTP_11);
+        let mut proxy;
+        if self.proxy {
+            if let Ok(proxy_url) = env::var("BURP_URL") {
+                proxy = Proxy::all(&proxy_url).unwrap();
+                request = request.proxy(proxy.clone());
+            }
+        }
+
+        let mut response = request.send().await?;
+    
+        if response.status().is_success() {
+            return Ok(response.text().await?);
+        } else {
+            let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
+            let mut flare = Flaresolverr::new(flare_url);
+            if self.proxy && env::var("BURP_URL").is_ok() {
+                flare.set_proxy(true);
+            }
+            let result = flare
+                .solve(FlareSolverrRequest {
+                    cmd: "request.get".to_string(),
+                    url: url.to_string(),
+                    maxTimeout: 60000,
+                })
+                .await;
+            match result {
+                Ok(res) => {
+                    let useragent = res.solution.userAgent;
+                    self.client.update()
+                        .headers(|headers| {
+                            headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
+                        })
+                        .apply()
+                        .unwrap();
+                    for cookie in res.solution.cookies {
+                        let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
+                        // Parse the domain string into a Url
+                        let cookie_url = url.split("/").collect::<Vec<&str>>()[..3].join("/");
+                        println!("{}", cookie_url);
+                        if let Ok(url) = url::Url::parse(cookie_url.as_str()) {
+                            println!("{:?}, {:?}", cookie_url, header);
+                            self.client.set_cookie(&url, header);
+                        }
+                    }
+                    request = self.client.get(url).version(Version::HTTP_11);
+                    if self.proxy {
+                        if let Ok(proxy_url) = env::var("BURP_URL") {
+                            proxy = Proxy::all(&proxy_url).unwrap();
+                            request = request.proxy(proxy.clone());
+                        }
+                    }
+
+                    response = request.send().await?;
+                    if response.status().is_success() {
+                        return Ok(response.text().await?);
+                    }
+
+                    Ok(res.solution.response)
+                }
+                Err(e) => {
+                    return Err(format!("Failed to solve FlareSolverr: {e}").into());
+                }
+            }
+        }
+    }
+}

src/videos.rs

@@ -266,11 +266,6 @@ impl VideoFormat {
             headers.insert(key, value);
         }
     }
-
-    pub fn protocol(mut self, protocol: String) -> Self {
-        self.protocol = Some(protocol);
-        self
-    }
 }
 #[derive(serde::Serialize, Debug)]
 pub struct Videos {