diff --git a/Cargo.toml b/Cargo.toml index cf5e710..4cf509d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,12 +11,13 @@ env_logger = "0.11.8" error-chain = "0.12.4" futures = "0.3.31" htmlentity = "1.3.2" -ntex = { version = "2.0", features = ["tokio"] } +ntex = { version = "2.15.1", features = ["tokio"] } ntex-files = "2.0.0" serde = "1.0.219" -serde_json = "1.0.140" -tokio = { version = "1", features = ["full"] } -wreq = { version = "5", features = ["full"] } +serde_json = "1.0.143" +tokio = { version = "1.47.1", features = ["full"] } +wreq = { version = "5.3.0", features = ["full", "cookies"] } wreq-util = "2" percent-encoding = "2.1" capitalize = "0.3.4" +url = "2.5.4" diff --git a/src/main.rs b/src/main.rs index 4260054..a4ea0e9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,10 +7,6 @@ use dotenvy::dotenv; use ntex_files as fs; use ntex::web; - -#[macro_use] -extern crate cute; - mod api; mod db; mod models; diff --git a/src/providers/missav.rs b/src/providers/missav.rs index 3247a53..ef8e765 100644 --- a/src/providers/missav.rs +++ b/src/providers/missav.rs @@ -13,6 +13,7 @@ use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::videos::ServerOptions; use crate::videos::{VideoItem}; use crate::DbPool; +use crate::util::requester::Requester; error_chain! { @@ -26,11 +27,15 @@ error_chain! { #[derive(Debug, Clone)] pub struct MissavProvider { url: String, + requester: Requester, } impl MissavProvider { pub fn new() -> Self { + let mut requester = Requester::new(); + requester.set_proxy(true); MissavProvider { url: "https://missav.ws".to_string(), + requester } } async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, sort: String, options: ServerOptions) -> Result> { @@ -52,57 +57,17 @@ impl MissavProvider { }; - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; - - let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - - let mut response = client.get(url_str.clone()) - // .proxy(proxy.clone()) - .send().await?; - if response.status().is_redirection(){ - response = client.get(response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; - } - if response.status().is_success() { - let text = response.text().await?; - let video_items: Vec = self.get_video_items_from_html(text.clone(), pool).await; - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); - } - Ok(video_items) - } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); - let flare = Flaresolverr::new(flare_url); - let result = flare - .solve(FlareSolverrRequest { - cmd: "request.get".to_string(), - url: url_str.clone(), - maxTimeout: 60000, - }) - .await; - let video_items = match result { - Ok(res) => { - // println!("FlareSolverr response: {}", res); - self.get_video_items_from_html(res.solution.response, pool).await - } - Err(e) => { - println!("Error solving FlareSolverr: {}", e); - return Err("Failed to solve FlareSolverr".into()); - } - }; - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + let text = self.requester.get(&url_str).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone(), pool).await; + if !video_items.is_empty() { + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); + } else{ + return Ok(old_items); } + Ok(video_items) } + async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str, sort: String, options: ServerOptions) -> Result> { let search_string = query.replace(" ", "%20"); let url_str = format!( @@ -124,55 +89,15 @@ impl MissavProvider { vec![] } }; - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; - - let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - - let mut response = client.get(url_str.clone()) - // .proxy(proxy.clone()) - .send().await?; - if response.status().is_redirection(){ - response = client.get(response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; - } - if response.status().is_success() { - let text = response.text().await?; - let video_items: Vec = self.get_video_items_from_html(text.clone(), pool).await; - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); - } - Ok(video_items) - } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); - let flare = Flaresolverr::new(flare_url); - let result = flare - .solve(FlareSolverrRequest { - cmd: "request.get".to_string(), - url: url_str.clone(), - maxTimeout: 60000, - }) - .await; - let video_items = match result { - Ok(res) => { - self.get_video_items_from_html(res.solution.response, pool).await - } - Err(e) => { - println!("Error solving FlareSolverr: {}", e); - return Err("Failed to solve FlareSolverr".into()); - } - }; - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); - } - Ok(video_items) + let text = self.requester.get(&url_str).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone(), pool).await; + if !video_items.is_empty() { + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); + } else{ + return Ok(old_items); } + Ok(video_items) } async fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec { @@ -223,26 +148,7 @@ impl MissavProvider { } } drop(conn); - - - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; - - let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - - let mut response = client.get(url_str.clone()) - // .proxy(proxy.clone()) - .send().await?; - if response.status().is_redirection(){ - response = client.get(response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; - } - let vid; - if response.status().is_success() { - vid = response.text().await?; - } else { - return Err("Failed to fetch video details".into()); - } + let vid = self.requester.get(&url_str).await.unwrap(); let mut title = vid.split(">()[1] .split("\"") .collect::>()[0].trim() diff --git a/src/providers/perfectgirls.rs b/src/providers/perfectgirls.rs index af2a5d5..1c7a065 100644 --- a/src/providers/perfectgirls.rs +++ b/src/providers/perfectgirls.rs @@ -221,11 +221,14 @@ impl PerfectgirlsProvider { .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::>()[1] + let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::>()[1] .split("data-original=\"").collect::>()[1] .split("\"") .collect::>()[0] .to_string(); + if thumb.starts_with("//"){ + thumb = format!("https:{}",thumb); + } let mut tags = vec![]; if video_segment.contains("href=\"/channels/"){ diff --git a/src/util/flaresolverr.rs b/src/util/flaresolverr.rs index 29ba7d2..5efbb84 100644 --- a/src/util/flaresolverr.rs +++ b/src/util/flaresolverr.rs @@ -1,7 +1,7 @@ -use std::collections::HashMap; +use std::{collections::HashMap, env}; use serde_json::json; -use wreq::Client; +use wreq::{Client, Proxy}; use wreq_util::Emulation; #[derive(serde::Serialize, serde::Deserialize, Debug)] @@ -13,31 +13,31 @@ pub struct FlareSolverrRequest { #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct FlaresolverrCookie { - name: String, //"cf_clearance", - value: String, //"lnKoXclrIp_mDrWJFfktPGm8GDyxjSpzy9dx0qDTiRg-1748689259-1.2.1.1-AIFERAPCdCSvvdu1mposNdUpKV9wHZXBpSI2L9k9TaKkPcqmomON_XEb6ZtRBtrmQu_DC8AzKllRg2vNzVKOUsvv9ndjQ.vv8Z7cNkgzpIbGFy96kXyAYH2mUk3Q7enZovDlEbK5kpV3Sbmd2M3_bUCBE1WjAMMdXlyNElH1LOpUm149O9hrluXjAffo4SwHI4HO0UckBPWBlBqhznKPgXxU0g8VHLDeYnQKViY8rP2ud4tyzKnJUxuYXzr4aWBNMp6TESp49vesRiel_Y5m.rlTY4zSb517S9iPbEQiYHRI.uH5mMHVI3jvJl0Mx94tPrpFnkhDdmzL3DRSllJe9k786Lf21I9WBoH2cCR3yHw", - domain: String, //".discord.com", - path: String, //"/", - expires: f64, //1780225259.237105, - size: u64, //438, - httpOnly: bool, //true, - secure: bool, //true, - session: bool, //false, - sameSite: Option, //"None", - priority: String, //"Medium", - sameParty: bool, //false, - sourceScheme: String, //"Secure", - sourcePort: u32, //443, - partitionKey: Option, //"https://perverzija.com" + pub name: String, //"cf_clearance", + pub value: String, //"lnKoXclrIp_mDrWJFfktPGm8GDyxjSpzy9dx0qDTiRg-1748689259-1.2.1.1-AIFERAPCdCSvvdu1mposNdUpKV9wHZXBpSI2L9k9TaKkPcqmomON_XEb6ZtRBtrmQu_DC8AzKllRg2vNzVKOUsvv9ndjQ.vv8Z7cNkgzpIbGFy96kXyAYH2mUk3Q7enZovDlEbK5kpV3Sbmd2M3_bUCBE1WjAMMdXlyNElH1LOpUm149O9hrluXjAffo4SwHI4HO0UckBPWBlBqhznKPgXxU0g8VHLDeYnQKViY8rP2ud4tyzKnJUxuYXzr4aWBNMp6TESp49vesRiel_Y5m.rlTY4zSb517S9iPbEQiYHRI.uH5mMHVI3jvJl0Mx94tPrpFnkhDdmzL3DRSllJe9k786Lf21I9WBoH2cCR3yHw", + pub domain: String, //".discord.com", + pub path: String, //"/", + pub expires: f64, //1780225259.237105, + pub size: u64, //438, + pub httpOnly: bool, //true, + pub secure: bool, //true, + pub session: bool, //false, + pub sameSite: Option, //"None", + pub priority: String, //"Medium", + pub sameParty: bool, //false, + pub sourceScheme: String, //"Secure", + pub sourcePort: u32, //443, + pub partitionKey: Option, } #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct FlareSolverrSolution { - url: String, - status: u32, + pub url: String, + pub status: u32, pub response: String, - headers: HashMap, - cookies: Vec, - userAgent: String, + pub headers: HashMap, + pub cookies: Vec, + pub userAgent: String, } // impl FlareSolverrSolution { // fn to_client(&self,){ @@ -65,15 +65,21 @@ pub struct FlareSolverrResponse { version: String, } pub struct Flaresolverr { - url: String + url: String, + proxy: bool, } impl Flaresolverr { pub fn new(url: String) -> Self { Flaresolverr { - url: url + url: url, + proxy: false, } } + pub fn set_proxy(&mut self, proxy: bool) { + self.proxy = proxy; + } + pub async fn solve( &self, request: FlareSolverrRequest, @@ -82,15 +88,23 @@ impl Flaresolverr { .emulation(Emulation::Firefox136) .build()?; - let response = client + let mut request = client .post(&self.url) .header("Content-Type", "application/json") .json(&json!({ "cmd": request.cmd, "url": request.url, "maxTimeout": request.maxTimeout, - })) - .send().await?; + })); + + if self.proxy { + if let Ok(proxy_url) = env::var("BURP_URL") { + let proxy = Proxy::all(&proxy_url).unwrap(); + request = request.proxy(proxy.clone()); + } + } + + let response = request.send().await?; let body: FlareSolverrResponse = response.json::().await?; Ok(body) diff --git a/src/util/mod.rs b/src/util/mod.rs index 1a01180..9f2e4ae 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,6 +1,9 @@ +use crate::providers::AnyProvider; + pub mod time; pub mod flaresolverr; pub mod cache; +pub mod requester; pub fn parse_abbreviated_number(s: &str) -> Option { let s = s.trim(); @@ -40,4 +43,11 @@ pub fn interleave(lists: &[Vec]) -> Vec { } result +} + +pub fn get_all_providers() -> Vec{ + + + + return vec![]; } \ No newline at end of file diff --git a/src/util/requester.rs b/src/util/requester.rs new file mode 100644 index 0000000..4d549a1 --- /dev/null +++ b/src/util/requester.rs @@ -0,0 +1,123 @@ +use wreq::header::HeaderValue; +use wreq::redirect::Policy; +use wreq::Client; +use wreq::Proxy; +use wreq::Version; +use wreq_util::Emulation; +use std::env; + +use crate::util::flaresolverr::FlareSolverrRequest; +use crate::util::flaresolverr::Flaresolverr; + +#[derive(Debug, Clone)] +pub struct Requester { + client: Client, + proxy: bool, + flaresolverr_session: Option, +} + +impl Requester { + pub fn new() -> Self { + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .cookie_store(true) + .redirect(Policy::default()) + .build() + .expect("Failed to create HTTP client"); + + Requester { + client, + proxy: false, + flaresolverr_session: None, + } + } + pub fn set_proxy(&mut self, proxy: bool) { + self.proxy = proxy; + } + + pub fn set_flaresolverr_session(&mut self, session: String) { + self.flaresolverr_session = Some(session); + } + + fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String { + if location.starts_with("http://") || location.starts_with("https://") { + location.to_string() + } else if location.starts_with("//") { + format!("{}{}", "https:", location) // Replace with your base URL + } else if location.starts_with("/") { + let base_url = prev_url.split('/').take(3).collect::>().join("/"); + format!("{}{}", base_url, location) + } else { + format!("{}/{}", prev_url, location) + } + } + + pub async fn get(&self, url: &str) -> Result> { + let mut request = self.client.get(url).version(Version::HTTP_11); + let mut proxy; + if self.proxy { + if let Ok(proxy_url) = env::var("BURP_URL") { + proxy = Proxy::all(&proxy_url).unwrap(); + request = request.proxy(proxy.clone()); + } + } + + let mut response = request.send().await?; + + if response.status().is_success() { + return Ok(response.text().await?); + } else { + let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let mut flare = Flaresolverr::new(flare_url); + if self.proxy && env::var("BURP_URL").is_ok() { + flare.set_proxy(true); + } + let result = flare + .solve(FlareSolverrRequest { + cmd: "request.get".to_string(), + url: url.to_string(), + maxTimeout: 60000, + }) + .await; + match result { + Ok(res) => { + let useragent = res.solution.userAgent; + self.client.update() + .headers(|headers| { + headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap()); + }) + .apply() + .unwrap(); + for cookie in res.solution.cookies { + let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap(); + // Parse the domain string into a Url + let cookie_url = url.split("/").collect::>()[..3].join("/"); + println!("{}", cookie_url); + if let Ok(url) = url::Url::parse(cookie_url.as_str()) { + println!("{:?}, {:?}", cookie_url, header); + self.client.set_cookie(&url, header); + } + } + request = self.client.get(url).version(Version::HTTP_11); + if self.proxy { + if let Ok(proxy_url) = env::var("BURP_URL") { + proxy = Proxy::all(&proxy_url).unwrap(); + request = request.proxy(proxy.clone()); + } + } + + response = request.send().await?; + if response.status().is_success() { + return Ok(response.text().await?); + } + + Ok(res.solution.response) + } + Err(e) => { + return Err(format!("Failed to solve FlareSolverr: {e}").into()); + } + } + } + } +} \ No newline at end of file diff --git a/src/videos.rs b/src/videos.rs index 688d0f2..11ea1a0 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -266,11 +266,6 @@ impl VideoFormat { headers.insert(key, value); } } - - pub fn protocol(mut self, protocol: String) -> Self { - self.protocol = Some(protocol); - self - } } #[derive(serde::Serialize, Debug)] pub struct Videos {