From 580751af03cf1be082c6d09f7fc4cdd18da583a6 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 31 May 2025 13:54:27 +0000 Subject: [PATCH] implemented query and flaresolverr --- src/main.rs | 2 + src/providers/perverzija.rs | 233 +++++++++++++++++++++++++++++------- src/util/flaresolverr.rs | 80 +++++++++++++ src/util/mod.rs | 3 +- 4 files changed, 274 insertions(+), 44 deletions(-) create mode 100644 src/util/flaresolverr.rs diff --git a/src/main.rs b/src/main.rs index 447d027..51e1a19 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,8 @@ mod util; #[ntex::main] async fn main() -> std::io::Result<()> { + std::env::set_var("RUST_LOG", "ntex=info"); + std::env::set_var("RUST_BACKTRACE", "1"); env_logger::init(); // You need this to actually see logs diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index 81a00b2..c1a236c 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -1,12 +1,13 @@ use std::vec; use error_chain::error_chain; -use htmlentity::entity::{decode, encode, CharacterSet, EncodeType, ICodedDataTrait}; -use htmlentity::types::{AnyhowResult, Byte}; +use htmlentity::entity::{decode, ICodedDataTrait}; +use reqwest::Proxy; use crate::providers::Provider; +use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; -use crate::videos::{self, PageInfo, Video_Embed, Video_Item, Videos}; // Make sure Provider trait is imported +use crate::videos::{self, Video_Embed, Video_Item}; // Make sure Provider trait is imported error_chain! { foreign_links { @@ -26,7 +27,7 @@ impl PerverzijaProvider { } async fn get(&self, page: &u8, featured: String) -> Result> { let mut prefix_uri = "".to_string(); - if featured == "featured"{ + if featured == "featured" { prefix_uri = "featured-scenes/".to_string(); } let mut url = format!("{}{}page/{}/", self.url, prefix_uri, page); @@ -35,58 +36,93 @@ impl PerverzijaProvider { } let client = reqwest::Client::builder() .user_agent("Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/33.0 Mobile/15E148 Safari/605.1.15") - // .proxy(Proxy::https("http://192.168.0.101:8080").unwrap()) - // .danger_accept_invalid_certs(true) + .proxy(Proxy::https("http://192.168.0.101:8080").unwrap()) + .danger_accept_invalid_certs(true) .build()?; - let response = client.get(url).send().await?; - + let response = client.get(url.clone()).send().await?; + // print!("Response: {:?}\n", response); if response.status().is_success() { let text = response.text().await?; let video_items = self.get_video_items_from_html(text.clone()); Ok(video_items) } else { - Err("Failed to fetch data".into()) + let flare = Flaresolverr::new("http://192.168.0.103:8191/v1".to_string()); + let result = flare + .solve(FlareSolverrRequest { + cmd: "request.get".to_string(), + url: url.clone(), + maxTimeout: 60000, + }) + .await; + println!("FlareSolverr result: {:?}", result); + let video_items = match result { + Err(e) => { + println!("Error solving FlareSolverr: {}", e); + return Err("Failed to solve FlareSolverr".into()); + } + Ok(res) => self.get_video_items_from_html(res), + }; + + Ok(video_items) } } - fn query(&self, query: &str) -> Result> { - println!("Searching for query: {}", query); - let url = format!("{}?s={}", self.url, query); - let client = reqwest::blocking::Client::new(); - let response = client.get(&url).send()?; + async fn query(&self, page: &u8, query: &str) -> Result> { + let search_string = query.replace(" ", "+"); + let mut url = format!("{}advanced-search/?_sf_s={}&sf_paged={}", self.url, search_string, page); + if page == &1 { + url = format!("{}advanced-search/?_sf_s={}", self.url, search_string); + } + let client = reqwest::Client::builder() + .user_agent("Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/33.0 Mobile/15E148 Safari/605.1.15") + .proxy(Proxy::https("http://192.168.0.101:8080").unwrap()) + .danger_accept_invalid_certs(true) + .build()?; + let response = client.get(url.clone()).send().await?; if response.status().is_success() { - let text = response.text().unwrap_or_default(); - - println!("{}", &text); - Ok(vec![]) + let text = response.text().await?; + let video_items = self.get_video_items_from_html_query(text.clone()); + Ok(video_items) } else { - Err("Failed to fetch data".into()) + let flare = Flaresolverr::new("http://192.168.0.103:8191/v1".to_string()); + let result = flare + .solve(FlareSolverrRequest { + cmd: "request.get".to_string(), + url: url.clone(), + maxTimeout: 60000, + }) + .await; + println!("FlareSolverr result: {:?}", result); + let video_items = match result { + Err(e) => { + println!("Error solving FlareSolverr: {}", e); + return Err("Failed to solve FlareSolverr".into()); + } + Ok(res) => self.get_video_items_from_html_query(res), + }; + + Ok(video_items) } } fn get_video_items_from_html(&self, html: String) -> Vec { let mut items: Vec = Vec::new(); - - let raw_html = html.split("video-listing-content").collect::>(); - - let video_listing_content = raw_html[1]; + let video_listing_content = html.split("video-listing-content").collect::>()[1]; let raw_videos = video_listing_content .split("video-item post") .collect::>()[1..] .to_vec(); - + // println!("Raw Videos: {:?}", raw_videos); for video_segment in &raw_videos { - let vid = video_segment.split("\n").collect::>(); let mut index = 0; - if vid.len() > 10 { - + if vid.len() > 20 { continue; } - for line in vid.clone(){ + for line in vid.clone() { println!("{}: {}\n\n", index, line); index += 1; } - + let mut title = vid[1].split(">").collect::>()[1] .split("<") .collect::>()[0] @@ -96,16 +132,16 @@ impl PerverzijaProvider { let url = vid[1].split("iframe src="").collect::>()[1] .split(""") .collect::>()[0] - .to_string().replace("index.php", "xs1.php");; + .to_string(); let id = url.split("data=").collect::>()[1] .split("&") .collect::>()[0] .to_string(); - let raw_duration = match vid.len(){ + let raw_duration = match vid.len() { 10 => vid[6].split("time_dur\">").collect::>()[1] - .split("<") - .collect::>()[0] - .to_string(), + .split("<") + .collect::>()[0] + .to_string(), _ => "00:00".to_string(), }; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; @@ -125,18 +161,129 @@ impl PerverzijaProvider { .collect::>()[0] .to_string(), }; - let mut embed_html = vid[1].split("data-embed='").collect::>()[1].split("'").collect::>()[0] + let mut embed_html = vid[1].split("data-embed='").collect::>()[1] + .split("'") + .collect::>()[0] + .to_string(); + let referer_url = vid[1].split("data-url='").collect::>()[1] + .split("'") + .collect::>()[0] .to_string(); - embed_html = embed_html.replace("index.php", "xs1.php"); - println!("Embed HTML: {}\n\n", embed_html); - println!("Url: {}\n\n", url.clone()); + // println!("Embed HTML: {}\n\n", embed_html); + // println!("Url: {}\n\n", url.clone()); let embed = Video_Embed::new(embed_html, url.clone()); - let mut video_item = - Video_Item::new(id, title, url.clone(), "perverzija".to_string(), thumb, duration); + let mut video_item = Video_Item::new( + id, + title, + url.clone(), + "perverzija".to_string(), + thumb, + duration, + ); video_item.embed = Some(embed); - let mut format = videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string()); - format.add_http_header("Referer".to_string(), url.clone().replace("xs1.php", "index.php")); + let mut format = + videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string()); + format.add_http_header("Referer".to_string(), referer_url.clone()); + if let Some(formats) = video_item.formats.as_mut() { + formats.push(format); + } else { + video_item.formats = Some(vec![format]); + } + items.push(video_item); + } + + return items; + } + + fn get_video_items_from_html_query(&self, html: String) -> Vec { + let mut items: Vec = Vec::new(); + let video_listing_content = html.split("search-filter-results-").collect::>()[1]; + let raw_videos = video_listing_content + .split("video-item post") + .collect::>()[1..] + .to_vec(); + // println!("Raw Videos: {:?}", raw_videos); + for video_segment in &raw_videos { + let vid = video_segment.split("\n").collect::>(); + if vid.len() > 20 { + continue; + } + // let mut index = 0; + // for line in vid.clone() { + // println!("{}: {}\n\n", index, line); + // index += 1; + // } + + let mut title = vid[3].split("title='").collect::>()[1] + .split("'") + .collect::>()[0] + .to_string(); + // html decode + title = decode(title.as_bytes()).to_string().unwrap_or(title); + let url = vid[4].split("iframe src="").collect::>()[1] + .split(""") + .collect::>()[0] + .to_string(); + let id = url.split("data=").collect::>()[1] + .split("&") + .collect::>()[0] + .to_string(); + let raw_duration = match vid.len() { + 18 => vid[16].split("time_dur\">").collect::>()[1] + .split("<") + .collect::>()[0] + .to_string(), + _ => "00:00".to_string(), + }; + let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; + let thumb_index = match vid.len() { + 18 => 14, + 13 => 8, + _=> { + println!("Unexpected video segment length: {}", vid.len()); + continue; + } + }; + let thumb = match vid[thumb_index].contains("srcset=") { + true => vid[thumb_index].split("sizes=").collect::>()[1] + .split("w, ") + .collect::>() + .last() + .unwrap() + .to_string() + .split(" ") + .collect::>()[0] + .to_string(), + false => vid[thumb_index].split("src=\"").collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(), + }; + let mut embed_html = vid[4].split("data-embed='").collect::>()[1] + .split("'") + .collect::>()[0] + .to_string(); + let referer_url = vid[4].split("data-url='").collect::>()[1] + .split("'") + .collect::>()[0] + .to_string(); + + // println!("Embed HTML: {}\n\n", embed_html); + // println!("Url: {}\n\n", url.clone()); + let embed = Video_Embed::new(embed_html, url.clone()); + let mut video_item = Video_Item::new( + id, + title, + url.clone(), + "perverzija".to_string(), + thumb, + duration, + ); + video_item.embed = Some(embed); + let mut format = + videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string()); + format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); } else { @@ -160,7 +307,7 @@ impl Provider for PerverzijaProvider { ) -> Vec { let _ = sort; let videos: std::result::Result, Error> = match query { - Some(q) => self.query(&q), + Some(q) => self.query(&page.parse::().unwrap_or(1), &q).await, None => self.get(&page.parse::().unwrap_or(1), featured).await, }; match videos { diff --git a/src/util/flaresolverr.rs b/src/util/flaresolverr.rs new file mode 100644 index 0000000..f7b8360 --- /dev/null +++ b/src/util/flaresolverr.rs @@ -0,0 +1,80 @@ +use std::collections::HashMap; + +use reqwest::Proxy; + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct FlareSolverrRequest { + pub cmd: String, + pub url: String, + pub maxTimeout: u32, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct FlaresolverrCookie { + name: String, //"cf_clearance", + value: String, //"lnKoXclrIp_mDrWJFfktPGm8GDyxjSpzy9dx0qDTiRg-1748689259-1.2.1.1-AIFERAPCdCSvvdu1mposNdUpKV9wHZXBpSI2L9k9TaKkPcqmomON_XEb6ZtRBtrmQu_DC8AzKllRg2vNzVKOUsvv9ndjQ.vv8Z7cNkgzpIbGFy96kXyAYH2mUk3Q7enZovDlEbK5kpV3Sbmd2M3_bUCBE1WjAMMdXlyNElH1LOpUm149O9hrluXjAffo4SwHI4HO0UckBPWBlBqhznKPgXxU0g8VHLDeYnQKViY8rP2ud4tyzKnJUxuYXzr4aWBNMp6TESp49vesRiel_Y5m.rlTY4zSb517S9iPbEQiYHRI.uH5mMHVI3jvJl0Mx94tPrpFnkhDdmzL3DRSllJe9k786Lf21I9WBoH2cCR3yHw", + domain: String, //".discord.com", + path: String, //"/", + expires: f64, //1780225259.237105, + size: u64, //438, + httpOnly: bool, //true, + secure: bool, //true, + session: bool, //false, + sameSite: String, //"None", + priority: String, //"Medium", + sameParty: bool, //false, + sourceScheme: String, //"Secure", + sourcePort: u8, //443, + partitionKey: String, //"https://perverzija.com" +} + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct FlareSolverrSolution { + url: String, //"https://pervl4.xtremestream.xyz/player/index.php?data=af8a224ded8ec0eadd5d93a746de9d97", + status: u8, + response: String, // "You can't access the video directly", + headers: HashMap, + cookies: Vec, //[], + userAgent: String, //"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" +} +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct FlareSolverrResponse { + status: String, + message: String, + solution: FlareSolverrSolution, + startTimestamp: u64, + endTimestamp: u64, + version: String, +} +pub struct Flaresolverr { + url: String, +} +impl Flaresolverr { + pub fn new(url: String) -> Self { + Flaresolverr { url } + } + + pub async fn solve( + &self, + request: FlareSolverrRequest, + ) -> Result> { + let client = reqwest::Client::builder() + // .user_agent("Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/33.0 Mobile/15E148 Safari/605.1.15") + .proxy(Proxy::https("http://192.168.0.101:8080").unwrap()) + .danger_accept_invalid_certs(true) + .build()?; + let response = client.post(&self.url).json(&request).send().await?; + println!("FlareSolverr response: {:?}", response); + if response.status().is_success() { + let json_response: FlareSolverrResponse = + response.json::().await?; + Ok(json_response.solution.response) + } else { + Err(format!( + "Failed to solve FlareSolverr request: HTTP {}", + response.status() + ) + .into()) + } + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index c25ca52..41aed28 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1 +1,2 @@ -pub mod time; \ No newline at end of file +pub mod time; +pub mod flaresolverr; \ No newline at end of file