use std::vec; use std::env; use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use reqwest::{Proxy}; use futures::future::join_all; use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; use crate::videos::{self, VideoEmbed, VideoItem}; use crate::DbPool; use crate::USER_AGENT; // Make sure Provider trait is imported error_chain! { foreign_links { Io(std::io::Error); HttpRequest(reqwest::Error); } } pub struct PerverzijaProvider { url: String, } impl PerverzijaProvider { pub fn new() -> Self { PerverzijaProvider { url: "https://tube.perverzija.com/".to_string(), } } async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result> { //TODO // let mut url = Url::parse("https://example.net")?; // url.query_pairs_mut().append_pair("foo", "bar"); // url.query_pairs_mut().append_pair("key", "dkhdsihdsaiufds"); // url.query_pairs_mut().append_pair("hello", "world"); // println!("{}", url.as_str()); let mut prefix_uri = "".to_string(); if featured == "featured" { prefix_uri = "featured-scenes/".to_string(); } let mut url = format!("{}{}page/{}/", self.url, prefix_uri, page); if page == 1 { url = format!("{}{}", self.url, prefix_uri); } let old_items = match cache.get(&url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { println!("Cache hit for URL: {}", url); return Ok(items.clone()); } else{ items.clone() } } None => { vec![] } }; let client = match env::var("BURP_URL").as_deref() { Ok(burp_url) => reqwest::Client::builder() .user_agent(USER_AGENT) .proxy(Proxy::https(burp_url).unwrap()) .danger_accept_invalid_certs(true) .build()?, Err(_) => reqwest::Client::builder() .user_agent(USER_AGENT) .danger_accept_invalid_certs(true) .build()?, }; let response = client.get(url.clone()).send().await?; // print!("Response: {:?}\n", response); if response.status().is_success() { let text = response.text().await?; let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); } else{ return Ok(old_items); } Ok(video_items) } else { let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), url: url.clone(), maxTimeout: 60000, }) .await; let video_items = match result { Ok(res) => { // println!("FlareSolverr response: {}", res); self.get_video_items_from_html(res.solution.response, pool) } Err(e) => { println!("Error solving FlareSolverr: {}", e); return Err("Failed to solve FlareSolverr".into()); } }; if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } } async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result> { let search_string = query.replace(" ", "+"); let mut url = format!( "{}page/{}/?s={}", self.url, page, search_string ); if page == 1 { url = format!("{}?s={}", self.url, search_string); } // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { return Ok(items.clone()); } else{ let _ = cache.check().await; return Ok(items.clone()) } } None => { vec![] } }; let client = match env::var("BURP_URL").as_deref() { Ok(burp_url) => reqwest::Client::builder() .user_agent(USER_AGENT) .proxy(Proxy::https(burp_url).unwrap()) .danger_accept_invalid_certs(true) .build()?, Err(_) => reqwest::Client::builder() .user_agent(USER_AGENT) .danger_accept_invalid_certs(true) .build()?, }; let response = client.get(url.clone()).send().await?; if response.status().is_success() { let text = response.text().await?; let video_items: Vec = self.get_video_items_from_html_query(text.clone(), pool).await; if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); } else{ return Ok(old_items); } Ok(video_items) } else { let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), url: url.clone(), maxTimeout: 60000, }) .await; let video_items = match result { Ok(res) => { self.get_video_items_from_html_query(res.solution.response, pool).await } Err(e) => { println!("Error solving FlareSolverr: {}", e); return Err("Failed to solve FlareSolverr".into()); } }; if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); } else{ return Ok(old_items); } Ok(video_items) } } fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec { if html.is_empty() { println!("HTML is empty"); return vec![]; } let mut items: Vec = Vec::new(); let video_listing_content = html.split("video-listing-content").collect::>()[1]; let raw_videos = video_listing_content .split("video-item post") .collect::>()[1..] .to_vec(); for video_segment in &raw_videos { let vid = video_segment.split("\n").collect::>(); if vid.len() > 20 { continue; } let mut title = vid[1].split(">").collect::>()[1] .split("<") .collect::>()[0] .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); let url = vid[1].split("iframe src="").collect::>()[1] .split(""") .collect::>()[0] .to_string().replace("index.php", "xs1.php"); let id = url.split("data=").collect::>()[1] .split("&") .collect::>()[0] .to_string(); let raw_duration = match vid.len() { 10 => vid[6].split("time_dur\">").collect::>()[1] .split("<") .collect::>()[0] .to_string(), _ => "00:00".to_string(), }; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; if !vid[4].contains("srcset=") && vid[4].split("src=\"").collect::>().len() == 1{ for (index, line) in vid.iter().enumerate(){ println!("Line {}: {}\n\n", index, line); } } let mut thumb = "".to_string(); for v in vid.clone(){ let line = v.trim(); if line.starts_with(">()[1] .split("\"") .collect::>()[0] .to_string(); } } let embed_html = vid[1].split("data-embed='").collect::>()[1] .split("'") .collect::>()[0] .to_string(); let id_url = vid[1].split("data-url='").collect::>()[1] .split("'") .collect::>()[0] .to_string(); let mut conn = pool.get().expect("couldn't get db connection from pool"); let _ = db::insert_video(&mut conn, &id_url, &url); drop(conn); let referer_url = "https://xtremestream.xyz/".to_string(); let embed = VideoEmbed::new(embed_html, url.clone()); let mut tags: Vec = Vec::new(); // Placeholder for tags, adjust as needed for tag in vid[0].split(" ").collect::>(){ if tag.starts_with("tag-") { let tag_name = tag.split("tag-").collect::>()[1] .to_string(); if !tag_name.is_empty() { tags.push(tag_name.replace("-", " ").to_string()); } } } let mut video_item = VideoItem::new( id, title, embed.source.clone(), "perverzija".to_string(), thumb, duration, ).tags(tags); // .embed(embed.clone()); let mut format = videos::VideoFormat::new(url.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); } else { video_item.formats = Some(vec![format]); } items.push(video_item); } return items; } async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec { let raw_videos = html .split("video-item post") .collect::>()[1..] .to_vec(); let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone())); let results: Vec> = join_all(futures).await; let items: Vec = results .into_iter() .filter_map(Result::ok) .collect(); return items; } async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result { let vid = snippet.split("\n").collect::>(); if vid.len() > 30 { return Err("Unexpected video snippet length".into()); } let mut title = vid[5].split(" title=\"").collect::>()[1] .split("\"") .collect::>()[0] .to_string(); title = decode(title.as_bytes()).to_string().unwrap_or(title); let thumb = match vid[6].split(" src=\"").collect::>().len(){ 1=>{ for (index,line) in vid.iter().enumerate() { println!("Line {}: {}", index, line.to_string().trim()); } return Err("Failed to parse thumbnail URL".into()); } _ => vid[6].split(" src=\"").collect::>()[1] .split("\"") .collect::>()[0] .to_string(), }; let duration = 0; let lookup_url = vid[5].split(" href=\"").collect::>()[1] .split("\"") .collect::>()[0] .to_string(); let referer_url = "https://xtremestream.xyz/".to_string(); let mut conn = pool.get().expect("couldn't get db connection from pool"); let db_result = db::get_video(&mut conn,lookup_url.clone()); match db_result { Ok(Some(url)) => { if url.starts_with("!"){ return Err("Video was removed".into()); } let mut id = url.split("data=").collect::>()[1] .to_string(); if id.contains("&"){ id = id.split("&").collect::>()[0].to_string() } let mut video_item = VideoItem::new( id, title, url.clone(), "perverzija".to_string(), thumb, duration, ); let mut format = videos::VideoFormat::new(url.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); } else { video_item.formats = Some(vec![format]); } return Ok(video_item); } Ok(None) => { }, Err(e) => { println!("Error fetching video from database: {}", e); // return Err(format!("Error fetching video from database: {}", e).into()); } } drop(conn); let client = match env::var("BURP_URL").as_deref() { Ok(burp_url) => reqwest::Client::builder() .user_agent(USER_AGENT) .proxy(Proxy::https(burp_url).unwrap()) .danger_accept_invalid_certs(true) .build()?, Err(_) => reqwest::Client::builder() .user_agent(USER_AGENT) .danger_accept_invalid_certs(true) .build()?, }; let response = client.get(lookup_url.clone()).send().await?; let text = match response.status().is_success(){ true => response.text().await?, false => { println!("Failed to fetch video details"); return Err("Failed to fetch video details".into()); } }; let mut url = text.split("