use std::vec; use std::env; use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use futures::future::join_all; use serde::Serialize; use wreq::Client; use wreq_util::Emulation; use serde::Deserialize; use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; use crate::videos::ServerOptions; use crate::videos::{self, VideoEmbed, VideoItem}; use crate::DbPool; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); JsonError(serde_json::Error); } } #[derive(Debug, Deserialize, Serialize)] struct PerverzijaDbEntry { url_string: String, tags_strings: Vec, } #[derive(Debug, Clone)] pub struct PerverzijaProvider { url: String, } impl PerverzijaProvider { pub fn new() -> Self { PerverzijaProvider { url: "https://tube.perverzija.com/".to_string(), } } async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result> { let mut prefix_uri = "".to_string(); if featured == "featured" { prefix_uri = "featured-scenes/".to_string(); } let mut url_str = format!("{}{}page/{}/", self.url, prefix_uri, page); if page == 1 { url_str = format!("{}{}", self.url, prefix_uri); } let old_items = match cache.get(&url_str) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { println!("Cache hit for URL: {}", url_str); return Ok(items.clone()); } else{ items.clone() } } None => { vec![] } }; let client = Client::builder() .emulation(Emulation::Firefox136) .build()?; let response = client.get(url_str.clone()).send().await?; // print!("Response: {:?}\n", response); if response.status().is_success() { let text = response.text().await?; let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); if !video_items.is_empty() { cache.remove(&url_str); cache.insert(url_str.clone(), video_items.clone()); } else{ return Ok(old_items); } Ok(video_items) } else { let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), url: url_str.clone(), maxTimeout: 60000, }) .await; let video_items = match result { Ok(res) => { // println!("FlareSolverr response: {}", res); self.get_video_items_from_html(res.solution.response, pool) } Err(e) => { println!("Error solving FlareSolverr: {}", e); return Err("Failed to solve FlareSolverr".into()); } }; if !video_items.is_empty() { cache.remove(&url_str); cache.insert(url_str.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } } async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result> { let mut query_parse = true; let search_string = query.replace(" ", "+"); let mut url_str = format!( "{}page/{}/?s={}", self.url, page, search_string ); if page == 1 { url_str = format!("{}?s={}", self.url, search_string); } if query.starts_with("@studio:") { let studio_name = query.replace("@studio:", ""); url_str = format!("{}studio/{}/page/{}/", self.url, studio_name, page); query_parse = false; } else if query.starts_with("@stars:") { let stars_name = query.replace("@stars:", ""); url_str = format!("{}stars/{}/page/{}/", self.url, stars_name, page); query_parse = false; } url_str = url_str.replace("page/1/", ""); // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&url_str) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { return Ok(items.clone()); } else{ let _ = cache.check().await; return Ok(items.clone()) } } None => { vec![] } }; let client = Client::builder() .emulation(Emulation::Firefox136) .build()?; let response = client.get(url_str.clone()).send().await?; if response.status().is_success() { let text = response.text().await?; let video_items: Vec = match query_parse{ true => {self.get_video_items_from_html_query(text.clone(), pool).await}, false => {self.get_video_items_from_html(text.clone(), pool)} }; if !video_items.is_empty() { cache.remove(&url_str); cache.insert(url_str.clone(), video_items.clone()); } else{ return Ok(old_items); } Ok(video_items) } else { let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), url: url_str.clone(), maxTimeout: 60000, }) .await; let video_items = match result { Ok(res) => { self.get_video_items_from_html_query(res.solution.response, pool).await } Err(e) => { println!("Error solving FlareSolverr: {}", e); return Err("Failed to solve FlareSolverr".into()); } }; if !video_items.is_empty() { cache.remove(&url_str); cache.insert(url_str.clone(), video_items.clone()); } else{ return Ok(old_items); } Ok(video_items) } } fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec { if html.is_empty() { println!("HTML is empty"); return vec![]; } let mut items: Vec = Vec::new(); let video_listing_content = html.split("video-listing-content").collect::>()[1]; let raw_videos = video_listing_content .split("video-item post") .collect::>()[1..] .to_vec(); for video_segment in &raw_videos { let vid = video_segment.split("\n").collect::>(); if vid.len() > 20 { continue; } // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line.to_string().trim()); // } let mut title = vid[1].split(">").collect::>()[1] .split("<") .collect::>()[0] .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); if !vid[1].contains("iframe src="") { continue; } let url_str = vid[1].split("iframe src="").collect::>()[1] .split(""") .collect::>()[0] .to_string().replace("index.php", "xs1.php"); if url_str.starts_with("https://streamtape.com/"){ continue; // Skip Streamtape links } let id = url_str.split("data=").collect::>()[1] .split("&") .collect::>()[0] .to_string(); let raw_duration = match vid.len() { 10 => vid[6].split("time_dur\">").collect::>()[1] .split("<") .collect::>()[0] .to_string(), _ => "00:00".to_string(), }; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; if !vid[4].contains("srcset=") && vid[4].split("src=\"").collect::>().len() == 1{ for (index, line) in vid.iter().enumerate(){ println!("Line {}: {}\n\n", index, line); } } let mut thumb = "".to_string(); for v in vid.clone(){ let line = v.trim(); if line.starts_with(">()[1] .split("\"") .collect::>()[0] .to_string(); } } let embed_html = vid[1].split("data-embed='").collect::>()[1] .split("'") .collect::>()[0] .to_string(); let id_url = vid[1].split("data-url='").collect::>()[1] .split("'") .collect::>()[0] .to_string(); let mut conn = pool.get().expect("couldn't get db connection from pool"); let _ = db::insert_video(&mut conn, &id_url, &url_str); drop(conn); let referer_url = "https://xtremestream.xyz/".to_string(); let embed = VideoEmbed::new(embed_html, url_str.clone()); let mut tags: Vec = Vec::new(); // Placeholder for tags, adjust as needed let studios_parts = vid[7].split("a href=\"").collect::>(); for studio in studios_parts.iter().skip(1) { if studio.starts_with("https://tube.perverzija.com/studio/"){ tags.push( studio.split("/\"").collect::>()[0] .replace("https://tube.perverzija.com/studio/", "@studio:") .to_string(), ); } } for tag in vid[0].split(" ").collect::>(){ if tag.starts_with("stars-") { let tag_name = tag.split("stars-").collect::>()[1].split("\"").collect::>()[0] .to_string(); if !tag_name.is_empty() { tags.push(format!("@stars:{}", tag_name)); } } } for tag in vid[0].split(" ").collect::>(){ if tag.starts_with("tag-") { let tag_name = tag.split("tag-").collect::>()[1] .to_string(); if !tag_name.is_empty() { tags.push(tag_name.replace("-", " ").to_string()); } } } let mut video_item = VideoItem::new( id, title, embed.source.clone(), "perverzija".to_string(), thumb, duration, ).tags(tags); // .embed(embed.clone()); let mut format = videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); } else { video_item.formats = Some(vec![format]); } items.push(video_item); } return items; } async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec { let raw_videos = html .split("video-item post") .collect::>()[1..] .to_vec(); let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone())); let results: Vec> = join_all(futures).await; let items: Vec = results .into_iter() .filter_map(Result::ok) .collect(); return items; } async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result { let vid = snippet.split("\n").collect::>(); if vid.len() > 30 { return Err("Unexpected video snippet length".into()); } let mut title = vid[5].split(" title=\"").collect::>()[1] .split("\"") .collect::>()[0] .to_string(); title = decode(title.as_bytes()).to_string().unwrap_or(title); let thumb = match vid[6].split(" src=\"").collect::>().len(){ 1=>{ for (index,line) in vid.iter().enumerate() { println!("Line {}: {}", index, line.to_string().trim()); } return Err("Failed to parse thumbnail URL".into()); } _ => vid[6].split(" src=\"").collect::>()[1] .split("\"") .collect::>()[0] .to_string(), }; let duration = 0; let lookup_url = vid[5].split(" href=\"").collect::>()[1] .split("\"") .collect::>()[0] .to_string(); let referer_url = "https://xtremestream.xyz/".to_string(); let mut conn = pool.get().expect("couldn't get db connection from pool"); let db_result = db::get_video(&mut conn,lookup_url.clone()); match db_result { Ok(Some(entry)) => { if entry.starts_with("{"){ // replace old urls with new json objects let entry = serde_json::from_str::(entry.as_str())?; let url_str = entry.url_string; let tags = entry.tags_strings; if url_str.starts_with("!"){ return Err("Video was removed".into()); } let mut id = url_str.split("data=").collect::>()[1] .to_string(); if id.contains("&"){ id = id.split("&").collect::>()[0].to_string() } let mut video_item = VideoItem::new( id, title, url_str.clone(), "perverzija".to_string(), thumb, duration, ) .tags(tags) ; let mut format = videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); } else { video_item.formats = Some(vec![format]); } return Ok(video_item) } else{ let _ = db::delete_video(&mut conn,lookup_url.clone()); }; } Ok(None) => { }, Err(e) => { println!("Error fetching video from database: {}", e); // return Err(format!("Error fetching video from database: {}", e).into()); } } drop(conn); let client = Client::builder() .emulation(Emulation::Firefox136) .build()?; let response = client.get(lookup_url.clone()).send().await?; let text = match response.status().is_success(){ true => response.text().await?, false => { println!("Failed to fetch video details"); return Err("Failed to fetch video details".into()); } }; let mut url_str = text.split("