diff --git a/src/api.rs b/src/api.rs index 8980f37..5e10a38 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1225,6 +1225,11 @@ async fn videos_post( .as_deref() .unwrap_or("") .to_string(); + let duration = video_request + .duration + .as_deref() + .unwrap_or("") + .to_string(); let options = ServerOptions { featured: Some(featured), category: Some(category), @@ -1235,6 +1240,7 @@ async fn videos_post( network: Some(network), stars: Some(stars), categories: Some(categories), + duration: Some(duration), }; let video_items = provider .get_videos( diff --git a/src/providers/hanime.rs b/src/providers/hanime.rs index 27d6d7d..4f3f816 100644 --- a/src/providers/hanime.rs +++ b/src/providers/hanime.rs @@ -2,8 +2,7 @@ use std::vec; use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; -use wreq::Client; -use wreq_util::Emulation; + use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 44307e8..9c23004 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -36,6 +36,7 @@ pub mod paradisehill; pub mod pornzog; pub mod youjizz; pub mod beeg; +pub mod tnaflix; // convenient alias pub type DynProvider = Arc; @@ -44,6 +45,7 @@ pub static ALL_PROVIDERS: Lazy> = Lazy::new(| let mut m = HashMap::default(); m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider); m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider); + m.insert("tnaflix", Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider); // add more here as you migrate them m }); diff --git a/src/providers/tnaflix.rs b/src/providers/tnaflix.rs new file mode 100644 index 0000000..7f66d31 --- /dev/null +++ b/src/providers/tnaflix.rs @@ -0,0 +1,561 @@ +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::Provider; +use crate::status::*; +use crate::util::cache::VideoCache; +use crate::util::parse_abbreviated_number; +use crate::util::time::parse_time_to_seconds; +use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; +use error_chain::error_chain; +use htmlentity::entity::{ICodedDataTrait, decode}; +// use std::sync::{Arc, RwLock}; +// use std::thread; +use std::vec; + +error_chain! { + foreign_links { + Io(std::io::Error); + HttpRequest(wreq::Error); + } +} + +#[derive(Debug, Clone)] +pub struct TnaflixProvider { + url: String, + // sites: Arc>>, + // categories: Arc>>, + // stars: Arc>>, +} +impl TnaflixProvider { + pub fn new() -> Self { + let provider = TnaflixProvider { + url: "https://www.tnaflix.com".to_string(), + // sites: Arc::new(RwLock::new(vec![FilterOption { + // id: "all".to_string(), + // title: "All".to_string(), + // }])), + // categories: Arc::new(RwLock::new(vec![FilterOption { + // id: "all".to_string(), + // title: "All".to_string(), + // }])), + // stars: Arc::new(RwLock::new(vec![FilterOption { + // id: "all".to_string(), + // title: "All".to_string(), + // }])), + }; + + // Kick off the background load but return immediately + // provider.spawn_initial_load(); + provider + } + + // fn spawn_initial_load(&self) { + // let url = self.url.clone(); + // let sites = Arc::clone(&self.sites); + // let categories = Arc::clone(&self.categories); + // let stars = Arc::clone(&self.stars); + + // thread::spawn(move || { + // // Create a tiny runtime just for these async tasks + // let rt = tokio::runtime::Builder::new_current_thread() + // .enable_all() + // .build() + // .expect("build tokio runtime"); + + // rt.block_on(async move { + // // If you have a streaming sites loader, call it here too + // if let Err(e) = Self::load_sites(&url, sites).await { + // eprintln!("load_sites_into failed: {e}"); + // } + + // if let Err(e) = Self::load_categories(&url, categories).await { + // eprintln!("load_categories failed: {e}"); + // } + + // if let Err(e) = Self::load_stars(&url, stars).await { + // eprintln!("load_stars failed: {e}"); + // } + // }); + // }); + // } + + // async fn load_stars(base_url: &str, stars: Arc>>) -> Result<()> { + // let mut requester = util::requester::Requester::new(); + // for page in [1..10].into_iter().flatten() { + // let text = requester + // .get(format!("{}/pornstars?page={}", &base_url, page).as_str()) + // .await + // .unwrap(); + // if text.contains("404 Not Found") || text.is_empty() { + // break; + // } + // let stars_div = text + // .split("Hall of Fame Pornstars") + // .collect::>()[1] + // .split("pagination") + // .collect::>()[0]; + // for stars_element in stars_div.split(">()[1..].to_vec() { + // let star_url = stars_element.split("href=\"").collect::>()[1] + // .split("\"") + // .collect::>()[0]; + // let star_id = star_url.split("/").collect::>()[4].to_string(); + // let star_name = stars_element.split("title=\"").collect::>()[1] + // .split("\"") + // .collect::>()[0] + // .to_string(); + // Self::push_unique( + // &stars, + // FilterOption { + // id: star_id, + // title: star_name, + // }, + // ); + // } + // } + // return Ok(()); + // } + + // async fn load_sites(base_url: &str, sites: Arc>>) -> Result<()> { + // let mut requester = util::requester::Requester::new(); + // let mut page = 0; + // loop { + // page += 1; + // let text = requester + // .get(format!("{}/sites/{}/", &base_url, page).as_str()) + // .await + // .unwrap(); + // if text.contains("404 Not Found") || text.is_empty() { + // break; + // } + // let sites_div = text + // .split("id=\"list_content_sources_sponsors_list_items\"") + // .collect::>()[1] + // .split("class=\"pagination\"") + // .collect::>()[0]; + // for sites_element in + // sites_div.split("class=\"headline\"").collect::>()[1..].to_vec() + // { + // let site_url = sites_element.split("href=\"").collect::>()[1] + // .split("\"") + // .collect::>()[0]; + // let site_id = site_url.split("/").collect::>()[4].to_string(); + // let site_name = sites_element.split("

").collect::>()[1] + // .split("<") + // .collect::>()[0] + // .to_string(); + // Self::push_unique( + // &sites, + // FilterOption { + // id: site_id, + // title: site_name, + // }, + // ); + // } + // } + // return Ok(()); + // } + + // async fn load_networks(base_url: &str, networks: Arc>>) -> Result<()> { + // let mut requester = util::requester::Requester::new(); + // let text = requester.get(&base_url).await.unwrap(); + // let networks_div = text.split("class=\"sites__list\"").collect::>()[1] + // .split("") + // .collect::>()[0]; + // for network_element in + // networks_div.split("sites__item").collect::>()[1..].to_vec() + // { + // if network_element.contains("sites__all") { + // continue; + // } + // let network_url = network_element.split("href=\"").collect::>()[1] + // .split("\"") + // .collect::>()[0]; + // let network_id = network_url.split("/").collect::>()[4].to_string(); + // let network_name = network_element.split(">").collect::>()[1] + // .split("<") + // .collect::>()[0] + // .to_string(); + // Self::push_unique( + // &networks, + // FilterOption { + // id: network_id, + // title: network_name, + // }, + // ); + // } + // return Ok(()); + // } + + // Push one item with minimal lock time and dedup by id + // fn push_unique(target: &Arc>>, item: FilterOption) { + // if let Ok(mut vec) = target.write() { + // if !vec.iter().any(|x| x.id == item.id) { + // vec.push(item); + // // Optional: keep it sorted for nicer UX + // // vec.sort_by(|a,b| a.title.cmp(&b.title)); + // } + // } + // } + + fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; + // let sites: Vec = self + // .sites + // .read() + // .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + // .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + // let networks: Vec = self + // .networks + // .read() + // .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + // .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + // let stars: Vec = self + // .stars + // .read() + // .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + // .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + Channel { + id: "omgxxx".to_string(), + name: "OMG XXX".to_string(), + description: "OMG look at that Collection!".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.omg.xxx".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ + ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "new".into(), + title: "New".into(), + }, + FilterOption { + id: "featured".into(), + title: "Most Viewed".into(), + }, + FilterOption { + id: "toprated".into(), + title: "Top Rated".into(), + }, + ], + multiSelect: false, + }, + ChannelOption { + id: "duration".to_string(), + title: "Duration".to_string(), + description: "Length of the Videos".to_string(), + systemImage: "timer".to_string(), + colorName: "green".to_string(), + options: vec![ + FilterOption { + id: "all".into(), + title: "All".into(), + }, + FilterOption { + id: "short".into(), + title: "Short (1-3 min)".into(), + }, + FilterOption { + id: "medium".into(), + title: "Medium (3-10 min)".into(), + }, + FilterOption { + id: "long".into(), + title: "Long (10-30 min)".into(), + }, + FilterOption { + id: "full".into(), + title: "Full length (30+ min)".into(), + }, + ], + multiSelect: false, + }, + // ChannelOption { + // id: "sites".to_string(), + // title: "Sites".to_string(), + // description: "Filter for different Sites".to_string(), + // systemImage: "rectangle.stack".to_string(), + // colorName: "green".to_string(), + // options: sites, + // multiSelect: false, + // }, + // ChannelOption { + // id: "networks".to_string(), + // title: "Networks".to_string(), + // description: "Filter for different Networks".to_string(), + // systemImage: "list.dash".to_string(), + // colorName: "purple".to_string(), + // options: networks, + // multiSelect: false, + // }, + // ChannelOption { + // id: "stars".to_string(), + // title: "Stars".to_string(), + // description: "Filter for different Pornstars".to_string(), + // systemImage: "star.fill".to_string(), + // colorName: "yellow".to_string(), + // options: stars, + // multiSelect: false, + // }, + ], + nsfw: true, + cacheDuration: None, + } + } + + async fn get( + &self, + cache: VideoCache, + page: u8, + sort: &str, + options: ServerOptions, + ) -> Result> { + let sort_string: String = match sort { + "featured" => "featured".to_string(), + "toprated" => "toprated".to_string(), + _ => "new".to_string(), + }; + let duration_string: String = match options.duration.unwrap_or("all".to_string()).as_str() { + "short" => "short".to_string(), + "medium" => "medium".to_string(), + "long" => "long".to_string(), + "full" => "full".to_string(), + _ => "all".to_string(), + }; + // if options.network.is_some() + // && !options.network.as_ref().unwrap().is_empty() + // && options.network.as_ref().unwrap() != "all" + // { + // sort_string = format!( + // "networks/{}{}", + // options.network.as_ref().unwrap(), + // alt_sort_string + // ); + // } + // if options.sites.is_some() + // && !options.sites.as_ref().unwrap().is_empty() + // && options.sites.as_ref().unwrap() != "all" + // { + // sort_string = format!( + // "sites/{}{}", + // options.sites.as_ref().unwrap(), + // alt_sort_string + // ); + // } + // if options.stars.is_some() + // && !options.stars.as_ref().unwrap().is_empty() + // && options.stars.as_ref().unwrap() != "all" + // { + // sort_string = format!( + // "models/{}{}", + // options.stars.as_ref().unwrap(), + // alt_sort_string + // ); + // } + let video_url = format!( + "{}/{}/{}?d={}", + self.url, sort_string, page, duration_string + ); + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + println!("Cache hit for URL: {}", video_url); + return Ok(items.clone()); + } else { + items.clone() + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&video_url).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone()); + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + async fn query( + &self, + cache: VideoCache, + page: u8, + query: &str, + options: ServerOptions, + ) -> Result> { + let search_string = query.to_string().to_lowercase().trim().replace(" ", "+"); + let duration_string: String = match options.duration.unwrap_or("all".to_string()).as_str() { + "short" => "short".to_string(), + "medium" => "medium".to_string(), + "long" => "long".to_string(), + "full" => "full".to_string(), + _ => "all".to_string(), + }; + let video_url = format!( + "{}/search?what={}&d={}&page={}", + self.url, search_string, duration_string, page + ); + // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } else { + let _ = cache.check().await; + return Ok(items.clone()); + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + + let text = requester.get(&video_url).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone()); + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + fn get_video_items_from_html(&self, html: String) -> Vec { + if html.is_empty() { + println!("HTML is empty"); + return vec![]; + } + let mut items: Vec = Vec::new(); + let raw_videos = html.split("pagination ").collect::>()[0] + .split("row video-list") + .collect::>()[1] + .split("col-xs-6 col-md-4 col-xl-3 mb-3") + .collect::>()[1..] + .to_vec(); + for video_segment in &raw_videos { + // let vid = video_segment.split("\n").collect::>(); + // for (index, line) in vid.iter().enumerate() { + // println!("Line {}: {}", index, line); + // } + let video_url: String = video_segment.split(" href=\"").collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(); + let mut title = video_segment + .split("class=\"video-title text-break\">") + .collect::>()[1] + .split("<") + .collect::>()[0] + .trim() + .to_string(); + // html decode + title = decode(title.as_bytes()).to_string().unwrap_or(title); + let id = video_url.split("/").collect::>()[5].to_string(); + + let thumb = match video_segment.contains("data-src=\""){ + true => video_segment.split("data-src=\"").collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(), + false => video_segment.split(">()[1] + .split("\"") + .collect::>()[0] + .to_string(), + }; + let raw_duration = video_segment + .split("thumb-icon video-duration\">") + .collect::>()[1] + .split("<") + .collect::>()[0] + .to_string(); + let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32; + let views = match video_segment.contains("icon-eye\">") { + true => parse_abbreviated_number( + video_segment + .split("icon-eye\">") + .collect::>()[1] + .split("<") + .collect::>()[0] + .trim(), + ) + .unwrap_or(0) as u32, + false => 0, + }; + + let preview = video_segment + .split("data-trailer=\"") + .collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(); + + let video_item = VideoItem::new( + id, + title, + video_url.to_string(), + "tnaflix".to_string(), + thumb, + duration, + ) + .views(views) + .preview(preview); + items.push(video_item); + } + return items; + } +} + +#[async_trait] +impl Provider for TnaflixProvider { + async fn get_videos( + &self, + cache: VideoCache, + pool: DbPool, + sort: String, + query: Option, + page: String, + per_page: String, + options: ServerOptions, + ) -> Vec { + let _ = per_page; + let _ = pool; + let videos: std::result::Result, Error> = match query { + Some(q) => { + self.query(cache, page.parse::().unwrap_or(1), &q, options) + .await + } + None => { + self.get(cache, page.parse::().unwrap_or(1), &sort, options) + .await + } + }; + match videos { + Ok(v) => v, + Err(e) => { + println!("Error fetching videos: {}", e); + vec![] + } + } + } + fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { + self.build_channel(clientversion) + } +} diff --git a/src/videos.rs b/src/videos.rs index d9b4b56..79f0f20 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -33,6 +33,7 @@ pub struct VideosRequest { pub networks: Option, // pub stars: Option, // pub categories: Option, + pub duration: Option, } #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] @@ -46,6 +47,7 @@ pub struct ServerOptions { pub network: Option, // pub stars: Option, // pub categories: Option, // + pub duration: Option, // } #[derive(serde::Serialize, Debug)]