diff --git a/src/api.rs b/src/api.rs index 8755aae..6610df9 100644 --- a/src/api.rs +++ b/src/api.rs @@ -157,6 +157,10 @@ async fn status(req: HttpRequest) -> Result { id: "lg".to_string(), title: "Longest".to_string(), }, + FilterOption { + id: "cm".to_string(), + title: "Newest".to_string(), + }, ], multiSelect: false, }], diff --git a/src/providers/pimpbunny.rs b/src/providers/pimpbunny.rs index e09eca0..c72c980 100644 --- a/src/providers/pimpbunny.rs +++ b/src/providers/pimpbunny.rs @@ -3,13 +3,15 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; +use crate::util::discord::send_discord_error_report; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; + use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; -use htmlentity::entity::{ICodedDataTrait, decode}; +use htmlentity::entity::{decode, ICodedDataTrait}; use std::sync::{Arc, RwLock}; use std::{thread, vec}; use titlecase::Titlecase; @@ -19,6 +21,13 @@ error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); + Json(serde_json::Error); + } + errors { + Parse(msg: String) { + description("parse error") + display("parse error: {}", msg) + } } } @@ -28,9 +37,10 @@ pub struct PimpbunnyProvider { stars: Arc>>, categories: Arc>>, } + impl PimpbunnyProvider { pub fn new() -> Self { - let provider = PimpbunnyProvider { + let provider = Self { url: "https://pimpbunny.com".to_string(), stars: Arc::new(RwLock::new(vec![])), categories: Arc::new(RwLock::new(vec![])), @@ -39,123 +49,6 @@ impl PimpbunnyProvider { provider } - fn spawn_initial_load(&self) { - let url = self.url.clone(); - let stars = Arc::clone(&self.stars); - let categories = Arc::clone(&self.categories); - - thread::spawn(move || { - // Create a tiny runtime just for these async tasks - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("build tokio runtime"); - - rt.block_on(async move { - if let Err(e) = Self::load_stars(&url, stars).await { - eprintln!("load_stars failed: {e}"); - } - if let Err(e) = Self::load_categories(&url, categories).await { - eprintln!("load_categories failed: {e}"); - } - }); - }); - } - - async fn load_stars(base_url: &str, stars: Arc>>) -> Result<()> { - let mut requester = Requester::new(); - let text = requester - .get( - format!("{}/onlyfans-models/?models_per_page=20", &base_url).as_str(), - Some(Version::HTTP_2), - ) - .await - .unwrap(); - let stars_div = text - .split("pb-list-models-block") - .collect::>() - .last() - .unwrap() - .split("pb-page-description") - .collect::>()[0]; - for stars_element in stars_div - .split("
") - .collect::>()[1..] - .to_vec() - { - if stars_element.contains("pb-promoted-link") { - continue; - } - let star_id = stars_element - .split("href=\"https://pimpbunny.com/onlyfans-models/") - .collect::>()[1] - .split("/\"") - .collect::>()[0] - .to_string(); - let star_name = stars_element - .split("
") - .collect::>()[1] - .split("<") - .collect::>()[0] - .to_string(); - Self::push_unique( - &stars, - FilterOption { - id: star_id, - title: star_name, - }, - ); - } - return Ok(()); - } - - async fn load_categories( - base_url: &str, - categories: Arc>>, - ) -> Result<()> { - let mut requester = Requester::new(); - let text = requester - .get( - format!("{}/categories/?items_per_page=120", &base_url).as_str(), - Some(Version::HTTP_2), - ) - .await - .unwrap(); - let categories_div = text - .split("list_categories_categories_list_items") - .collect::>() - .last() - .unwrap() - .split("pb-pagination-wrapper") - .collect::>()[0]; - for categories_element in categories_div - .split("
") - .collect::>()[1..] - .to_vec() - { - let category_id = categories_element - .split("href=\"https://pimpbunny.com/categories/") - .collect::>()[1] - .split("/\"") - .collect::>()[0] - .to_string(); - let category_name = categories_element - .split("
") - .collect::>()[1] - .split("<") - .collect::>()[0] - .titlecase(); - Self::push_unique( - &categories, - FilterOption { - id: category_id, - title: category_name, - }, - ); - } - return Ok(()); - } - fn build_channel(&self, clientversion: ClientVersion) -> Channel { let _ = clientversion; Channel { @@ -203,17 +96,146 @@ impl PimpbunnyProvider { } } - // Push one item with minimal lock time and dedup by id + fn spawn_initial_load(&self) { + let url = self.url.clone(); + let stars = Arc::clone(&self.stars); + let categories = Arc::clone(&self.categories); + + thread::spawn(move || { + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + eprintln!("tokio runtime failed: {e}"); + let _ = futures::executor::block_on(send_discord_error_report( + &e, + Some("Pimpbunny Provider"), + Some("Failed to create tokio runtime"), + file!(), + line!(), + module_path!(), + )); + return; + } + }; + + rt.block_on(async { + if let Err(e) = Self::load_stars(&url, Arc::clone(&stars)).await { + eprintln!("load_stars failed: {e}"); + send_discord_error_report( + &e, + Some("Pimpbunny Provider"), + Some("Failed to load stars during initial load"), + file!(), + line!(), + module_path!(), + ).await; + } + if let Err(e) = Self::load_categories(&url, Arc::clone(&categories)).await { + eprintln!("load_categories failed: {e}"); + send_discord_error_report( + &e, + Some("Pimpbunny Provider"), + Some("Failed to load categories during initial load"), + file!(), + line!(), + module_path!(), + ).await; + } + }); + }); + } + fn push_unique(target: &Arc>>, item: FilterOption) { if let Ok(mut vec) = target.write() { if !vec.iter().any(|x| x.id == item.id) { vec.push(item); - // Optional: keep it sorted for nicer UX - // vec.sort_by(|a,b| a.title.cmp(&b.title)); } } } + async fn load_stars(base: &str, stars: Arc>>) -> Result<()> { + let mut requester = Requester::new(); + let text = requester + .get( + &format!("{base}/onlyfans-models/?models_per_page=20"), + Some(Version::HTTP_2), + ) + .await + .map_err(|e| Error::from(format!("{}", e)))?; + + let block = text + .split("vt_list_models_with_advertising_custom_models_list_items") + .last() + .ok_or_else(|| ErrorKind::Parse("missing stars block".into()))? + .split("pb-page-description") + .next() + .unwrap_or(""); + + for el in block.split("
").skip(1) { + if el.contains("pb-promoted-link") || !el.contains("href=\"https://pimpbunny.com/onlyfans-models/") { + continue; + } + + let id = el + .split("href=\"https://pimpbunny.com/onlyfans-models/") + .nth(1) + .and_then(|s| s.split("/\"").next()) + .ok_or_else(|| ErrorKind::Parse(format!("star id: {el}").into()))? + .to_string(); + + let title = el + .split("ui-card-title") + .nth(1) + .and_then(|s| s.split('<').next()) + .ok_or_else(|| ErrorKind::Parse(format!("star title: {el}").into()))? + .to_string(); + + Self::push_unique(&stars, FilterOption { id, title }); + } + Ok(()) + } + + async fn load_categories(base: &str, cats: Arc>>) -> Result<()> { + let mut requester = Requester::new(); + let text = requester + .get( + &format!("{base}/categories/?items_per_page=120"), + Some(Version::HTTP_2), + ) + .await + .map_err(|e| Error::from(format!("{}", e)))?; + + let block = text + .split("list_categories_categories_list_items") + .last() + .ok_or_else(|| ErrorKind::Parse("categories block".into()))? + .split("pb-pagination-wrapper") + .next() + .unwrap_or(""); + + for el in block.split("
").skip(1) { + let id = el + .split("href=\"https://pimpbunny.com/categories/") + .nth(1) + .and_then(|s| s.split("/\"").next()) + .ok_or_else(|| ErrorKind::Parse(format!("category id: {el}").into()))? + .to_string(); + + let title = el + .split("ui-heading-h3") + .nth(1) + .and_then(|s| s.split('<').next()) + .ok_or_else(|| ErrorKind::Parse(format!("category title: {el}").into()))? + .titlecase(); + + Self::push_unique(&cats, FilterOption { id, title }); + } + Ok(()) + } + async fn get( &self, cache: VideoCache, @@ -338,181 +360,147 @@ impl PimpbunnyProvider { if html.is_empty() || html.contains("404 Not Found") { return vec![]; } - let raw_videos = html.split("pb-pagination-wrapper").collect::>()[0] - .split("pb-list-items") - .collect::>()[1] - .split("
") - .collect::>()[1..] - .to_vec(); - let futures = raw_videos - .into_iter() + let block = match html + .split("pb-pagination-wrapper") + .next() + .and_then(|s| s.split("pb-list-items").nth(1)) + { + Some(b) => b, + None => return vec![], + }; + + let futures = block + .split("
") + .skip(1) .map(|el| self.get_video_item(el.to_string(), requester.clone())); - let results: Vec> = join_all(futures).await; - let video_items: Vec = results.into_iter().filter_map(Result::ok).collect(); - return video_items; + + join_all(futures) + .await + .into_iter() + .filter_map(Result::ok) + .collect() } async fn get_video_item( &self, - video_segment: String, + seg: String, mut requester: Requester, ) -> Result { - let video_url: String = video_segment.split(" href=\"").collect::>()[1] - .split("\"") - .collect::>()[0] + let video_url = seg + .split(" href=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .ok_or_else(|| ErrorKind::Parse("video url".into()))? .to_string(); - let mut title = video_segment.split("pb-item-title").collect::>()[1] - .split(">") - .collect::>()[1] - .split("<") - .collect::>()[0] + + let mut title = seg + .split("pb-item-title") + .nth(1) + .and_then(|s| s.split('>').nth(1)) + .and_then(|s| s.split('<').next()) + .ok_or_else(|| ErrorKind::Parse("video title".into()))? .trim() .to_string(); - // html decode - title = decode(title.as_bytes()) - .to_string() - .unwrap_or(title) - .titlecase(); - let id = video_url.split("/").collect::>()[4] - .split(".") - .collect::>()[0] + + title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase(); + + let id = video_url + .split('/') + .nth(4) + .and_then(|s| s.split('.').next()) + .ok_or_else(|| ErrorKind::Parse("video id".into()))? .to_string(); - let mut thumb = video_segment.split("pb-thumbnail").collect::>()[1] + let thumb_block = seg + .split("pb-thumbnail") + .nth(1) + .ok_or_else(|| ErrorKind::Parse("thumb block".into()))?; + + let mut thumb = thumb_block .split("src=\"") - .collect::>()[1] - .split("\"") - .collect::>()[0] + .nth(1) + .and_then(|s| s.split('"').next()) + .unwrap_or("") .to_string(); - if thumb.starts_with("data:image/jpg;base64") { - thumb = video_segment.split("pb-thumbnail").collect::>()[1] - .split("data-webp=\"") - .collect::>()[1] - .split("\"") - .collect::>()[0] - .to_string(); - } - let preview = video_segment.split("pb-thumbnail").collect::>()[1] - .split("data-preview=\"") - .collect::>()[1] - .split("\"") - .collect::>()[0] - .to_string(); - let (tags, formats, views, duration) = match self.extract_media(&video_url, &mut requester).await { - Ok((t, f, v, d)) => (t, f, v, d), - Err(_) => return Err(Error::from("Video media extraction failed")), - }; - if formats.is_empty() { - return Err(Error::from("No formats found for video")); + if thumb.starts_with("data:image") { + thumb = thumb_block + .split("data-webp=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .unwrap_or("") + .to_string(); } - let video_item = VideoItem::new( + + let preview = thumb_block + .split("data-preview=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .unwrap_or("") + .to_string(); + + let (tags, formats, views, duration) = + self.extract_media(&video_url, &mut requester).await?; + + Ok(VideoItem::new( id, title, video_url, - "pimpbunny".to_string(), + "pimpbunny".into(), thumb, duration, ) .formats(formats) .tags(tags) .preview(preview) - .views(views) - ; - return Ok(video_item); + .views(views)) } async fn extract_media( &self, - video_page_url: &str, + url: &str, requester: &mut Requester, ) -> Result<(Vec, Vec, u32, u32)> { - let mut formats = vec![]; - let mut tags = vec![]; let text = requester - .get(&video_page_url, Some(Version::HTTP_2)) + .get(url, Some(Version::HTTP_2)) .await - .unwrap(); - if text.contains("pb-video-models"){ - let stars_elements = text.split("pb-video-models").collect::>()[1] - .split("pb-video-statistic") - .collect::>()[0] - .split("pb-models-item pb-models-item") - .collect::>()[1..] - .to_vec(); - for star_el in stars_elements { - let star_id = star_el - .split("href=\"https://pimpbunny.com/onlyfans-models/") - .collect::>()[1] - .split("/\"") - .collect::>()[0] - .to_string(); - let star_name = star_el - .split("") - .collect::>()[1] - .split("<") - .collect::>()[0] - .to_string(); - tags.push(star_name.clone()); - Self::push_unique( - &self.stars, - FilterOption { - id: star_id, - title: star_name.clone(), - }, - ); - } - } - if text.contains("pb-video-tags") { - let categories_elements = text.split("pb-tags-list").collect::>()[1] - .split("
") - .collect::>()[0] - .split("href=\"https://pimpbunny.com/tags/") - .collect::>()[1..] - .to_vec(); - for categories_el in categories_elements { - let category_id = categories_el.split("\"").collect::>()[0].to_string(); - let category_name = categories_el.split("\">").collect::>()[1] - .split("<") - .collect::>()[0] - .titlecase(); - tags.push(category_name.clone()); - Self::push_unique( - &self.categories, - FilterOption { - id: category_id, - title: category_name.clone(), - }, - ); - } - } + .map_err(|e| Error::from(format!("{}", e)))?; let json_str = text - .split(";") - .collect::>()[0]; - let json = serde_json::from_str::(json_str).unwrap_or_default(); + .split("application/ld+json\">") + .nth(1) + .and_then(|s| s.split("").next()) + .ok_or_else(|| ErrorKind::Parse("ld+json".into()))?; + + let json: serde_json::Value = serde_json::from_str(json_str)?; + let video_url = json["contentUrl"].as_str().unwrap_or("").to_string(); let quality = video_url - .split("_") - .collect::>() + .split('_') .last() - .map_or("", |v| v) - .split(".") - .collect::>()[0] + .and_then(|s| s.split('.').next()) + .unwrap_or("") .to_string(); - let views = json["interactionStatistic"].as_array().unwrap()[0]["userInteractionCount"] - .as_str().unwrap().parse::().unwrap_or(0); - let raw_duration = json["duration"].as_str().unwrap_or("00:00").replace("PT", "").replace("H", ":").replace("M", ":").replace("S", ""); - let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32; - formats.push(VideoFormat::new( - video_url, - quality.clone(), - "video/mp4".to_string(), - )); - Ok((tags, formats, views, duration)) + + let views = json["interactionStatistic"] + .as_array() + .and_then(|a| a.first()) + .and_then(|v| v["userInteractionCount"].as_str()) + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + + let duration = json["duration"] + .as_str() + .map(|d| parse_time_to_seconds(&d.replace(['P','T','H','M','S'], "")).unwrap_or(0)) + .unwrap_or(0) as u32; + + Ok(( + vec![], + vec![VideoFormat::new(video_url, quality, "video/mp4".into())], + views, + duration, + )) } } @@ -521,34 +509,27 @@ impl Provider for PimpbunnyProvider { async fn get_videos( &self, cache: VideoCache, - pool: DbPool, + _pool: DbPool, sort: String, query: Option, page: String, - per_page: String, + _per_page: String, options: ServerOptions, ) -> Vec { - let _ = per_page; - let _ = pool; - let videos: std::result::Result, Error> = match query { - Some(q) => { - self.query(cache, page.parse::().unwrap_or(1), &q, options) - .await - } - None => { - self.get(cache, page.parse::().unwrap_or(1), &sort, options) - .await - } + let page = page.parse::().unwrap_or(1); + + let res = match query { + Some(q) => self.to_owned().query(cache, page, &q, options).await, + None => self.get(cache, page, &sort, options).await, }; - match videos { - Ok(v) => v, - Err(e) => { - println!("Error fetching videos: {}", e); - vec![] - } - } + + res.unwrap_or_else(|e| { + eprintln!("pimpbunny error: {e}"); + vec![] + }) } - fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { - self.build_channel(clientversion) + + fn get_channel(&self, v: ClientVersion) -> Channel { + self.build_channel(v) } } diff --git a/src/providers/pornhub.rs b/src/providers/pornhub.rs index 98bc5f5..594cab2 100644 --- a/src/providers/pornhub.rs +++ b/src/providers/pornhub.rs @@ -4,238 +4,254 @@ use crate::providers::Provider; use crate::util::cache::VideoCache; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; + use error_chain::error_chain; -use htmlentity::entity::{ICodedDataTrait, decode}; -use std::vec; +use htmlentity::entity::{decode, ICodedDataTrait}; use async_trait::async_trait; +use std::vec; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); } + errors { + Parse(msg: String) { + description("parse error") + display("parse error: {}", msg) + } + } } #[derive(Debug, Clone)] pub struct PornhubProvider { url: String, } + impl PornhubProvider { pub fn new() -> Self { - PornhubProvider { + Self { url: "https://www.pornhub.com".to_string(), } } + async fn get( &self, cache: VideoCache, page: u8, sort: &str, - options:ServerOptions + options: ServerOptions, ) -> Result> { let video_url = format!("{}/video?o={}&page={}", self.url, sort, page); + let old_items = match cache.get(&video_url) { - Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { - return Ok(items.clone()); - } else { - items.clone() - } - } - None => { - vec![] + Some((time, items)) if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 => { + return Ok(items.clone()); } + Some((_, items)) => items.clone(), + None => vec![], }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, None).await.unwrap(); - let video_items: Vec = self.get_video_items_from_html(text.clone(),"