diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 5184bf1..f10f600 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -4,11 +4,7 @@ use rustc_hash::FxHashMap as HashMap; use std::sync::Arc; use crate::{ - DbPool, - api::ClientVersion, - status::Channel, - util::cache::VideoCache, - videos::{ServerOptions, VideoItem}, + DbPool, api::ClientVersion, status::Channel, util::cache::VideoCache, videos::{ServerOptions, VideoItem} }; pub mod all; @@ -38,6 +34,7 @@ pub mod youjizz; pub mod beeg; pub mod tnaflix; pub mod pornxp; +pub mod rule34gen; // convenient alias pub type DynProvider = Arc; @@ -48,6 +45,7 @@ pub static ALL_PROVIDERS: Lazy> = Lazy::new(| m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider); m.insert("tnaflix", Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider); m.insert("pornxp", Arc::new(pornxp::PornxpProvider::new()) as DynProvider); + m.insert("rule34gen", Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider); // add more here as you migrate them m }); diff --git a/src/providers/rule34gen.rs b/src/providers/rule34gen.rs new file mode 100644 index 0000000..f6bd722 --- /dev/null +++ b/src/providers/rule34gen.rs @@ -0,0 +1,288 @@ +use crate::api::*; +use crate::status::*; +use crate::util::parse_abbreviated_number; +use crate::DbPool; +use crate::providers::Provider; +use crate::util::cache::VideoCache; +use crate::util::time::parse_time_to_seconds; +use crate::videos::{ServerOptions, VideoItem}; +use error_chain::error_chain; +use htmlentity::entity::{ICodedDataTrait, decode}; +use std::vec; +use std::time::{SystemTime, UNIX_EPOCH}; +use async_trait::async_trait; + +error_chain! { + foreign_links { + Io(std::io::Error); + HttpRequest(wreq::Error); + } +} + +#[derive(Debug, Clone)] +pub struct Rule34genProvider { + url: String, +} +impl Rule34genProvider { + pub fn new() -> Self { + Rule34genProvider { + url: "https://rule34gen.com".to_string(), + } + } + +fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; + Channel { + id: "rule34gen".to_string(), + name: "Rule34Gen".to_string(), + description: "If it exists, here might be an AI generated video of it".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34gen.com".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "post_date".to_string(), + title: "Newest".to_string(), + }, + FilterOption { + id: "video_viewed".to_string(), + title: "Most Viewed".to_string(), + }, + FilterOption { + id: "rating".to_string(), + title: "Top Rated".to_string(), + }, + FilterOption { + id: "duration".to_string(), + title: "Longest".to_string(), + }, + FilterOption { + id: "pseudo_random".to_string(), + title: "Random".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: Some(1800), + } + } + + async fn get( + &self, + cache: VideoCache, + page: u8, + sort: &str, + options: ServerOptions + ) -> Result> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + + let timestamp_millis = now.as_millis(); // u128 + let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"]; + let sort = if expected_sorts.contains(&sort) { + sort + } else { + "post_date" + }; + + let index = format!("rule34gen:{}:{}", page, sort); + + let url = format!("{}/?mode=async&function=get_block&block_id=custom_list_videos_most_recent_videos&tag_ids=&sort_by={}&from={}&_={}", self.url, sort, page, timestamp_millis); + + let mut old_items: Vec = vec![]; + if !(sort == "pseudo_random") { + old_items = match cache.get(&index) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + println!("Cache hit for URL: {}", url); + return Ok(items.clone()); + } else { + items.clone() + } + } + None => { + vec![] + } + }; + } + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&url).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone()); + if !video_items.is_empty() { + cache.remove(&url); + cache.insert(url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + async fn query( + &self, + cache: VideoCache, + page: u8, + query: &str, + sort: &str, + options: ServerOptions + ) -> Result> { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"); + let timestamp_millis = now.as_millis(); // u128 + let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"]; + let sort = if expected_sorts.contains(&sort) { + sort + } else { + "post_date" + }; + + let index = format!("rule34gen:{}:{}:{}", page, sort, query); + + let url = format!("{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}", self.url, query.replace(" ","-"), sort, page, page, timestamp_millis); + + // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&index) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } else { + let _ = cache.check().await; + return Ok(items.clone()); + } + } + None => { + vec![] + } + }; + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&url).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone()); + if !video_items.is_empty() { + cache.remove(&url); + cache.insert(url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + fn get_video_items_from_html(&self, html: String) -> Vec { + if html.is_empty() { + println!("HTML is empty"); + return vec![]; + } + let mut items: Vec = Vec::new(); + let video_listing_content = html.split("
>()[1].split("
>()[0].to_string(); + let raw_videos = video_listing_content + .split("
>()[1..] + .to_vec(); + for video_segment in &raw_videos { + // let vid = video_segment.split("\n").collect::>()[1] + // for (index, line) in vid.iter().enumerate() { + // println!("Line {}: {}", index, line); + // } + + if video_segment.contains("https://rule34gen.com/images/advertisements"){ + continue; + } + + let mut title = video_segment.split("
").collect::>()[1] + .split("<") + .collect::>()[0] + .to_string(); + // html decode + title = decode(title.as_bytes()).to_string().unwrap_or(title); + let id = video_segment.split("https://rule34gen.com/video/").collect::>()[1].split("/").collect::>()[0].to_string(); + let raw_duration = video_segment.split("
").collect::>()[1] + .split("<") + .collect::>()[0] + .to_string(); + let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; + let views = parse_abbreviated_number(&video_segment + .split("
").collect::>()[1].split("").collect::>()[1] + .split("<") + .collect::>()[0]).unwrap_or(0); +//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/ +//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/ + let thumb = video_segment.split(">()[1].split("data-original=\"").collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(); + let url = video_segment.split(">()[1] + .split("\"") + .collect::>()[0] + .to_string(); + // let preview = video_segment.split("
>()[1] + // .split("\"") + // .collect::>()[0] + // .to_string(); + + + let video_item = VideoItem::new( + id, + title, + url.to_string(), + "rule34gen".to_string(), + thumb, + duration, + ) + .views(views) + // .preview(preview) + ; + + + items.push(video_item); + } + return items; + } + + +} + +#[async_trait] +impl Provider for Rule34genProvider { + async fn get_videos( + &self, + cache: VideoCache, + pool: DbPool, + sort: String, + query: Option, + page: String, + per_page: String, + options: ServerOptions, + ) -> Vec { + let _ = options; + let _ = per_page; + let _ = pool; // Ignored in this implementation + let videos: std::result::Result, Error> = match query { + Some(q) => { + self.query(cache, page.parse::().unwrap_or(1), &q, &sort, options) + .await + } + None => { + self.get(cache, page.parse::().unwrap_or(1), &sort, options) + .await + } + }; + match videos { + Ok(v) => v, + Err(e) => { + println!("Error fetching videos: {}", e); + vec![] + } + } + } + + fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { + self.build_channel(clientversion) + } +}