use crate::DbPool; use crate::api::*; use crate::providers::{Provider, report_provider_error}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::vec; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); } } #[derive(Debug, Clone)] pub struct Rule34genProvider { url: String, } impl Rule34genProvider { pub fn new() -> Self { Rule34genProvider { url: "https://rule34gen.com".to_string(), } } fn build_channel(&self, clientversion: ClientVersion) -> Channel { let _ = clientversion; Channel { id: "rule34gen".to_string(), name: "Rule34Gen".to_string(), description: "If it exists, here might be an AI generated video of it".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34gen.com".to_string(), status: "active".to_string(), categories: vec![], options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "post_date".to_string(), title: "Newest".to_string(), }, FilterOption { id: "video_viewed".to_string(), title: "Most Viewed".to_string(), }, FilterOption { id: "rating".to_string(), title: "Top Rated".to_string(), }, FilterOption { id: "duration".to_string(), title: "Longest".to_string(), }, FilterOption { id: "pseudo_random".to_string(), title: "Random".to_string(), }, ], multiSelect: false, }], nsfw: true, cacheDuration: Some(1800), } } async fn get( &self, cache: VideoCache, page: u8, sort: &str, options: ServerOptions, ) -> Result> { let expected_sorts = vec![ "post_date", "video_viewed", "rating", "duration", "pseudo_random", ]; let sort = if expected_sorts.contains(&sort) { sort } else { "post_date" }; let index = format!("rule34gen:{}:{}", page, sort); let url = if page <= 1 { format!("{}/?sort_by={}", self.url, sort) } else { format!("{}/{}/?sort_by={}", self.url, page, sort) }; let mut old_items: Vec = vec![]; if !(sort == "pseudo_random") { old_items = match cache.get(&index) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { // println!("Cache hit for URL: {}", url); return Ok(items.clone()); } else { items.clone() } } None => { vec![] } }; } let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let text = match requester.get(&url, None).await { Ok(text) => text, Err(e) => { report_provider_error("rule34gen", "get.request", &format!("url={url}; error={e}")) .await; return Ok(old_items); } }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } async fn query( &self, cache: VideoCache, page: u8, query: &str, sort: &str, options: ServerOptions, ) -> Result> { let expected_sorts = vec![ "post_date", "video_viewed", "rating", "duration", "pseudo_random", ]; let sort = if expected_sorts.contains(&sort) { sort } else { "post_date" }; let index = format!("rule34gen:{}:{}:{}", page, sort, query); let search_slug = query.replace(" ", "-"); let url = if page <= 1 { format!("{}/search/{}/?sort_by={}", self.url, search_slug, sort) } else { format!( "{}/search/{}/{}/?sort_by={}", self.url, search_slug, page, sort ) }; // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&index) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { return Ok(items.clone()); } else { let _ = cache.check().await; return Ok(items.clone()); } } None => { vec![] } }; let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let text = match requester.get(&url, None).await { Ok(text) => text, Err(e) => { report_provider_error( "rule34gen", "query.request", &format!("url={url}; error={e}"), ) .await; return Ok(old_items); } }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } fn get_video_items_from_html(&self, html: String) -> Vec { if html.is_empty() { println!("HTML is empty"); return vec![]; } let mut items: Vec = Vec::new(); if html.contains("cards__item") { for video_segment in html.split("
") .nth(1) .and_then(|s| s.split('<').next()) .unwrap_or_default() .trim() .to_string(); } title = decode(title.as_bytes()).to_string().unwrap_or(title); let thumb = video_segment .split("data-original=\"") .nth(1) .and_then(|s| s.split('"').next()) .unwrap_or_default() .to_string(); let preview = video_segment .split("data-preview=\"") .nth(1) .and_then(|s| s.split('"').next()) .unwrap_or_default() .to_string(); let raw_duration = video_segment .split("card__label card__label--primary\">") .nth(1) .and_then(|s| s.split('<').next()) .unwrap_or_default() .trim() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let views_text = video_segment .split("") .nth(2) .and_then(|s| s.split("").next()) .unwrap_or_default() .replace("views", "") .replace(' ', "") .trim() .to_string(); let views = parse_abbreviated_number(views_text.as_str()).unwrap_or(0) as u32; items.push( VideoItem::new( id, title, video_url, "rule34gen".to_string(), thumb, duration, ) .views(views) .preview(preview), ); } return items; } let video_listing_content = html .split("
>() .get(1) .copied() .unwrap_or_default() .split("
>() .get(0) .copied() .unwrap_or_default() .to_string(); let raw_videos = video_listing_content .split("
>()[1..] .to_vec(); for video_segment in &raw_videos { // let vid = video_segment.split("\n").collect::>().get(1).copied().unwrap_or_default() // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } if video_segment.contains("https://rule34gen.com/images/advertisements") { continue; } let mut title = video_segment .split("
") .collect::>() .get(1) .copied() .unwrap_or_default() .split("<") .collect::>() .get(0) .copied() .unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); let id = video_segment .split("https://rule34gen.com/video/") .collect::>() .get(1) .copied() .unwrap_or_default() .split("/") .collect::>() .get(0) .copied() .unwrap_or_default() .to_string(); let raw_duration = video_segment .split("
") .collect::>() .get(1) .copied() .unwrap_or_default() .split("<") .collect::>() .get(0) .copied() .unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let views = parse_abbreviated_number( &video_segment .split("
") .collect::>() .get(1) .copied() .unwrap_or_default() .split("") .collect::>() .get(1) .copied() .unwrap_or_default() .split("<") .collect::>() .get(0) .copied() .unwrap_or_default(), ) .unwrap_or(0); //https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/ //https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/ let thumb = video_segment .split(">() .get(1) .copied() .unwrap_or_default() .split("data-original=\"") .collect::>() .get(1) .copied() .unwrap_or_default() .split("\"") .collect::>() .get(0) .copied() .unwrap_or_default() .to_string(); let url = video_segment .split(">() .get(1) .copied() .unwrap_or_default() .split("\"") .collect::>() .get(0) .copied() .unwrap_or_default() .to_string(); // let preview = video_segment.split("
>().get(1).copied().unwrap_or_default() // .split("\"") // .collect::>().get(0).copied().unwrap_or_default() // .to_string(); let video_item = VideoItem::new( id, title, url.to_string(), "rule34gen".to_string(), thumb, duration, ) .views(views) // .preview(preview) ; items.push(video_item); } return items; } } #[async_trait] impl Provider for Rule34genProvider { async fn get_videos( &self, cache: VideoCache, pool: DbPool, sort: String, query: Option, page: String, per_page: String, options: ServerOptions, ) -> Vec { let _ = options; let _ = per_page; let _ = pool; // Ignored in this implementation let videos: std::result::Result, Error> = match query { Some(q) => { self.query(cache, page.parse::().unwrap_or(1), &q, &sort, options) .await } None => { self.get(cache, page.parse::().unwrap_or(1), &sort, options) .await } }; match videos { Ok(v) => v, Err(e) => { println!("Error fetching videos: {}", e); vec![] } } } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } }