use crate::DbPool; use crate::api::ClientVersion; use crate::providers::{Provider, requester_or_default, slug_dashed}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::send_discord_error_report; use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use scraper::{Html, Selector}; use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use std::vec; /// Hands out `/proxy/rule34video/` URLs instead of the page URLs. Switched off; /// the proxy and resolver stay in place (`proxies/rule34video.rs`). const USE_MEDIA_PROXY: bool = false; const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60); const TARGET_CACHE_SIZE: usize = 2000; /// A probed archive smaller than this loses to keyword search: "elf girl" is a /// one-video tag, while the search finds hundreds. const MIN_PROBED_ARCHIVE_ITEMS: usize = 4; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "hentai-animation", tags: &["rule34", "animation", "fandom"], }; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); } errors { ParsingError(t: String) { description("html parsing error") display("HTML parsing error: '{}'", t) } } } /// What a request resolves to. Artists live under `/models/`, franchises and /// genres under `/categories/`, characters and everything else under `/tags/`. #[derive(Debug, Clone, PartialEq, Eq)] enum Target { Latest, Search(String), Artist(String), Category(String), Tag(String), } impl Target { /// Archive path (`models/{slug}`, ...) for the archive targets. fn archive_path(&self) -> Option { match self { Target::Artist(slug) => Some(format!("models/{slug}")), Target::Category(slug) => Some(format!("categories/{slug}")), Target::Tag(slug) => Some(format!("tags/{slug}")), _ => None, } } } #[derive(Clone)] pub struct Rule34videoProvider { url: String, /// Bare query -> resolved target, so later pages skip the archive probes. targets: Arc>>, } impl Rule34videoProvider { pub fn new() -> Self { Rule34videoProvider { url: "https://rule34video.com".to_string(), targets: Arc::new(Mutex::new(HashMap::new())), } } fn remembered_target(&self, query: &str) -> Option { let targets = self.targets.lock().ok()?; let (time, target) = targets.get(&slug_dashed(query))?; (time.elapsed() < TARGET_TTL).then(|| target.clone()) } fn remember_target(&self, query: &str, target: &Target) { if let Ok(mut targets) = self.targets.lock() { if targets.len() >= TARGET_CACHE_SIZE { targets.retain(|_, (time, _)| time.elapsed() < TARGET_TTL); if targets.len() >= TARGET_CACHE_SIZE { targets.clear(); } } targets.insert(slug_dashed(query), (Instant::now(), target.clone())); } } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { Channel { id: "rule34video".to_string(), name: "Rule34Video".to_string(), description: "If it exists, there is porn".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34video.com".to_string(), status: "active".to_string(), categories: vec![], options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Sort the Videos".to_string(), systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "post_date".to_string(), title: "Newest".to_string(), }, FilterOption { id: "video_viewed".to_string(), title: "Most Viewed".to_string(), }, FilterOption { id: "rating".to_string(), title: "Top Rated".to_string(), }, FilterOption { id: "duration".to_string(), title: "Longest".to_string(), }, FilterOption { id: "pseudo_random".to_string(), title: "Random".to_string(), }, ], multiSelect: false, }], nsfw: true, cacheDuration: Some(1800), ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()) } } /// Explicit `artist:` / `category:` / `tag:` style prefixes (the server has /// already stripped a leading `#`). fn parse_prefixed(query: &str) -> Option { let (kind, value) = query.trim().split_once(':')?; let slug = slug_dashed(value); if slug.is_empty() { return None; } match kind.trim().to_ascii_lowercase().as_str() { "artist" | "artists" | "model" | "models" | "author" | "creator" | "uploader" => { Some(Target::Artist(slug)) } "cat" | "category" | "categories" | "franchise" | "series" | "game" => { Some(Target::Category(slug)) } "tag" | "tags" | "character" | "char" => Some(Target::Tag(slug)), _ => None, } } /// Titles of the tags offered by `/search_ajax.php?tag=`. fn parse_tag_suggestions(html: &str) -> Vec { let document = Html::parse_fragment(html); let Ok(selector) = Selector::parse("input[data-title]") else { return vec![]; }; document .select(&selector) .filter_map(|input| input.value().attr("data-title")) .map(|title| title.trim().to_string()) .filter(|title| !title.is_empty()) .collect() } /// The suggested tag whose name (ignoring a trailing "(franchise)") equals /// the query, e.g. "samus aran" -> "samus aran (metroid)". fn best_tag_title(titles: &[String], query: &str) -> Option { let wanted = slug_dashed(query); if wanted.is_empty() { return None; } let mut exact = titles.iter().filter(|title| { let base = title.split('(').next().unwrap_or(title); slug_dashed(base) == wanted || slug_dashed(title) == wanted }); let first = exact.next()?; // Several franchises share a character name: ambiguous, leave it to search. exact.next().is_none().then(|| first.clone()) } /// A free-form query that names an artist, category or tag goes to that /// archive (probed on the site; unknown names 404), else keyword search. /// The second value is false when a probe failed (rate limit, outage) and /// the answer must not be remembered. async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> (Target, bool) { let slug = slug_dashed(query); if slug.is_empty() { return (Target::Search(query.trim().to_string()), true); } // Artist slugs are inconsistent on the site: "general-butch" but "biggreen". let mut candidates = vec![Target::Artist(slug.clone())]; if slug.contains('-') { candidates.push(Target::Artist(slug.replace('-', ""))); } candidates.push(Target::Category(slug.clone())); candidates.push(Target::Tag(slug)); let mut requester = requester_or_default(options, "rule34video", "resolve_keyword"); let referer = format!("{}/", self.url); let mut conclusive = true; for candidate in &candidates { let Some(path) = candidate.archive_path() else { continue; }; let url = format!("{}/{path}/", self.url); let headers = vec![("Referer".to_string(), referer.clone())]; match requester.get_probe(&url, headers).await { Ok(Some(body)) => { let items = self.get_video_items_from_html(body).len(); if items >= MIN_PROBED_ARCHIVE_ITEMS { return (candidate.clone(), true); } } Ok(None) => {} Err(_) => conclusive = false, } } // Characters are tagged "name (franchise)": ask the site's own tag search. let suggest_url = format!( "{}/search_ajax.php?tag={}", self.url, utf8_percent_encode(query.trim(), NON_ALPHANUMERIC) ); let headers = vec![ ("Referer".to_string(), referer), ("X-Requested-With".to_string(), "XMLHttpRequest".to_string()), ]; match requester.get_probe(&suggest_url, headers).await { Ok(Some(body)) => { let titles = Self::parse_tag_suggestions(&body); if let Some(title) = Self::best_tag_title(&titles, query) { return (Target::Tag(slug_dashed(&title)), conclusive); } } Ok(None) => {} Err(_) => conclusive = false, } (Target::Search(query.trim().to_string()), conclusive) } async fn resolve_target(&self, query: Option<&str>, options: &ServerOptions) -> Target { let Some(query) = query.map(str::trim).filter(|q| !q.is_empty()) else { return Target::Latest; }; if let Some(target) = Self::parse_prefixed(query) { return target; } if let Some(target) = self.remembered_target(query) { return target; } let (target, conclusive) = self.resolve_keyword(query, options).await; if conclusive { self.remember_target(query, &target); } target } async fn archive( &self, cache: VideoCache, target: &Target, page: u8, sort: &str, options: ServerOptions, ) -> Result> { let Some(path) = target.archive_path() else { return Ok(vec![]); }; let sort_val = match sort { "post_date" | "video_viewed" | "rating" | "duration" | "pseudo_random" => sort, _ => "post_date", }; let index = format!("rule34video:{path}:{page}:{sort_val}"); if sort_val != "pseudo_random" { if let Some((time, items)) = cache.get(&index) { if time.elapsed().unwrap_or_default().as_secs() < 300 { return Ok(items.clone()); } } } let mut requester = options.requester.clone().ok_or("Requester missing")?; let url = format!( "{}/{path}/?mode=async&function=get_block&block_id=custom_list_videos_common_videos&sort_by={sort_val}&from={page}", self.url ); let headers = vec![("Referer".to_string(), format!("{}/{path}/", self.url))]; let stale = || cache.get(&index).map(|(_, items)| items).unwrap_or_default(); let text = match requester.get_probe(&url, headers).await { Ok(Some(text)) => text, // A 404 is an unknown archive (or a page past the end). Ok(None) => return Ok(vec![]), Err(e) => { eprintln!("rule34video archive fetch failed for {url}: {e}"); return Ok(stale()); } }; let video_items = self.get_video_items_from_html(text); let video_items = Self::proxy_items(video_items, &options); if video_items.is_empty() { return Ok(stale()); } cache.insert(index, video_items.clone()); Ok(video_items) } /// `/search/{q}/` takes dashes for spaces. A raw `/` splits the path (404), /// `%20` is refused (403) and `&` cuts the query short, so keep only words. fn search_path_segment(query: &str) -> String { slug_dashed(query) .split('-') .map(|word| utf8_percent_encode(word, NON_ALPHANUMERIC).to_string()) .collect::>() .join("-") } /// Helper to safely extract a string between two delimiters fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> { let start_idx = content.find(start_pat)? + start_pat.len(); let sub = &content[start_idx..]; let end_idx = sub.find(end_pat)?; Some(&sub[..end_idx]) } async fn get( &self, cache: VideoCache, page: u8, sort: &str, options: ServerOptions, ) -> Result> { let timestamp_millis = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis()) .unwrap_or(0); let expected_sorts = vec![ "post_date", "video_viewed", "rating", "duration", "pseudo_random", ]; let sort_val = if expected_sorts.contains(&sort) { sort } else { "post_date" }; let index = format!("rule34video:{}:{}", page, sort_val); if sort_val != "pseudo_random" { if let Some((time, items)) = cache.get(&index) { if time.elapsed().unwrap_or_default().as_secs() < 300 { return Ok(items.clone()); } } } let mut requester = options.requester.clone().ok_or("Requester missing")?; let url = format!( "{}/?mode=async&function=get_block&block_id=custom_list_videos_most_recent_videos&tag_ids=&sort_by={}&from={}&_={}", self.url, sort_val, page, timestamp_millis ); let text = requester.get(&url, None).await.unwrap_or_else(|e| { eprintln!("Error fetching rule34video URL {}: {}", url, e); let _ = send_discord_error_report( e.to_string(), None, Some(&url), None, file!(), line!(), module_path!(), ); "".to_string() }); let video_items = self.get_video_items_from_html(text); let video_items = Self::proxy_items(video_items, &options); if !video_items.is_empty() { cache.insert(index, video_items.clone()); Ok(video_items) } else { // Return empty or old items if available Ok(cache .get(&index) .map(|(_, items)| items) .unwrap_or_default()) } } async fn query( &self, cache: VideoCache, page: u8, query: &str, sort: &str, options: ServerOptions, ) -> Result> { let timestamp_millis = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis()) .unwrap_or(0); let expected_sorts = vec![ "post_date", "video_viewed", "rating", "duration", "pseudo_random", ]; let sort_val = if expected_sorts.contains(&sort) { sort } else { "post_date" }; let index = format!("rule34video:{}:{}:{}", page, sort_val, query); if let Some((time, items)) = cache.get(&index) { if time.elapsed().unwrap_or_default().as_secs() < 300 { return Ok(items.clone()); } } let mut requester = options.requester.clone().ok_or("Requester missing")?; let url = format!( "{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}", self.url, Self::search_path_segment(query), sort_val, page, page, timestamp_millis ); let text = requester.get(&url, None).await.unwrap_or_else(|e| { eprintln!("Error fetching rule34video URL {}: {}", url, e); let _ = send_discord_error_report( e.to_string(), None, Some(&url), None, file!(), line!(), module_path!(), ); "".to_string() }); let video_items = self.get_video_items_from_html(text); let video_items = Self::proxy_items(video_items, &options); if !video_items.is_empty() { cache.insert(index, video_items.clone()); Ok(video_items) } else { Ok(cache .get(&index) .map(|(_, items)| items) .unwrap_or_default()) } } /// Playback resolves the video page on demand through `/proxy/rule34video/`, /// instead of fetching every listed page up front (which ran into 429s). fn proxy_items(mut items: Vec, options: &ServerOptions) -> Vec { if !USE_MEDIA_PROXY { return items; } for item in &mut items { if let Some(path) = item.url.split("/video/").nth(1).filter(|p| !p.is_empty()) { item.url = crate::providers::build_proxy_url(options, "rule34video", path); } } items } fn get_video_items_from_html(&self, html: String) -> Vec { if html.is_empty() { return vec![]; } // Isolate the video listing section. A result that fits on one page has // no pagination block, so the listing then runs to the end. let Some(start) = html.find("id=\"custom_list_videos") else { return vec![]; }; let video_listing = &html[start..]; let video_listing = video_listing .find("
", "<") .unwrap_or("Unknown"); let title = decode(title_raw.as_bytes()) .to_string() .unwrap_or_else(|_| title_raw.to_string()); // ID extraction let id = Self::extract_between(video_segment, "https://rule34video.com/video/", "/") .unwrap_or("0") .to_string(); // Duration extraction let raw_duration = Self::extract_between(video_segment, "
", "<").unwrap_or("0:00"); let duration = parse_time_to_seconds(raw_duration).unwrap_or(0) as u32; // Views extraction: `
`. The sibling `views` div is the comment // count, so match the title rather than the bare class. let views = Self::extract_between(video_segment, "title=\"Views: ", "\"") .and_then(|v| parse_abbreviated_number(v.trim())) .or_else(|| { Self::extract_between(video_segment, "
", "
") .map(|s| s.rsplit("").next().unwrap_or(s)) .and_then(|v| parse_abbreviated_number(v.trim())) }) .unwrap_or(0); // Rating extraction: `72% (47)` inside the rating div. let rating = Self::extract_between(video_segment, "custom-thumbs-up", "
") .and_then(|s| s.rsplit("").next()) .and_then(|s| s.trim().split('%').next()) .and_then(|s| s.trim().parse::().ok()); // Thumbnail extraction: the site dropped `data-original`. Lazy // cards are ``; eager ones carry the real `src`. let thumb = Self::extract_between(video_segment, "") .and_then(|tag| { Self::extract_between(tag, "data-rd-src=\"", "\"") .or_else(|| { Self::extract_between(tag, " src=\"", "\"") .filter(|src| src.starts_with("http")) }) .or_else(|| Self::extract_between(tag, "data-rd-jpg=\"", "\"")) }) .or_else(|| Self::extract_between(video_segment, "data-original=\"", "\"")) .unwrap_or("") .to_string(); // URL extraction let url = Self::extract_between(video_segment, ", page: String, _per_page: String, options: ServerOptions, ) -> Vec { let page_num = page.parse::().unwrap_or(1); let target = self.resolve_target(query.as_deref(), &options).await; let result = match &target { Target::Latest => self.get(cache, page_num, &sort, options).await, Target::Search(q) => self.query(cache, page_num, q, &sort, options).await, archive => self.archive(cache, archive, page_num, &sort, options).await, }; match result { Ok(v) => v, Err(e) => { eprintln!("Error fetching videos: {}", e); vec![] } } } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } } #[cfg(test)] mod tests { use super::*; #[test] fn parses_card_thumb_from_img_src_and_skips_ads() { let html = r#"
"#; let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string()); assert_eq!(items.len(), 1); assert_eq!(items[0].id, "7"); } #[test] fn search_segment_uses_dashes_and_drops_path_breaking_punctuation() { assert_eq!(Rule34videoProvider::search_path_segment("elf girl"), "elf-girl"); assert_eq!(Rule34videoProvider::search_path_segment("2b/9s"), "2b-9s"); assert_eq!(Rule34videoProvider::search_path_segment("Tifa & Aerith"), "tifa-aerith"); assert_eq!(Rule34videoProvider::search_path_segment("Pokémon"), "pok%C3%A9mon"); } #[test] fn remembers_resolved_targets_by_slug() { let provider = Rule34videoProvider::new(); provider.remember_target("Samus Aran", &Target::Tag("samus-aran-metroid".into())); assert_eq!( provider.remembered_target("samus aran"), Some(Target::Tag("samus-aran-metroid".into())) ); assert_eq!(provider.remembered_target("samus"), None); } }