From 316c317c815bdfcd52db11b016fa4bc7aecd307c Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 9 Oct 2026 22:27:29 +0000 Subject: [PATCH] Fix rule34video: thumbs, views and playback Thumbs moved to data-rd-src, views to title="Views: ..", and yt-dlp no longer resolves the page URL, so formats and tags come from the detail page flashvars. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi --- src/providers/rule34video.rs | 203 ++++++++++++++++++++++++++++++++--- 1 file changed, 191 insertions(+), 12 deletions(-) diff --git a/src/providers/rule34video.rs b/src/providers/rule34video.rs index 05f7252..8c81bb1 100644 --- a/src/providers/rule34video.rs +++ b/src/providers/rule34video.rs @@ -5,14 +5,18 @@ use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::send_discord_error_report; use crate::util::parse_abbreviated_number; +use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; -use crate::videos::{ServerOptions, VideoItem}; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; +use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use std::time::{SystemTime, UNIX_EPOCH}; use std::vec; +const ENRICH_CONCURRENCY: usize = 6; + pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "hentai-animation", @@ -152,6 +156,7 @@ impl Rule34videoProvider { "".to_string() }); let video_items = self.get_video_items_from_html(text); + let video_items = self.enrich_items(video_items, requester).await; if !video_items.is_empty() { cache.insert(index, video_items.clone()); @@ -224,6 +229,7 @@ impl Rule34videoProvider { "".to_string() }); let video_items = self.get_video_items_from_html(text); + let video_items = self.enrich_items(video_items, requester).await; if !video_items.is_empty() { cache.insert(index, video_items.clone()); @@ -236,6 +242,103 @@ impl Rule34videoProvider { } } + /// Reads a single-quoted JS string value (`key: '...'`) from the + /// `flashvars` object, honouring `\'` escapes. + fn flashvar(block: &str, key: &str) -> Option { + let marker = format!("{}: '", key); + let start = block.find(&marker)? + marker.len(); + let mut out = String::new(); + let mut chars = block[start..].chars(); + while let Some(c) = chars.next() { + match c { + '\\' => out.push(chars.next()?), + '\'' => return Some(out), + other => out.push(other), + } + } + None + } + + /// The page URL is no longer resolvable by yt-dlp's Rule34Video + /// extractor ("No video formats found"), so pull the signed mp4 URLs + /// (360p/480p/720p, valid ~12h, no cookie/Referer needed) and the tags + /// straight from the detail page's `flashvars`. + fn parse_detail(html: &str) -> (Vec, Vec) { + let Some(start) = html.find("var flashvars") else { + return (vec![], vec![]); + }; + let block = &html[start..]; + let block = &block[..block.len().min(6000)]; + + let mut formats: Vec<(u32, VideoFormat)> = Vec::new(); + for (url_key, text_key) in [ + ("video_url", "video_url_text"), + ("video_alt_url", "video_alt_url_text"), + ("video_alt_url2", "video_alt_url2_text"), + ("video_alt_url3", "video_alt_url3_text"), + ] { + let Some(url) = Self::flashvar(block, url_key) else { + continue; + }; + if !url.starts_with("http") { + continue; + } + let quality = Self::flashvar(block, text_key).unwrap_or_else(|| "auto".to_string()); + let height = quality.trim_end_matches('p').parse::().unwrap_or(0); + let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string()) + .format_id(format!("mp4-{}", quality)); + if height > 0 { + format = format.height(height); + } + formats.push((height, format)); + } + formats.sort_by(|a, b| b.0.cmp(&a.0)); + + let mut tags: Vec = Vec::new(); + for key in ["video_categories", "video_tags"] { + for tag in Self::flashvar(block, key).unwrap_or_default().split(',') { + let tag = tag.trim(); + if !tag.is_empty() && !tags.iter().any(|t| t.eq_ignore_ascii_case(tag)) { + tags.push(tag.to_string()); + } + } + } + + (formats.into_iter().map(|(_, f)| f).collect(), tags) + } + + async fn enrich_items(&self, items: Vec, requester: Requester) -> Vec { + let referer = format!("{}/", self.url); + stream::iter(items.into_iter().map(|mut item| { + let mut requester = requester.clone(); + let referer = referer.clone(); + async move { + if item.url.is_empty() { + return item; + } + let html = requester + .get_with_headers( + &item.url, + vec![("Referer".to_string(), referer)], + None, + ) + .await + .unwrap_or_default(); + let (formats, tags) = Self::parse_detail(&html); + if !formats.is_empty() { + item.formats = Some(formats); + } + if !tags.is_empty() { + item.tags = Some(tags); + } + item + } + })) + .buffered(ENRICH_CONCURRENCY) + .collect::>() + .await + } + fn get_video_items_from_html(&self, html: String) -> Vec { if html.is_empty() { return vec![]; @@ -280,15 +383,37 @@ impl Rule34videoProvider { Self::extract_between(video_segment, "
", "<").unwrap_or("0:00"); let duration = parse_time_to_seconds(raw_duration).unwrap_or(0) as u32; - // Views extraction - let views_segment = Self::extract_between(video_segment, "
", "<"); - let views_count_str = views_segment - .and_then(|s| s.split("").nth(1)) - .unwrap_or("0"); - let views = parse_abbreviated_number(views_count_str.trim()).unwrap_or(0); + // Views extraction: `
`. The sibling `views` div is the comment + // count, so match the title rather than the bare class. + let views = Self::extract_between(video_segment, "title=\"Views: ", "\"") + .and_then(|v| parse_abbreviated_number(v.trim())) + .or_else(|| { + Self::extract_between(video_segment, "
", "
") + .map(|s| s.rsplit("").next().unwrap_or(s)) + .and_then(|v| parse_abbreviated_number(v.trim())) + }) + .unwrap_or(0); - // Thumbnail extraction - let thumb = Self::extract_between(video_segment, "data-original=\"", "\"") + // Rating extraction: `72% (47)` inside the rating div. + let rating = Self::extract_between(video_segment, "custom-thumbs-up", "
") + .and_then(|s| s.rsplit("").next()) + .and_then(|s| s.trim().split('%').next()) + .and_then(|s| s.trim().parse::().ok()); + + // Thumbnail extraction: the site dropped `data-original`. Lazy + // cards are ``; eager ones carry the real `src`. + let thumb = Self::extract_between(video_segment, "") + .and_then(|tag| { + Self::extract_between(tag, "data-rd-src=\"", "\"") + .or_else(|| { + Self::extract_between(tag, " src=\"", "\"") + .filter(|src| src.starts_with("http")) + }) + .or_else(|| Self::extract_between(tag, "data-rd-jpg=\"", "\"")) + }) + .or_else(|| Self::extract_between(video_segment, "data-original=\"", "\"")) .unwrap_or("") .to_string(); @@ -298,10 +423,13 @@ impl Rule34videoProvider { .unwrap_or("") .to_string(); - items.push( + let mut item = VideoItem::new(id, title, url, "Rule34video".to_string(), thumb, duration) - .views(views), - ); + .views(views); + if let Some(rating) = rating { + item = item.rating(rating); + } + items.push(item); } items } @@ -339,3 +467,54 @@ impl Provider for Rule34videoProvider { Some(self.build_channel(clientversion)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_card_thumb_from_img_src_and_skips_ads() { + let html = r#"
+ +
a +
1:28
Some Title
+
72% (47)
+
0
+
6.9K
+
+
b +
0:30
Lazy
+
+