simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rule34video: make queries robust (single-page results, safe search paths, 429-proof enrichment)

Browse Source 
- Parser no longer requires a pagination block: single-page results
  (small tags, short searches) used to parse to zero items.
- Search path is slugged like the site's own form: "2b/9s" 404'd, "%20"
  is refused with 403 and "&" truncated the query.
- Probed archives need >= 4 videos to beat keyword search ("elf girl" was a
  one-video tag); conclusive query -> target resolutions are remembered for 6h
  so later pages and the prefetch skip the probes.
- Enriched details are cached per video URL for 2h (tokens last ~12h);
  detail fetches go through get_probe so a lasting 429 stops the batch instead
  of escalating each page to Jina/FlareSolverr. Archive failures serve stale
  cache.
- requester: get_probe waits before retrying 429/5xx and compares redirect
  paths percent-decoded (non-ASCII archives like /tags/pokémon-species/ were
  treated as redirected away); get_with_headers backs off 1/2/4s on 429.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
2026-10-21 20:17:58 +00:00
parent 921ad93149
commit b131910ea2
2 changed files with 208 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
src/providers/rule34video.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, probe_first_archive, requester_or_default, slug_dashed};
use crate::providers::{Provider, requester_or_default, slug_dashed};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
@@ -13,11 +13,22 @@ use error_chain::error_chain;
use futures::stream::{self, StreamExt};
use htmlentity::entity::{ICodedDataTrait, decode};
use scraper::{Html, Selector};
use std::time::{SystemTime, UNIX_EPOCH};
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
use std::vec;
const ENRICH_CONCURRENCY: usize = 6;
const ENRICH_CONCURRENCY: usize = 4;
/// The signed `v-acctoken` format URLs last ~12h; keep details well inside that.
const DETAIL_TTL: Duration = Duration::from_secs(2 * 60 * 60);
const DETAIL_CACHE_SIZE: usize = 5000;
const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60);
const TARGET_CACHE_SIZE: usize = 2000;
/// A probed archive smaller than this loses to keyword search: "elf girl" is a
/// one-video tag, while the search finds hundreds.
const MIN_PROBED_ARCHIVE_ITEMS: usize = 4;
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
crate::providers::ProviderChannelMetadata {
@@ -61,15 +72,43 @@ impl Target {
}
}
#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct Rule34videoProvider {
url: String,
/// Enriched items (formats, tags, artist) by video URL. Every listing costs
/// ~23 detail pages, again for the prefetched next page; refetching them
/// on each request is what runs into the site's 429 rate limit.
details: VideoCache,
/// Bare query -> resolved target, so later pages skip the archive probes.
targets: Arc<Mutex<HashMap<String, (Instant, Target)>>>,
}
impl Rule34videoProvider {
pub fn new() -> Self {
let mut details = VideoCache::new();
details.max_size(DETAIL_CACHE_SIZE);
Rule34videoProvider {
url: "https://rule34video.com".to_string(),
details,
targets: Arc::new(Mutex::new(HashMap::new())),
}
}
fn remembered_target(&self, query: &str) -> Option<Target> {
let targets = self.targets.lock().ok()?;
let (time, target) = targets.get(&slug_dashed(query))?;
(time.elapsed() < TARGET_TTL).then(|| target.clone())
}
fn remember_target(&self, query: &str, target: &Target) {
if let Ok(mut targets) = self.targets.lock() {
if targets.len() >= TARGET_CACHE_SIZE {
targets.retain(|_, (time, _)| time.elapsed() < TARGET_TTL);
if targets.len() >= TARGET_CACHE_SIZE {
targets.clear();
}
}
targets.insert(slug_dashed(query), (Instant::now(), target.clone()));
}
}
@@ -170,29 +209,37 @@ impl Rule34videoProvider {
/// A free-form query that names an artist, category or tag goes to that
/// archive (probed on the site; unknown names 404), else keyword search.
async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> Target {
/// The second value is false when a probe failed (rate limit, outage) and
/// the answer must not be remembered.
async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> (Target, bool) {
let slug = slug_dashed(query);
if slug.is_empty() {
return Target::Search(query.trim().to_string());
return (Target::Search(query.trim().to_string()), true);
}
let candidates = [
Target::Artist(slug.clone()),
Target::Category(slug.clone()),
Target::Tag(slug),
];
let urls: Vec<String> = candidates
.iter()
.filter_map(|t| t.archive_path())
.map(|path| format!("{}/{path}/", self.url))
.collect();
let mut requester = requester_or_default(options, "rule34video", "resolve_keyword");
let referer = format!("{}/", self.url);
if let Some((index, _)) = probe_first_archive(&mut requester, &urls, &referer, |body| {
body.contains("item thumb video_")
})
.await
{
return candidates[index].clone();
let mut conclusive = true;
for candidate in &candidates {
let Some(path) = candidate.archive_path() else {
continue;
};
let url = format!("{}/{path}/", self.url);
let headers = vec![("Referer".to_string(), referer.clone())];
match requester.get_probe(&url, headers).await {
Ok(Some(body)) => {
let items = self.get_video_items_from_html(body).len();
if items >= MIN_PROBED_ARCHIVE_ITEMS {
return (candidate.clone(), true);
}
}
Ok(None) => {}
Err(_) => conclusive = false,
}
}
// Characters are tagged "name (franchise)": ask the site's own tag search.
@@ -205,13 +252,17 @@ impl Rule34videoProvider {
("Referer".to_string(), referer),
("X-Requested-With".to_string(), "XMLHttpRequest".to_string()),
];
if let Ok(Some(body)) = requester.get_probe(&suggest_url, headers).await {
match requester.get_probe(&suggest_url, headers).await {
Ok(Some(body)) => {
let titles = Self::parse_tag_suggestions(&body);
if let Some(title) = Self::best_tag_title(&titles, query) {
return Target::Tag(slug_dashed(&title));
return (Target::Tag(slug_dashed(&title)), conclusive);
}
}
Target::Search(query.trim().to_string())
Ok(None) => {}
Err(_) => conclusive = false,
}
(Target::Search(query.trim().to_string()), conclusive)
}
async fn resolve_target(&self, query: Option<&str>, options: &ServerOptions) -> Target {
@@ -221,7 +272,14 @@ impl Rule34videoProvider {
if let Some(target) = Self::parse_prefixed(query) {
return target;
}
self.resolve_keyword(query, options).await
if let Some(target) = self.remembered_target(query) {
return target;
}
let (target, conclusive) = self.resolve_keyword(query, options).await;
if conclusive {
self.remember_target(query, &target);
}
target
}
async fn archive(
@@ -254,20 +312,35 @@ impl Rule34videoProvider {
self.url
);
let headers = vec![("Referer".to_string(), format!("{}/{path}/", self.url))];
let stale = || cache.get(&index).map(|(_, items)| items).unwrap_or_default();
let text = match requester.get_probe(&url, headers).await {
Ok(Some(text)) => text,
// A 404 is an unknown archive (or a page past the end).
Ok(None) => return Ok(vec![]),
Err(e) => return Err(format!("archive fetch failed for {url}: {e}").into()),
Err(e) => {
eprintln!("rule34video archive fetch failed for {url}: {e}");
return Ok(stale());
}
};
let video_items = self.get_video_items_from_html(text);
let video_items = self.enrich_items(video_items, requester).await;
if !video_items.is_empty() {
cache.insert(index, video_items.clone());
if video_items.is_empty() {
return Ok(stale());
}
cache.insert(index, video_items.clone());
Ok(video_items)
}
/// `/search/{q}/` takes dashes for spaces. A raw `/` splits the path (404),
/// `%20` is refused (403) and `&` cuts the query short, so keep only words.
fn search_path_segment(query: &str) -> String {
slug_dashed(query)
.split('-')
.map(|word| utf8_percent_encode(word, NON_ALPHANUMERIC).to_string())
.collect::<Vec<_>>()
.join("-")
}
/// Helper to safely extract a string between two delimiters
fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> {
let start_idx = content.find(start_pat)? + start_pat.len();
@@ -383,7 +456,7 @@ impl Rule34videoProvider {
let url = format!(
"{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}",
self.url,
query.replace(" ", "-"),
Self::search_path_segment(query),
sort_val,
page,
page,
@@ -513,25 +586,54 @@ impl Rule34videoProvider {
artists
}
fn cached_detail(&self, item: &mut VideoItem) -> bool {
let Some((time, cached)) = self.details.get(&item.url) else {
return false;
};
let Some(cached) = cached.into_iter().next() else {
return false;
};
if time.elapsed().unwrap_or(DETAIL_TTL) >= DETAIL_TTL {
return false;
}
item.formats = cached.formats;
item.tags = cached.tags;
item.uploader = cached.uploader;
item.uploaderUrl = cached.uploaderUrl;
item.uploaderId = cached.uploaderId;
true
}
async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
let referer = format!("{}/", self.url);
let base = self.url.clone();
// Once the site rate-limits a detail page, the rest of the batch keeps
// the listing data instead of digging the hole deeper.
let throttled = Arc::new(AtomicBool::new(false));
stream::iter(items.into_iter().map(|mut item| {
let mut requester = requester.clone();
let referer = referer.clone();
let base = base.clone();
let throttled = throttled.clone();
async move {
if item.url.is_empty() {
if item.url.is_empty() || self.cached_detail(&mut item) {
return item;
}
let html = requester
.get_with_headers(
&item.url,
vec![("Referer".to_string(), referer)],
None,
)
.await
.unwrap_or_default();
if throttled.load(Ordering::Relaxed) {
return item;
}
// `get_probe` hands challenges to the full fallback chain but
// reports a lasting 429 as an error rather than escalating every
// detail page to FlareSolverr.
let headers = vec![("Referer".to_string(), referer)];
let html = match requester.get_probe(&item.url, headers).await {
Ok(Some(html)) => html,
Ok(None) => return item,
Err(_) => {
throttled.store(true, Ordering::Relaxed);
return item;
}
};
let (formats, mut tags) = Self::parse_detail(&html);
if !formats.is_empty() {
item.formats = Some(formats);
@@ -550,6 +652,9 @@ impl Rule34videoProvider {
if !tags.is_empty() {
item.tags = Some(tags);
}
if item.formats.is_some() {
self.details.insert(item.url.clone(), vec![item.clone()]);
}
item
}
}))
@@ -563,15 +668,15 @@ impl Rule34videoProvider {
return vec![];
}
// Safely isolate the video listing section
let video_listing = match Self::extract_between(
&html,
"id=\"custom_list_videos",
"<div class=\"pagination\"",
) {
Some(content) => content,
None => return vec![],
// Isolate the video listing section. A result that fits on one page has
// no pagination block, so the listing then runs to the end.
let Some(start) = html.find("id=\"custom_list_videos") else {
return vec![];
};
let video_listing = &html[start..];
let video_listing = video_listing
.find("<div class=\"pagination\"")
.map_or(video_listing, |end| &video_listing[..end]);
let mut items = Vec::new();
// Skip the first split result as it's the preamble
@@ -785,4 +890,34 @@ mod tests {
vec![("JOIGang (VA)".to_string(), "joigang-va".to_string())]
);
}
#[test]
fn parses_a_single_page_listing_without_pagination() {
let html = r#"<div id="custom_list_videos_common_videos_items">
<div class="item thumb video_7"><a class="th js-open-popup" href="https://rule34video.com/video/7/elf/">
<img class="thumb" src="https://rule34video.com/t/7.jpg"><div class="time">0:30</div>
<div class="thumb_title">Elf</div></a></div></div></div>"#;
let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string());
assert_eq!(items.len(), 1);
assert_eq!(items[0].id, "7");
}
#[test]
fn search_segment_uses_dashes_and_drops_path_breaking_punctuation() {
assert_eq!(Rule34videoProvider::search_path_segment("elf girl"), "elf-girl");
assert_eq!(Rule34videoProvider::search_path_segment("2b/9s"), "2b-9s");
assert_eq!(Rule34videoProvider::search_path_segment("Tifa & Aerith"), "tifa-aerith");
assert_eq!(Rule34videoProvider::search_path_segment("Pokémon"), "pok%C3%A9mon");
}
#[test]
fn remembers_resolved_targets_by_slug() {
let provider = Rule34videoProvider::new();
provider.remember_target("Samus Aran", &Target::Tag("samus-aran-metroid".into()));
assert_eq!(
provider.remembered_target("samus aran"),
Some(Target::Tag("samus-aran-metroid".into()))
);
assert_eq!(provider.remembered_target("samus"), None);
}
}

34
src/util/requester.rs
View File

@@ -674,6 +674,18 @@ impl Requester {
url::Url::parse(url).ok().map(|u| u.path().to_string())
}
/// `Url` percent-encodes a non-ASCII path ("/tags/pok%C3%A9mon/") while the
/// response URI keeps it raw ("/tags/pokémon/"): compare them decoded.
fn same_path(asked: &str, landed: &str) -> bool {
let normalize = |path: &str| {
percent_encoding::percent_decode_str(path)
.decode_utf8_lossy()
.trim_end_matches('/')
.to_lowercase()
};
normalize(asked) == normalize(landed)
}
/// Existence probe for archive pages (pornstar / studio / tag / category).
///
/// Reads the raw response so a real 404/410 answers immediately with
@@ -687,7 +699,11 @@ impl Requester {
headers: Vec<(String, String)>,
) -> Result<Option<String>, AnyErr> {
let mut last_err = String::new();
for _ in 0..2 {
for attempt in 0..2 {
if attempt > 0 {
// An immediate retry of a 429 is all but certain to hit it again.
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
}
match self.get_raw_with_headers(url, headers.clone()).await {
Ok(response) => {
let status = response.status().as_u16();
@@ -696,9 +712,7 @@ impl Requester {
// lands on a different path is a miss.
let redirected_away = Self::path_of(url)
.zip(Some(response.uri().path().to_string()))
.is_some_and(|(asked, landed)| {
!asked.trim_end_matches('/').eq_ignore_ascii_case(landed.trim_end_matches('/'))
});
.is_some_and(|(asked, landed)| !Self::same_path(&asked, &landed));
let body = response.text().await.unwrap_or_default();
if status == 404 || status == 410 {
return Ok(None);
@@ -893,7 +907,9 @@ impl Requester {
rate_limit_retries,
MAX_RATE_LIMIT_RETRIES
);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
// 1s, 2s, 4s: a site-wide limit needs longer than a second to lift.
tokio::time::sleep(std::time::Duration::from_secs(1 << (rate_limit_retries - 1)))
.await;
continue;
} else {
crate::flow_debug!(
@@ -1027,6 +1043,14 @@ impl Requester {
mod tests {
use super::Requester;
#[test]
fn probe_paths_compare_percent_decoded() {
let asked = Requester::path_of("https://rule34video.com/tags/pokémon-species/").unwrap();
assert!(Requester::same_path(&asked, "/tags/pokémon-species/"));
assert!(Requester::same_path("/Models/X", "/models/x/"));
assert!(!Requester::same_path(&asked, "/search/"));
}
#[test]
fn new_requesters_share_cookie_jar() {
let a = Requester::new();