rule34video: make queries robust (single-page results, safe search paths, 429-proof enrichment)

- Parser no longer requires a pagination block: single-page results
  (small tags, short searches) used to parse to zero items.
- Search path is slugged like the site's own form: "2b/9s" 404'd, "%20"
  is refused with 403 and "&" truncated the query.
- Probed archives need >= 4 videos to beat keyword search ("elf girl" was a
  one-video tag); conclusive query -> target resolutions are remembered for 6h
  so later pages and the prefetch skip the probes.
- Enriched details are cached per video URL for 2h (tokens last ~12h);
  detail fetches go through get_probe so a lasting 429 stops the batch instead
  of escalating each page to Jina/FlareSolverr. Archive failures serve stale
  cache.
- requester: get_probe waits before retrying 429/5xx and compares redirect
  paths percent-decoded (non-ASCII archives like /tags/pokémon-species/ were
  treated as redirected away); get_with_headers backs off 1/2/4s on 429.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi

src/providers/rule34video.rs

@@ -1,6 +1,6 @@
 use crate::DbPool;
 use crate::api::ClientVersion;
-use crate::providers::{Provider, probe_first_archive, requester_or_default, slug_dashed};
+use crate::providers::{Provider, requester_or_default, slug_dashed};
 use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::discord::send_discord_error_report;
@@ -13,11 +13,22 @@ use error_chain::error_chain;
 use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 use scraper::{Html, Selector};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
 use std::vec;
 
-const ENRICH_CONCURRENCY: usize = 6;
+const ENRICH_CONCURRENCY: usize = 4;
+/// The signed `v-acctoken` format URLs last ~12h; keep details well inside that.
+const DETAIL_TTL: Duration = Duration::from_secs(2 * 60 * 60);
+const DETAIL_CACHE_SIZE: usize = 5000;
+const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60);
+const TARGET_CACHE_SIZE: usize = 2000;
+/// A probed archive smaller than this loses to keyword search: "elf girl" is a
+/// one-video tag, while the search finds hundreds.
+const MIN_PROBED_ARCHIVE_ITEMS: usize = 4;
 
 pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
     crate::providers::ProviderChannelMetadata {
@@ -61,15 +72,43 @@ impl Target {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Clone)]
 pub struct Rule34videoProvider {
     url: String,
+    /// Enriched items (formats, tags, artist) by video URL. Every listing costs
+    /// ~23 detail pages, again for the prefetched next page; refetching them
+    /// on each request is what runs into the site's 429 rate limit.
+    details: VideoCache,
+    /// Bare query -> resolved target, so later pages skip the archive probes.
+    targets: Arc<Mutex<HashMap<String, (Instant, Target)>>>,
 }
 
 impl Rule34videoProvider {
     pub fn new() -> Self {
+        let mut details = VideoCache::new();
+        details.max_size(DETAIL_CACHE_SIZE);
         Rule34videoProvider {
             url: "https://rule34video.com".to_string(),
+            details,
+            targets: Arc::new(Mutex::new(HashMap::new())),
+        }
+    }
+
+    fn remembered_target(&self, query: &str) -> Option<Target> {
+        let targets = self.targets.lock().ok()?;
+        let (time, target) = targets.get(&slug_dashed(query))?;
+        (time.elapsed() < TARGET_TTL).then(|| target.clone())
+    }
+
+    fn remember_target(&self, query: &str, target: &Target) {
+        if let Ok(mut targets) = self.targets.lock() {
+            if targets.len() >= TARGET_CACHE_SIZE {
+                targets.retain(|_, (time, _)| time.elapsed() < TARGET_TTL);
+                if targets.len() >= TARGET_CACHE_SIZE {
+                    targets.clear();
+                }
+            }
+            targets.insert(slug_dashed(query), (Instant::now(), target.clone()));
         }
     }
 
@@ -170,29 +209,37 @@ impl Rule34videoProvider {
 
     /// A free-form query that names an artist, category or tag goes to that
     /// archive (probed on the site; unknown names 404), else keyword search.
-    async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> Target {
+    /// The second value is false when a probe failed (rate limit, outage) and
+    /// the answer must not be remembered.
+    async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> (Target, bool) {
         let slug = slug_dashed(query);
         if slug.is_empty() {
-            return Target::Search(query.trim().to_string());
+            return (Target::Search(query.trim().to_string()), true);
         }
         let candidates = [
             Target::Artist(slug.clone()),
             Target::Category(slug.clone()),
             Target::Tag(slug),
         ];
-        let urls: Vec<String> = candidates
-            .iter()
-            .filter_map(|t| t.archive_path())
-            .map(|path| format!("{}/{path}/", self.url))
-            .collect();
         let mut requester = requester_or_default(options, "rule34video", "resolve_keyword");
         let referer = format!("{}/", self.url);
-        if let Some((index, _)) = probe_first_archive(&mut requester, &urls, &referer, |body| {
+        let mut conclusive = true;
-            body.contains("item thumb video_")
+        for candidate in &candidates {
-        })
+            let Some(path) = candidate.archive_path() else {
-        .await
+                continue;
-        {
+            };
-            return candidates[index].clone();
+            let url = format!("{}/{path}/", self.url);
+            let headers = vec![("Referer".to_string(), referer.clone())];
+            match requester.get_probe(&url, headers).await {
+                Ok(Some(body)) => {
+                    let items = self.get_video_items_from_html(body).len();
+                    if items >= MIN_PROBED_ARCHIVE_ITEMS {
+                        return (candidate.clone(), true);
+                    }
+                }
+                Ok(None) => {}
+                Err(_) => conclusive = false,
+            }
         }
 
         // Characters are tagged "name (franchise)": ask the site's own tag search.
@@ -205,13 +252,17 @@ impl Rule34videoProvider {
             ("Referer".to_string(), referer),
             ("X-Requested-With".to_string(), "XMLHttpRequest".to_string()),
         ];
-        if let Ok(Some(body)) = requester.get_probe(&suggest_url, headers).await {
+        match requester.get_probe(&suggest_url, headers).await {
-            let titles = Self::parse_tag_suggestions(&body);
+            Ok(Some(body)) => {
-            if let Some(title) = Self::best_tag_title(&titles, query) {
+                let titles = Self::parse_tag_suggestions(&body);
-                return Target::Tag(slug_dashed(&title));
+                if let Some(title) = Self::best_tag_title(&titles, query) {
+                    return (Target::Tag(slug_dashed(&title)), conclusive);
+                }
             }
+            Ok(None) => {}
+            Err(_) => conclusive = false,
         }
-        Target::Search(query.trim().to_string())
+        (Target::Search(query.trim().to_string()), conclusive)
     }
 
     async fn resolve_target(&self, query: Option<&str>, options: &ServerOptions) -> Target {
@@ -221,7 +272,14 @@ impl Rule34videoProvider {
         if let Some(target) = Self::parse_prefixed(query) {
             return target;
         }
-        self.resolve_keyword(query, options).await
+        if let Some(target) = self.remembered_target(query) {
+            return target;
+        }
+        let (target, conclusive) = self.resolve_keyword(query, options).await;
+        if conclusive {
+            self.remember_target(query, &target);
+        }
+        target
     }
 
     async fn archive(
@@ -254,20 +312,35 @@ impl Rule34videoProvider {
             self.url
         );
         let headers = vec![("Referer".to_string(), format!("{}/{path}/", self.url))];
+        let stale = || cache.get(&index).map(|(_, items)| items).unwrap_or_default();
         let text = match requester.get_probe(&url, headers).await {
             Ok(Some(text)) => text,
             // A 404 is an unknown archive (or a page past the end).
             Ok(None) => return Ok(vec![]),
-            Err(e) => return Err(format!("archive fetch failed for {url}: {e}").into()),
+            Err(e) => {
+                eprintln!("rule34video archive fetch failed for {url}: {e}");
+                return Ok(stale());
+            }
         };
         let video_items = self.get_video_items_from_html(text);
         let video_items = self.enrich_items(video_items, requester).await;
-        if !video_items.is_empty() {
+        if video_items.is_empty() {
-            cache.insert(index, video_items.clone());
+            return Ok(stale());
         }
+        cache.insert(index, video_items.clone());
         Ok(video_items)
     }
 
+    /// `/search/{q}/` takes dashes for spaces. A raw `/` splits the path (404),
+    /// `%20` is refused (403) and `&` cuts the query short, so keep only words.
+    fn search_path_segment(query: &str) -> String {
+        slug_dashed(query)
+            .split('-')
+            .map(|word| utf8_percent_encode(word, NON_ALPHANUMERIC).to_string())
+            .collect::<Vec<_>>()
+            .join("-")
+    }
+
     /// Helper to safely extract a string between two delimiters
     fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> {
         let start_idx = content.find(start_pat)? + start_pat.len();
@@ -383,7 +456,7 @@ impl Rule34videoProvider {
         let url = format!(
             "{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}",
             self.url,
-            query.replace(" ", "-"),
+            Self::search_path_segment(query),
             sort_val,
             page,
             page,
@@ -513,25 +586,54 @@ impl Rule34videoProvider {
         artists
     }
 
+    fn cached_detail(&self, item: &mut VideoItem) -> bool {
+        let Some((time, cached)) = self.details.get(&item.url) else {
+            return false;
+        };
+        let Some(cached) = cached.into_iter().next() else {
+            return false;
+        };
+        if time.elapsed().unwrap_or(DETAIL_TTL) >= DETAIL_TTL {
+            return false;
+        }
+        item.formats = cached.formats;
+        item.tags = cached.tags;
+        item.uploader = cached.uploader;
+        item.uploaderUrl = cached.uploaderUrl;
+        item.uploaderId = cached.uploaderId;
+        true
+    }
+
     async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
         let referer = format!("{}/", self.url);
         let base = self.url.clone();
+        // Once the site rate-limits a detail page, the rest of the batch keeps
+        // the listing data instead of digging the hole deeper.
+        let throttled = Arc::new(AtomicBool::new(false));
         stream::iter(items.into_iter().map(|mut item| {
             let mut requester = requester.clone();
             let referer = referer.clone();
             let base = base.clone();
+            let throttled = throttled.clone();
             async move {
-                if item.url.is_empty() {
+                if item.url.is_empty() || self.cached_detail(&mut item) {
                     return item;
                 }
-                let html = requester
+                if throttled.load(Ordering::Relaxed) {
-                    .get_with_headers(
+                    return item;
-                        &item.url,
+                }
-                        vec![("Referer".to_string(), referer)],
+                // `get_probe` hands challenges to the full fallback chain but
-                        None,
+                // reports a lasting 429 as an error rather than escalating every
-                    )
+                // detail page to FlareSolverr.
-                    .await
+                let headers = vec![("Referer".to_string(), referer)];
-                    .unwrap_or_default();
+                let html = match requester.get_probe(&item.url, headers).await {
+                    Ok(Some(html)) => html,
+                    Ok(None) => return item,
+                    Err(_) => {
+                        throttled.store(true, Ordering::Relaxed);
+                        return item;
+                    }
+                };
                 let (formats, mut tags) = Self::parse_detail(&html);
                 if !formats.is_empty() {
                     item.formats = Some(formats);
@@ -550,6 +652,9 @@ impl Rule34videoProvider {
                 if !tags.is_empty() {
                     item.tags = Some(tags);
                 }
+                if item.formats.is_some() {
+                    self.details.insert(item.url.clone(), vec![item.clone()]);
+                }
                 item
             }
         }))
@@ -563,15 +668,15 @@ impl Rule34videoProvider {
             return vec![];
         }
 
-        // Safely isolate the video listing section
+        // Isolate the video listing section. A result that fits on one page has
-        let video_listing = match Self::extract_between(
+        // no pagination block, so the listing then runs to the end.
-            &html,
+        let Some(start) = html.find("id=\"custom_list_videos") else {
-            "id=\"custom_list_videos",
+            return vec![];
-            "<div class=\"pagination\"",
-        ) {
-            Some(content) => content,
-            None => return vec![],
         };
+        let video_listing = &html[start..];
+        let video_listing = video_listing
+            .find("<div class=\"pagination\"")
+            .map_or(video_listing, |end| &video_listing[..end]);
 
         let mut items = Vec::new();
         // Skip the first split result as it's the preamble
@@ -785,4 +890,34 @@ mod tests {
             vec![("JOIGang (VA)".to_string(), "joigang-va".to_string())]
         );
     }
+
+    #[test]
+    fn parses_a_single_page_listing_without_pagination() {
+        let html = r#"<div id="custom_list_videos_common_videos_items">
+<div class="item thumb video_7"><a class="th js-open-popup" href="https://rule34video.com/video/7/elf/">
+<img class="thumb" src="https://rule34video.com/t/7.jpg"><div class="time">0:30</div>
+<div class="thumb_title">Elf</div></a></div></div></div>"#;
+        let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string());
+        assert_eq!(items.len(), 1);
+        assert_eq!(items[0].id, "7");
+    }
+
+    #[test]
+    fn search_segment_uses_dashes_and_drops_path_breaking_punctuation() {
+        assert_eq!(Rule34videoProvider::search_path_segment("elf  girl"), "elf-girl");
+        assert_eq!(Rule34videoProvider::search_path_segment("2b/9s"), "2b-9s");
+        assert_eq!(Rule34videoProvider::search_path_segment("Tifa & Aerith"), "tifa-aerith");
+        assert_eq!(Rule34videoProvider::search_path_segment("Pokémon"), "pok%C3%A9mon");
+    }
+
+    #[test]
+    fn remembers_resolved_targets_by_slug() {
+        let provider = Rule34videoProvider::new();
+        provider.remember_target("Samus Aran", &Target::Tag("samus-aran-metroid".into()));
+        assert_eq!(
+            provider.remembered_target("samus  aran"),
+            Some(Target::Tag("samus-aran-metroid".into()))
+        );
+        assert_eq!(provider.remembered_target("samus"), None);
+    }
 }

src/util/requester.rs

@@ -674,6 +674,18 @@ impl Requester {
         url::Url::parse(url).ok().map(|u| u.path().to_string())
     }
 
+    /// `Url` percent-encodes a non-ASCII path ("/tags/pok%C3%A9mon/") while the
+    /// response URI keeps it raw ("/tags/pokémon/"): compare them decoded.
+    fn same_path(asked: &str, landed: &str) -> bool {
+        let normalize = |path: &str| {
+            percent_encoding::percent_decode_str(path)
+                .decode_utf8_lossy()
+                .trim_end_matches('/')
+                .to_lowercase()
+        };
+        normalize(asked) == normalize(landed)
+    }
+
     /// Existence probe for archive pages (pornstar / studio / tag / category).
     ///
     /// Reads the raw response so a real 404/410 answers immediately with
@@ -687,7 +699,11 @@ impl Requester {
         headers: Vec<(String, String)>,
     ) -> Result<Option<String>, AnyErr> {
         let mut last_err = String::new();
-        for _ in 0..2 {
+        for attempt in 0..2 {
+            if attempt > 0 {
+                // An immediate retry of a 429 is all but certain to hit it again.
+                tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
+            }
             match self.get_raw_with_headers(url, headers.clone()).await {
                 Ok(response) => {
                     let status = response.status().as_u16();
@@ -696,9 +712,7 @@ impl Requester {
                     // lands on a different path is a miss.
                     let redirected_away = Self::path_of(url)
                         .zip(Some(response.uri().path().to_string()))
-                        .is_some_and(|(asked, landed)| {
+                        .is_some_and(|(asked, landed)| !Self::same_path(&asked, &landed));
-                            !asked.trim_end_matches('/').eq_ignore_ascii_case(landed.trim_end_matches('/'))
-                        });
                     let body = response.text().await.unwrap_or_default();
                     if status == 404 || status == 410 {
                         return Ok(None);
@@ -893,7 +907,9 @@ impl Requester {
                     rate_limit_retries,
                     MAX_RATE_LIMIT_RETRIES
                 );
-                tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                // 1s, 2s, 4s: a site-wide limit needs longer than a second to lift.
+                tokio::time::sleep(std::time::Duration::from_secs(1 << (rate_limit_retries - 1)))
+                    .await;
                 continue;
             } else {
                 crate::flow_debug!(
@@ -1027,6 +1043,14 @@ impl Requester {
 mod tests {
     use super::Requester;
 
+    #[test]
+    fn probe_paths_compare_percent_decoded() {
+        let asked = Requester::path_of("https://rule34video.com/tags/pokémon-species/").unwrap();
+        assert!(Requester::same_path(&asked, "/tags/pokémon-species/"));
+        assert!(Requester::same_path("/Models/X", "/models/x/"));
+        assert!(!Requester::same_path(&asked, "/search/"));
+    }
+
     #[test]
     fn new_requesters_share_cookie_jar() {
         let a = Requester::new();