Fix xxxtik empty results; replay solved identity on raw requests

Raw requester calls did not adopt the remembered FlareSolverr user agent,
so a RedGifs token minted under it was rejected on every gif lookup.
Tag/creator pages that mix non-RedGifs sources are refilled from later
cursor batches instead of coming back empty.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
2026-10-10 09:33:36 +00:00
parent 5176178607
commit 2d35f29f44
3 changed files with 54 additions and 5 deletions

View File

@@ -30,6 +30,7 @@ const RESOLVE_CONCURRENCY: usize = 8;
// xxxtik's listing endpoints are cursor-based (the numeric `id` of the last item
// returned), not page-number based, so reaching page N requires walking N
// sequential requests from the start of the feed.
const MAX_REFILL_FETCHES: usize = 4;
const MAX_PAGE_WALK: u16 = 25;
// xxxtik exposes ~60k tags through /tag/options, far too many to background-load
@@ -423,18 +424,43 @@ impl XxxtikProvider {
let mut items = Vec::new();
for _ in 0..page {
let batch = self.fetch_one_page(path, cursor, per_page, options).await?;
if batch.is_empty() {
// Tag and creator pages mix in posts whose source is not RedGifs
// (erome links, empty sources); those cannot be resolved here and
// used to leave whole pages empty ("girl" returned nothing). Keep
// pulling raw batches, bounded, until a page has enough
// resolvable posts.
let mut collected: Vec<Post> = Vec::new();
let mut exhausted = false;
for _ in 0..MAX_REFILL_FETCHES {
let batch = self.fetch_one_page(path, cursor, per_page, options).await?;
if batch.is_empty() {
exhausted = true;
break;
}
cursor = batch.last().map(|post| post.id).unwrap_or(cursor);
let short_batch = batch.len() < per_page;
collected.extend(batch.into_iter().filter(Self::is_redgifs_post));
if collected.len() >= per_page || short_batch {
break;
}
}
if exhausted && collected.is_empty() {
items = Vec::new();
break;
}
cursor = batch.last().map(|post| post.id).unwrap_or(cursor);
items = batch;
items = collected;
}
Ok(items)
}
fn is_redgifs_post(post: &Post) -> bool {
post.source
.as_deref()
.map(|source| source.contains("redgifs.com/"))
.unwrap_or(false)
}
async fn fetch_target_posts(
&self,
target: &Target,
@@ -791,6 +817,25 @@ impl Provider for XxxtikProvider {
#[cfg(test)]
mod tests {
#[test]
fn only_redgifs_sources_are_resolvable() {
let mk = |source: Option<&str>| super::Post {
id: 1,
uuid: "u".to_string(),
description: None,
source: source.map(str::to_string),
status: None,
created_at: String::new(),
author: Default::default(),
tags: vec![],
};
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://www.redgifs.com/watch/abc"))));
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://redgifs.com/watch/abc"))));
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("https://www.erome.com/a/CE5z46ne#39"))));
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some(""))));
assert!(!XxxtikProvider::is_redgifs_post(&mk(None)));
}
use super::*;
#[test]

View File

@@ -496,6 +496,10 @@ impl Requester {
timeout: Option<Duration>,
http_version: Option<Version>,
) -> Result<Response, wreq::Error> {
// Replay the solved FlareSolverr identity here too: a token minted
// through `get_with_headers` under that user agent is rejected (401)
// when the follow-up raw request goes out with the default one.
self.adopt_remembered_identity(url);
let cookie_preview = self.debug_cookie_preview_from_owned_headers(url, &headers);
#[cfg(not(feature = "debug"))]
let _ = &cookie_preview;