Fix xxdbx site-relative thumbs; cumlouder answers 404 immediately
xxdbx assets are now /id.jpg, not //host/..; prefixing https: produced https:/id.jpg. cumlouder's unknown-tag 404 went through the requester's mirror/FlareSolverr fallbacks (146s); read the raw response and return an empty list at once while still retrying 5xx. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
@@ -29,7 +29,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
|
||||
| [x] | `fullpornerorg` | `lulustream` proxy 403 / 502 | three separate causes: (1) hoster files deleted ("File is no longer available") but still listed -> now probed and dropped; (2) `myvidplay.com` embeds lost their host (bare ID sent to lulustream, wrong namespace) -> routed to the doodstream proxy with host kept, which now allows myvidplay/playmogo; (3) tnmr.org m3u8 is TLS-fingerprint gated (works with chrome TLS = the channel's `--impersonate chrome-120`; plain yt-dlp 403s are the checker's expected CF warnings). myvidplay playback could not be verified: the embed now returns a Turnstile captcha shell to this client |
|
||||
| [-] | `ripnsfw` | `doodstream` format 502 | lulustream format on the same items resolves (302). The doodstream format cannot be verified here: `*.dood.video` resolves to 127.0.0.1 on this host's resolver, so the CDN is unreachable. Needs the production network |
|
||||
| [-] | `freepornvideosxxx` | yt-dlp 502 on page URL | listing/search/p2 fine (24 items). Page URL is yt-dlp-native (html5 `<source>` 480/720/2160). The 502 is the site's edge answering plain yt-dlp's TLS fingerprint (3 of 4 items 502; 4/4 fine with `--impersonate chrome-120`; curl always 200). The channel declares that command but see "Open decision" below: status never delivers it |
|
||||
| [ ] | `hdporn92` | m3u8 formats time out (curl + yt-dlp) | morencius.com streams |
|
||||
| [-] | `hdporn92` | m3u8 formats time out (curl + yt-dlp) | provider works (embed deobfuscated, signed m3u8 returned, yt-dlp resolves the first item). Every morencius.com request (embed page AND playlist) carries a constant ~20.1s server-side delay from this egress (detail page on hdporn92.com is 0.25s), which exceeds check.py's 15s HEAD / 20s yt-dlp socket timeouts. Raising enrichment concurrency 4 -> 10 made it worse (108s, then timeouts), so it was reverted. Likely a host tarpit for this network; verify from the production IP |
|
||||
|
||||
## 3. Thumbnails
|
||||
|
||||
@@ -55,7 +55,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
|
||||
| | Channel | Symptom | Notes |
|
||||
|---|---|---|---|
|
||||
| [ ] | `aps` | 30-98s per request; check.py 30s timeout | |
|
||||
| [ ] | `hdporn92` | feed about 66s per request | shares item with section 2 |
|
||||
| [-] | `hdporn92` | feed about 66s per request | consequence of the 20s per-embed host delay above (10 embeds per page); not fixable in the provider without moving to play-time resolution, which would put the 20s on every play and every HLS playlist request |
|
||||
| [ ] | `porndish` | 14-23s feed/search | |
|
||||
|
||||
## 6. Tooling
|
||||
|
||||
@@ -357,22 +357,33 @@ impl CumlouderProvider {
|
||||
let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))];
|
||||
|
||||
// The origin sporadically answers its first uncached request with a
|
||||
// Cloudflare 522; a single retry succeeds.
|
||||
// Cloudflare 522; a retry succeeds. Use the raw response so a real
|
||||
// 404 (unknown tag/category) answers immediately instead of
|
||||
// escalating through the requester's mirror/FlareSolverr fallbacks,
|
||||
// which cost minutes.
|
||||
let mut last_err = String::new();
|
||||
let mut html = None;
|
||||
for _ in 0..3 {
|
||||
match requester
|
||||
.get_with_headers(&url, headers.clone(), None)
|
||||
.get_raw_with_headers(&url, headers.clone())
|
||||
.await
|
||||
{
|
||||
Ok(body) if body.contains("muestra-escena") => {
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
Ok(body) => {
|
||||
Ok(response) => {
|
||||
let status = response.status().as_u16();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
if status == 404 {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
if body.contains("muestra-escena") {
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
if status >= 500 || status == 429 {
|
||||
last_err = format!("HTTP {status}");
|
||||
continue;
|
||||
}
|
||||
// A page with no cards is a real empty archive.
|
||||
last_err = format!("no cards in {} bytes", body.len());
|
||||
// A page with no cards is a real empty archive (404 body
|
||||
// included); only keep retrying on transport errors.
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -52,6 +52,23 @@ impl XxdbxProvider {
|
||||
provider
|
||||
}
|
||||
|
||||
/// Resolves the card's asset URLs, which the site serves protocol-relative
|
||||
/// (`//prev.xxdbx.com/..`) or, now, site-relative (`/1234.jpg`).
|
||||
fn absolute_url(raw: &str) -> String {
|
||||
let raw = raw.trim();
|
||||
if raw.is_empty() {
|
||||
String::new()
|
||||
} else if raw.starts_with("http://") || raw.starts_with("https://") {
|
||||
raw.to_string()
|
||||
} else if let Some(rest) = raw.strip_prefix("//") {
|
||||
format!("https://{rest}")
|
||||
} else if raw.starts_with('/') {
|
||||
format!("https://xxdbx.com{raw}")
|
||||
} else {
|
||||
format!("https://xxdbx.com/{raw}")
|
||||
}
|
||||
}
|
||||
|
||||
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
|
||||
let _ = clientversion;
|
||||
Channel {
|
||||
@@ -272,26 +289,24 @@ impl XxdbxProvider {
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
|
||||
let thumb = format!(
|
||||
"https:{}",
|
||||
video_segment
|
||||
.split("<img ")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("src=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.last()
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
);
|
||||
let thumb_raw = video_segment
|
||||
.split("<img ")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("src=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.last()
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let thumb = Self::absolute_url(&thumb_raw);
|
||||
let raw_duration = video_segment
|
||||
.split("<div class=\"v_dur\">")
|
||||
.collect::<Vec<&str>>()
|
||||
@@ -306,21 +321,19 @@ impl XxdbxProvider {
|
||||
.to_string();
|
||||
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
|
||||
|
||||
let preview = format!(
|
||||
"https:{}",
|
||||
video_segment
|
||||
.split("data-preview=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
);
|
||||
let preview_raw = video_segment
|
||||
.split("data-preview=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let preview = Self::absolute_url(&preview_raw);
|
||||
let tags = video_segment
|
||||
.split("<div class=\"v_tags\">")
|
||||
.collect::<Vec<&str>>()
|
||||
@@ -439,3 +452,25 @@ impl Provider for XxdbxProvider {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::XxdbxProvider;
|
||||
|
||||
#[test]
|
||||
fn resolves_site_relative_protocol_relative_and_absolute_assets() {
|
||||
assert_eq!(
|
||||
XxdbxProvider::absolute_url("/7004779764736.jpg"),
|
||||
"https://xxdbx.com/7004779764736.jpg"
|
||||
);
|
||||
assert_eq!(
|
||||
XxdbxProvider::absolute_url("//prev.xxdbx.com/700477973230.mp4"),
|
||||
"https://prev.xxdbx.com/700477973230.mp4"
|
||||
);
|
||||
assert_eq!(
|
||||
XxdbxProvider::absolute_url("https://cdn.example/a.jpg"),
|
||||
"https://cdn.example/a.jpg"
|
||||
);
|
||||
assert_eq!(XxdbxProvider::absolute_url(""), "");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user