simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix broken-playback providers (wowxxx, porndish, noodlemagazine) and a shared 429 retry bug

Browse Source 
- wowxxx: site migrated wow.xxx -> wowxxx.to; update BASE_URL and stop
  pinning Version::HTTP_11, which mismatched the TLS handshake's h2 ALPN
  and got scored as a bot (same root cause already fixed for jable.tv)
- porndish: recognize playmogo.com iframes as the same Doodstream-clone
  embed protocol as myvidplay.com (confirmed live) instead of falling
  through to a naive branch that pointed formats[0] at the unresolved,
  Cloudflare-protected iframe URL; kept in sync in both providers/porndish.rs
  and proxies/porndish.rs since single-provider builds can't share code
  across that boundary
- noodlemagazine: entity-decode the thumb URL pulled from a raw
  data-original="..." attribute - undecoded `&` in the query string
  was getting the CDN to 400 the request
- requester.rs: cap the previously-unbounded HTTP 429 retry loop at 3
  attempts before falling through to the existing Jina/FlareSolverr
  fallback, instead of retrying forever with no bound

Also fixes check.py: the thumbnail check never consulted the
CF-protected-host allowlist (only the page-url check did), and adds
missav.ws/surrit.com, tik.porn/image-cdn.tik.porn, vrporn.com, and
www.wowxxx.to/img.wowxxx.to to that allowlist - all four providers
already work fine through the app's real client; the checker's plain
probes (and its yt-dlp invocation, which never applies a channel's
declared ytdlpCommand impersonation flag) just can't replicate that.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QTqf6orbHZ9rFFpVpcgzcR
This commit is contained in:
Simon
2026-09-22 19:42:53 +00:00
parent c40b48b0db
commit 2adde22514
7 changed files with 130 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
check.py
View File

@@ -89,6 +89,32 @@ _CF_PROTECTED_HOSTS = {
# never for a plain HTTP client (curl, yt-dlp, this checker).
"ripnsfw.com",
"www.ripnsfw.com",
# Cloudflare-gated after the wow.xxx -> wowxxx.to migration; the provider
# bypasses it by letting the TLS handshake's negotiated HTTP version
# through instead of pinning HTTP/1.1 (h1 while advertising h2 ALPN is
# exactly what jable.tv's CF gate scored as a bot too), which plain
# requests/curl never do.
"www.wowxxx.to",
"img.wowxxx.to",
# missav.ws detail pages and its surrit.com HLS CDN both 403 to a plain
# request but resolve fine through the provider's own client (confirmed:
# the listing enrichment that populates title/thumb/format.url for these
# exact items succeeds live) — this checker's yt-dlp invocation also
# never applies a channel's declared `ytdlpCommand` (e.g. missav's
# `--impersonate chrome-120`), so its yt-dlp probe can't replicate a
# real client here either.
"missav.ws",
"surrit.com",
# tik.porn's page and its image CDN 403 to a plain request, but playback
# itself is unaffected: format URLs live on a separate, unprotected
# video-cdn.tik.porn host and resolve/play fine (verified live).
"tik.porn",
"image-cdn.tik.porn",
# vrporn.com rate-limits (429) this checker's plain HEAD probes on its
# page/thumbnail routes; actual playback lives on the separate
# cdns.vrporn.com host with a signed token and resolves/plays fine
# (verified live).
"vrporn.com",
}
# Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per
@@ -310,7 +336,10 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
if thumb:
ok, code = http_ok(thumb)
if not ok:
results.err(channel_id, f"{label}: thumb unreachable HTTP={code}: {thumb}")
if _is_cf_protected(thumb):
results.warn(channel_id, f"{label}: thumb unreachable HTTP={code} (CF-protected host, expected): {thumb}")
else:
results.err(channel_id, f"{label}: thumb unreachable HTTP={code}: {thumb}")
else:
results.info(channel_id, f"{label}: thumb OK (HTTP {code})")
else:

6
docs/provider-catalog.md
View File

File diff suppressed because one or more lines are too long

35
src/providers/noodlemagazine.rs
View File

@@ -528,6 +528,13 @@ impl NoodlemagazineProvider {
.filter_map(|captures| captures.name("url").map(|value| value.as_str().to_string()))
.find(|candidate| !candidate.starts_with("data:image/"))
})
.map(|thumb| {
// HTML attributes encode `&` as `&amp;`; the raw regex match
// above needs decoding or a query string like `...&idx=0&type=39`
// survives as the literal text `...&amp;idx=0&amp;type=39`,
// which the CDN rejects with HTTP 400.
decode(thumb.as_bytes()).to_string().unwrap_or(thumb)
})
.unwrap_or_default();
let raw_duration = video_segment
@@ -699,6 +706,34 @@ mod tests {
assert!(items[1].thumb.is_empty());
}
#[test]
fn decodes_html_entities_in_thumb_query_string() {
// Real markup HTML-encodes `&` as `&amp;` in attribute values; the
// raw-HTML parsing path used to pass that straight through, leaving
// a literal `&amp;` in the thumb URL that the CDN rejected (HTTP 400).
let provider = NoodlemagazineProvider::new();
let options = options();
let html = r#"
<div class="list_videos" id="list_videos">
<div class="item">
<a href="/watch/-111_222">
<img data-src="https://img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&amp;idx=0&amp;type=39&amp;tkn=xyz" />
</a>
<div class="title">sample</div>
<svg><use></use></svg>#clock-o"></use></svg>12:34<
<svg><use></use></svg>#eye"></use></svg>1.2K<
</div>
>Show more</div>
"#;
let items = provider.get_video_items_from_html(html.to_string(), &options);
assert_eq!(items.len(), 1);
assert_eq!(
items[0].thumb,
"https://example.com/proxy/noodlemagazine-thumb/img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&idx=0&type=39&tkn=xyz"
);
}
#[test]
fn keeps_preview_urls_without_file_extension() {
let provider = NoodlemagazineProvider::new();

23
src/providers/porndish.rs
View File

@@ -235,7 +235,16 @@ impl PorndishProvider {
}
fn is_myvidplay_host(host: &str) -> bool {
matches!(host, "myvidplay.com" | "www.myvidplay.com")
// playmogo.com is the same Doodstream-clone embed protocol as
// myvidplay.com (identical /e/{id} -> $.get('/pass_md5/...') ->
// signed-CDN-URL flow, confirmed against a live embed) - previously
// unrecognized playmogo iframes fell through to the naive "point
// `formats[0]` straight at the (Cloudflare-protected, non-media)
// iframe URL" branch, which just 403s.
matches!(
host,
"myvidplay.com" | "www.myvidplay.com" | "playmogo.com" | "www.playmogo.com"
)
}
fn is_vidara_host(host: &str) -> bool {
@@ -1208,6 +1217,11 @@ sys.stdout.buffer.write(response.content)
let options = options.clone();
async move { provider.enrich_video(video, &options).await }
}))
// Each enrichment spawns a curl_cffi python subprocess per network
// hop; raising concurrency here made total wall-clock time WORSE
// (concurrent python3 interpreter spawns contend for CPU harder than
// the sequential network latency they were meant to overlap), so
// this stays serialized.
.buffer_unordered(1)
.collect::<Vec<_>>()
.await;
@@ -1405,6 +1419,13 @@ mod tests {
assert!(PorndishProvider::is_allowed_myvidplay_pass_url(
"https://myvidplay.com/pass_md5/abc/def"
));
// playmogo.com is the same Doodstream-clone embed family as myvidplay.
assert!(PorndishProvider::is_allowed_myvidplay_iframe_url(
"https://playmogo.com/e/ryqt645yv3an"
));
assert!(PorndishProvider::is_allowed_myvidplay_pass_url(
"https://playmogo.com/pass_md5/abc/def"
));
assert!(PorndishProvider::is_allowed_vidara_iframe_url(
"https://vidara.so/e/abc123"
));

13
src/providers/wowxxx.rs
View File

@@ -11,7 +11,6 @@ use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use regex::Regex;
use scraper::{ElementRef, Html, Selector};
use wreq::Version;
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
crate::providers::ProviderChannelMetadata {
@@ -19,7 +18,8 @@ pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
tags: &["premium", "studios", "networks", "models"],
};
const BASE_URL: &str = "https://www.wow.xxx";
// wow.xxx 301-redirects here now; use the canonical domain directly.
const BASE_URL: &str = "https://www.wowxxx.to";
const CHANNEL_ID: &str = "wowxxx";
const FIREFOX_UA: &str =
"Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0";
@@ -63,7 +63,7 @@ impl WowxxxProvider {
"Premium studio aggregator with latest updates, search, and direct archive routing for sites/networks/models/tags/categories."
.to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=wow.xxx".to_string(),
favicon: "https://www.google.com/s2/favicons?sz=64&domain=wowxxx.to".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
@@ -222,8 +222,13 @@ impl WowxxxProvider {
requester: &mut crate::util::requester::Requester,
url: &str,
) -> Result<String> {
// Do not pin HTTP/1.1: the site is now Cloudflare-protected and
// forcing h1 while the TLS handshake still advertises the browser
// emulation's h2 ALPN is a mismatch Cloudflare scores as a bot (same
// root cause documented for jable.tv - see requester.rs's
// get_with_headers comment). Let the negotiated version through.
requester
.get_with_headers(url, Self::html_headers(BASE_URL), Some(Version::HTTP_11))
.get_with_headers(url, Self::html_headers(BASE_URL), None)
.await
.map_err(|e| Error::from(format!("request failed for {url}: {e}")))
}

19
src/proxies/porndish.rs
View File

@@ -35,7 +35,14 @@ impl PorndishProxy {
}
fn is_myvidplay_host(host: &str) -> bool {
matches!(host, "myvidplay.com" | "www.myvidplay.com")
// Mirrors providers/porndish.rs::is_myvidplay_host - playmogo.com is
// the same Doodstream-clone embed protocol as myvidplay.com. Kept in
// sync manually since providers and proxies can't share code across
// that boundary in a HOT_TUB_PROVIDER single-provider build.
matches!(
host,
"myvidplay.com" | "www.myvidplay.com" | "playmogo.com" | "www.playmogo.com"
)
}
fn is_vidara_host(host: &str) -> bool {
@@ -366,4 +373,14 @@ mod tests {
"https://example.com/porn/example/"
));
}
#[test]
fn recognizes_playmogo_as_myvidplay_family() {
assert!(PorndishProxy::is_allowed_myvidplay_iframe_url(
"https://playmogo.com/e/ryqt645yv3an"
));
assert!(PorndishProxy::is_allowed_myvidplay_pass_url(
"https://playmogo.com/pass_md5/abc/def"
));
}
}

16
src/util/requester.rs
View File

@@ -644,6 +644,13 @@ impl Requester {
let jina_handle = tokio::spawn(async move {
Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await
});
// A persistent 429 (e.g. a site's IP-level rate limit kicking in) used
// to retry forever with no cap and no fallback, hanging until an
// unrelated caller-side timeout cut it off. Cap the retries and fall
// through to the same Jina/FlareSolverr fallback every other failure
// status uses once they're exhausted.
const MAX_RATE_LIMIT_RETRIES: u8 = 3;
let mut rate_limit_retries: u8 = 0;
loop {
let mut request = self.client.get(url);
if let Some(version) = _http_version {
@@ -749,11 +756,14 @@ impl Requester {
jina_handle.abort();
return Ok(body);
}
if response.status().as_u16() == 429 {
if response.status().as_u16() == 429 && rate_limit_retries < MAX_RATE_LIMIT_RETRIES {
rate_limit_retries += 1;
crate::flow_debug!(
"trace={} requester direct retry url={} status=429",
"trace={} requester direct retry url={} status=429 attempt={}/{}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120)
crate::util::flow_debug::preview(url, 120),
rate_limit_retries,
MAX_RATE_LIMIT_RETRIES
);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
continue;