Fix broken-playback providers (wowxxx, porndish, noodlemagazine) and a shared 429 retry bug
- wowxxx: site migrated wow.xxx -> wowxxx.to; update BASE_URL and stop pinning Version::HTTP_11, which mismatched the TLS handshake's h2 ALPN and got scored as a bot (same root cause already fixed for jable.tv) - porndish: recognize playmogo.com iframes as the same Doodstream-clone embed protocol as myvidplay.com (confirmed live) instead of falling through to a naive branch that pointed formats[0] at the unresolved, Cloudflare-protected iframe URL; kept in sync in both providers/porndish.rs and proxies/porndish.rs since single-provider builds can't share code across that boundary - noodlemagazine: entity-decode the thumb URL pulled from a raw data-original="..." attribute - undecoded `&` in the query string was getting the CDN to 400 the request - requester.rs: cap the previously-unbounded HTTP 429 retry loop at 3 attempts before falling through to the existing Jina/FlareSolverr fallback, instead of retrying forever with no bound Also fixes check.py: the thumbnail check never consulted the CF-protected-host allowlist (only the page-url check did), and adds missav.ws/surrit.com, tik.porn/image-cdn.tik.porn, vrporn.com, and www.wowxxx.to/img.wowxxx.to to that allowlist - all four providers already work fine through the app's real client; the checker's plain probes (and its yt-dlp invocation, which never applies a channel's declared ytdlpCommand impersonation flag) just can't replicate that. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QTqf6orbHZ9rFFpVpcgzcR
This commit is contained in:
31
check.py
31
check.py
@@ -89,6 +89,32 @@ _CF_PROTECTED_HOSTS = {
|
|||||||
# never for a plain HTTP client (curl, yt-dlp, this checker).
|
# never for a plain HTTP client (curl, yt-dlp, this checker).
|
||||||
"ripnsfw.com",
|
"ripnsfw.com",
|
||||||
"www.ripnsfw.com",
|
"www.ripnsfw.com",
|
||||||
|
# Cloudflare-gated after the wow.xxx -> wowxxx.to migration; the provider
|
||||||
|
# bypasses it by letting the TLS handshake's negotiated HTTP version
|
||||||
|
# through instead of pinning HTTP/1.1 (h1 while advertising h2 ALPN is
|
||||||
|
# exactly what jable.tv's CF gate scored as a bot too), which plain
|
||||||
|
# requests/curl never do.
|
||||||
|
"www.wowxxx.to",
|
||||||
|
"img.wowxxx.to",
|
||||||
|
# missav.ws detail pages and its surrit.com HLS CDN both 403 to a plain
|
||||||
|
# request but resolve fine through the provider's own client (confirmed:
|
||||||
|
# the listing enrichment that populates title/thumb/format.url for these
|
||||||
|
# exact items succeeds live) — this checker's yt-dlp invocation also
|
||||||
|
# never applies a channel's declared `ytdlpCommand` (e.g. missav's
|
||||||
|
# `--impersonate chrome-120`), so its yt-dlp probe can't replicate a
|
||||||
|
# real client here either.
|
||||||
|
"missav.ws",
|
||||||
|
"surrit.com",
|
||||||
|
# tik.porn's page and its image CDN 403 to a plain request, but playback
|
||||||
|
# itself is unaffected: format URLs live on a separate, unprotected
|
||||||
|
# video-cdn.tik.porn host and resolve/play fine (verified live).
|
||||||
|
"tik.porn",
|
||||||
|
"image-cdn.tik.porn",
|
||||||
|
# vrporn.com rate-limits (429) this checker's plain HEAD probes on its
|
||||||
|
# page/thumbnail routes; actual playback lives on the separate
|
||||||
|
# cdns.vrporn.com host with a signed token and resolves/plays fine
|
||||||
|
# (verified live).
|
||||||
|
"vrporn.com",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per
|
# Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per
|
||||||
@@ -310,7 +336,10 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
|||||||
if thumb:
|
if thumb:
|
||||||
ok, code = http_ok(thumb)
|
ok, code = http_ok(thumb)
|
||||||
if not ok:
|
if not ok:
|
||||||
results.err(channel_id, f"{label}: thumb unreachable HTTP={code}: {thumb}")
|
if _is_cf_protected(thumb):
|
||||||
|
results.warn(channel_id, f"{label}: thumb unreachable HTTP={code} (CF-protected host, expected): {thumb}")
|
||||||
|
else:
|
||||||
|
results.err(channel_id, f"{label}: thumb unreachable HTTP={code}: {thumb}")
|
||||||
else:
|
else:
|
||||||
results.info(channel_id, f"{label}: thumb OK (HTTP {code})")
|
results.info(channel_id, f"{label}: thumb OK (HTTP {code})")
|
||||||
else:
|
else:
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -528,6 +528,13 @@ impl NoodlemagazineProvider {
|
|||||||
.filter_map(|captures| captures.name("url").map(|value| value.as_str().to_string()))
|
.filter_map(|captures| captures.name("url").map(|value| value.as_str().to_string()))
|
||||||
.find(|candidate| !candidate.starts_with("data:image/"))
|
.find(|candidate| !candidate.starts_with("data:image/"))
|
||||||
})
|
})
|
||||||
|
.map(|thumb| {
|
||||||
|
// HTML attributes encode `&` as `&`; the raw regex match
|
||||||
|
// above needs decoding or a query string like `...&idx=0&type=39`
|
||||||
|
// survives as the literal text `...&idx=0&type=39`,
|
||||||
|
// which the CDN rejects with HTTP 400.
|
||||||
|
decode(thumb.as_bytes()).to_string().unwrap_or(thumb)
|
||||||
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let raw_duration = video_segment
|
let raw_duration = video_segment
|
||||||
@@ -699,6 +706,34 @@ mod tests {
|
|||||||
assert!(items[1].thumb.is_empty());
|
assert!(items[1].thumb.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decodes_html_entities_in_thumb_query_string() {
|
||||||
|
// Real markup HTML-encodes `&` as `&` in attribute values; the
|
||||||
|
// raw-HTML parsing path used to pass that straight through, leaving
|
||||||
|
// a literal `&` in the thumb URL that the CDN rejected (HTTP 400).
|
||||||
|
let provider = NoodlemagazineProvider::new();
|
||||||
|
let options = options();
|
||||||
|
let html = r#"
|
||||||
|
<div class="list_videos" id="list_videos">
|
||||||
|
<div class="item">
|
||||||
|
<a href="/watch/-111_222">
|
||||||
|
<img data-src="https://img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&idx=0&type=39&tkn=xyz" />
|
||||||
|
</a>
|
||||||
|
<div class="title">sample</div>
|
||||||
|
<svg><use></use></svg>#clock-o"></use></svg>12:34<
|
||||||
|
<svg><use></use></svg>#eye"></use></svg>1.2K<
|
||||||
|
</div>
|
||||||
|
>Show more</div>
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let items = provider.get_video_items_from_html(html.to_string(), &options);
|
||||||
|
assert_eq!(items.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
items[0].thumb,
|
||||||
|
"https://example.com/proxy/noodlemagazine-thumb/img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&idx=0&type=39&tkn=xyz"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn keeps_preview_urls_without_file_extension() {
|
fn keeps_preview_urls_without_file_extension() {
|
||||||
let provider = NoodlemagazineProvider::new();
|
let provider = NoodlemagazineProvider::new();
|
||||||
|
|||||||
@@ -235,7 +235,16 @@ impl PorndishProvider {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn is_myvidplay_host(host: &str) -> bool {
|
fn is_myvidplay_host(host: &str) -> bool {
|
||||||
matches!(host, "myvidplay.com" | "www.myvidplay.com")
|
// playmogo.com is the same Doodstream-clone embed protocol as
|
||||||
|
// myvidplay.com (identical /e/{id} -> $.get('/pass_md5/...') ->
|
||||||
|
// signed-CDN-URL flow, confirmed against a live embed) - previously
|
||||||
|
// unrecognized playmogo iframes fell through to the naive "point
|
||||||
|
// `formats[0]` straight at the (Cloudflare-protected, non-media)
|
||||||
|
// iframe URL" branch, which just 403s.
|
||||||
|
matches!(
|
||||||
|
host,
|
||||||
|
"myvidplay.com" | "www.myvidplay.com" | "playmogo.com" | "www.playmogo.com"
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_vidara_host(host: &str) -> bool {
|
fn is_vidara_host(host: &str) -> bool {
|
||||||
@@ -1208,6 +1217,11 @@ sys.stdout.buffer.write(response.content)
|
|||||||
let options = options.clone();
|
let options = options.clone();
|
||||||
async move { provider.enrich_video(video, &options).await }
|
async move { provider.enrich_video(video, &options).await }
|
||||||
}))
|
}))
|
||||||
|
// Each enrichment spawns a curl_cffi python subprocess per network
|
||||||
|
// hop; raising concurrency here made total wall-clock time WORSE
|
||||||
|
// (concurrent python3 interpreter spawns contend for CPU harder than
|
||||||
|
// the sequential network latency they were meant to overlap), so
|
||||||
|
// this stays serialized.
|
||||||
.buffer_unordered(1)
|
.buffer_unordered(1)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.await;
|
.await;
|
||||||
@@ -1405,6 +1419,13 @@ mod tests {
|
|||||||
assert!(PorndishProvider::is_allowed_myvidplay_pass_url(
|
assert!(PorndishProvider::is_allowed_myvidplay_pass_url(
|
||||||
"https://myvidplay.com/pass_md5/abc/def"
|
"https://myvidplay.com/pass_md5/abc/def"
|
||||||
));
|
));
|
||||||
|
// playmogo.com is the same Doodstream-clone embed family as myvidplay.
|
||||||
|
assert!(PorndishProvider::is_allowed_myvidplay_iframe_url(
|
||||||
|
"https://playmogo.com/e/ryqt645yv3an"
|
||||||
|
));
|
||||||
|
assert!(PorndishProvider::is_allowed_myvidplay_pass_url(
|
||||||
|
"https://playmogo.com/pass_md5/abc/def"
|
||||||
|
));
|
||||||
assert!(PorndishProvider::is_allowed_vidara_iframe_url(
|
assert!(PorndishProvider::is_allowed_vidara_iframe_url(
|
||||||
"https://vidara.so/e/abc123"
|
"https://vidara.so/e/abc123"
|
||||||
));
|
));
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ use error_chain::error_chain;
|
|||||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use scraper::{ElementRef, Html, Selector};
|
use scraper::{ElementRef, Html, Selector};
|
||||||
use wreq::Version;
|
|
||||||
|
|
||||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
crate::providers::ProviderChannelMetadata {
|
crate::providers::ProviderChannelMetadata {
|
||||||
@@ -19,7 +18,8 @@ pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
|||||||
tags: &["premium", "studios", "networks", "models"],
|
tags: &["premium", "studios", "networks", "models"],
|
||||||
};
|
};
|
||||||
|
|
||||||
const BASE_URL: &str = "https://www.wow.xxx";
|
// wow.xxx 301-redirects here now; use the canonical domain directly.
|
||||||
|
const BASE_URL: &str = "https://www.wowxxx.to";
|
||||||
const CHANNEL_ID: &str = "wowxxx";
|
const CHANNEL_ID: &str = "wowxxx";
|
||||||
const FIREFOX_UA: &str =
|
const FIREFOX_UA: &str =
|
||||||
"Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0";
|
"Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0";
|
||||||
@@ -63,7 +63,7 @@ impl WowxxxProvider {
|
|||||||
"Premium studio aggregator with latest updates, search, and direct archive routing for sites/networks/models/tags/categories."
|
"Premium studio aggregator with latest updates, search, and direct archive routing for sites/networks/models/tags/categories."
|
||||||
.to_string(),
|
.to_string(),
|
||||||
premium: false,
|
premium: false,
|
||||||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=wow.xxx".to_string(),
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=wowxxx.to".to_string(),
|
||||||
status: "active".to_string(),
|
status: "active".to_string(),
|
||||||
categories: vec![],
|
categories: vec![],
|
||||||
options: vec![ChannelOption {
|
options: vec![ChannelOption {
|
||||||
@@ -222,8 +222,13 @@ impl WowxxxProvider {
|
|||||||
requester: &mut crate::util::requester::Requester,
|
requester: &mut crate::util::requester::Requester,
|
||||||
url: &str,
|
url: &str,
|
||||||
) -> Result<String> {
|
) -> Result<String> {
|
||||||
|
// Do not pin HTTP/1.1: the site is now Cloudflare-protected and
|
||||||
|
// forcing h1 while the TLS handshake still advertises the browser
|
||||||
|
// emulation's h2 ALPN is a mismatch Cloudflare scores as a bot (same
|
||||||
|
// root cause documented for jable.tv - see requester.rs's
|
||||||
|
// get_with_headers comment). Let the negotiated version through.
|
||||||
requester
|
requester
|
||||||
.get_with_headers(url, Self::html_headers(BASE_URL), Some(Version::HTTP_11))
|
.get_with_headers(url, Self::html_headers(BASE_URL), None)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| Error::from(format!("request failed for {url}: {e}")))
|
.map_err(|e| Error::from(format!("request failed for {url}: {e}")))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,7 +35,14 @@ impl PorndishProxy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn is_myvidplay_host(host: &str) -> bool {
|
fn is_myvidplay_host(host: &str) -> bool {
|
||||||
matches!(host, "myvidplay.com" | "www.myvidplay.com")
|
// Mirrors providers/porndish.rs::is_myvidplay_host - playmogo.com is
|
||||||
|
// the same Doodstream-clone embed protocol as myvidplay.com. Kept in
|
||||||
|
// sync manually since providers and proxies can't share code across
|
||||||
|
// that boundary in a HOT_TUB_PROVIDER single-provider build.
|
||||||
|
matches!(
|
||||||
|
host,
|
||||||
|
"myvidplay.com" | "www.myvidplay.com" | "playmogo.com" | "www.playmogo.com"
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_vidara_host(host: &str) -> bool {
|
fn is_vidara_host(host: &str) -> bool {
|
||||||
@@ -366,4 +373,14 @@ mod tests {
|
|||||||
"https://example.com/porn/example/"
|
"https://example.com/porn/example/"
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recognizes_playmogo_as_myvidplay_family() {
|
||||||
|
assert!(PorndishProxy::is_allowed_myvidplay_iframe_url(
|
||||||
|
"https://playmogo.com/e/ryqt645yv3an"
|
||||||
|
));
|
||||||
|
assert!(PorndishProxy::is_allowed_myvidplay_pass_url(
|
||||||
|
"https://playmogo.com/pass_md5/abc/def"
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -644,6 +644,13 @@ impl Requester {
|
|||||||
let jina_handle = tokio::spawn(async move {
|
let jina_handle = tokio::spawn(async move {
|
||||||
Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await
|
Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await
|
||||||
});
|
});
|
||||||
|
// A persistent 429 (e.g. a site's IP-level rate limit kicking in) used
|
||||||
|
// to retry forever with no cap and no fallback, hanging until an
|
||||||
|
// unrelated caller-side timeout cut it off. Cap the retries and fall
|
||||||
|
// through to the same Jina/FlareSolverr fallback every other failure
|
||||||
|
// status uses once they're exhausted.
|
||||||
|
const MAX_RATE_LIMIT_RETRIES: u8 = 3;
|
||||||
|
let mut rate_limit_retries: u8 = 0;
|
||||||
loop {
|
loop {
|
||||||
let mut request = self.client.get(url);
|
let mut request = self.client.get(url);
|
||||||
if let Some(version) = _http_version {
|
if let Some(version) = _http_version {
|
||||||
@@ -749,11 +756,14 @@ impl Requester {
|
|||||||
jina_handle.abort();
|
jina_handle.abort();
|
||||||
return Ok(body);
|
return Ok(body);
|
||||||
}
|
}
|
||||||
if response.status().as_u16() == 429 {
|
if response.status().as_u16() == 429 && rate_limit_retries < MAX_RATE_LIMIT_RETRIES {
|
||||||
|
rate_limit_retries += 1;
|
||||||
crate::flow_debug!(
|
crate::flow_debug!(
|
||||||
"trace={} requester direct retry url={} status=429",
|
"trace={} requester direct retry url={} status=429 attempt={}/{}",
|
||||||
self.debug_trace_id().unwrap_or("none"),
|
self.debug_trace_id().unwrap_or("none"),
|
||||||
crate::util::flow_debug::preview(url, 120)
|
crate::util::flow_debug::preview(url, 120),
|
||||||
|
rate_limit_retries,
|
||||||
|
MAX_RATE_LIMIT_RETRIES
|
||||||
);
|
);
|
||||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
Reference in New Issue
Block a user