diff --git a/check.py b/check.py index 61f66e8..3414de8 100644 --- a/check.py +++ b/check.py @@ -89,6 +89,32 @@ _CF_PROTECTED_HOSTS = { # never for a plain HTTP client (curl, yt-dlp, this checker). "ripnsfw.com", "www.ripnsfw.com", + # Cloudflare-gated after the wow.xxx -> wowxxx.to migration; the provider + # bypasses it by letting the TLS handshake's negotiated HTTP version + # through instead of pinning HTTP/1.1 (h1 while advertising h2 ALPN is + # exactly what jable.tv's CF gate scored as a bot too), which plain + # requests/curl never do. + "www.wowxxx.to", + "img.wowxxx.to", + # missav.ws detail pages and its surrit.com HLS CDN both 403 to a plain + # request but resolve fine through the provider's own client (confirmed: + # the listing enrichment that populates title/thumb/format.url for these + # exact items succeeds live) — this checker's yt-dlp invocation also + # never applies a channel's declared `ytdlpCommand` (e.g. missav's + # `--impersonate chrome-120`), so its yt-dlp probe can't replicate a + # real client here either. + "missav.ws", + "surrit.com", + # tik.porn's page and its image CDN 403 to a plain request, but playback + # itself is unaffected: format URLs live on a separate, unprotected + # video-cdn.tik.porn host and resolve/play fine (verified live). + "tik.porn", + "image-cdn.tik.porn", + # vrporn.com rate-limits (429) this checker's plain HEAD probes on its + # page/thumbnail routes; actual playback lives on the separate + # cdns.vrporn.com host with a signed token and resolves/plays fine + # (verified live). + "vrporn.com", } # Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per @@ -310,7 +336,10 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool) if thumb: ok, code = http_ok(thumb) if not ok: - results.err(channel_id, f"{label}: thumb unreachable HTTP={code}: {thumb}") + if _is_cf_protected(thumb): + results.warn(channel_id, f"{label}: thumb unreachable HTTP={code} (CF-protected host, expected): {thumb}") + else: + results.err(channel_id, f"{label}: thumb unreachable HTTP={code}: {thumb}") else: results.info(channel_id, f"{label}: thumb OK (HTTP {code})") else: diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index c402f78..17e8c84 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -35,7 +35,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `javtiful` | `jav` | no | no | JAV channel family. | | `melonstube` | `mainstream-tube` | no | yes | Meta-search aggregator for melonstube.com — every card is an `/out/?l=&c=&v=3` redirect link whose destination (a third-party host) is embedded, zero-network, in a MessagePack blob under the `l` param (mixed binary/string fields: view/click counts, a JSON date-range blob, an array of related-video ids, and the destination URL itself as one msgpack string). Decoding is local: base64-decode `l`, then regex-extract the `https?://...` destination directly out of the raw (lossy-UTF8-converted) msgpack bytes — **the regex must be a positive allowlist of legal URI characters** (`[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`), not a denylist of a few excluded characters: the byte immediately following the msgpack string (a length-prefix marker for the next field, e.g. `\xcd`) lossy-converts to U+FFFD (code point 0xFFFD), which a denylist limited to ASCII control chars (`[^\x00-\x1f\\"']`) fails to exclude, silently appending a corrupt trailing character to the URL. Feeds: `/new` (latest), `/popular`, `/rating` (all three are static curated lists, ~120 items, true pagination via `?page=N`); search is NOT the vestigial `/search?q=` GET endpoint (returns generic filler content behind a fake static result counter regardless of query) — the real mechanism mirrors the site's own search form: POST `search_query[query]=` to `/searching/by-form`, which 303-redirects (cookies + redirect auto-followed by the shared `Requester`) to a resolved, genuinely query-relevant destination depending on entity match — `/pornstar/`, `/category/`, `/search/a/` (studio), or `/search/` (generic free text) — all of which support `?page=N` pagination; a non-matching query resolves to a page with zero result cards, which `parse_listing` naturally returns as empty. Destination hosts fall into three buckets: (1) the common case — yt-dlp (`--impersonate chrome-120`) resolves the destination directly, so `video.url` is just the decoded destination URL, no proxy; (2) `JUNK_HOSTS` (currently `fhgte.com`) — observed dead-end paywall/signup funnels with no free playable video, so cards pointing there are dropped entirely rather than surfaced as false-positive results; (3) `HARD_HOSTS` (`manysex.com`, `videomanysex.com`) — yt-dlp cannot resolve these, so `video.url` is routed through `/proxy/melonstube/{host}/{path}`. The proxy (`src/proxies/melonstube.rs`) ports vjav.rs's Cyrillic-homoglyph-obfuscated base64 decode chain to pull the real `get_file` path/query out of `videofile.php`'s `video_url` field, then makes two manual (non-auto-redirect) hops to the final CDN URL: hop 1 needs the manysex.com/videomanysex.com page as `Referer`; hop 2 (to the signed `ahcdn.com` URL) must be sent with **no** Referer at all, because the CDN's signed URL embeds a literal `referer=none,.manysex.com,.gstatic.com` allow-list that rejects the videomanysex.com Referer used on hop 1 — auto-redirect clients that forward the same Referer to every hop get a 403 at hop 2. Before returning the 302, an anti-false-positive check confirms the resolved URL string contains the requested numeric video id and that a ranged GET (`Range: bytes=0-65535`) returns 200/206 with a `video/*`/`octet-stream` content-type, so a paywall/ad/error page can't masquerade as the real stream. Thumbnails (`ttcache.com`) load directly, no proxy. No `/api/uploaders` (aggregator has no stable uploader identity). **2026-09-21 fix:** melonstube.com started fronting itself with Cloudflare bot management (a real Turnstile "challenge" page, `cf-mitigated: challenge`), which the shared Requester's Firefox151 emulation gets on every path — this made the provider return 0 items (direct fetch, Jina mirror, and FlareSolverr all failed/unreachable). A/B probing many `curl_cffi` fingerprints found Chrome120/124 pass the plain `/new`/`/popular`/`/rating` listing pages but still 403 on `/category/*` and the `/searching/by-form` search POST, while **Safari18** passes all of them; listing and search fetches now try a dedicated `wreq::Client` with `Emulation::Safari18` first and fall back to the shared Requester if Cloudflare's rules change again. Also fixed: `get_videos` previously ignored `perPage` entirely (`let _ = per_page;`), returning the full ~120-card page regardless of the requested size — it now truncates to the requested `perPage` after fetching. **2026-09-21 sub-provider resolution:** a live crawl of `/new`/`/popular`/`/rating` (~3,350 out-links, 117 distinct destination hosts) found the third bucket above ("yt-dlp resolves it directly") was hiding real failures — `eporner.com` alone is ~34% of all traffic and, like the `eporner` channel itself, serves a restricted preview clip without the site's hash/xhr dance; `fapcat.com`/`momvids.com` 403 a plain fetch (Cloudflare); a few others just needed the right extraction. Per-card enrichment (bounded `buffer_unordered`, `ENRICH_CONCURRENCY=8`, each resolution capped at `ENRICH_TIMEOUT_SECS=10` so one slow/unreachable host can't stall the page, and only run for the first `per_page` pending cards — not the full ~240-card page) now dispatches by destination host (`classify_source`) to: **Eporner** — `crate::util::eporner_resolve::resolve_public_formats` (moved out of `eporner.rs` into `src/util/` specifically so both providers get it without breaking either one's `HOT_TUB_PROVIDER=` single-provider build, which only compiles the one selected provider module). **Kvs** (`pornfreenow.com`, `analdin.xxx`, `babepump.com`, `w1mp.com`, `fetishshrine.com`, `pervclips.com`, `xozilla.xxx`, `ok.xxx`, `ebonypeek.com`, `xgroovy.com`) — a generic regex scan for the `get_file/{N}/{hash}/.../{id}[_variant].mp4` shape already used by porn4fans.rs/xgroovy.rs/thepornbunny.rs, ranking by resolution suffix and falling back to a qualitative `hd`-suffix check for sites (analdin, xozilla) that don't tag an explicit height; teaser variants (`_s.mp4`, `_preview.mp4`) excluded. **KvsCfGated** (`fapcat.com`, `momvids.com`) — same KVS extraction, fetched through the Safari18 bypass client melonstube.com itself needs. **XhamsterMirror** (`xh.partners`) — an affiliate short-link into a genuine xHamster mirror instance (same `xhcdn.com` CDN, same embedded `"sources":{"mp4":{...}}` JSON as xhamster.com); the Referer is built from the mirror's actual resolved host, not assumed, since the signed CDN URL's allow-list is mirror-specific. **SomePorn** (`some.porn`) — no page fetch at all; its HLS master playlist is addressable directly from the numeric id in the page URL (`s.cdn3x.com/get-playlist?id={id}&q=720p`). **Pornone** (`pornone.com`) — direct unauthenticated `` mp4 tags at multiple resolutions right on the detail page. Hosts outside this set keep the original fallback (bare destination URL, no formats). `www.flirt4free.com` (a live-cam profile page, not a VOD) moved into the same drop bucket as `JUNK_HOSTS`. | | `missav` | `jav` | no | no | HLS format pattern. | -| `noodlemagazine` | `mainstream-tube` | no | yes | Best template for media and thumbnail proxying. | +| `noodlemagazine` | `mainstream-tube` | no | yes | Best template for media and thumbnail proxying. The raw-HTML card parser pulls the thumb URL straight out of a `data-original="..."` attribute — real markup HTML-encodes `&` as `&` there, so the extracted query string must be entity-decoded (`htmlentity::entity::decode`) before proxying, or a literal `&` survives into the URL and the CDN 400s it. Some thumbs are OK.ru's `i.mycdn.me/getVideoPreview?...&tkn=...` signed preview endpoint, which has been observed to expire within seconds of the listing fetch that generated it — an inherent CDN token-TTL characteristic, not fixable by faster requests. | | `okporn` | `mainstream-tube` | no | no | Simple mainstream archive. | | `okxxx` | `mainstream-tube` | no | no | Mainstream search/archive pattern. | | `omgxxx` | `studio-network` | yes | no | Best template for sites/networks/stars filter catalogs. | @@ -47,7 +47,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `pmvhaven` | `pmv-compilation` | no | no | PMV grouping example. | | `porn00` | `mainstream-tube` | no | no | Lightweight scraper. | | `porn4fans` | `onlyfans` | no | no | KVS (Kernel Video Sharing) scraper for porn4fans.com (OnlyFans creator clips); Cloudflare-fronted but serves direct requests (no JS challenge), so the shared requester works without Jina/FlareSolverr and detail-page enrichment is safe; all feeds are fetched as KVS `?mode=async&function=get_block` HTML fragments (cleaner + properly paginated vs the JS-filled full pages), parsed with `scraper` over `div.item` cards (`a.img-wrap.video` href→id/title, `img.thumb` data-webp/src, `div.duration`, `li.video-item.views span`, `li.video-item.model a` for uploader, `div.preview-video[data-src]` preview clip); latest feed is `/onlyfans-videos/` block `custom_list_videos_latest_videos_list` paginated by `from=N` (12/page — note: NOT `/latest-updates/`, which 404s); search is `/search/{dashed-query}/` block `custom_list_videos_videos_list_search_result` with `q={query}&category_ids=&from_videos=N` (24/page); category `/categories/{slug}/` and tag `/tags/{slug}/` share block `custom_list_videos_common_videos_list` (`from=N`, 12/page); model `/models/{slug}/` uses block `custom_list_videos_models_videos_list`; sort maps new→post_date, popular→video_viewed, rated→rating, longest→duration; `cat:`/`category:`, `tag:`, and `model:`/`uploader:`/`pornstar:`/`star:` query prefixes route to the matching archive, and a bare query that exactly matches a background-loaded category title goes to that archive instead of keyword search; background-loads the 55-entry category title→slug map from `/categories/` (`#list_categories_categories_list_items a.item`) for the `categories` filter option (sanitized out of `/api/status` like `stars`/`networks`, but honored in `/api/videos`); `video.url` is the `/video/{id}/{slug}/` page URL (NOT yt-dlp-resolvable — yt-dlp's generic KVS extractor fails on this site's flashvars), so per-card enrichment fetches the detail page and pulls the direct `video_url`/`video_alt_url` flashvars (480p/720p) into `formats` (bounded `buffered(8)` concurrency); KVS `get_file` MP4 URLs come as `…/ID.mp4/?v-acctoken=…` with a trailing slash before the query — the provider strips it to `…/ID.mp4?v-acctoken=…` so the path ends in `.mp4` (health-check/yt-dlp media detection keys off the extension); formats carry a `Referer` header (works with or without it); thumbnails (`/contents/videos_screenshots/…`) need no proxy or referer; uploader name on a card is the OnlyFans handle while the `/models/{slug}/` URL slug is the canonical model name (they legitimately differ — e.g. handle "Blasianflexcouple" at slug `nina-lee`), so `uploader` uses the display handle and `uploaderUrl`/`uploaderId` (`porn4fans:`) use the slug; no `/api/uploaders` profile, no proxy; note "teen" and similar are compliance-blocked keywords that the site itself returns empty for. | -| `porndish` | `studio-network` | no | yes | Redirect proxy plus thumb proxy usage. | +| `porndish` | `studio-network` | no | yes | Redirect proxy plus thumb proxy usage. Detail pages embed one of three iframe hosts: vidara.so (`/api/stream?filecode=`), myvidplay.com, or playmogo.com — the latter two are the *same* Doodstream-clone embed protocol (`/e/{id}` page → `$.get('/pass_md5/...')` → signed CDN URL with a random suffix + token + expiry), confirmed live against a real playmogo embed, so playmogo.com is just added to `is_myvidplay_host`'s match arm (duplicated in both `providers/porndish.rs` and `proxies/porndish.rs` — HOT_TUB_PROVIDER single-provider builds can't share code across that boundary, so keep both in sync manually). Previously-unrecognized playmogo iframes fell through to a naive branch that pointed `formats[0]` straight at the (Cloudflare-protected, non-media) iframe URL, which just 403s. Per-item enrichment fetches are deliberately sequential (`buffer_unordered(1)`) — each hop spawns a `curl_cffi` python subprocess, and raising concurrency there made total wall-clock time *worse* (concurrent interpreter spawns contend for CPU harder than the network latency they'd overlap). | | `pornhat` | `mainstream-tube` | no | no | Basic tube provider. | | `pornhd3x` | `studio-network` | no | yes | Best template for complex catalogs and redirect proxy generation. | | `pornhub` | `mainstream-tube` | no | no | Rich metadata and format examples. | @@ -88,7 +88,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `xhamster` | `mainstream-tube` | no | no | HTML scraper for xhamster.com; card selector `div[data-video-type="video"]` with `data-video-id`; thumbnails via `img[data-role="thumb-preview-img"]` at `ic-vt-nss.xhcdn.com` (no proxy, no Referer needed); pagination via `?page=N` query param (browse feeds use infinite-scroll so only search reliably returns different content per page); feeds: `/newest` (default), `/most-viewed`, `/best`; categories via `/categories/{slug}`; channels via `/channels/{slug}`; 43 hardcoded categories as `categories` option; uploader type inferred from URL path (`/channels/` → channel, `/creators/` → creator, `/pornstars/` → pornstar); supports `cat:`/`category:` and `channel:` query shortcuts, plus static category name matching; preview mp4 clips from `data-previewvideo` attribute; yt-dlp resolves `video.url` natively (xHamster extractor, 28 formats); no proxy needed. | | `camsoda` | `live-cams` | no | no | Live-cam provider for camsoda.com (chaturbate-style — `live` performers streaming now, `video.url` = the room page, `is_live=true`, no `formats`). camsoda.com is hard Cloudflare-protected: direct requests and yt-dlp both get HTTP 403, and FlareSolverr was unreliable during development, so the live-browse API is reached through the shared requester's Jina mirror fallback (`r.jina.ai/http://...`, `X-Return-Format: html`); Jina rate-limits per IP, so the provider caches each fetched feed URL for 60s (and serves stale items on a 429 rather than emptying the feed), and a single-provider build (`HOT_TUB_PROVIDER=camsoda`) validates most cleanly (one fetch at a time). Endpoint (found in the non-CF static `main.js` bundle): `GET https://www.camsoda.com/api/v1/browse/react{route}?p=N` returning a body with a top-level `userList` array (Jina wraps it in `
`, so the provider scans for the first balanced top-level `{...}` object — tracking string/escape state rather than just pairing the first `{` with the last `}` in the body, since a naive last-`}` pairing pulls in any trailing markup the wrapper appends after `
` and made `serde_json` fail with "key must be a string" — then parses it with `serde_json::Value`, like the chaturbate provider). Per-cam fields: `username`→id + room URL (`/{username}`), `subjectText`→title (html-decoded, falls back to `displayName`), `displayName`→uploader, `connectionCount`→views (string or number tolerated), `thumbUrl`→thumb (direct `media.livemediahost.com` CDN, no proxy/referer), `status` (skip `offline`), `vr`/`private` surfaced as tags. Category option `category` uses verified `browse/react` routes — `all`(featured)/`girls`/`trans`/`couples`/`voyeur-cams`/`new` (`/male` is NOT a path route, camsoda gates male via `gender-hide`); `cat:`/`category:` prefixes and a bare keyword matching a category id route there too. Search: `GET browse/react/search/{dashed-query}?sortByConnection=1` (single connection-sorted result set, no real paging). Playback: `video.url` is the live room page; the room and the token-gated edge HLS (`*.livemediahost.com`) are both Cloudflare-protected, so HLS can't be resolved server-side and no `formats` are populated — yt-dlp has a `Camsoda` live extractor that resolves the room on a non-CF-blocked client, and `check.py` reports the sandbox's CF 403s as expected warnings (`www.camsoda.com` is in its CF allowlist), not errors. The earlier recorded-`/media` JSON scrape was replaced because clips were token-gated/non-playable; live cams are the site's actual product. No proxy needed. | | `xvideos` | `mainstream-tube` | no | no | HTML scraper for xvideos.com; handles two card formats: homepage (`div.thumb-block[data-id][data-eid]`) uses `p.title a[title]` + `data-pvv` on img, best-of-month page uses `div.thumb-block.video[data-video=JSON]` with `div.title a` text + `previewVideo` JSON key; thumbnails at `thumb-cdn77.xvideos-cdn.com` / `thumbs-gcore.xvideos-cdn.com` (no proxy needed); latest: `/` (page 1) / `/new/{N-1}` (page N≥2); best-of-month: `/best/{YYYY-MM}` (previous calendar month), page N: `/best/{YYYY-MM}/{N-1}`; search: `/?k={query}` / `/?k={query}&p={N-1}` (0-indexed); tag shortcuts: `/tags/{slug}/{N-1}`; category shortcuts: `/c/{Name}-{ID}/{N-1}` (38 hardcoded categories); `cat:`, `tag:`, `uploader:` query prefix routing; yt-dlp resolves `video.url` natively (XVideos extractor → HLS formats); CDN preview mp4 in `preview` field; no proxy needed. | -| `wowxxx` | `studio-network` | no | no | HTML scraper for wow.xxx premium aggregator; default feed `/latest-updates/`, page 2 `/{N}/` suffix (for example `/latest-updates/2/`), search `/search/{query}/relevance/` with the same page suffix; supports `site:`/`studio:`/`network:`/`model:`/`pornstar:`/`tag:`/`cat:` query shortcuts to direct archive routes; list cards expose preview clips (`cast.wow.xxx/preview/*.mp4`), thumbnails (`img.wow.xxx/.../medium@2x/1.jpg`), duration, rating, views, site (as uploader), and model tags; `video.url` is the detail page URL and yt-dlp resolves HTML5 MP4 formats dynamically; no proxy needed. | +| `wowxxx` | `studio-network` | no | no | HTML scraper for the wowxxx.to premium aggregator (wow.xxx 301-redirects here now; `BASE_URL` points at the canonical domain directly). The site is Cloudflare-protected — `fetch_html` must NOT pin `Version::HTTP_11` (removed; now `None`), since forcing h1 while the TLS handshake still advertises the browser emulation's h2 ALPN is exactly the mismatch documented for jable.tv's CF gate. Default feed `/latest-updates/`, page 2 `/{N}/` suffix (for example `/latest-updates/2/`), search `/search/{query}/relevance/` with the same page suffix; supports `site:`/`studio:`/`network:`/`model:`/`pornstar:`/`tag:`/`cat:` query shortcuts to direct archive routes; list cards expose preview clips (`cast.wowxxx.to/preview/*.mp4`), thumbnails (`img.wowxxx.to/.../medium@2x/1.jpg`), duration, rating, views, site (as uploader), and model tags; `video.url` is the detail page URL and yt-dlp resolves HTML5 MP4 formats dynamically; no proxy needed. `www.wowxxx.to`/`img.wowxxx.to` are in `check.py`'s CF-protected allowlist since a plain probe still 403s even though the app's own client succeeds. | | `xlecx` | `hentai-animation` | no | no | DLE (DataLife Engine)-style HTML scraper for xlecx.one, following the `thaiporntv` template. Latest feed `/f/cat={id}/sort={mode}/order=desc/` (page N≥2 appends `page/{N}/`); `sort` options new/popular/commented/rated map to DLE's `date`/`news_read`/`comm_num`/`rating` sort params; `category` option is the 2 hardcoded top-level archives (`videos`, `hentai-video`). Search via `/index.php?do=search&subaction=search&story={query}` (page N adds `search_start={N}`). Tag shortcut (`#slug` query) routes to `/tags/{slug}/` (paginated via `/tags/{slug}/page/{N}/`); the ~1,044-entry tag catalog is background-loaded once at startup from `/tags/` (`.tags-list1 a[href*='/tags/']`, absolute-URL anchors) into an `Arc>>` and exposed as the `filter`/"Tags" `ChannelOption` — note this option (and any `categories`) is deliberately stripped from the live `/api/status` response by `should_hide_status_option`/`decorate_channel` in `providers/mod.rs` (any `id=="filter"` option titled "Tags" is hidden site-wide), so its absence there is expected, not a bug; it is still honored by `/api/videos`. Cards parsed from `h3.thumb__title` + `img` inside standard DLE thumb blocks; id extracted from the `/{id}-{slug}.html` URL suffix. Thumbnails are direct `xlecx.one/uploads/posts/...` webp (no proxy/Referer needed). `video.url` is the detail page URL, which yt-dlp resolves natively via the generic HTML5 embed extractor; no `formats`/proxy needed. No `/api/uploaders` (no stable uploader identity on the site). | | `xxxtik` | `tiktok` | yes | no | JSON-API short-form aggregator for xxxtik.com — every post is a moderated repost of a RedGifs clip, so the real media backend is the public **RedGifs v2 API**, not xxxtik itself (new pattern; no other provider currently resolves through a third-party media API). Listing API (`xxxtik-api-iw98m.ondigitalocean.app`, found by grepping the Angular `main-es2015.js` bundle): `GET /post/new`, `/post/top/{week,month,year,all}`, `/post/tag/{name}`, `/post/creator/{username}` — all **cursor**-paginated (`?cursor={lastItemId}&limit=N`; the `cursor` is the numeric `id` of the last item from the previous batch, not an offset/count — reaching page N walks N sequential requests, mirroring `fikfap`'s `fetch_cursor_page`). `GET /search?query=Q` returns tag/profile autocomplete suggestions only (no posts), so free-text search is routed through it to resolve a `Tag`/`Creator` target before a second listing call; `tag:`/`category:`/`cat:` and `user:`/`uploader:`/`creator:` query prefixes skip that lookup. Each post's `source` field is a `redgifs.com/watch/{id}` URL; the provider fetches a short-lived anonymous bearer token from `POST api.redgifs.com/v2/auth/temporary` (cached, refreshed once on a 401) and resolves `GET api.redgifs.com/v2/gifs/{id}` (bounded to 8-way concurrency via `buffer_unordered`) for the real `media.redgifs.com/*.mp4` + poster, both fetchable with zero auth/Referer. `video.url` is the xxxtik page (`https://xxxtik.com/feed/{uuid}`, not yt-dlp-resolvable — Angular SPA, generic extractor fails), with `formats` populated from the resolved redgifs mp4; tags merge xxxtik's own tags with RedGifs' tags. 20 curated tags exposed via `categories` (xxxtik has ~62k tags total, too many to background-load). `/api/uploaders` works via `GET /user/by-username/{name}` + `/post/creator/{name}`, but xxxtik's "creator" accounts are inconsistent: some (e.g. `dlhoodninja`, `besttits`) return real posts matching their profile `_count.posts`; others with a nonzero `_count.posts` (e.g. `bigboobsgw`, count 472) return an empty `/post/creator/` list — likely synthetic curation accounts (`name@default` emails) rather than real uploaders. The provider degrades gracefully (returns the profile with `videos: []`, no error) rather than guessing which accounts are "real". No proxy needed — all media/thumb URLs are publicly fetchable with no Referer or auth. | diff --git a/src/providers/noodlemagazine.rs b/src/providers/noodlemagazine.rs index 5588664..cee1491 100644 --- a/src/providers/noodlemagazine.rs +++ b/src/providers/noodlemagazine.rs @@ -528,6 +528,13 @@ impl NoodlemagazineProvider { .filter_map(|captures| captures.name("url").map(|value| value.as_str().to_string())) .find(|candidate| !candidate.starts_with("data:image/")) }) + .map(|thumb| { + // HTML attributes encode `&` as `&`; the raw regex match + // above needs decoding or a query string like `...&idx=0&type=39` + // survives as the literal text `...&idx=0&type=39`, + // which the CDN rejects with HTTP 400. + decode(thumb.as_bytes()).to_string().unwrap_or(thumb) + }) .unwrap_or_default(); let raw_duration = video_segment @@ -699,6 +706,34 @@ mod tests { assert!(items[1].thumb.is_empty()); } + #[test] + fn decodes_html_entities_in_thumb_query_string() { + // Real markup HTML-encodes `&` as `&` in attribute values; the + // raw-HTML parsing path used to pass that straight through, leaving + // a literal `&` in the thumb URL that the CDN rejected (HTTP 400). + let provider = NoodlemagazineProvider::new(); + let options = options(); + let html = r#" +
+
+ + + +
sample
+ #clock-o">12:34< + #eye">1.2K< +
+ >Show more
+ "#; + + let items = provider.get_video_items_from_html(html.to_string(), &options); + assert_eq!(items.len(), 1); + assert_eq!( + items[0].thumb, + "https://example.com/proxy/noodlemagazine-thumb/img.pvvstream.pro/preview/abc/-111_222/240/iv.okcdn.ru/getVideoPreview?id=1&idx=0&type=39&tkn=xyz" + ); + } + #[test] fn keeps_preview_urls_without_file_extension() { let provider = NoodlemagazineProvider::new(); diff --git a/src/providers/porndish.rs b/src/providers/porndish.rs index 7c41dd5..e7e54e8 100644 --- a/src/providers/porndish.rs +++ b/src/providers/porndish.rs @@ -235,7 +235,16 @@ impl PorndishProvider { } fn is_myvidplay_host(host: &str) -> bool { - matches!(host, "myvidplay.com" | "www.myvidplay.com") + // playmogo.com is the same Doodstream-clone embed protocol as + // myvidplay.com (identical /e/{id} -> $.get('/pass_md5/...') -> + // signed-CDN-URL flow, confirmed against a live embed) - previously + // unrecognized playmogo iframes fell through to the naive "point + // `formats[0]` straight at the (Cloudflare-protected, non-media) + // iframe URL" branch, which just 403s. + matches!( + host, + "myvidplay.com" | "www.myvidplay.com" | "playmogo.com" | "www.playmogo.com" + ) } fn is_vidara_host(host: &str) -> bool { @@ -1208,6 +1217,11 @@ sys.stdout.buffer.write(response.content) let options = options.clone(); async move { provider.enrich_video(video, &options).await } })) + // Each enrichment spawns a curl_cffi python subprocess per network + // hop; raising concurrency here made total wall-clock time WORSE + // (concurrent python3 interpreter spawns contend for CPU harder than + // the sequential network latency they were meant to overlap), so + // this stays serialized. .buffer_unordered(1) .collect::>() .await; @@ -1405,6 +1419,13 @@ mod tests { assert!(PorndishProvider::is_allowed_myvidplay_pass_url( "https://myvidplay.com/pass_md5/abc/def" )); + // playmogo.com is the same Doodstream-clone embed family as myvidplay. + assert!(PorndishProvider::is_allowed_myvidplay_iframe_url( + "https://playmogo.com/e/ryqt645yv3an" + )); + assert!(PorndishProvider::is_allowed_myvidplay_pass_url( + "https://playmogo.com/pass_md5/abc/def" + )); assert!(PorndishProvider::is_allowed_vidara_iframe_url( "https://vidara.so/e/abc123" )); diff --git a/src/providers/wowxxx.rs b/src/providers/wowxxx.rs index dd15c84..237fc2b 100644 --- a/src/providers/wowxxx.rs +++ b/src/providers/wowxxx.rs @@ -11,7 +11,6 @@ use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use scraper::{ElementRef, Html, Selector}; -use wreq::Version; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { @@ -19,7 +18,8 @@ pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = tags: &["premium", "studios", "networks", "models"], }; -const BASE_URL: &str = "https://www.wow.xxx"; +// wow.xxx 301-redirects here now; use the canonical domain directly. +const BASE_URL: &str = "https://www.wowxxx.to"; const CHANNEL_ID: &str = "wowxxx"; const FIREFOX_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0"; @@ -63,7 +63,7 @@ impl WowxxxProvider { "Premium studio aggregator with latest updates, search, and direct archive routing for sites/networks/models/tags/categories." .to_string(), premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=wow.xxx".to_string(), + favicon: "https://www.google.com/s2/favicons?sz=64&domain=wowxxx.to".to_string(), status: "active".to_string(), categories: vec![], options: vec![ChannelOption { @@ -222,8 +222,13 @@ impl WowxxxProvider { requester: &mut crate::util::requester::Requester, url: &str, ) -> Result { + // Do not pin HTTP/1.1: the site is now Cloudflare-protected and + // forcing h1 while the TLS handshake still advertises the browser + // emulation's h2 ALPN is a mismatch Cloudflare scores as a bot (same + // root cause documented for jable.tv - see requester.rs's + // get_with_headers comment). Let the negotiated version through. requester - .get_with_headers(url, Self::html_headers(BASE_URL), Some(Version::HTTP_11)) + .get_with_headers(url, Self::html_headers(BASE_URL), None) .await .map_err(|e| Error::from(format!("request failed for {url}: {e}"))) } diff --git a/src/proxies/porndish.rs b/src/proxies/porndish.rs index 3b02b9c..902b940 100644 --- a/src/proxies/porndish.rs +++ b/src/proxies/porndish.rs @@ -35,7 +35,14 @@ impl PorndishProxy { } fn is_myvidplay_host(host: &str) -> bool { - matches!(host, "myvidplay.com" | "www.myvidplay.com") + // Mirrors providers/porndish.rs::is_myvidplay_host - playmogo.com is + // the same Doodstream-clone embed protocol as myvidplay.com. Kept in + // sync manually since providers and proxies can't share code across + // that boundary in a HOT_TUB_PROVIDER single-provider build. + matches!( + host, + "myvidplay.com" | "www.myvidplay.com" | "playmogo.com" | "www.playmogo.com" + ) } fn is_vidara_host(host: &str) -> bool { @@ -366,4 +373,14 @@ mod tests { "https://example.com/porn/example/" )); } + + #[test] + fn recognizes_playmogo_as_myvidplay_family() { + assert!(PorndishProxy::is_allowed_myvidplay_iframe_url( + "https://playmogo.com/e/ryqt645yv3an" + )); + assert!(PorndishProxy::is_allowed_myvidplay_pass_url( + "https://playmogo.com/pass_md5/abc/def" + )); + } } diff --git a/src/util/requester.rs b/src/util/requester.rs index 624553d..062fa8b 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -644,6 +644,13 @@ impl Requester { let jina_handle = tokio::spawn(async move { Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await }); + // A persistent 429 (e.g. a site's IP-level rate limit kicking in) used + // to retry forever with no cap and no fallback, hanging until an + // unrelated caller-side timeout cut it off. Cap the retries and fall + // through to the same Jina/FlareSolverr fallback every other failure + // status uses once they're exhausted. + const MAX_RATE_LIMIT_RETRIES: u8 = 3; + let mut rate_limit_retries: u8 = 0; loop { let mut request = self.client.get(url); if let Some(version) = _http_version { @@ -749,11 +756,14 @@ impl Requester { jina_handle.abort(); return Ok(body); } - if response.status().as_u16() == 429 { + if response.status().as_u16() == 429 && rate_limit_retries < MAX_RATE_LIMIT_RETRIES { + rate_limit_retries += 1; crate::flow_debug!( - "trace={} requester direct retry url={} status=429", + "trace={} requester direct retry url={} status=429 attempt={}/{}", self.debug_trace_id().unwrap_or("none"), - crate::util::flow_debug::preview(url, 120) + crate::util::flow_debug::preview(url, 120), + rate_limit_retries, + MAX_RATE_LIMIT_RETRIES ); tokio::time::sleep(std::time::Duration::from_secs(1)).await; continue;