From c244461d0b50a979637f9bc80b7a81a032a3f4c3 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 22 Sep 2026 08:28:05 +0000 Subject: [PATCH] Resolve melonstube's top-20 destination hosts into direct formats melonstube is a meta-search aggregator that redirects out to ~120 third-party hosts. A live crawl of /new, /popular, and /rating (~3,350 out-links) found the previous "assume yt-dlp can resolve it" fallback was hiding real failures on the highest-traffic hosts: - eporner.com is ~34% of all melonstube traffic by itself, and like the eporner channel (fixed earlier), serves a restricted preview clip without its hash/xhr dance. - fapcat.com/momvids.com 403 a plain fetch (Cloudflare). - Several others just needed real extraction instead of a guess. Per-card enrichment now dispatches by destination host to: - Eporner: reuses eporner's own hash/xhr + Referer resolution, moved to src/util/eporner_resolve.rs so both providers' single-provider builds still compile (HOT_TUB_PROVIDER= only compiles the one selected provider module; src/util/* always compiles). - Kvs (pornfreenow, analdin, babepump, w1mp, fetishshrine, pervclips, xozilla, ok.xxx, ebonypeek, xgroovy): a shared get_file extractor, the same KVS backend several other Hot Tub providers already scrape. - KvsCfGated (fapcat, momvids): same extractor via the Safari18 bypass client melonstube.com itself already needs. - XhamsterMirror (xh.partners): a genuine xHamster mirror instance, Referer built from the actual resolved host. - SomePorn: HLS playlist URL built from the page's video id directly. - Pornone: direct tag extraction. flirt4free.com (a live-cam page, not a VOD) moved into the dropped- host bucket alongside the existing junk/join.* hosts. Enrichment is bounded (concurrency, a 10s per-card timeout, and capped to the requested perPage rather than the full ~240-card page) so a slow or rate-limited destination can't stall the response. Co-Authored-By: Claude Sonnet 5 --- docs/provider-catalog.md | 4 +- src/providers/eporner.rs | 170 +---------- src/providers/melonstube.rs | 576 +++++++++++++++++++++++++++++++++--- src/util/eporner_resolve.rs | 198 +++++++++++++ src/util/mod.rs | 1 + 5 files changed, 746 insertions(+), 203 deletions(-) create mode 100644 src/util/eporner_resolve.rs diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index 22734c6..fa26706 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -33,7 +33,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `hsex` | `chinese` | yes | no | Strong template for tags, uploaders, and direct HLS formats. | | `hypnotube` | `fetish-kink` | no | no | Fetish/tube hybrid. | | `javtiful` | `jav` | no | no | JAV channel family. | -| `melonstube` | `mainstream-tube` | no | yes | Meta-search aggregator for melonstube.com — every card is an `/out/?l=&c=&v=3` redirect link whose destination (a third-party host) is embedded, zero-network, in a MessagePack blob under the `l` param (mixed binary/string fields: view/click counts, a JSON date-range blob, an array of related-video ids, and the destination URL itself as one msgpack string). Decoding is local: base64-decode `l`, then regex-extract the `https?://...` destination directly out of the raw (lossy-UTF8-converted) msgpack bytes — **the regex must be a positive allowlist of legal URI characters** (`[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`), not a denylist of a few excluded characters: the byte immediately following the msgpack string (a length-prefix marker for the next field, e.g. `\xcd`) lossy-converts to U+FFFD (code point 0xFFFD), which a denylist limited to ASCII control chars (`[^\x00-\x1f\\"']`) fails to exclude, silently appending a corrupt trailing character to the URL. Feeds: `/new` (latest), `/popular`, `/rating` (all three are static curated lists, ~120 items, true pagination via `?page=N`); search is NOT the vestigial `/search?q=` GET endpoint (returns generic filler content behind a fake static result counter regardless of query) — the real mechanism mirrors the site's own search form: POST `search_query[query]=` to `/searching/by-form`, which 303-redirects (cookies + redirect auto-followed by the shared `Requester`) to a resolved, genuinely query-relevant destination depending on entity match — `/pornstar/`, `/category/`, `/search/a/` (studio), or `/search/` (generic free text) — all of which support `?page=N` pagination; a non-matching query resolves to a page with zero result cards, which `parse_listing` naturally returns as empty. Destination hosts fall into three buckets: (1) the common case — yt-dlp (`--impersonate chrome-120`) resolves the destination directly, so `video.url` is just the decoded destination URL, no proxy; (2) `JUNK_HOSTS` (currently `fhgte.com`) — observed dead-end paywall/signup funnels with no free playable video, so cards pointing there are dropped entirely rather than surfaced as false-positive results; (3) `HARD_HOSTS` (`manysex.com`, `videomanysex.com`) — yt-dlp cannot resolve these, so `video.url` is routed through `/proxy/melonstube/{host}/{path}`. The proxy (`src/proxies/melonstube.rs`) ports vjav.rs's Cyrillic-homoglyph-obfuscated base64 decode chain to pull the real `get_file` path/query out of `videofile.php`'s `video_url` field, then makes two manual (non-auto-redirect) hops to the final CDN URL: hop 1 needs the manysex.com/videomanysex.com page as `Referer`; hop 2 (to the signed `ahcdn.com` URL) must be sent with **no** Referer at all, because the CDN's signed URL embeds a literal `referer=none,.manysex.com,.gstatic.com` allow-list that rejects the videomanysex.com Referer used on hop 1 — auto-redirect clients that forward the same Referer to every hop get a 403 at hop 2. Before returning the 302, an anti-false-positive check confirms the resolved URL string contains the requested numeric video id and that a ranged GET (`Range: bytes=0-65535`) returns 200/206 with a `video/*`/`octet-stream` content-type, so a paywall/ad/error page can't masquerade as the real stream. Thumbnails (`ttcache.com`) load directly, no proxy. No `/api/uploaders` (aggregator has no stable uploader identity). **2026-09-21 fix:** melonstube.com started fronting itself with Cloudflare bot management (a real Turnstile "challenge" page, `cf-mitigated: challenge`), which the shared Requester's Firefox151 emulation gets on every path — this made the provider return 0 items (direct fetch, Jina mirror, and FlareSolverr all failed/unreachable). A/B probing many `curl_cffi` fingerprints found Chrome120/124 pass the plain `/new`/`/popular`/`/rating` listing pages but still 403 on `/category/*` and the `/searching/by-form` search POST, while **Safari18** passes all of them; listing and search fetches now try a dedicated `wreq::Client` with `Emulation::Safari18` first and fall back to the shared Requester if Cloudflare's rules change again. Also fixed: `get_videos` previously ignored `perPage` entirely (`let _ = per_page;`), returning the full ~120-card page regardless of the requested size — it now truncates to the requested `perPage` after fetching. | +| `melonstube` | `mainstream-tube` | no | yes | Meta-search aggregator for melonstube.com — every card is an `/out/?l=&c=&v=3` redirect link whose destination (a third-party host) is embedded, zero-network, in a MessagePack blob under the `l` param (mixed binary/string fields: view/click counts, a JSON date-range blob, an array of related-video ids, and the destination URL itself as one msgpack string). Decoding is local: base64-decode `l`, then regex-extract the `https?://...` destination directly out of the raw (lossy-UTF8-converted) msgpack bytes — **the regex must be a positive allowlist of legal URI characters** (`[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`), not a denylist of a few excluded characters: the byte immediately following the msgpack string (a length-prefix marker for the next field, e.g. `\xcd`) lossy-converts to U+FFFD (code point 0xFFFD), which a denylist limited to ASCII control chars (`[^\x00-\x1f\\"']`) fails to exclude, silently appending a corrupt trailing character to the URL. Feeds: `/new` (latest), `/popular`, `/rating` (all three are static curated lists, ~120 items, true pagination via `?page=N`); search is NOT the vestigial `/search?q=` GET endpoint (returns generic filler content behind a fake static result counter regardless of query) — the real mechanism mirrors the site's own search form: POST `search_query[query]=` to `/searching/by-form`, which 303-redirects (cookies + redirect auto-followed by the shared `Requester`) to a resolved, genuinely query-relevant destination depending on entity match — `/pornstar/`, `/category/`, `/search/a/` (studio), or `/search/` (generic free text) — all of which support `?page=N` pagination; a non-matching query resolves to a page with zero result cards, which `parse_listing` naturally returns as empty. Destination hosts fall into three buckets: (1) the common case — yt-dlp (`--impersonate chrome-120`) resolves the destination directly, so `video.url` is just the decoded destination URL, no proxy; (2) `JUNK_HOSTS` (currently `fhgte.com`) — observed dead-end paywall/signup funnels with no free playable video, so cards pointing there are dropped entirely rather than surfaced as false-positive results; (3) `HARD_HOSTS` (`manysex.com`, `videomanysex.com`) — yt-dlp cannot resolve these, so `video.url` is routed through `/proxy/melonstube/{host}/{path}`. The proxy (`src/proxies/melonstube.rs`) ports vjav.rs's Cyrillic-homoglyph-obfuscated base64 decode chain to pull the real `get_file` path/query out of `videofile.php`'s `video_url` field, then makes two manual (non-auto-redirect) hops to the final CDN URL: hop 1 needs the manysex.com/videomanysex.com page as `Referer`; hop 2 (to the signed `ahcdn.com` URL) must be sent with **no** Referer at all, because the CDN's signed URL embeds a literal `referer=none,.manysex.com,.gstatic.com` allow-list that rejects the videomanysex.com Referer used on hop 1 — auto-redirect clients that forward the same Referer to every hop get a 403 at hop 2. Before returning the 302, an anti-false-positive check confirms the resolved URL string contains the requested numeric video id and that a ranged GET (`Range: bytes=0-65535`) returns 200/206 with a `video/*`/`octet-stream` content-type, so a paywall/ad/error page can't masquerade as the real stream. Thumbnails (`ttcache.com`) load directly, no proxy. No `/api/uploaders` (aggregator has no stable uploader identity). **2026-09-21 fix:** melonstube.com started fronting itself with Cloudflare bot management (a real Turnstile "challenge" page, `cf-mitigated: challenge`), which the shared Requester's Firefox151 emulation gets on every path — this made the provider return 0 items (direct fetch, Jina mirror, and FlareSolverr all failed/unreachable). A/B probing many `curl_cffi` fingerprints found Chrome120/124 pass the plain `/new`/`/popular`/`/rating` listing pages but still 403 on `/category/*` and the `/searching/by-form` search POST, while **Safari18** passes all of them; listing and search fetches now try a dedicated `wreq::Client` with `Emulation::Safari18` first and fall back to the shared Requester if Cloudflare's rules change again. Also fixed: `get_videos` previously ignored `perPage` entirely (`let _ = per_page;`), returning the full ~120-card page regardless of the requested size — it now truncates to the requested `perPage` after fetching. **2026-09-21 sub-provider resolution:** a live crawl of `/new`/`/popular`/`/rating` (~3,350 out-links, 117 distinct destination hosts) found the third bucket above ("yt-dlp resolves it directly") was hiding real failures — `eporner.com` alone is ~34% of all traffic and, like the `eporner` channel itself, serves a restricted preview clip without the site's hash/xhr dance; `fapcat.com`/`momvids.com` 403 a plain fetch (Cloudflare); a few others just needed the right extraction. Per-card enrichment (bounded `buffer_unordered`, `ENRICH_CONCURRENCY=8`, each resolution capped at `ENRICH_TIMEOUT_SECS=10` so one slow/unreachable host can't stall the page, and only run for the first `per_page` pending cards — not the full ~240-card page) now dispatches by destination host (`classify_source`) to: **Eporner** — `crate::util::eporner_resolve::resolve_public_formats` (moved out of `eporner.rs` into `src/util/` specifically so both providers get it without breaking either one's `HOT_TUB_PROVIDER=` single-provider build, which only compiles the one selected provider module). **Kvs** (`pornfreenow.com`, `analdin.xxx`, `babepump.com`, `w1mp.com`, `fetishshrine.com`, `pervclips.com`, `xozilla.xxx`, `ok.xxx`, `ebonypeek.com`, `xgroovy.com`) — a generic regex scan for the `get_file/{N}/{hash}/.../{id}[_variant].mp4` shape already used by porn4fans.rs/xgroovy.rs/thepornbunny.rs, ranking by resolution suffix and falling back to a qualitative `hd`-suffix check for sites (analdin, xozilla) that don't tag an explicit height; teaser variants (`_s.mp4`, `_preview.mp4`) excluded. **KvsCfGated** (`fapcat.com`, `momvids.com`) — same KVS extraction, fetched through the Safari18 bypass client melonstube.com itself needs. **XhamsterMirror** (`xh.partners`) — an affiliate short-link into a genuine xHamster mirror instance (same `xhcdn.com` CDN, same embedded `"sources":{"mp4":{...}}` JSON as xhamster.com); the Referer is built from the mirror's actual resolved host, not assumed, since the signed CDN URL's allow-list is mirror-specific. **SomePorn** (`some.porn`) — no page fetch at all; its HLS master playlist is addressable directly from the numeric id in the page URL (`s.cdn3x.com/get-playlist?id={id}&q=720p`). **Pornone** (`pornone.com`) — direct unauthenticated `` mp4 tags at multiple resolutions right on the detail page. Hosts outside this set keep the original fallback (bare destination URL, no formats). `www.flirt4free.com` (a live-cam profile page, not a VOD) moved into the same drop bucket as `JUNK_HOSTS`. | | `missav` | `jav` | no | no | HLS format pattern. | | `noodlemagazine` | `mainstream-tube` | no | yes | Best template for media and thumbnail proxying. | | `okporn` | `mainstream-tube` | no | no | Simple mainstream archive. | @@ -82,7 +82,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `kwiky` | `tiktok` | no | no | Short-form cam clip provider for kwiky.com (ICF/StreamMate platform). Two-step fetch: ID list then metadata batch. Auth: XSRF token extracted from `meta.content = ''` injected by `/api/state/v1/preload`, cached in an `Arc>>` with double-checked locking (same pattern as `xxxtik` bearer token); all API calls also require `x-gateway: 76d59e04-97fa-4ced-aa99-b86ffaf756a2` (platform-identifying fixed UUID observed in browser JS, not stored in cookies or localStorage). Default feed: `GET /api/v1/gateway/v1/quickies/recommended?source=` (cursor-paged, walks up to 5 cursors for higher page numbers). All tag/keyword queries go through `keywordSearch?keyword=...` (offset-paged: `pageToken=(page-1)*perPage`) — the recommended endpoint's `&tag=` param does not reliably filter, so even curated-tag requests use keywordSearch. Media: `media.url` = direct `media.icfcdn.com/*.mp4` (publicly accessible, no Referer/auth), `media.thumbUrl` thumbnail (same CDN, direct), `media.previewUrl` preview clip; both kwiky page URLs (`kwiky.com/quickies/{id}`) and the media CDN URLs pass direct `curl -I` health checks but the page URL is NOT yt-dlp-resolvable (React SPA, generic extractor fails), so `formats` are populated with the direct MP4 URL and `video.url` is the page. Aspect ratio set from `media.width`/`media.height`; `uploadedAt` from RFC3339 `created` field; uploader name/URL/ID from `creator.name` and `creator.id`. 48 curated tags from `/api/v1/gateway/v1/keywords/populartags?gender=f` exposed via `categories` filter option. No proxy needed. | | `fullporner` | `mainstream-tube` | no | no | HTML scraper for fullporner.com; thumbnail IDs derived from `/thumb/{id}.jpg` URLs and used to build direct `xiaoshenke.net/vid/{id}/720` media redirect URLs (Referer + User-Agent headers required); supports cat:/category:/pornstar:/star: shortcut queries; no proxy needed. | | `thepornbunny` | `mainstream-tube` | no | yes | KVS-style HTML scraper for thepornbunny.com; 24 items per site page; thumbnails at `https://www.thepornbunny.com/images/thumb/{id}.webp` from `data-original` attribute (no proxy needed); studio exposed as uploader; pornstar names in tags; `/proxy/thepornbunny/{slug}` fetches the video page, extracts `generate_mp4(enc_data, key, rnd, video_id)` args, decrypts `enc_data` via PBKDF2-HMAC-SHA512+AES-256-CBC to get an OK.ru session key, calls `api.ok.ru/fb.do?method=video.get&session_key=KEY&vids=RND` to get signed CDN URLs, and returns 302 to the best-quality okcdn.ru/vkuser.net MP4 URL (no special client headers needed); supports sort: new/popular/rated, 20 hardcoded categories via `categories` option, and tag:/category:/studio:/pornstar: query shortcuts. | -| `eporner` | `mainstream-tube` | no | no | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); supports sort: new/popular/rated/best; 65 hardcoded categories, plus `cat:`/`category:`, `tag:`, `pornstar:`/`star:`, `uploader:`/`profile:` query-prefix routing to the matching archive, a bare-keyword category-name match, and a background-loaded pornstar name→URL map from `/pornstar-list/` (falls back to `/tag/{slug}` keyword search otherwise). `video.url` is the detail page — **not** yt-dlp-resolvable and not directly playable on its own: opening it bare serves a small restricted/preview clip. Real playback needs per-card enrichment (bounded `buffer_unordered(6)`): fetch the detail page for its 32-hex `hash`, derive the site's own hashed token (`calc_hash` — split into four 8-hex chunks, each parsed as u32 and base36-encoded, concatenated; reverse-engineered from the player JS), then call the site's own `GET /xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false` JSON API for direct per-resolution mp4 URLs into `formats[]`. Separately from the hash dance, the resolved CDN mp4 URLs enforce their own Referer check: a request with no Referer or one pointing at eporner.com gets the real file, but any other non-empty Referer gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the exact "this video is only available at eporner.com"-style short clip a client sees if its default (or app-injected) Referer doesn't match, so every format explicitly declares `http_headers: {Referer: "https://www.eporner.com/"}` rather than relying on the client's default. No proxy needed (formats are direct CDN URLs, just header-gated). | +| `eporner` | `mainstream-tube` | no | no | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); supports sort: new/popular/rated/best; 65 hardcoded categories, plus `cat:`/`category:`, `tag:`, `pornstar:`/`star:`, `uploader:`/`profile:` query-prefix routing to the matching archive, a bare-keyword category-name match, and a background-loaded pornstar name→URL map from `/pornstar-list/` (falls back to `/tag/{slug}` keyword search otherwise). `video.url` is the detail page — **not** yt-dlp-resolvable and not directly playable on its own: opening it bare serves a small restricted/preview clip. Real playback needs per-card enrichment (bounded `buffer_unordered(6)`): fetch the detail page for its 32-hex `hash`, derive the site's own hashed token (`calc_hash` — split into four 8-hex chunks, each parsed as u32 and base36-encoded, concatenated; reverse-engineered from the player JS), then call the site's own `GET /xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false` JSON API for direct per-resolution mp4 URLs into `formats[]`. Separately from the hash dance, the resolved CDN mp4 URLs enforce their own Referer check: a request with no Referer or one pointing at eporner.com gets the real file, but any other non-empty Referer gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the exact "this video is only available at eporner.com"-style short clip a client sees if its default (or app-injected) Referer doesn't match, so every format explicitly declares `http_headers: {Referer: "https://www.eporner.com/"}` rather than relying on the client's default. No proxy needed (formats are direct CDN URLs, just header-gated). **2026-09-21:** the `calc_hash`/hash-regex/`alnum_id_from_href`/`resolve_formats` logic moved to `src/util/eporner_resolve.rs` so `melonstube` (a meta-search aggregator that redirects out to eporner.com for ~34% of its own cards) can reuse the exact same resolution without a cross-provider `use`, which would break either provider's `HOT_TUB_PROVIDER=` single-provider build (only the one selected provider module gets compiled; `src/util/*` always does). `eporner.rs` itself now just calls `crate::util::eporner_resolve::resolve_formats(...)`. | | `xnxx` | `mainstream-tube` | no | no | HTML scraper for xnxx.com (10M+ videos); unified card parser handles two formats: `div.thumb-block[data-eid]` (search) and `div.thumb-block.video[data-video='{"id":...}']` (hits); eid extracted from `/video-{eid}/{slug}` URL path; thumbnails at `thumb-cdn77.xnxx-cdn.com` and `thumbs-gcore.xnxx-cdn.com` (no proxy, no Referer needed); 0-indexed pagination (page 1 = `/hits`, page N = `/hits/{N-1}`); default feed is `/hits` (most-viewed — xnxx has no chronological listing); search via `/search/{slug}` (works for keywords and tags); supports `tag:`, `cat:`, `category:` query shortcuts; yt-dlp resolves `video.url` natively (XNXX extractor, returns 4-7 HLS formats); no proxy needed. | | `xgroovy` | `mainstream-tube` | no | no | KVS (Kernel Video Sharing)-style HTML scraper for xgroovy.com; card selector `div.item[data-video-id]` with `data-views`/`data-rating` as clean numeric attributes directly on the card (no text-parsing needed, unlike xhamster); title/url from `a.popito[href] > strong.title`; thumbnails at `i.xgroovy.com` (no proxy, no Referer needed) from `img.thumb[data-jpg|src]`, preview clip from `img[data-preview]`; duration rendered as free text ("N min"/"N sec", not colon-separated) via a bespoke `parse_duration` helper; path-segment pagination (`/new/` = page 1, `/new/2/` = page 2, unlike xhamster's `?page=N` query string); feeds: `/new/` (default), categories via `/categories/{slug}/` (73 hardcoded categories as `categories` option), pornstars via `/pornstars/{slug}/`, channels via `/channels/{slug}/`; search via `/search/{dashed-query}/` (multi-word queries are lowercased and dash-joined); uploader kind inferred from URL path (`/pornstars/` vs `/channels/`) via `div.author-link > a`; supports `cat:`/`category:` query shortcuts plus static category name matching; yt-dlp resolves `video.url` via the generic html5-embed extractor (no dedicated xgroovy extractor, but resolves cleanly to a 1080p format); no proxy needed. | | `xhamster` | `mainstream-tube` | no | no | HTML scraper for xhamster.com; card selector `div[data-video-type="video"]` with `data-video-id`; thumbnails via `img[data-role="thumb-preview-img"]` at `ic-vt-nss.xhcdn.com` (no proxy, no Referer needed); pagination via `?page=N` query param (browse feeds use infinite-scroll so only search reliably returns different content per page); feeds: `/newest` (default), `/most-viewed`, `/best`; categories via `/categories/{slug}`; channels via `/channels/{slug}`; 43 hardcoded categories as `categories` option; uploader type inferred from URL path (`/channels/` → channel, `/creators/` → creator, `/pornstars/` → pornstar); supports `cat:`/`category:` and `channel:` query shortcuts, plus static category name matching; preview mp4 clips from `data-previewvideo` attribute; yt-dlp resolves `video.url` natively (xHamster extractor, 28 formats); no proxy needed. | diff --git a/src/providers/eporner.rs b/src/providers/eporner.rs index 3fd52be..60c6ec5 100644 --- a/src/providers/eporner.rs +++ b/src/providers/eporner.rs @@ -8,14 +8,13 @@ use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; -use crate::videos::{ServerOptions, VideoFormat, VideoItem}; +use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; -use regex::Regex; use scraper::{ElementRef, Html, Selector}; -use std::sync::{Arc, OnceLock, RwLock}; +use std::sync::{Arc, RwLock}; use std::{collections::HashMap, thread, vec}; use wreq::Version; @@ -351,18 +350,6 @@ impl EpornerProvider { digits.parse::().ok().map(|v| v / 100.0) } - fn alnum_id_regex() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"/video-([A-Za-z0-9]+)").expect("valid regex")) - } - - fn alnum_id_from_href(href: &str) -> Option { - Self::alnum_id_regex() - .captures(href) - .and_then(|c| c.get(1)) - .map(|m| m.as_str().to_string()) - } - fn parse_list_cards(html: &str) -> Result> { let document = Html::parse_document(html); let card_sel = Self::selector("div.mb[data-id]")?; @@ -390,7 +377,7 @@ impl EpornerProvider { if page_url.is_empty() { continue; } - let Some(alnum_id) = Self::alnum_id_from_href(href) else { + let Some(alnum_id) = crate::util::eporner_resolve::alnum_id_from_href(href) else { continue; }; @@ -453,115 +440,15 @@ impl EpornerProvider { Ok(cards) } - fn hash_regex() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r#"hash\s*[:=]\s*['"]([0-9a-f]{32})"#).expect("valid regex")) - } - - fn height_regex() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"^(\d+)").expect("valid regex")) - } - - fn base36(mut n: u32) -> String { - const DIGITS: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; - if n == 0 { - return "0".to_string(); - } - let mut buf = Vec::new(); - while n > 0 { - buf.push(DIGITS[(n % 36) as usize]); - n /= 36; - } - buf.reverse(); - String::from_utf8(buf).unwrap_or_default() - } - - /// Reverse-engineered from the site's player JS: the page's 32-hex-char - /// `hash` is split into four 8-hex-char chunks, each parsed as a u32 and - /// base36-encoded, then concatenated. The `/xhr/video/{id}` format API - /// requires this derived value or it serves a restricted/preview - /// response instead of the real sources. - fn calc_hash(hash: &str) -> Option { - if hash.len() != 32 || !hash.is_ascii() { - return None; - } - let mut out = String::new(); - for chunk_start in (0..32).step_by(8) { - let chunk = &hash[chunk_start..chunk_start + 8]; - let n = u32::from_str_radix(chunk, 16).ok()?; - out.push_str(&Self::base36(n)); - } - Some(out) - } - - /// Fetches the detail page for its `hash`, then calls the site's own - /// `/xhr/video/{alnum_id}` JSON API (the same one the site's own player - /// uses) to get real direct mp4 URLs across all available resolutions. - /// Without this, the client is left to open the bare page URL, which - /// eporner serves as a small restricted/preview clip rather than the - /// full video. - /// - /// The resolved CDN mp4 URLs also enforce a Referer check independent of - /// the hash/xhr dance above: a request whose Referer is absent or points - /// at eporner.com is served the real file, but any other non-empty - /// Referer gets a 302 to a static `.../na.mp4` ("not available") decoy - /// clip — the exact "this video is only available at eporner.com"-style - /// clip users see when playback doesn't carry the header the CDN wants. - /// Each format therefore declares an explicit `Referer` so the client's - /// own default (or lack thereof) can't accidentally trip this check. - async fn resolve_formats( - requester: &mut Requester, - alnum_id: &str, - page_url: &str, - ) -> Option> { - let detail_html = Self::fetch_html(requester, page_url).await.ok()?; - let hash = &Self::hash_regex().captures(&detail_html)?[1]; - let calc = Self::calc_hash(hash)?; - - let api_url = format!( - "{BASE_URL}/xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false" - ); - let json_text = requester - .get_with_headers(&api_url, Self::html_headers(page_url), Some(Version::HTTP_11)) - .await - .ok()?; - let value: serde_json::Value = serde_json::from_str(&json_text).ok()?; - - if value.get("available").and_then(|v| v.as_bool()) == Some(false) { - return None; - } - - let mp4_sources = value.get("sources")?.get("mp4")?.as_object()?; - let mut ranked: Vec<(u32, VideoFormat)> = Vec::new(); - for (key, entry) in mp4_sources { - let Some(src) = entry.get("src").and_then(|v| v.as_str()) else { - continue; - }; - let label = entry - .get("labelShort") - .and_then(|v| v.as_str()) - .unwrap_or(key.as_str()); - let height = Self::height_regex() - .captures(label) - .and_then(|c| c[1].parse::().ok()) - .unwrap_or(0); - let format = VideoFormat::new(src.to_string(), label.to_string(), "mp4".to_string()) - .format_id(key.clone()) - .format_note(key.clone()) - .height(height) - .http_header("Referer".to_string(), format!("{BASE_URL}/")); - ranked.push((height, format)); - } - if ranked.is_empty() { - return None; - } - ranked.sort_by(|a, b| b.0.cmp(&a.0)); - Some(ranked.into_iter().map(|(_, f)| f).collect()) - } - async fn build_item(requester: &mut Requester, card: EpornerCard) -> VideoItem { - let formats = Self::resolve_formats(requester, &card.alnum_id, &card.page_url).await; + // The hash/xhr resolution + Referer handling live in + // `crate::util::eporner_resolve` (shared with other providers that + // redirect out to eporner.com, e.g. `melonstube`) rather than here, + // since single-provider builds can't depend on another provider's + // module — see that module's doc comment. + let formats = + crate::util::eporner_resolve::resolve_formats(requester, &card.alnum_id, &card.page_url) + .await; let mut item = VideoItem::new( card.id, @@ -813,36 +700,5 @@ impl Provider for EpornerProvider { } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn calc_hash_matches_site_algorithm() { - // Reverse-engineered from vjs.js; verified against a live page/xhr pair. - let hash = "06aadb04e6f6e853e25f5a583f173617"; - assert_eq!(hash.len(), 32); - assert_eq!( - EpornerProvider::calc_hash(hash).unwrap(), - "1ulk3o1s31fmb1qt66aghi70uv" - ); - } - - #[test] - fn calc_hash_rejects_wrong_length() { - assert!(EpornerProvider::calc_hash("abc").is_none()); - } - - #[test] - fn alnum_id_from_href_extracts_path_segment() { - assert_eq!( - EpornerProvider::alnum_id_from_href("/video-jQDTzcHvtjv/vj-blacked/"), - Some("jQDTzcHvtjv".to_string()) - ); - assert_eq!( - EpornerProvider::alnum_id_from_href("https://www.eporner.com/video-5Ssyrtrs9FG/-/"), - Some("5Ssyrtrs9FG".to_string()) - ); - assert_eq!(EpornerProvider::alnum_id_from_href("/hd-porn/95008/foo/"), None); - } -} +// The calc_hash / alnum_id_from_href unit tests moved to +// `crate::util::eporner_resolve` along with the functions themselves. diff --git a/src/providers/melonstube.rs b/src/providers/melonstube.rs index 3063341..04a6b92 100644 --- a/src/providers/melonstube.rs +++ b/src/providers/melonstube.rs @@ -5,14 +5,17 @@ use crate::providers::{ }; use crate::status::*; use crate::util::cache::VideoCache; -use crate::videos::{ServerOptions, VideoItem}; +use crate::util::requester::Requester; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD}; use chrono::{Duration as ChronoDuration, Utc}; use error_chain::error_chain; +use futures::stream::{self, StreamExt}; use regex::Regex; use scraper::{Html, Selector}; -use std::sync::Arc; +use std::collections::HashSet; +use std::sync::{Arc, OnceLock}; use url::form_urlencoded::Serializer; use wreq::cookie::Jar; use wreq::redirect::Policy; @@ -39,6 +42,75 @@ const HARD_HOSTS: &[&str] = &["manysex.com", "videomanysex.com"]; // false-positive "ad" results. const JUNK_HOSTS: &[&str] = &["fhgte.com"]; +// Live-cam profile pages, not a completed VOD — dropped the same way as +// junk hosts rather than surfaced as a broken "video". +const LIVE_CAM_HOSTS: &[&str] = &["flirt4free.com"]; + +// Destination hosts that share the classic KVS (Kernel Video Sharing) +// `get_file/{N}/{hash}/{bucket}/{id}/{id}[_variant].mp4[?token]` pattern +// already used by several other Hot Tub KVS-style providers (porn4fans.rs, +// xgroovy.rs, thepornbunny.rs), reachable directly through the shared +// Requester with no anti-bot bypass needed. Verified live against a real +// card from each host. +const KVS_HOSTS: &[&str] = &[ + "pornfreenow.com", + "analdin.xxx", + "babepump.com", + "w1mp.com", + "fetishshrine.com", + "pervclips.com", + "xozilla.xxx", + "ok.xxx", + "ebonypeek.com", + "xgroovy.com", +]; + +// Same KVS backend as `KVS_HOSTS`, but Cloudflare-fronted: a plain fetch +// gets a 403 challenge (verified live), so these go through the same +// Safari18 bypass client `melonstube.com` itself needs before the KVS +// extraction applies. +const KVS_CF_GATED_HOSTS: &[&str] = &["fapcat.com", "momvids.com"]; + +// Redirect gateways into an xHamster mirror instance — verified live to be +// genuine xHamster infrastructure (same xhcdn.com CDN, same embedded +// `"sources":{"mp4":{...}}` JSON xhamster.com itself uses), just reached +// through an affiliate short-link rather than xhamster.com directly. +const XHAMSTER_MIRROR_HOSTS: &[&str] = &["xh.partners"]; + +// `some.porn`'s HLS master playlist can be built directly from the numeric +// id in the page URL — no page fetch required. +const SOME_PORN_HOSTS: &[&str] = &["some.porn"]; + +// pornone.com embeds direct, unauthenticated `` mp4 tags +// at multiple resolutions right on the detail page. +const PORNONE_HOSTS: &[&str] = &["pornone.com"]; + +// Bounded concurrency for the per-card destination-page enrichment pass. +// Deliberately higher than most other providers' enrichment concurrency: +// this fans out to many *different* third-party hosts (not repeated calls +// to one site), so there's no single-origin rate limit to respect, and a +// low value compounds badly with `ENRICH_TIMEOUT_SECS` when several cards +// in one page happen to hit a slow/unreachable host back to back. +const ENRICH_CONCURRENCY: usize = 8; +// Per-card cap so one slow or unreachable destination host can't stall the +// whole listing response. +const ENRICH_TIMEOUT_SECS: u64 = 10; +// Upper bound on how many cards get enriched per listing fetch, regardless +// of how many cards the page actually has (~240 for /new) — matches the +// max `perPage` `get_videos` accepts, so any valid request is fully +// enriched while still bounding worst-case fetch fan-out. +const MAX_ENRICH_PER_FETCH: usize = 120; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SourceKind { + Eporner, + Kvs, + KvsCfGated, + XhamsterMirror, + SomePorn, + Pornone, +} + error_chain! { foreign_links { Io(std::io::Error); @@ -244,7 +316,7 @@ impl MelonstubeProvider { .ok() } - async fn fetch_html_bypass(url: &str) -> Option { + async fn fetch_html_bypass(url: &str, referer: &str) -> Option { let client = Self::build_bypass_client()?; let response = client .get(url) @@ -253,7 +325,7 @@ impl MelonstubeProvider { "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ) .header("accept-language", "en-US,en;q=0.5") - .header("referer", BASE_URL) + .header("referer", referer) .send() .await .ok()?; @@ -281,48 +353,336 @@ impl MelonstubeProvider { Some(response.uri().to_string().trim_end_matches('/').to_string()) } + fn bare_host(host: &str) -> &str { + host.strip_prefix("www.").unwrap_or(host) + } + fn is_junk_host(host: &str) -> bool { - JUNK_HOSTS.contains(&host) || host.starts_with("join.") + let bare = Self::bare_host(host); + JUNK_HOSTS.contains(&bare) || LIVE_CAM_HOSTS.contains(&bare) || host.starts_with("join.") } fn is_hard_host(host: &str) -> bool { - HARD_HOSTS.contains(&host) + HARD_HOSTS.contains(&Self::bare_host(host)) } - fn parse_listing(&self, html: &str, options: &ServerOptions) -> Vec { + fn classify_source(host: &str) -> Option { + let bare = Self::bare_host(host); + if bare == "eporner.com" { + Some(SourceKind::Eporner) + } else if KVS_HOSTS.contains(&bare) { + Some(SourceKind::Kvs) + } else if KVS_CF_GATED_HOSTS.contains(&bare) { + Some(SourceKind::KvsCfGated) + } else if XHAMSTER_MIRROR_HOSTS.contains(&bare) { + Some(SourceKind::XhamsterMirror) + } else if SOME_PORN_HOSTS.contains(&bare) { + Some(SourceKind::SomePorn) + } else if PORNONE_HOSTS.contains(&bare) { + Some(SourceKind::Pornone) + } else { + None + } + } + + fn kvs_mp4_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new( + r#"(?:https?://[^"'\s\\<>]+?)?get_file/\d+/[0-9a-f]+/\d+/\d+/[^"'\s\\<>]+\.mp4[^"'\s\\<>]*"#, + ) + .expect("valid regex") + }) + } + + fn kvs_height_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"_(\d{3,4})p\.mp4").expect("valid regex")) + } + + /// Ranks a KVS `get_file` URL for sorting only — not every site tags its + /// higher-quality variant with a numeric resolution; several (analdin, + /// xozilla) instead suffix it with a bare "hd" (`{id}hd.mp4` vs the + /// default `{id}.mp4`), which carries no exact pixel height. + fn kvs_quality_score(url: &str) -> u32 { + if let Some(height) = Self::kvs_height_regex() + .captures(url) + .and_then(|c| c[1].parse::().ok()) + { + return height; + } + if url.contains("hd.mp4") { 1 } else { 0 } + } + + /// Extracts every distinct `get_file` mp4 URL from a KVS-CMS detail page + /// (the same backend porn4fans.rs/xgroovy.rs/thepornbunny.rs already + /// scrape), regardless of which JS variable name the site wraps them in + /// — some expose named `video_url`/`video_alt_url` flashvars, others just + /// inline the URL directly, so this scans the raw HTML for the URL shape + /// itself. Short teaser/thumbnail-preview variants (`_s.mp4`, + /// `_preview.mp4`) are excluded so a card never resolves to a decoy clip. + fn parse_kvs_formats(html: &str, origin: &str) -> Option> { + let mut seen = HashSet::new(); + let mut ranked: Vec<(u32, VideoFormat)> = Vec::new(); + + for m in Self::kvs_mp4_regex().find_iter(html) { + let raw = m.as_str(); + if raw.contains("_s.mp4") || raw.contains("_preview.mp4") || raw.contains("_thumb") { + continue; + } + let url = if raw.starts_with("http") { + raw.to_string() + } else { + format!("{origin}/{raw}") + }; + if !seen.insert(url.clone()) { + continue; + } + let score = Self::kvs_quality_score(&url); + let height = Self::kvs_height_regex() + .captures(&url) + .and_then(|c| c[1].parse::().ok()) + .unwrap_or(0); + let quality = if height > 0 { + format!("{height}p") + } else if url.contains("hd.mp4") { + "hd".to_string() + } else { + "source".to_string() + }; + let format = VideoFormat::new(url, quality.clone(), "mp4".to_string()) + .format_note(quality) + .height(height) + .http_header("Referer".to_string(), format!("{origin}/")); + ranked.push((score, format)); + } + + if ranked.is_empty() { + return None; + } + ranked.sort_by(|a, b| b.0.cmp(&a.0)); + Some(ranked.into_iter().map(|(_, f)| f).collect()) + } + + async fn resolve_kvs( + requester: &mut Requester, + detail_url: &str, + cf_gated: bool, + ) -> Option> { + let origin = url::Url::parse(detail_url).ok()?.origin().ascii_serialization(); + + let html = if cf_gated { + Self::fetch_html_bypass(detail_url, &format!("{origin}/")).await? + } else { + requester + .get_with_headers( + detail_url, + vec![ + ("user-agent".to_string(), BROWSER_UA.to_string()), + ( + "accept".to_string(), + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + .to_string(), + ), + ], + None, + ) + .await + .ok()? + }; + + Self::parse_kvs_formats(&html, &origin) + } + + async fn resolve_eporner( + requester: &mut Requester, + detail_url: &str, + ) -> Option> { + crate::util::eporner_resolve::resolve_public_formats(requester, detail_url).await + } + + fn xhamster_mirror_mp4_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new(r#""(\d{3,4})p":"(https:\\/\\/[^"]+?\.mp4[^"]*)""#).expect("valid regex") + }) + } + + /// `xh.partners`-style gateways redirect into a live xHamster mirror + /// instance whose page embeds a `"sources":{"mp4":{"144p":"...",...}}` + /// JSON block identical in shape to xhamster.com's own. The signed CDN + /// URLs require a Referer from the exact mirror host that served them + /// (verified live), so the Referer is built from the actual final URL + /// after following the gateway's redirect rather than assumed. + async fn resolve_xhamster_mirror( + requester: &mut Requester, + gateway_url: &str, + ) -> Option> { + let response = requester + .get_raw_with_headers( + gateway_url, + vec![("User-Agent".to_string(), BROWSER_UA.to_string())], + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + let final_host = response.uri().host()?.to_string(); + let html = response.text().await.ok()?; + + let anchor = html.find(r#""sources":{"mp4":{"#)?; + let window_end = (anchor + 6000).min(html.len()); + let window = &html[anchor..window_end]; + let referer = format!("https://{final_host}/"); + + let mut ranked: Vec<(u32, VideoFormat)> = Vec::new(); + for cap in Self::xhamster_mirror_mp4_regex().captures_iter(window) { + let Ok(height) = cap[1].parse::() else { + continue; + }; + let url = cap[2].replace(r"\/", "/"); + let format = VideoFormat::new(url, format!("{height}p"), "mp4".to_string()) + .format_note(format!("{height}p")) + .height(height) + .http_header("Referer".to_string(), referer.clone()); + ranked.push((height, format)); + } + if ranked.is_empty() { + return None; + } + ranked.sort_by(|a, b| b.0.cmp(&a.0)); + Some(ranked.into_iter().map(|(_, f)| f).collect()) + } + + fn some_porn_id_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"/video/(\d+)").expect("valid regex")) + } + + /// `some.porn`'s HLS master playlist is addressable directly from the + /// numeric video id in the page URL — no page fetch needed. Verified + /// live: `s.cdn3x.com/get-playlist` returns a real segmented `.m3u8` + /// with no session/cookie requirement. + fn resolve_some_porn(detail_url: &str) -> Option> { + let id = Self::some_porn_id_regex().captures(detail_url)?.get(1)?.as_str(); + let url = format!("https://s.cdn3x.com/get-playlist?id={id}&q=720p"); + Some(vec![ + VideoFormat::m3u8(url, "720p".to_string(), "hls".to_string()) + .format_note("720p".to_string()), + ]) + } + + fn pornone_source_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new(r#" &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"_(\d+)x(\d+)_").expect("valid regex")) + } + + /// pornone.com embeds direct, unauthenticated `` mp4 + /// tags at multiple resolutions right on the detail page — no Referer + /// required (verified live). + async fn resolve_pornone( + requester: &mut Requester, + detail_url: &str, + ) -> Option> { + let html = requester + .get_with_headers( + detail_url, + vec![("user-agent".to_string(), BROWSER_UA.to_string())], + None, + ) + .await + .ok()?; + + let mut seen = HashSet::new(); + let mut ranked: Vec<(u32, VideoFormat)> = Vec::new(); + for cap in Self::pornone_source_regex().captures_iter(&html) { + let url = cap[1].to_string(); + if !seen.insert(url.clone()) { + continue; + } + let height = Self::pornone_resolution_regex() + .captures(&url) + .and_then(|c| c[2].parse::().ok()) + .unwrap_or(0); + let quality = if height > 0 { + format!("{height}p") + } else { + "source".to_string() + }; + let format = VideoFormat::new(url, quality.clone(), "mp4".to_string()) + .format_note(quality) + .height(height); + ranked.push((height, format)); + } + if ranked.is_empty() { + return None; + } + ranked.sort_by(|a, b| b.0.cmp(&a.0)); + Some(ranked.into_iter().map(|(_, f)| f).collect()) + } + + async fn resolve_source_formats( + kind: SourceKind, + requester: &mut Requester, + detail_url: &str, + ) -> Option> { + match kind { + SourceKind::Eporner => Self::resolve_eporner(requester, detail_url).await, + SourceKind::Kvs => Self::resolve_kvs(requester, detail_url, false).await, + SourceKind::KvsCfGated => Self::resolve_kvs(requester, detail_url, true).await, + SourceKind::XhamsterMirror => Self::resolve_xhamster_mirror(requester, detail_url).await, + SourceKind::SomePorn => Self::resolve_some_porn(detail_url), + SourceKind::Pornone => Self::resolve_pornone(requester, detail_url).await, + } + } + + fn parse_listing( + &self, + html: &str, + options: &ServerOptions, + ) -> (Vec, Vec<(usize, SourceKind, String)>) { let document = Html::parse_document(html); let card_sel = match Selector::parse("div.card[data-public-id]") { Ok(s) => s, - Err(_) => return vec![], + Err(_) => return (vec![], vec![]), }; let link_sel = match Selector::parse("a.item-link[href]") { Ok(s) => s, - Err(_) => return vec![], + Err(_) => return (vec![], vec![]), }; let img_sel = match Selector::parse("img.item-image") { Ok(s) => s, - Err(_) => return vec![], + Err(_) => return (vec![], vec![]), }; let model_sel = match Selector::parse("a.pornstar-label") { Ok(s) => s, - Err(_) => return vec![], + Err(_) => return (vec![], vec![]), }; let source_sel = match Selector::parse(r#"a.item-source[href^="/source/"]"#) { Ok(s) => s, - Err(_) => return vec![], + Err(_) => return (vec![], vec![]), }; let dur_re = match Regex::new(r#"class="badge[^"]*"[^>]*>\s*(\d{1,2}:\d{2}(?::\d{2})?)\s*"#) { Ok(r) => r, - Err(_) => return vec![], + Err(_) => return (vec![], vec![]), }; let ago_re = match Regex::new(r#"([^<]+)"#) { Ok(r) => r, - Err(_) => return vec![], + Err(_) => return (vec![], vec![]), }; let mut items = Vec::new(); + let mut pending: Vec<(usize, SourceKind, String)> = Vec::new(); for card in document.select(&card_sel) { let public_id = match card.value().attr("data-public-id") { @@ -381,14 +741,13 @@ impl MelonstubeProvider { let proxy_target = strip_url_scheme(&destination_url); let proxy_url = build_proxy_url(options, CHANNEL_ID, &proxy_target); if !proxy_url.is_empty() { - let mut format = crate::videos::VideoFormat::new( - proxy_url, - "auto".to_string(), - "video/mp4".to_string(), - ); + let mut format = + VideoFormat::new(proxy_url, "auto".to_string(), "video/mp4".to_string()); format.add_http_header("Referer".to_string(), destination_url.clone()); item = item.formats(vec![format]); } + } else if let Some(kind) = Self::classify_source(host) { + pending.push((items.len(), kind, destination_url.clone())); } if let Some(ts) = ago_re @@ -427,7 +786,7 @@ impl MelonstubeProvider { items.push(item); } - items + (items, pending) } fn resolve_target(query: &str, sort: &str, page: u32) -> Target { @@ -480,6 +839,7 @@ impl MelonstubeProvider { &self, cache: VideoCache, target: Target, + per_page: usize, options: ServerOptions, ) -> Result> { let url = match &target { @@ -507,34 +867,70 @@ impl MelonstubeProvider { } } - let html = match Self::fetch_html_bypass(&url).await { + let mut requester = requester_or_default(&options, CHANNEL_ID, "fetch_and_parse"); + + let html = match Self::fetch_html_bypass(&url, BASE_URL).await { Some(html) if !html.is_empty() => html, - _ => { - let mut requester = requester_or_default(&options, CHANNEL_ID, "fetch_and_parse"); - requester - .get_with_headers( - &url, - vec![ - ("user-agent".to_string(), BROWSER_UA.to_string()), - ( - "accept".to_string(), - "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - .to_string(), - ), - ("accept-language".to_string(), "en-US,en;q=0.5".to_string()), - ], - Some(wreq::Version::HTTP_11), - ) - .await - .map_err(|e| Error::from(format!("request failed url={url}: {e}")))? - } + _ => requester + .get_with_headers( + &url, + vec![ + ("user-agent".to_string(), BROWSER_UA.to_string()), + ( + "accept".to_string(), + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + .to_string(), + ), + ("accept-language".to_string(), "en-US,en;q=0.5".to_string()), + ], + Some(wreq::Version::HTTP_11), + ) + .await + .map_err(|e| Error::from(format!("request failed url={url}: {e}")))?, }; if html.is_empty() { return Ok(vec![]); } - let items = self.parse_listing(&html, &options); + let (mut items, mut pending) = self.parse_listing(&html, &options); + // Only enrich as many cards as this request can actually use — the + // page itself may have ~240 cards, but per_page is what the caller + // asked for. Cached results still hold the full parsed list for a + // later, larger per_page request; only the cards beyond this bound + // stay on the plain-destination-URL fallback for *this* response. + pending.truncate(per_page.min(MAX_ENRICH_PER_FETCH)); + + if !pending.is_empty() { + let resolved = stream::iter(pending.into_iter().map(|(idx, kind, dest_url)| { + let mut req = requester.clone(); + async move { + // Bounded so one slow/hung third-party destination (e.g. + // an unreachable FlareSolverr fallback) can't stall the + // whole listing response — the card just falls back to + // its plain destination URL with no formats. + let formats = tokio::time::timeout( + std::time::Duration::from_secs(ENRICH_TIMEOUT_SECS), + Self::resolve_source_formats(kind, &mut req, &dest_url), + ) + .await + .ok() + .flatten(); + (idx, formats) + } + })) + .buffer_unordered(ENRICH_CONCURRENCY) + .collect::>() + .await; + + for (idx, formats) in resolved { + if let Some(formats) = formats { + if let Some(item) = items.get_mut(idx) { + item.formats = Some(formats); + } + } + } + } if !items.is_empty() { cache.insert(url, items.clone()); @@ -562,7 +958,7 @@ impl Provider for MelonstubeProvider { let query_str = query.unwrap_or_default(); let target = Self::resolve_target(&query_str, &sort, page); - match self.fetch_and_parse(cache, target, options).await { + match self.fetch_and_parse(cache, target, per_page, options).await { Ok(mut items) => { items.truncate(per_page); items @@ -662,9 +1058,99 @@ mod tests { assert!(!MelonstubeProvider::is_hard_host("www.429men.com")); assert!(MelonstubeProvider::is_junk_host("fhgte.com")); assert!(MelonstubeProvider::is_junk_host("join.nastyclub.com")); + assert!(MelonstubeProvider::is_junk_host("www.flirt4free.com")); assert!(!MelonstubeProvider::is_junk_host("www.429men.com")); } + #[test] + fn classifies_source_hosts() { + use super::SourceKind; + + assert_eq!( + MelonstubeProvider::classify_source("www.eporner.com"), + Some(SourceKind::Eporner) + ); + assert_eq!( + MelonstubeProvider::classify_source("eporner.com"), + Some(SourceKind::Eporner) + ); + assert_eq!( + MelonstubeProvider::classify_source("xgroovy.com"), + Some(SourceKind::Kvs) + ); + assert_eq!( + MelonstubeProvider::classify_source("www.analdin.xxx"), + Some(SourceKind::Kvs) + ); + assert_eq!( + MelonstubeProvider::classify_source("www.fapcat.com"), + Some(SourceKind::KvsCfGated) + ); + assert_eq!( + MelonstubeProvider::classify_source("xh.partners"), + Some(SourceKind::XhamsterMirror) + ); + assert_eq!( + MelonstubeProvider::classify_source("some.porn"), + Some(SourceKind::SomePorn) + ); + assert_eq!( + MelonstubeProvider::classify_source("pornone.com"), + Some(SourceKind::Pornone) + ); + assert_eq!(MelonstubeProvider::classify_source("www.429men.com"), None); + // Hosts still resolved via manysex.com's dedicated proxy must not + // also be picked up here — the two paths are mutually exclusive in + // `parse_listing`. + assert_eq!(MelonstubeProvider::classify_source("manysex.com"), None); + } + + #[test] + fn parses_kvs_formats_from_html() { + let html = r#" + var flashvars = { + video_url: 'https://www.analdin.xxx/get_file/20/6d9d5daae562d9c7a41c88b6ac9210e5/812000/812240/812240.mp4/', + video_alt_url: 'https://www.analdin.xxx/get_file/20/28c406693124ceee1097b4d31cd0fdbb/812000/812240/812240hd.mp4/', + video_url_text: '480p', + video_alt_url_text: '720p', + }; + var preview = 'https://www.analdin.xxx/get_file/20/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/812000/812240/812240_preview.mp4/'; + "#; + + let formats = MelonstubeProvider::parse_kvs_formats(html, "https://www.analdin.xxx") + .expect("finds formats"); + assert_eq!(formats.len(), 2); + // Higher-labelled resolution ranked first. + assert!(formats[0].url.contains("812240hd.mp4")); + assert!(formats[1].url.contains("812240.mp4")); + for format in &formats { + assert!(!format.url.contains("_preview.mp4")); + assert_eq!( + format.http_headers_pairs(), + vec![("Referer".to_string(), "https://www.analdin.xxx/".to_string())] + ); + } + } + + #[test] + fn parses_kvs_formats_excludes_short_teaser_variant() { + let html = r#"video_url: 'https://w1mp.com/get_file/6/4a69e7a72396b25724d177cb557a2b68/561000/561838/561838_s.mp4/'"#; + assert!(MelonstubeProvider::parse_kvs_formats(html, "https://w1mp.com").is_none()); + } + + #[test] + fn resolves_some_porn_playlist_url_from_page_id() { + let formats = MelonstubeProvider::resolve_some_porn( + "https://some.porn/video/278305/the-worlds-hottest-italian?utm_campaign=933ab2ff", + ) + .expect("builds a format"); + assert_eq!(formats.len(), 1); + assert_eq!( + formats[0].url, + "https://s.cdn3x.com/get-playlist?id=278305&q=720p" + ); + } + #[test] fn parses_cards_from_html() { let html = r##" @@ -684,7 +1170,8 @@ mod tests { let provider = MelonstubeProvider::new(); let options = make_options(); - let items = provider.parse_listing(html, &options); + let (items, pending) = provider.parse_listing(html, &options); + assert!(pending.is_empty()); assert_eq!(items.len(), 1); let item = &items[0]; assert_eq!(item.id, "rrUbb7lXkQG"); @@ -712,7 +1199,8 @@ mod tests { let provider = MelonstubeProvider::new(); let options = make_options(); - let items = provider.parse_listing(html, &options); + let (items, pending) = provider.parse_listing(html, &options); + assert!(pending.is_empty()); assert_eq!(items.len(), 1); let item = &items[0]; assert!(item.url.starts_with("https://videomanysex.com/video/3519093/")); diff --git a/src/util/eporner_resolve.rs b/src/util/eporner_resolve.rs new file mode 100644 index 0000000..b16d283 --- /dev/null +++ b/src/util/eporner_resolve.rs @@ -0,0 +1,198 @@ +//! eporner.com direct-mp4 resolution, shared across providers. +//! +//! `eporner.rs` is the primary user of this (the `eporner` channel itself), +//! but other providers that redirect out to eporner.com (e.g. `melonstube`, +//! a meta-search aggregator) need the exact same handling to avoid leaving +//! the client with a bare page URL — see [`resolve_formats`] for why. +//! +//! Lives under `src/util/` rather than being imported directly from +//! `eporner.rs` because `HOT_TUB_PROVIDER=` single-provider builds only +//! compile the one selected provider module; a cross-provider `use` would +//! break every other provider's single-provider build. `src/util/*` is +//! always compiled regardless of provider selection. + +use crate::util::requester::Requester; +use crate::videos::VideoFormat; +use regex::Regex; +use std::sync::OnceLock; +use wreq::Version; + +const BASE_URL: &str = "https://www.eporner.com"; +const FIREFOX_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0"; +const HTML_ACCEPT: &str = + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"; + +fn html_headers(referer: &str) -> Vec<(String, String)> { + vec![ + ("User-Agent".to_string(), FIREFOX_UA.to_string()), + ("Accept".to_string(), HTML_ACCEPT.to_string()), + ("Referer".to_string(), referer.to_string()), + ] +} + +fn alnum_id_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"/video-([A-Za-z0-9]+)").expect("valid regex")) +} + +/// Extracts the alphanumeric video id eporner uses in its page URL +/// (`/video-{alnum_id}/...`) and its `/xhr/video/{alnum_id}` format API. +pub fn alnum_id_from_href(href: &str) -> Option { + alnum_id_regex() + .captures(href) + .and_then(|c| c.get(1)) + .map(|m| m.as_str().to_string()) +} + +fn hash_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r#"hash\s*[:=]\s*['"]([0-9a-f]{32})"#).expect("valid regex")) +} + +fn height_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"^(\d+)").expect("valid regex")) +} + +fn base36(mut n: u32) -> String { + const DIGITS: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + if n == 0 { + return "0".to_string(); + } + let mut buf = Vec::new(); + while n > 0 { + buf.push(DIGITS[(n % 36) as usize]); + n /= 36; + } + buf.reverse(); + String::from_utf8(buf).unwrap_or_default() +} + +/// Reverse-engineered from the site's player JS: the page's 32-hex-char +/// `hash` is split into four 8-hex-char chunks, each parsed as a u32 and +/// base36-encoded, then concatenated. The `/xhr/video/{id}` format API +/// requires this derived value or it serves a restricted/preview response +/// instead of the real sources. +fn calc_hash(hash: &str) -> Option { + if hash.len() != 32 || !hash.is_ascii() { + return None; + } + let mut out = String::new(); + for chunk_start in (0..32).step_by(8) { + let chunk = &hash[chunk_start..chunk_start + 8]; + let n = u32::from_str_radix(chunk, 16).ok()?; + out.push_str(&base36(n)); + } + Some(out) +} + +/// Fetches the detail page for its `hash`, then calls the site's own +/// `/xhr/video/{alnum_id}` JSON API (the same one the site's own player +/// uses) to get real direct mp4 URLs across all available resolutions. +/// Without this, the client is left to open the bare page URL, which +/// eporner serves as a small restricted/preview clip rather than the full +/// video. +/// +/// The resolved CDN mp4 URLs also enforce a Referer check independent of +/// the hash/xhr dance above: a request whose Referer is absent or points at +/// eporner.com is served the real file, but any other non-empty Referer +/// gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the +/// exact "this video is only available at eporner.com"-style clip users see +/// when playback doesn't carry the header the CDN wants. Each format +/// therefore declares an explicit `Referer` so the client's own default (or +/// lack thereof) can't accidentally trip this check. +pub async fn resolve_formats( + requester: &mut Requester, + alnum_id: &str, + page_url: &str, +) -> Option> { + let detail_html = requester + .get_with_headers(page_url, html_headers(page_url), Some(Version::HTTP_11)) + .await + .ok()?; + let hash = &hash_regex().captures(&detail_html)?[1]; + let calc = calc_hash(hash)?; + + let api_url = format!( + "{BASE_URL}/xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false" + ); + let json_text = requester + .get_with_headers(&api_url, html_headers(page_url), Some(Version::HTTP_11)) + .await + .ok()?; + let value: serde_json::Value = serde_json::from_str(&json_text).ok()?; + + if value.get("available").and_then(|v| v.as_bool()) == Some(false) { + return None; + } + + let mp4_sources = value.get("sources")?.get("mp4")?.as_object()?; + let mut ranked: Vec<(u32, VideoFormat)> = Vec::new(); + for (key, entry) in mp4_sources { + let Some(src) = entry.get("src").and_then(|v| v.as_str()) else { + continue; + }; + let label = entry + .get("labelShort") + .and_then(|v| v.as_str()) + .unwrap_or(key.as_str()); + let height = height_regex() + .captures(label) + .and_then(|c| c[1].parse::().ok()) + .unwrap_or(0); + let format = VideoFormat::new(src.to_string(), label.to_string(), "mp4".to_string()) + .format_id(key.clone()) + .format_note(key.clone()) + .height(height) + .http_header("Referer".to_string(), format!("{BASE_URL}/")); + ranked.push((height, format)); + } + if ranked.is_empty() { + return None; + } + ranked.sort_by(|a, b| b.0.cmp(&a.0)); + Some(ranked.into_iter().map(|(_, f)| f).collect()) +} + +/// Resolves eporner.com direct mp4 formats for an arbitrary eporner detail +/// page URL, extracting the alnum id from the URL itself. For use by +/// providers that only have a raw destination URL (e.g. a meta-search +/// aggregator's out-link), not a pre-parsed listing card. +pub async fn resolve_public_formats( + requester: &mut Requester, + page_url: &str, +) -> Option> { + let alnum_id = alnum_id_from_href(page_url)?; + resolve_formats(requester, &alnum_id, page_url).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn calc_hash_matches_site_algorithm() { + // Reverse-engineered from vjs.js; verified against a live page/xhr pair. + let hash = "06aadb04e6f6e853e25f5a583f173617"; + assert_eq!(hash.len(), 32); + assert_eq!(calc_hash(hash).unwrap(), "1ulk3o1s31fmb1qt66aghi70uv"); + } + + #[test] + fn calc_hash_rejects_wrong_length() { + assert!(calc_hash("abc").is_none()); + } + + #[test] + fn alnum_id_from_href_extracts_path_segment() { + assert_eq!( + alnum_id_from_href("/video-jQDTzcHvtjv/vj-blacked/"), + Some("jQDTzcHvtjv".to_string()) + ); + assert_eq!( + alnum_id_from_href("https://www.eporner.com/video-5Ssyrtrs9FG/-/"), + Some("5Ssyrtrs9FG".to_string()) + ); + assert_eq!(alnum_id_from_href("/hd-porn/95008/foo/"), None); + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 8db7574..870998b 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -3,6 +3,7 @@ use std::process::Command; pub mod cache; pub mod discord; +pub mod eporner_resolve; pub mod flaresolverr; pub mod flow_debug; pub mod hoster_proxy;