From 7fbf6cfd1527f7c770b764667710a4bcdc623b03 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 9 Oct 2026 22:27:28 +0000 Subject: [PATCH] Add cumlouder provider Listing scraper with category/tag/series routing; /search/ is behind a Cloudflare challenge so keywords map to archives. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi --- build.rs | 5 + docs/provider-catalog.md | 1 + src/providers/cumlouder.rs | 508 +++++++++++++++++++++++++++++++++++++ 3 files changed, 514 insertions(+) create mode 100644 src/providers/cumlouder.rs diff --git a/build.rs b/build.rs index a52de2d..165d54b 100644 --- a/build.rs +++ b/build.rs @@ -356,6 +356,11 @@ const PROVIDERS: &[ProviderDef] = &[ module: "jable", ty: "JableProvider", }, + ProviderDef { + id: "cumlouder", + module: "cumlouder", + ty: "CumlouderProvider", + }, ProviderDef { id: "fullporner", module: "fullporner", diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index 17e8c84..b2d9693 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -81,6 +81,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `jable` | `jav` | no | yes | HTML JAV archive scraper; extracts `var hlsUrl` from detail pages; m3u8 format requires Referer + browser User-Agent; proxy route handles HEAD (200 OK) and GET (redirect to watch page) since yt-dlp blocks jable.tv; tag/category/model shortcut queries. | | `kwiky` | `tiktok` | no | no | Short-form cam clip provider for kwiky.com (ICF/StreamMate platform). Two-step fetch: ID list then metadata batch. Auth: XSRF token extracted from `meta.content = ''` injected by `/api/state/v1/preload`, cached in an `Arc>>` with double-checked locking (same pattern as `xxxtik` bearer token); all API calls also require `x-gateway: 76d59e04-97fa-4ced-aa99-b86ffaf756a2` (platform-identifying fixed UUID observed in browser JS, not stored in cookies or localStorage). Default feed: `GET /api/v1/gateway/v1/quickies/recommended?source=` (cursor-paged, walks up to 5 cursors for higher page numbers). All tag/keyword queries go through `keywordSearch?keyword=...` (offset-paged: `pageToken=(page-1)*perPage`) — the recommended endpoint's `&tag=` param does not reliably filter, so even curated-tag requests use keywordSearch. Media: `media.url` = direct `media.icfcdn.com/*.mp4` (publicly accessible, no Referer/auth), `media.thumbUrl` thumbnail (same CDN, direct), `media.previewUrl` preview clip; both kwiky page URLs (`kwiky.com/quickies/{id}`) and the media CDN URLs pass direct `curl -I` health checks but the page URL is NOT yt-dlp-resolvable (React SPA, generic extractor fails), so `formats` are populated with the direct MP4 URL and `video.url` is the page. Aspect ratio set from `media.width`/`media.height`; `uploadedAt` from RFC3339 `created` field; uploader name/URL/ID from `creator.name` and `creator.id`. 48 curated tags from `/api/v1/gateway/v1/keywords/populartags?gender=f` exposed via `categories` filter option. No proxy needed. | | `fullporner` | `mainstream-tube` | no | no | HTML scraper for fullporner.com; thumbnail IDs derived from `/thumb/{id}.jpg` URLs and used to build direct `xiaoshenke.net/vid/{id}/720` media redirect URLs (Referer + User-Agent headers required); supports cat:/category:/pornstar:/star: shortcut queries; no proxy needed. | +| `cumlouder` | `mainstream-tube` | no | no | HTML scraper for cumlouder.com (Spanish/amateur tube, same platform family as superporn). Cards are `a.muestra-escena[href^="/porn-video/"]` with lazy thumb in `img[data-src]` (protocol-relative `//th0cdn.cumlouder.com/...`, direct, no Referer/proxy), `h2` title, `.vistas` views, `.minutos` duration (`06:14 m`); relative date only ("2 years ago"), so `uploadedAt` is not set. Default feed `/porn/` (page N≥2 is the bare `/N/` path, not `/porn/N/`); category `/porn-videos/{slug}/{N}/`; tag `/tag/{slug}/{N}/`; series `/series/{slug}/{N}/`. **The site's own `/search/` (POST `form[search]` or `?q=`) is behind a Cloudflare managed challenge — 403 even in a real browser — so keyword queries are routed instead: a query matching a hardcoded category (72, scraped from `/categories/`) goes to the category archive, anything else to `/tag/{slug}/` (unknown tags 404 → empty list).** Supports `cat:`/`category:`, `tag:`, `series:` prefixes and the `categories` option. `video.url` is the detail page, which yt-dlp resolves natively via the generic html5 extractor (the page embeds a signed `m5cdnst.cumlouder.com/*.mp4?secure=…` ``, no Referer needed), so no `formats`/enrichment. The origin intermittently answers a first, uncached request with a Cloudflare 522 (a retry succeeds) — the listing fetch retries up to 3×, and `check.py`/yt-dlp may show a one-off `HTTP=-1`/522 on a cold detail page that is not a provider fault. No proxy needed. | | `thepornbunny` | `mainstream-tube` | no | yes | KVS-style HTML scraper for thepornbunny.com; 24 items per site page; thumbnails at `https://www.thepornbunny.com/images/thumb/{id}.webp` from `data-original` attribute (no proxy needed); studio exposed as uploader; pornstar names in tags; `/proxy/thepornbunny/{slug}` fetches the video page, extracts `generate_mp4(enc_data, key, rnd, video_id)` args, decrypts `enc_data` via PBKDF2-HMAC-SHA512+AES-256-CBC to get an OK.ru session key, calls `api.ok.ru/fb.do?method=video.get&session_key=KEY&vids=RND` to get signed CDN URLs, and returns 302 to the best-quality okcdn.ru/vkuser.net MP4 URL (no special client headers needed); supports sort: new/popular/rated, 20 hardcoded categories via `categories` option, and tag:/category:/studio:/pornstar: query shortcuts. | | `eporner` | `mainstream-tube` | no | no | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); supports sort: new/popular/rated/best; 65 hardcoded categories, plus `cat:`/`category:`, `tag:`, `pornstar:`/`star:`, `uploader:`/`profile:` query-prefix routing to the matching archive, a bare-keyword category-name match, and a background-loaded pornstar name→URL map from `/pornstar-list/` (falls back to `/tag/{slug}` keyword search otherwise). `video.url` is the detail page — **not** yt-dlp-resolvable and not directly playable on its own: opening it bare serves a small restricted/preview clip. Real playback needs per-card enrichment (bounded `buffer_unordered(6)`): fetch the detail page for its 32-hex `hash`, derive the site's own hashed token (`calc_hash` — split into four 8-hex chunks, each parsed as u32 and base36-encoded, concatenated; reverse-engineered from the player JS), then call the site's own `GET /xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false` JSON API for direct per-resolution mp4 URLs into `formats[]`. Separately from the hash dance, the resolved CDN mp4 URLs enforce their own Referer check: a request with no Referer or one pointing at eporner.com gets the real file, but any other non-empty Referer gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the exact "this video is only available at eporner.com"-style short clip a client sees if its default (or app-injected) Referer doesn't match, so every format explicitly declares `http_headers: {Referer: "https://www.eporner.com/"}` rather than relying on the client's default. No proxy needed (formats are direct CDN URLs, just header-gated). **2026-09-21:** the `calc_hash`/hash-regex/`alnum_id_from_href`/`resolve_formats` logic moved to `src/util/eporner_resolve.rs` so `melonstube` (a meta-search aggregator that redirects out to eporner.com for ~34% of its own cards) can reuse the exact same resolution without a cross-provider `use`, which would break either provider's `HOT_TUB_PROVIDER=` single-provider build (only the one selected provider module gets compiled; `src/util/*` always does). `eporner.rs` itself now just calls `crate::util::eporner_resolve::resolve_formats(...)`. | | `xnxx` | `mainstream-tube` | no | no | HTML scraper for xnxx.com (10M+ videos); unified card parser handles two formats: `div.thumb-block[data-eid]` (search) and `div.thumb-block.video[data-video='{"id":...}']` (hits); eid extracted from `/video-{eid}/{slug}` URL path; thumbnails at `thumb-cdn77.xnxx-cdn.com` and `thumbs-gcore.xnxx-cdn.com` (no proxy, no Referer needed); 0-indexed pagination (page 1 = `/hits`, page N = `/hits/{N-1}`); default feed is `/hits` (most-viewed — xnxx has no chronological listing); search via `/search/{slug}` (works for keywords and tags); supports `tag:`, `cat:`, `category:` query shortcuts; yt-dlp resolves `video.url` natively (XNXX extractor, returns 4-7 HLS formats); no proxy needed. | diff --git a/src/providers/cumlouder.rs b/src/providers/cumlouder.rs new file mode 100644 index 0000000..d77cf5a --- /dev/null +++ b/src/providers/cumlouder.rs @@ -0,0 +1,508 @@ +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::{Provider, report_provider_error, requester_or_default}; +use crate::status::*; +use crate::util::cache::VideoCache; +use crate::util::time::parse_time_to_seconds; +use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; +use error_chain::error_chain; +use scraper::{Html, Selector}; + +pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = + crate::providers::ProviderChannelMetadata { + group_id: "mainstream-tube", + tags: &["amateur", "spanish", "hd", "free"], + }; + +const BASE_URL: &str = "https://www.cumlouder.com"; +const CHANNEL_ID: &str = "cumlouder"; + +// Scraped from /categories/ (`a.muestra-categoria[tag-url]`). The site's own +// /search/ endpoint sits behind a Cloudflare managed challenge, so keyword +// queries are routed to these archives or to `/tag/{slug}/` instead. +const CATEGORIES: &[(&str, &str)] = &[ + ("anal", "Anal"), + ("big-ass", "Big ass"), + ("spain", "Spain"), + ("latina", "Latina"), + ("mature", "Mature"), + ("teen", "Teen"), + ("black", "Black"), + ("squirting", "Squirting"), + ("hardcore", "Hardcore"), + ("big-dick", "Big dick"), + ("natural-tits", "Natural tits"), + ("cowgirl", "Cowgirl"), + ("busty", "Busty"), + ("ass", "Ass"), + ("facial", "Facial"), + ("reality", "Reality"), + ("orgasm", "Orgasm"), + ("casting", "Casting"), + ("milf", "Milf"), + ("tattoo", "Tattoo"), + ("huge-boobs", "Huge boobs"), + ("fake-tits", "Fake tits"), + ("brunette", "Brunette"), + ("public", "Public"), + ("cumshot", "Cumshot"), + ("deep-throat", "Deep throat"), + ("big-tits", "Big tits"), + ("party", "Party"), + ("interracial", "Interracial"), + ("parody", "Parody"), + ("amateur", "Amateur"), + ("pornstar", "Pornstar"), + ("petite", "Petite"), + ("rough-sex", "Rough sex"), + ("blonde", "Blonde"), + ("doggystyle", "Doggystyle"), + ("schoolgirl", "Schoolgirl"), + ("double-penetration", "Double penetration"), + ("van", "Van"), + ("creampie", "Creampie"), + ("barcelona", "Barcelona"), + ("exhibitionist", "Exhibitionist"), + ("threesome", "Threesome"), + ("blowjob", "Blowjob"), + ("mulatta", "Mulatta"), + ("money", "Money"), + ("masturbation", "Masturbation"), + ("tits", "Tits"), + ("handjob", "Handjob"), + ("comedy", "Comedy"), + ("european", "European"), + ("beach", "Beach"), + ("redhead", "Redhead"), + ("scandal", "Scandal"), + ("orgy", "Orgy"), + ("freak", "Freak"), + ("pov", "POV"), + ("lesbian", "Lesbian"), + ("russian", "Russian"), + ("ass-to-mouth", "Ass to mouth"), + ("los-angeles", "Los Angeles"), + ("fuck", "Fuck"), + ("slapping", "Slapping"), + ("nice-eyes", "Nice eyes"), + ("asian", "Asian"), + ("asslicking", "Asslicking"), + ("mixer", "Mixer"), + ("sport", "Sport"), + ("busted", "Busted"), + ("piercing", "Piercing"), + ("gape", "Gape"), + ("first-time", "First time"), +]; + +error_chain! { + foreign_links { + Io(std::io::Error); + Url(url::ParseError); + } + errors { + Parse(msg: String) { + description("parse error") + display("parse error: {}", msg) + } + } +} + +#[derive(Debug, Clone)] +pub struct CumlouderProvider; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Target { + Latest, + Category(String), + Tag(String), + Series(String), +} + +impl CumlouderProvider { + pub fn new() -> Self { + Self + } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + let mut category_options = vec![FilterOption { + id: "all".to_string(), + title: "All".to_string(), + }]; + category_options.extend(CATEGORIES.iter().map(|(slug, title)| FilterOption { + id: (*slug).to_string(), + title: (*title).to_string(), + })); + + Channel { + id: CHANNEL_ID.to_string(), + name: "Cumlouder".to_string(), + description: "Free amateur and Spanish-scene porn videos from Cumlouder, with category and tag archives.".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=cumlouder.com".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ + ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Browse Cumlouder by newest videos.".to_string(), + systemImage: "arrow.up.arrow.down".to_string(), + colorName: "blue".to_string(), + options: vec![FilterOption { + id: "new".to_string(), + title: "Latest".to_string(), + }], + multiSelect: false, + }, + ChannelOption { + id: "categories".to_string(), + title: "Categories".to_string(), + description: "Browse a Cumlouder category archive.".to_string(), + systemImage: "square.grid.2x2".to_string(), + colorName: "orange".to_string(), + options: category_options, + multiSelect: false, + }, + ], + nsfw: true, + cacheDuration: Some(1800), + ytdlpCommand: None, + } + } + + fn slugify(value: &str) -> String { + let mut out = String::new(); + let mut last_dash = true; + for c in value.trim().chars() { + if c.is_alphanumeric() { + out.extend(c.to_lowercase()); + last_dash = false; + } else if !last_dash { + out.push('-'); + last_dash = true; + } + } + out.trim_matches('-').to_string() + } + + fn category_slug(value: &str) -> Option<&'static str> { + let slug = Self::slugify(value); + CATEGORIES.iter().find_map(|(s, title)| { + (*s == slug || Self::slugify(title) == slug).then_some(*s) + }) + } + + fn pick_target(query: Option<&str>, options: &ServerOptions) -> Target { + if let Some(q) = query.map(str::trim).filter(|q| !q.is_empty()) { + let lower = q.to_ascii_lowercase(); + for prefix in ["cat:", "category:"] { + if let Some(rest) = lower.strip_prefix(prefix) { + let slug = Self::category_slug(rest).map(str::to_string); + return Target::Category(slug.unwrap_or_else(|| Self::slugify(rest))); + } + } + if let Some(rest) = lower.strip_prefix("tag:") { + return Target::Tag(Self::slugify(rest)); + } + if let Some(rest) = lower.strip_prefix("series:") { + return Target::Series(Self::slugify(rest)); + } + // The site search is behind a Cloudflare challenge: fall back to a + // category archive, then to the free-form tag archive. + if let Some(slug) = Self::category_slug(q) { + return Target::Category(slug.to_string()); + } + return Target::Tag(Self::slugify(q)); + } + + if let Some(cat) = options.categories.as_deref() { + if cat != "all" && !cat.is_empty() { + return Target::Category(Self::slugify(cat)); + } + } + Target::Latest + } + + fn build_listing_url(target: &Target, page: u32) -> String { + let suffix = if page > 1 { + format!("{page}/") + } else { + String::new() + }; + match target { + // Page 1 is /porn/, page N is the bare /N/ path. + Target::Latest => { + if page > 1 { + format!("{BASE_URL}/{page}/") + } else { + format!("{BASE_URL}/porn/") + } + } + Target::Category(slug) => format!("{BASE_URL}/porn-videos/{slug}/{suffix}"), + Target::Tag(slug) => format!("{BASE_URL}/tag/{slug}/{suffix}"), + Target::Series(slug) => format!("{BASE_URL}/series/{slug}/{suffix}"), + } + } + + fn selector(value: &str) -> Result { + Selector::parse(value).map_err(|e| Error::from(format!("selector `{value}` failed: {e}"))) + } + + fn absolute(url: &str) -> String { + if let Some(rest) = url.strip_prefix("//") { + format!("https://{rest}") + } else if url.starts_with('/') { + format!("{BASE_URL}{url}") + } else { + url.to_string() + } + } + + fn parse_views(text: &str) -> Option { + let token = text.split_whitespace().next()?.replace(',', ""); + let (num, mult) = match token.chars().last()? { + 'K' | 'k' => (&token[..token.len() - 1], 1_000.0), + 'M' | 'm' => (&token[..token.len() - 1], 1_000_000.0), + _ => (token.as_str(), 1.0), + }; + num.parse::().ok().map(|n| (n * mult) as u32) + } + + fn parse_listing_page(html: &str) -> Result> { + let document = Html::parse_document(html); + let card_sel = Self::selector("a.muestra-escena[href]")?; + let img_sel = Self::selector("img")?; + let title_sel = Self::selector("h2")?; + let views_sel = Self::selector(".vistas")?; + let minutes_sel = Self::selector(".minutos")?; + + let mut items = Vec::new(); + let mut seen = std::collections::HashSet::new(); + + for card in document.select(&card_sel) { + let href = card.value().attr("href").unwrap_or_default(); + let Some(slug) = href + .strip_prefix("/porn-video/") + .map(|s| s.trim_matches('/').to_string()) + .filter(|s| !s.is_empty()) + else { + continue; + }; + if !seen.insert(slug.clone()) { + continue; + } + + let img = card.select(&img_sel).next(); + let thumb = img + .and_then(|i| i.value().attr("data-src")) + .map(Self::absolute) + .unwrap_or_default(); + + let title = card + .select(&title_sel) + .next() + .map(|h| h.text().collect::().trim().to_string()) + .filter(|t| !t.is_empty()) + .or_else(|| { + img.and_then(|i| i.value().attr("alt")) + .map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()) + }) + .unwrap_or_else(|| slug.replace('-', " ")); + + let duration = card + .select(&minutes_sel) + .next() + .map(|m| m.text().collect::()) + .and_then(|t| { + let t = t.trim().trim_end_matches('m').trim().to_string(); + parse_time_to_seconds(&t) + }) + .and_then(|s| u32::try_from(s).ok()) + .unwrap_or(0); + + let views = card + .select(&views_sel) + .next() + .map(|v| v.text().collect::()) + .and_then(|t| Self::parse_views(t.trim())); + + let url = format!("{BASE_URL}/porn-video/{slug}/"); + let mut item = VideoItem::new( + slug, + title, + url, + CHANNEL_ID.to_string(), + thumb, + duration, + ); + item.views = views; + item.aspectRatio = Some(16.0 / 9.0); + items.push(item); + } + + Ok(items) + } + + async fn fetch_listing( + target: &Target, + page: u32, + per_page: usize, + options: &ServerOptions, + ) -> Result> { + let url = Self::build_listing_url(target, page); + let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_listing"); + let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))]; + + // The origin sporadically answers its first uncached request with a + // Cloudflare 522; a single retry succeeds. + let mut last_err = String::new(); + let mut html = None; + for _ in 0..3 { + match requester + .get_with_headers(&url, headers.clone(), None) + .await + { + Ok(body) if body.contains("muestra-escena") => { + html = Some(body); + break; + } + Ok(body) => { + last_err = format!("no cards in {} bytes", body.len()); + // A page with no cards is a real empty archive (404 body + // included); only keep retrying on transport errors. + html = Some(body); + break; + } + Err(e) => last_err = e.to_string(), + } + } + let html = html.ok_or_else(|| Error::from(format!("fetch failed for {url}: {last_err}")))?; + + let mut items = Self::parse_listing_page(&html)?; + items.truncate(per_page.max(1)); + Ok(items) + } +} + +#[async_trait] +impl Provider for CumlouderProvider { + async fn get_videos( + &self, + _cache: VideoCache, + _pool: DbPool, + _sort: String, + query: Option, + page: String, + per_page: String, + options: ServerOptions, + ) -> Vec { + let page = page.parse::().unwrap_or(1).max(1); + let per_page = per_page.parse::().unwrap_or(30).clamp(1, 60); + let target = Self::pick_target(query.as_deref(), &options); + + match Self::fetch_listing(&target, page, per_page, &options).await { + Ok(items) => items, + Err(e) => { + report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await; + vec![] + } + } + } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts() -> ServerOptions { + ServerOptions { + featured: None, + category: None, + sites: None, + filter: None, + language: None, + public_url_base: None, + requester: None, + network: None, + stars: None, + categories: None, + duration: None, + sort: None, + sexuality: None, + client_version: None, + } + } + + #[test] + fn builds_urls() { + assert_eq!( + CumlouderProvider::build_listing_url(&Target::Latest, 1), + "https://www.cumlouder.com/porn/" + ); + assert_eq!( + CumlouderProvider::build_listing_url(&Target::Latest, 3), + "https://www.cumlouder.com/3/" + ); + assert_eq!( + CumlouderProvider::build_listing_url(&Target::Tag("deepthroat".into()), 2), + "https://www.cumlouder.com/tag/deepthroat/2/" + ); + assert_eq!( + CumlouderProvider::build_listing_url(&Target::Category("anal".into()), 1), + "https://www.cumlouder.com/porn-videos/anal/" + ); + } + + #[test] + fn routes_queries() { + let o = opts(); + assert_eq!( + CumlouderProvider::pick_target(Some("Big Tits"), &o), + Target::Category("big-tits".into()) + ); + assert_eq!( + CumlouderProvider::pick_target(Some("deepthroat"), &o), + Target::Tag("deepthroat".into()) + ); + assert_eq!( + CumlouderProvider::pick_target(Some("series:Nacho Vidal"), &o), + Target::Series("nacho-vidal".into()) + ); + assert_eq!(CumlouderProvider::pick_target(None, &o), Target::Latest); + } + + #[test] + fn parses_cards() { + let html = r#" + Some scene +

Some scene

+ + 1,154 views + 2 years ago + + 06:14 m + +

Category

"#; + let items = CumlouderProvider::parse_listing_page(html).unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].id, "some-scene"); + assert_eq!(items[0].title, "Some scene"); + assert_eq!(items[0].duration, 374); + assert_eq!(items[0].views, Some(1154)); + assert_eq!( + items[0].thumb, + "https://th0cdn.cumlouder.com/abc/thumbnail_001.jpg" + ); + assert_eq!( + items[0].url, + "https://www.cumlouder.com/porn-video/some-scene/" + ); + } +}