Add cumlouder provider
Listing scraper with category/tag/series routing; /search/ is behind a Cloudflare challenge so keywords map to archives. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
508
src/providers/cumlouder.rs
Normal file
508
src/providers/cumlouder.rs
Normal file
@@ -0,0 +1,508 @@
|
||||
use crate::DbPool;
|
||||
use crate::api::ClientVersion;
|
||||
use crate::providers::{Provider, report_provider_error, requester_or_default};
|
||||
use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::time::parse_time_to_seconds;
|
||||
use crate::videos::{ServerOptions, VideoItem};
|
||||
use async_trait::async_trait;
|
||||
use error_chain::error_chain;
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||
crate::providers::ProviderChannelMetadata {
|
||||
group_id: "mainstream-tube",
|
||||
tags: &["amateur", "spanish", "hd", "free"],
|
||||
};
|
||||
|
||||
const BASE_URL: &str = "https://www.cumlouder.com";
|
||||
const CHANNEL_ID: &str = "cumlouder";
|
||||
|
||||
// Scraped from /categories/ (`a.muestra-categoria[tag-url]`). The site's own
|
||||
// /search/ endpoint sits behind a Cloudflare managed challenge, so keyword
|
||||
// queries are routed to these archives or to `/tag/{slug}/` instead.
|
||||
const CATEGORIES: &[(&str, &str)] = &[
|
||||
("anal", "Anal"),
|
||||
("big-ass", "Big ass"),
|
||||
("spain", "Spain"),
|
||||
("latina", "Latina"),
|
||||
("mature", "Mature"),
|
||||
("teen", "Teen"),
|
||||
("black", "Black"),
|
||||
("squirting", "Squirting"),
|
||||
("hardcore", "Hardcore"),
|
||||
("big-dick", "Big dick"),
|
||||
("natural-tits", "Natural tits"),
|
||||
("cowgirl", "Cowgirl"),
|
||||
("busty", "Busty"),
|
||||
("ass", "Ass"),
|
||||
("facial", "Facial"),
|
||||
("reality", "Reality"),
|
||||
("orgasm", "Orgasm"),
|
||||
("casting", "Casting"),
|
||||
("milf", "Milf"),
|
||||
("tattoo", "Tattoo"),
|
||||
("huge-boobs", "Huge boobs"),
|
||||
("fake-tits", "Fake tits"),
|
||||
("brunette", "Brunette"),
|
||||
("public", "Public"),
|
||||
("cumshot", "Cumshot"),
|
||||
("deep-throat", "Deep throat"),
|
||||
("big-tits", "Big tits"),
|
||||
("party", "Party"),
|
||||
("interracial", "Interracial"),
|
||||
("parody", "Parody"),
|
||||
("amateur", "Amateur"),
|
||||
("pornstar", "Pornstar"),
|
||||
("petite", "Petite"),
|
||||
("rough-sex", "Rough sex"),
|
||||
("blonde", "Blonde"),
|
||||
("doggystyle", "Doggystyle"),
|
||||
("schoolgirl", "Schoolgirl"),
|
||||
("double-penetration", "Double penetration"),
|
||||
("van", "Van"),
|
||||
("creampie", "Creampie"),
|
||||
("barcelona", "Barcelona"),
|
||||
("exhibitionist", "Exhibitionist"),
|
||||
("threesome", "Threesome"),
|
||||
("blowjob", "Blowjob"),
|
||||
("mulatta", "Mulatta"),
|
||||
("money", "Money"),
|
||||
("masturbation", "Masturbation"),
|
||||
("tits", "Tits"),
|
||||
("handjob", "Handjob"),
|
||||
("comedy", "Comedy"),
|
||||
("european", "European"),
|
||||
("beach", "Beach"),
|
||||
("redhead", "Redhead"),
|
||||
("scandal", "Scandal"),
|
||||
("orgy", "Orgy"),
|
||||
("freak", "Freak"),
|
||||
("pov", "POV"),
|
||||
("lesbian", "Lesbian"),
|
||||
("russian", "Russian"),
|
||||
("ass-to-mouth", "Ass to mouth"),
|
||||
("los-angeles", "Los Angeles"),
|
||||
("fuck", "Fuck"),
|
||||
("slapping", "Slapping"),
|
||||
("nice-eyes", "Nice eyes"),
|
||||
("asian", "Asian"),
|
||||
("asslicking", "Asslicking"),
|
||||
("mixer", "Mixer"),
|
||||
("sport", "Sport"),
|
||||
("busted", "Busted"),
|
||||
("piercing", "Piercing"),
|
||||
("gape", "Gape"),
|
||||
("first-time", "First time"),
|
||||
];
|
||||
|
||||
error_chain! {
|
||||
foreign_links {
|
||||
Io(std::io::Error);
|
||||
Url(url::ParseError);
|
||||
}
|
||||
errors {
|
||||
Parse(msg: String) {
|
||||
description("parse error")
|
||||
display("parse error: {}", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CumlouderProvider;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum Target {
|
||||
Latest,
|
||||
Category(String),
|
||||
Tag(String),
|
||||
Series(String),
|
||||
}
|
||||
|
||||
impl CumlouderProvider {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||
let mut category_options = vec![FilterOption {
|
||||
id: "all".to_string(),
|
||||
title: "All".to_string(),
|
||||
}];
|
||||
category_options.extend(CATEGORIES.iter().map(|(slug, title)| FilterOption {
|
||||
id: (*slug).to_string(),
|
||||
title: (*title).to_string(),
|
||||
}));
|
||||
|
||||
Channel {
|
||||
id: CHANNEL_ID.to_string(),
|
||||
name: "Cumlouder".to_string(),
|
||||
description: "Free amateur and Spanish-scene porn videos from Cumlouder, with category and tag archives.".to_string(),
|
||||
premium: false,
|
||||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=cumlouder.com".to_string(),
|
||||
status: "active".to_string(),
|
||||
categories: vec![],
|
||||
options: vec![
|
||||
ChannelOption {
|
||||
id: "sort".to_string(),
|
||||
title: "Sort".to_string(),
|
||||
description: "Browse Cumlouder by newest videos.".to_string(),
|
||||
systemImage: "arrow.up.arrow.down".to_string(),
|
||||
colorName: "blue".to_string(),
|
||||
options: vec![FilterOption {
|
||||
id: "new".to_string(),
|
||||
title: "Latest".to_string(),
|
||||
}],
|
||||
multiSelect: false,
|
||||
},
|
||||
ChannelOption {
|
||||
id: "categories".to_string(),
|
||||
title: "Categories".to_string(),
|
||||
description: "Browse a Cumlouder category archive.".to_string(),
|
||||
systemImage: "square.grid.2x2".to_string(),
|
||||
colorName: "orange".to_string(),
|
||||
options: category_options,
|
||||
multiSelect: false,
|
||||
},
|
||||
],
|
||||
nsfw: true,
|
||||
cacheDuration: Some(1800),
|
||||
ytdlpCommand: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn slugify(value: &str) -> String {
|
||||
let mut out = String::new();
|
||||
let mut last_dash = true;
|
||||
for c in value.trim().chars() {
|
||||
if c.is_alphanumeric() {
|
||||
out.extend(c.to_lowercase());
|
||||
last_dash = false;
|
||||
} else if !last_dash {
|
||||
out.push('-');
|
||||
last_dash = true;
|
||||
}
|
||||
}
|
||||
out.trim_matches('-').to_string()
|
||||
}
|
||||
|
||||
fn category_slug(value: &str) -> Option<&'static str> {
|
||||
let slug = Self::slugify(value);
|
||||
CATEGORIES.iter().find_map(|(s, title)| {
|
||||
(*s == slug || Self::slugify(title) == slug).then_some(*s)
|
||||
})
|
||||
}
|
||||
|
||||
fn pick_target(query: Option<&str>, options: &ServerOptions) -> Target {
|
||||
if let Some(q) = query.map(str::trim).filter(|q| !q.is_empty()) {
|
||||
let lower = q.to_ascii_lowercase();
|
||||
for prefix in ["cat:", "category:"] {
|
||||
if let Some(rest) = lower.strip_prefix(prefix) {
|
||||
let slug = Self::category_slug(rest).map(str::to_string);
|
||||
return Target::Category(slug.unwrap_or_else(|| Self::slugify(rest)));
|
||||
}
|
||||
}
|
||||
if let Some(rest) = lower.strip_prefix("tag:") {
|
||||
return Target::Tag(Self::slugify(rest));
|
||||
}
|
||||
if let Some(rest) = lower.strip_prefix("series:") {
|
||||
return Target::Series(Self::slugify(rest));
|
||||
}
|
||||
// The site search is behind a Cloudflare challenge: fall back to a
|
||||
// category archive, then to the free-form tag archive.
|
||||
if let Some(slug) = Self::category_slug(q) {
|
||||
return Target::Category(slug.to_string());
|
||||
}
|
||||
return Target::Tag(Self::slugify(q));
|
||||
}
|
||||
|
||||
if let Some(cat) = options.categories.as_deref() {
|
||||
if cat != "all" && !cat.is_empty() {
|
||||
return Target::Category(Self::slugify(cat));
|
||||
}
|
||||
}
|
||||
Target::Latest
|
||||
}
|
||||
|
||||
fn build_listing_url(target: &Target, page: u32) -> String {
|
||||
let suffix = if page > 1 {
|
||||
format!("{page}/")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
match target {
|
||||
// Page 1 is /porn/, page N is the bare /N/ path.
|
||||
Target::Latest => {
|
||||
if page > 1 {
|
||||
format!("{BASE_URL}/{page}/")
|
||||
} else {
|
||||
format!("{BASE_URL}/porn/")
|
||||
}
|
||||
}
|
||||
Target::Category(slug) => format!("{BASE_URL}/porn-videos/{slug}/{suffix}"),
|
||||
Target::Tag(slug) => format!("{BASE_URL}/tag/{slug}/{suffix}"),
|
||||
Target::Series(slug) => format!("{BASE_URL}/series/{slug}/{suffix}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn selector(value: &str) -> Result<Selector> {
|
||||
Selector::parse(value).map_err(|e| Error::from(format!("selector `{value}` failed: {e}")))
|
||||
}
|
||||
|
||||
fn absolute(url: &str) -> String {
|
||||
if let Some(rest) = url.strip_prefix("//") {
|
||||
format!("https://{rest}")
|
||||
} else if url.starts_with('/') {
|
||||
format!("{BASE_URL}{url}")
|
||||
} else {
|
||||
url.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_views(text: &str) -> Option<u32> {
|
||||
let token = text.split_whitespace().next()?.replace(',', "");
|
||||
let (num, mult) = match token.chars().last()? {
|
||||
'K' | 'k' => (&token[..token.len() - 1], 1_000.0),
|
||||
'M' | 'm' => (&token[..token.len() - 1], 1_000_000.0),
|
||||
_ => (token.as_str(), 1.0),
|
||||
};
|
||||
num.parse::<f64>().ok().map(|n| (n * mult) as u32)
|
||||
}
|
||||
|
||||
fn parse_listing_page(html: &str) -> Result<Vec<VideoItem>> {
|
||||
let document = Html::parse_document(html);
|
||||
let card_sel = Self::selector("a.muestra-escena[href]")?;
|
||||
let img_sel = Self::selector("img")?;
|
||||
let title_sel = Self::selector("h2")?;
|
||||
let views_sel = Self::selector(".vistas")?;
|
||||
let minutes_sel = Self::selector(".minutos")?;
|
||||
|
||||
let mut items = Vec::new();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
|
||||
for card in document.select(&card_sel) {
|
||||
let href = card.value().attr("href").unwrap_or_default();
|
||||
let Some(slug) = href
|
||||
.strip_prefix("/porn-video/")
|
||||
.map(|s| s.trim_matches('/').to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !seen.insert(slug.clone()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let img = card.select(&img_sel).next();
|
||||
let thumb = img
|
||||
.and_then(|i| i.value().attr("data-src"))
|
||||
.map(Self::absolute)
|
||||
.unwrap_or_default();
|
||||
|
||||
let title = card
|
||||
.select(&title_sel)
|
||||
.next()
|
||||
.map(|h| h.text().collect::<String>().trim().to_string())
|
||||
.filter(|t| !t.is_empty())
|
||||
.or_else(|| {
|
||||
img.and_then(|i| i.value().attr("alt"))
|
||||
.map(|t| t.trim().to_string())
|
||||
.filter(|t| !t.is_empty())
|
||||
})
|
||||
.unwrap_or_else(|| slug.replace('-', " "));
|
||||
|
||||
let duration = card
|
||||
.select(&minutes_sel)
|
||||
.next()
|
||||
.map(|m| m.text().collect::<String>())
|
||||
.and_then(|t| {
|
||||
let t = t.trim().trim_end_matches('m').trim().to_string();
|
||||
parse_time_to_seconds(&t)
|
||||
})
|
||||
.and_then(|s| u32::try_from(s).ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let views = card
|
||||
.select(&views_sel)
|
||||
.next()
|
||||
.map(|v| v.text().collect::<String>())
|
||||
.and_then(|t| Self::parse_views(t.trim()));
|
||||
|
||||
let url = format!("{BASE_URL}/porn-video/{slug}/");
|
||||
let mut item = VideoItem::new(
|
||||
slug,
|
||||
title,
|
||||
url,
|
||||
CHANNEL_ID.to_string(),
|
||||
thumb,
|
||||
duration,
|
||||
);
|
||||
item.views = views;
|
||||
item.aspectRatio = Some(16.0 / 9.0);
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
async fn fetch_listing(
|
||||
target: &Target,
|
||||
page: u32,
|
||||
per_page: usize,
|
||||
options: &ServerOptions,
|
||||
) -> Result<Vec<VideoItem>> {
|
||||
let url = Self::build_listing_url(target, page);
|
||||
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_listing");
|
||||
let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))];
|
||||
|
||||
// The origin sporadically answers its first uncached request with a
|
||||
// Cloudflare 522; a single retry succeeds.
|
||||
let mut last_err = String::new();
|
||||
let mut html = None;
|
||||
for _ in 0..3 {
|
||||
match requester
|
||||
.get_with_headers(&url, headers.clone(), None)
|
||||
.await
|
||||
{
|
||||
Ok(body) if body.contains("muestra-escena") => {
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
Ok(body) => {
|
||||
last_err = format!("no cards in {} bytes", body.len());
|
||||
// A page with no cards is a real empty archive (404 body
|
||||
// included); only keep retrying on transport errors.
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
Err(e) => last_err = e.to_string(),
|
||||
}
|
||||
}
|
||||
let html = html.ok_or_else(|| Error::from(format!("fetch failed for {url}: {last_err}")))?;
|
||||
|
||||
let mut items = Self::parse_listing_page(&html)?;
|
||||
items.truncate(per_page.max(1));
|
||||
Ok(items)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for CumlouderProvider {
|
||||
async fn get_videos(
|
||||
&self,
|
||||
_cache: VideoCache,
|
||||
_pool: DbPool,
|
||||
_sort: String,
|
||||
query: Option<String>,
|
||||
page: String,
|
||||
per_page: String,
|
||||
options: ServerOptions,
|
||||
) -> Vec<VideoItem> {
|
||||
let page = page.parse::<u32>().unwrap_or(1).max(1);
|
||||
let per_page = per_page.parse::<usize>().unwrap_or(30).clamp(1, 60);
|
||||
let target = Self::pick_target(query.as_deref(), &options);
|
||||
|
||||
match Self::fetch_listing(&target, page, per_page, &options).await {
|
||||
Ok(items) => items,
|
||||
Err(e) => {
|
||||
report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await;
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn opts() -> ServerOptions {
|
||||
ServerOptions {
|
||||
featured: None,
|
||||
category: None,
|
||||
sites: None,
|
||||
filter: None,
|
||||
language: None,
|
||||
public_url_base: None,
|
||||
requester: None,
|
||||
network: None,
|
||||
stars: None,
|
||||
categories: None,
|
||||
duration: None,
|
||||
sort: None,
|
||||
sexuality: None,
|
||||
client_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_urls() {
|
||||
assert_eq!(
|
||||
CumlouderProvider::build_listing_url(&Target::Latest, 1),
|
||||
"https://www.cumlouder.com/porn/"
|
||||
);
|
||||
assert_eq!(
|
||||
CumlouderProvider::build_listing_url(&Target::Latest, 3),
|
||||
"https://www.cumlouder.com/3/"
|
||||
);
|
||||
assert_eq!(
|
||||
CumlouderProvider::build_listing_url(&Target::Tag("deepthroat".into()), 2),
|
||||
"https://www.cumlouder.com/tag/deepthroat/2/"
|
||||
);
|
||||
assert_eq!(
|
||||
CumlouderProvider::build_listing_url(&Target::Category("anal".into()), 1),
|
||||
"https://www.cumlouder.com/porn-videos/anal/"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routes_queries() {
|
||||
let o = opts();
|
||||
assert_eq!(
|
||||
CumlouderProvider::pick_target(Some("Big Tits"), &o),
|
||||
Target::Category("big-tits".into())
|
||||
);
|
||||
assert_eq!(
|
||||
CumlouderProvider::pick_target(Some("deepthroat"), &o),
|
||||
Target::Tag("deepthroat".into())
|
||||
);
|
||||
assert_eq!(
|
||||
CumlouderProvider::pick_target(Some("series:Nacho Vidal"), &o),
|
||||
Target::Series("nacho-vidal".into())
|
||||
);
|
||||
assert_eq!(CumlouderProvider::pick_target(None, &o), Target::Latest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_cards() {
|
||||
let html = r#"<a class="muestra-escena" href="/porn-video/some-scene/">
|
||||
<img class="thumb" src="data:image/gif;base64,AA" data-src="//th0cdn.cumlouder.com/abc/thumbnail_001.jpg" alt="Some scene">
|
||||
<h2> <span class="ico-h2 sprite"></span> Some scene</h2>
|
||||
<span class="box-fecha-mins">
|
||||
<span class="vistas"> <span class="ico-vistas sprite"></span> 1,154 views</span>
|
||||
<span class="fecha"> <span class="ico-fecha sprite"></span> 2 years ago</span>
|
||||
</span>
|
||||
<span class="minutos"> <span class="ico-minutos sprite"></span> 06:14 m</span>
|
||||
</a>
|
||||
<a class="muestra-escena" href="/porn-videos/anal/"><h2>Category</h2></a>"#;
|
||||
let items = CumlouderProvider::parse_listing_page(html).unwrap();
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0].id, "some-scene");
|
||||
assert_eq!(items[0].title, "Some scene");
|
||||
assert_eq!(items[0].duration, 374);
|
||||
assert_eq!(items[0].views, Some(1154));
|
||||
assert_eq!(
|
||||
items[0].thumb,
|
||||
"https://th0cdn.cumlouder.com/abc/thumbnail_001.jpg"
|
||||
);
|
||||
assert_eq!(
|
||||
items[0].url,
|
||||
"https://www.cumlouder.com/porn-video/some-scene/"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user