use crate::DbPool; use crate::api::ClientVersion; use crate::providers::{ Provider, build_proxy_url, report_provider_error, requester_or_default, strip_url_scheme, }; use crate::status::*; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use chrono::NaiveDate; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use scraper::{ElementRef, Html, Selector}; use serde::Deserialize; use std::collections::HashSet; use wreq::Version; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "amateur-homemade", tags: &["amateur", "homemade", "interracial"], }; const BASE_URL: &str = "https://www.clapdat.com"; const SEARCH_URL: &str = "https://search.clapdat.com/indexes/videos/search"; const SEARCH_KEY: &str = "36ce9a190ca0e797debc3f0a2a311749dbd76262c389531c3a37e9dd74ab9df5"; const CHANNEL_ID: &str = "clapdat"; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); Json(serde_json::Error); } } #[derive(Debug, Clone)] pub struct ClapdatProvider { url: String, } #[derive(Debug, Clone)] enum Target { Trending, Recent, Search { query: String }, Tag { slug: String }, User { username: String }, } #[derive(Debug, Clone)] struct StubVideo { id: String, title: String, url: String, thumb: String, duration: u32, } #[derive(Debug, Deserialize)] struct SearchResponse { #[serde(default)] hits: Vec, } #[derive(Debug, Deserialize)] struct SearchHit { #[serde(rename = "_id", default)] id: String, #[serde(default)] title: String, #[serde(default)] slug: String, #[serde(default)] image: String, } impl ClapdatProvider { pub fn new() -> Self { Self { url: BASE_URL.to_string(), } } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { Channel { id: CHANNEL_ID.to_string(), name: "ClapDat".to_string(), description: "ClapDat trending/recent feeds with tag and uploader shortcuts." .to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=clapdat.com".to_string(), status: "active".to_string(), categories: vec![], options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Trending or latest ClapDat feed.".to_string(), systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "trending".to_string(), title: "Trending".to_string(), }, FilterOption { id: "new".to_string(), title: "Recent".to_string(), }, ], multiSelect: false, }], nsfw: true, cacheDuration: Some(1800), ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()) } } fn resolve_target(&self, query: &str, sort: &str) -> Target { let q = query.trim(); if let Some(value) = q.strip_prefix("tag:").or_else(|| q.strip_prefix('#')) { let slug = value.trim().to_lowercase().replace(' ', "-"); if !slug.is_empty() { return Target::Tag { slug }; } } if let Some(value) = q .strip_prefix("user:") .or_else(|| q.strip_prefix("uploader:")) { let username = value.trim().to_lowercase().replace(' ', "-"); if !username.is_empty() { return Target::User { username }; } } if !q.is_empty() { return Target::Search { query: q.to_string(), }; } match sort { "recent" | "new" | "latest" => Target::Recent, _ => Target::Trending, } } fn listing_url(&self, target: &Target, page: u16) -> Option { let page = page.max(1); match target { Target::Trending => Some(if page == 1 { self.url.clone() } else { format!("{}/trending/{page}", self.url) }), Target::Recent => Some(if page == 1 { self.url.clone() } else { format!("{}/recent/{page}", self.url) }), Target::Tag { slug } => Some(if page == 1 { format!("{}/tag/{slug}", self.url) } else { format!("{}/tag/{slug}/{page}", self.url) }), Target::User { username } => Some(if page == 1 { format!("{}/user/{username}", self.url) } else { format!("{}/user/{username}/{page}", self.url) }), Target::Search { .. } => None, } } fn selector(value: &str) -> Result { Selector::parse(value).map_err(|error| Error::from(format!("selector `{value}`: {error}"))) } fn regex(value: &str) -> Result { Regex::new(value).map_err(|error| Error::from(format!("regex `{value}`: {error}"))) } fn normalize_text(value: &str) -> String { decode(value.as_bytes()) .to_string() .unwrap_or_else(|_| value.to_string()) .replace('\u{a0}', " ") .split_whitespace() .collect::>() .join(" ") .trim() .to_string() } fn normalize_url(&self, value: &str) -> String { let trimmed = value.trim(); if trimmed.starts_with("http://") || trimmed.starts_with("https://") { return trimmed.to_string(); } if trimmed.starts_with("//") { return format!("https:{trimmed}"); } format!( "{}/{}", self.url.trim_end_matches('/'), trimmed.trim_start_matches('/') ) } fn extract_video_id(url: &str) -> Option { let re = Regex::new(r"-([a-z0-9]+)(?:/|$)").ok()?; re.captures(url) .and_then(|caps| caps.get(1).map(|m| m.as_str().to_string())) } fn parse_duration(value: &str) -> u32 { parse_time_to_seconds(value) .and_then(|seconds| u32::try_from(seconds).ok()) .unwrap_or(0) } fn parse_card( &self, card: &ElementRef<'_>, link_sel: &Selector, img_sel: &Selector, ) -> Option { let link = card.select(link_sel).next()?; let href = link.value().attr("href")?; let url = self.normalize_url(href); let id = Self::extract_video_id(&url)?; let title = card .select(&Self::selector("h3").ok()?) .next() .map(|node| Self::normalize_text(&node.text().collect::>().join(" "))) .unwrap_or_default(); if title.is_empty() { return None; } let duration_text = card .select(&Self::selector("span").ok()?) .filter_map(|node| { let value = Self::normalize_text(&node.text().collect::>().join(" ")); if value.contains(':') { Some(value) } else { None } }) .next() .unwrap_or_default(); let thumb = card .select(img_sel) .filter_map(|img| { // Collect the attributes we are looking for img.value() .attr("src") .or_else(|| img.value().attr("data-src")) }) .nth(1) // Skips the first match (index 0) and takes the second match (index 1) .map(|value| self.normalize_url(value)) .unwrap_or_default(); Some(StubVideo { id, title, url, thumb, duration: Self::parse_duration(&duration_text), }) } fn parse_listing_html(&self, html: &str) -> Result> { let doc = Html::parse_document(html); let card_sel = Self::selector("div.video-card")?; let link_sel = Self::selector("a[href*='/video/']")?; let img_sel = Self::selector("img")?; let mut out = Vec::new(); let mut seen = HashSet::new(); for card in doc.select(&card_sel) { if let Some(stub) = self.parse_card(&card, &link_sel, &img_sel) { if seen.insert(stub.id.clone()) { out.push(stub); } } } Ok(out) } fn parse_home_section_html(&self, html: &str, section_id: &str) -> Result> { let doc = Html::parse_document(html); let section_sel = Self::selector(&format!("section#{section_id}"))?; let card_sel = Self::selector("div.video-card")?; let link_sel = Self::selector("a[href*='/video/']")?; let img_sel = Self::selector("img")?; let mut out = Vec::new(); let mut seen = HashSet::new(); if let Some(section) = doc.select(§ion_sel).next() { for card in section.select(&card_sel) { if let Some(stub) = self.parse_card(&card, &link_sel, &img_sel) { if seen.insert(stub.id.clone()) { out.push(stub); } } } } Ok(out) } fn html_headers(&self) -> Vec<(String, String)> { vec![ ("accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8".to_string()), ("accept-language".to_string(), "en-US,en;q=0.8".to_string()), ("user-agent".to_string(), "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36".to_string()), ("referer".to_string(), self.url.clone()), ] } async fn fetch_html(&self, options: &ServerOptions, url: &str) -> Result { let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_html"); requester .get_with_headers(url, self.html_headers(), Some(Version::HTTP_11)) .await .map_err(|error| Error::from(format!("request failed for {url}: {error}"))) } async fn search_videos( &self, options: &ServerOptions, query: &str, page: u16, per_page: usize, ) -> Result> { let mut requester = requester_or_default(options, CHANNEL_ID, "search_videos"); let offset = page.saturating_sub(1) as usize * per_page; let query_encoded = url::form_urlencoded::byte_serialize(query.as_bytes()).collect::(); let search_url = format!("{SEARCH_URL}?q={query_encoded}&limit={per_page}&offset={offset}"); let auth_header = format!("Bearer {SEARCH_KEY}"); let headers = vec![ ("accept".to_string(), "application/json".to_string()), ("authorization".to_string(), auth_header), ( "user-agent".to_string(), "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36".to_string(), ), ]; let text = requester .get_with_headers(&search_url, headers, Some(Version::HTTP_11)) .await .map_err(|error| Error::from(format!("search request failed: {error}")))?; let parsed: SearchResponse = serde_json::from_str(&text)?; Ok(parsed .hits .into_iter() .filter_map(|hit| { let slug = hit.slug.trim(); if hit.id.is_empty() || slug.is_empty() || hit.title.trim().is_empty() { return None; } Some(StubVideo { id: hit.id, title: Self::normalize_text(&hit.title), url: format!("{}/video/{}", self.url, slug), thumb: hit.image, duration: 0, }) }) .collect()) } fn extract_detail_metadata( &self, html: &str, ) -> ( Vec, Option, Option, Option, Option, ) { let uploader_name = Self::regex(r#"]*>[^<]*]*>\s*]*>([^<]+)

"#) .ok() .and_then(|re| re.captures(html)) .and_then(|caps| { let slug = caps.get(1)?.as_str().to_string(); let name = Self::normalize_text(caps.get(2)?.as_str()); if name.is_empty() { return None; } Some((name, slug)) }); let uploader = uploader_name.as_ref().map(|v| v.0.clone()); let uploader_url = uploader_name .as_ref() .map(|v| format!("{}/user/{}", self.url, v.1)); let uploader_id = uploader_name .as_ref() .map(|v| format!("{CHANNEL_ID}:{}", v.1)); let uploaded_at = Self::regex(r#"

([A-Za-z]{3}\s+\d{1,2},\s+\d{4})

"#) .ok() .and_then(|re| re.captures(html)) .and_then(|caps| caps.get(1).map(|m| m.as_str().to_string())) .and_then(|value| NaiveDate::parse_from_str(&value, "%b %e, %Y").ok()) .and_then(|date| date.and_hms_opt(0, 0, 0)) .and_then(|dt| u64::try_from(dt.and_utc().timestamp()).ok()); let tag_re = Self::regex(r#"
]*>([^<]+)"#).ok(); let tags = tag_re .map(|re| { re.captures_iter(html) .filter_map(|caps| caps.get(1).map(|m| Self::normalize_text(m.as_str()))) .filter(|t| !t.is_empty()) .collect::>() }) .unwrap_or_default(); (tags, uploader, uploader_url, uploader_id, uploaded_at) } async fn enrich_video(&self, options: &ServerOptions, stub: StubVideo) -> VideoItem { let proxy_url = build_proxy_url(&options, CHANNEL_ID, &strip_url_scheme(&stub.url)); let mut item = VideoItem::new( stub.id, stub.title, proxy_url, CHANNEL_ID.to_string(), stub.thumb, stub.duration, ); if let Ok(detail_html) = self.fetch_html(options, &stub.url).await { let (tags, uploader, uploader_url, uploader_id, uploaded_at) = self.extract_detail_metadata(&detail_html); if !tags.is_empty() { item.tags = Some(tags); } if let Some(value) = uploader { item = item.uploader(value); } if let Some(value) = uploader_url { item = item.uploader_url(value); } if let Some(value) = uploader_id { item.uploaderId = Some(value); } if let Some(value) = uploaded_at { item.uploadedAt = Some(value); } } item } } #[async_trait] impl Provider for ClapdatProvider { async fn get_videos( &self, _cache: crate::util::cache::VideoCache, _pool: DbPool, sort: String, query: Option, page: String, per_page: String, options: ServerOptions, ) -> Vec { let page_num = page.parse::().unwrap_or(1).max(1); let per_page_num = per_page.parse::().unwrap_or(20).clamp(1, 60); let sort_value = if sort.trim().is_empty() { options.sort.as_deref().unwrap_or("trending").to_string() } else { sort }; let query_value = query.unwrap_or_default(); let target = self.resolve_target(&query_value, &sort_value); let stubs = match &target { Target::Search { query } => match self .search_videos(&options, query, page_num, per_page_num) .await { Ok(items) => items, Err(error) => { report_provider_error(CHANNEL_ID, "search_videos", &error.to_string()).await; vec![] } }, _ => { let Some(url) = self.listing_url(&target, page_num) else { return vec![]; }; match self.fetch_html(&options, &url).await { Ok(html) => { let parsed = match (&target, page_num) { (Target::Trending, 1) => { self.parse_home_section_html(&html, "trending-videos") } (Target::Recent, 1) => { self.parse_home_section_html(&html, "recent-videos") } _ => self.parse_listing_html(&html), }; match parsed { Ok(items) => items, Err(error) => { report_provider_error( CHANNEL_ID, "parse_listing_html", &error.to_string(), ) .await; vec![] } } } Err(error) => { report_provider_error(CHANNEL_ID, "fetch_html", &error.to_string()).await; vec![] } } } }; let mut output = Vec::with_capacity(stubs.len()); for stub in stubs.into_iter().take(per_page_num) { output.push(self.enrich_video(&options, stub).await); } output } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } }