554 lines
18 KiB
Rust
554 lines
18 KiB
Rust
use crate::DbPool;
|
|
use crate::api::ClientVersion;
|
|
use crate::providers::{
|
|
Provider, build_proxy_url, report_provider_error, requester_or_default, strip_url_scheme,
|
|
};
|
|
use crate::status::*;
|
|
use crate::util::time::parse_time_to_seconds;
|
|
use crate::videos::{ServerOptions, VideoItem};
|
|
|
|
use async_trait::async_trait;
|
|
use chrono::NaiveDate;
|
|
use error_chain::error_chain;
|
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
|
use regex::Regex;
|
|
use scraper::{ElementRef, Html, Selector};
|
|
use serde::Deserialize;
|
|
use std::collections::HashSet;
|
|
use wreq::Version;
|
|
|
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
|
crate::providers::ProviderChannelMetadata {
|
|
group_id: "amateur-homemade",
|
|
tags: &["amateur", "homemade", "interracial"],
|
|
};
|
|
|
|
const BASE_URL: &str = "https://www.clapdat.com";
|
|
const SEARCH_URL: &str = "https://search.clapdat.com/indexes/videos/search";
|
|
const SEARCH_KEY: &str = "36ce9a190ca0e797debc3f0a2a311749dbd76262c389531c3a37e9dd74ab9df5";
|
|
const CHANNEL_ID: &str = "clapdat";
|
|
|
|
error_chain! {
|
|
foreign_links {
|
|
Io(std::io::Error);
|
|
HttpRequest(wreq::Error);
|
|
Json(serde_json::Error);
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ClapdatProvider {
|
|
url: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
enum Target {
|
|
Trending,
|
|
Recent,
|
|
Search { query: String },
|
|
Tag { slug: String },
|
|
User { username: String },
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct StubVideo {
|
|
id: String,
|
|
title: String,
|
|
url: String,
|
|
thumb: String,
|
|
duration: u32,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct SearchResponse {
|
|
#[serde(default)]
|
|
hits: Vec<SearchHit>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct SearchHit {
|
|
#[serde(rename = "_id", default)]
|
|
id: String,
|
|
#[serde(default)]
|
|
title: String,
|
|
#[serde(default)]
|
|
slug: String,
|
|
#[serde(default)]
|
|
image: String,
|
|
}
|
|
|
|
impl ClapdatProvider {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
url: BASE_URL.to_string(),
|
|
}
|
|
}
|
|
|
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
|
Channel {
|
|
id: CHANNEL_ID.to_string(),
|
|
name: "ClapDat".to_string(),
|
|
description: "ClapDat trending/recent feeds with tag and uploader shortcuts."
|
|
.to_string(),
|
|
premium: false,
|
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=clapdat.com".to_string(),
|
|
status: "active".to_string(),
|
|
categories: vec![],
|
|
options: vec![ChannelOption {
|
|
id: "sort".to_string(),
|
|
title: "Sort".to_string(),
|
|
description: "Trending or latest ClapDat feed.".to_string(),
|
|
systemImage: "list.number".to_string(),
|
|
colorName: "blue".to_string(),
|
|
options: vec![
|
|
FilterOption {
|
|
id: "trending".to_string(),
|
|
title: "Trending".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "new".to_string(),
|
|
title: "Recent".to_string(),
|
|
},
|
|
],
|
|
multiSelect: false,
|
|
}],
|
|
nsfw: true,
|
|
cacheDuration: Some(1800),
|
|
ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string())
|
|
}
|
|
}
|
|
|
|
fn resolve_target(&self, query: &str, sort: &str) -> Target {
|
|
let q = query.trim();
|
|
if let Some(value) = q.strip_prefix("tag:").or_else(|| q.strip_prefix('#')) {
|
|
let slug = value.trim().to_lowercase().replace(' ', "-");
|
|
if !slug.is_empty() {
|
|
return Target::Tag { slug };
|
|
}
|
|
}
|
|
|
|
if let Some(value) = q
|
|
.strip_prefix("user:")
|
|
.or_else(|| q.strip_prefix("uploader:"))
|
|
{
|
|
let username = value.trim().to_lowercase().replace(' ', "-");
|
|
if !username.is_empty() {
|
|
return Target::User { username };
|
|
}
|
|
}
|
|
|
|
if !q.is_empty() {
|
|
return Target::Search {
|
|
query: q.to_string(),
|
|
};
|
|
}
|
|
|
|
match sort {
|
|
"recent" | "new" | "latest" => Target::Recent,
|
|
_ => Target::Trending,
|
|
}
|
|
}
|
|
|
|
fn listing_url(&self, target: &Target, page: u16) -> Option<String> {
|
|
let page = page.max(1);
|
|
match target {
|
|
Target::Trending => Some(if page == 1 {
|
|
self.url.clone()
|
|
} else {
|
|
format!("{}/trending/{page}", self.url)
|
|
}),
|
|
Target::Recent => Some(if page == 1 {
|
|
self.url.clone()
|
|
} else {
|
|
format!("{}/recent/{page}", self.url)
|
|
}),
|
|
Target::Tag { slug } => Some(if page == 1 {
|
|
format!("{}/tag/{slug}", self.url)
|
|
} else {
|
|
format!("{}/tag/{slug}/{page}", self.url)
|
|
}),
|
|
Target::User { username } => Some(if page == 1 {
|
|
format!("{}/user/{username}", self.url)
|
|
} else {
|
|
format!("{}/user/{username}/{page}", self.url)
|
|
}),
|
|
Target::Search { .. } => None,
|
|
}
|
|
}
|
|
|
|
fn selector(value: &str) -> Result<Selector> {
|
|
Selector::parse(value).map_err(|error| Error::from(format!("selector `{value}`: {error}")))
|
|
}
|
|
|
|
fn regex(value: &str) -> Result<Regex> {
|
|
Regex::new(value).map_err(|error| Error::from(format!("regex `{value}`: {error}")))
|
|
}
|
|
|
|
fn normalize_text(value: &str) -> String {
|
|
decode(value.as_bytes())
|
|
.to_string()
|
|
.unwrap_or_else(|_| value.to_string())
|
|
.replace('\u{a0}', " ")
|
|
.split_whitespace()
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
.trim()
|
|
.to_string()
|
|
}
|
|
|
|
fn normalize_url(&self, value: &str) -> String {
|
|
let trimmed = value.trim();
|
|
if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
|
|
return trimmed.to_string();
|
|
}
|
|
if trimmed.starts_with("//") {
|
|
return format!("https:{trimmed}");
|
|
}
|
|
format!(
|
|
"{}/{}",
|
|
self.url.trim_end_matches('/'),
|
|
trimmed.trim_start_matches('/')
|
|
)
|
|
}
|
|
|
|
fn extract_video_id(url: &str) -> Option<String> {
|
|
let re = Regex::new(r"-([a-z0-9]+)(?:/|$)").ok()?;
|
|
re.captures(url)
|
|
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()))
|
|
}
|
|
|
|
fn parse_duration(value: &str) -> u32 {
|
|
parse_time_to_seconds(value)
|
|
.and_then(|seconds| u32::try_from(seconds).ok())
|
|
.unwrap_or(0)
|
|
}
|
|
|
|
fn parse_card(
|
|
&self,
|
|
card: &ElementRef<'_>,
|
|
link_sel: &Selector,
|
|
img_sel: &Selector,
|
|
) -> Option<StubVideo> {
|
|
let link = card.select(link_sel).next()?;
|
|
let href = link.value().attr("href")?;
|
|
let url = self.normalize_url(href);
|
|
let id = Self::extract_video_id(&url)?;
|
|
|
|
let title = card
|
|
.select(&Self::selector("h3").ok()?)
|
|
.next()
|
|
.map(|node| Self::normalize_text(&node.text().collect::<Vec<_>>().join(" ")))
|
|
.unwrap_or_default();
|
|
if title.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let duration_text = card
|
|
.select(&Self::selector("span").ok()?)
|
|
.filter_map(|node| {
|
|
let value = Self::normalize_text(&node.text().collect::<Vec<_>>().join(" "));
|
|
if value.contains(':') {
|
|
Some(value)
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
.next()
|
|
.unwrap_or_default();
|
|
|
|
let thumb = card
|
|
.select(img_sel)
|
|
.filter_map(|img| {
|
|
// Collect the attributes we are looking for
|
|
img.value()
|
|
.attr("src")
|
|
.or_else(|| img.value().attr("data-src"))
|
|
})
|
|
.nth(1) // Skips the first match (index 0) and takes the second match (index 1)
|
|
.map(|value| self.normalize_url(value))
|
|
.unwrap_or_default();
|
|
|
|
Some(StubVideo {
|
|
id,
|
|
title,
|
|
url,
|
|
thumb,
|
|
duration: Self::parse_duration(&duration_text),
|
|
})
|
|
}
|
|
|
|
fn parse_listing_html(&self, html: &str) -> Result<Vec<StubVideo>> {
|
|
let doc = Html::parse_document(html);
|
|
let card_sel = Self::selector("div.video-card")?;
|
|
let link_sel = Self::selector("a[href*='/video/']")?;
|
|
let img_sel = Self::selector("img")?;
|
|
let mut out = Vec::new();
|
|
let mut seen = HashSet::new();
|
|
|
|
for card in doc.select(&card_sel) {
|
|
if let Some(stub) = self.parse_card(&card, &link_sel, &img_sel) {
|
|
if seen.insert(stub.id.clone()) {
|
|
out.push(stub);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(out)
|
|
}
|
|
|
|
fn parse_home_section_html(&self, html: &str, section_id: &str) -> Result<Vec<StubVideo>> {
|
|
let doc = Html::parse_document(html);
|
|
let section_sel = Self::selector(&format!("section#{section_id}"))?;
|
|
let card_sel = Self::selector("div.video-card")?;
|
|
let link_sel = Self::selector("a[href*='/video/']")?;
|
|
let img_sel = Self::selector("img")?;
|
|
let mut out = Vec::new();
|
|
let mut seen = HashSet::new();
|
|
|
|
if let Some(section) = doc.select(§ion_sel).next() {
|
|
for card in section.select(&card_sel) {
|
|
if let Some(stub) = self.parse_card(&card, &link_sel, &img_sel) {
|
|
if seen.insert(stub.id.clone()) {
|
|
out.push(stub);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(out)
|
|
}
|
|
|
|
fn html_headers(&self) -> Vec<(String, String)> {
|
|
vec![
|
|
("accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8".to_string()),
|
|
("accept-language".to_string(), "en-US,en;q=0.8".to_string()),
|
|
("user-agent".to_string(), "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36".to_string()),
|
|
("referer".to_string(), self.url.clone()),
|
|
]
|
|
}
|
|
|
|
async fn fetch_html(&self, options: &ServerOptions, url: &str) -> Result<String> {
|
|
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_html");
|
|
requester
|
|
.get_with_headers(url, self.html_headers(), Some(Version::HTTP_11))
|
|
.await
|
|
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))
|
|
}
|
|
|
|
async fn search_videos(
|
|
&self,
|
|
options: &ServerOptions,
|
|
query: &str,
|
|
page: u16,
|
|
per_page: usize,
|
|
) -> Result<Vec<StubVideo>> {
|
|
let mut requester = requester_or_default(options, CHANNEL_ID, "search_videos");
|
|
let offset = page.saturating_sub(1) as usize * per_page;
|
|
let query_encoded =
|
|
url::form_urlencoded::byte_serialize(query.as_bytes()).collect::<String>();
|
|
let search_url = format!("{SEARCH_URL}?q={query_encoded}&limit={per_page}&offset={offset}");
|
|
let auth_header = format!("Bearer {SEARCH_KEY}");
|
|
let headers = vec![
|
|
("accept".to_string(), "application/json".to_string()),
|
|
("authorization".to_string(), auth_header),
|
|
(
|
|
"user-agent".to_string(),
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36".to_string(),
|
|
),
|
|
];
|
|
|
|
let text = requester
|
|
.get_with_headers(&search_url, headers, Some(Version::HTTP_11))
|
|
.await
|
|
.map_err(|error| Error::from(format!("search request failed: {error}")))?;
|
|
|
|
let parsed: SearchResponse = serde_json::from_str(&text)?;
|
|
Ok(parsed
|
|
.hits
|
|
.into_iter()
|
|
.filter_map(|hit| {
|
|
let slug = hit.slug.trim();
|
|
if hit.id.is_empty() || slug.is_empty() || hit.title.trim().is_empty() {
|
|
return None;
|
|
}
|
|
Some(StubVideo {
|
|
id: hit.id,
|
|
title: Self::normalize_text(&hit.title),
|
|
url: format!("{}/video/{}", self.url, slug),
|
|
thumb: hit.image,
|
|
duration: 0,
|
|
})
|
|
})
|
|
.collect())
|
|
}
|
|
|
|
fn extract_detail_metadata(
|
|
&self,
|
|
html: &str,
|
|
) -> (
|
|
Vec<String>,
|
|
Option<String>,
|
|
Option<String>,
|
|
Option<String>,
|
|
Option<u64>,
|
|
) {
|
|
let uploader_name =
|
|
Self::regex(r#"<a href="/user/([^"]+)"[^>]*>[^<]*<img[^>]*>\s*<p[^>]*>([^<]+)</p>"#)
|
|
.ok()
|
|
.and_then(|re| re.captures(html))
|
|
.and_then(|caps| {
|
|
let slug = caps.get(1)?.as_str().to_string();
|
|
let name = Self::normalize_text(caps.get(2)?.as_str());
|
|
if name.is_empty() {
|
|
return None;
|
|
}
|
|
Some((name, slug))
|
|
});
|
|
|
|
let uploader = uploader_name.as_ref().map(|v| v.0.clone());
|
|
let uploader_url = uploader_name
|
|
.as_ref()
|
|
.map(|v| format!("{}/user/{}", self.url, v.1));
|
|
let uploader_id = uploader_name
|
|
.as_ref()
|
|
.map(|v| format!("{CHANNEL_ID}:{}", v.1));
|
|
|
|
let uploaded_at =
|
|
Self::regex(r#"<p class="s-1he9h8j">([A-Za-z]{3}\s+\d{1,2},\s+\d{4})</p>"#)
|
|
.ok()
|
|
.and_then(|re| re.captures(html))
|
|
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()))
|
|
.and_then(|value| NaiveDate::parse_from_str(&value, "%b %e, %Y").ok())
|
|
.and_then(|date| date.and_hms_opt(0, 0, 0))
|
|
.and_then(|dt| u64::try_from(dt.and_utc().timestamp()).ok());
|
|
|
|
let tag_re = Self::regex(r#"<a href="/tag/[^"]+"[^>]*>([^<]+)</a>"#).ok();
|
|
let tags = tag_re
|
|
.map(|re| {
|
|
re.captures_iter(html)
|
|
.filter_map(|caps| caps.get(1).map(|m| Self::normalize_text(m.as_str())))
|
|
.filter(|t| !t.is_empty())
|
|
.collect::<Vec<_>>()
|
|
})
|
|
.unwrap_or_default();
|
|
|
|
(tags, uploader, uploader_url, uploader_id, uploaded_at)
|
|
}
|
|
|
|
async fn enrich_video(&self, options: &ServerOptions, stub: StubVideo) -> VideoItem {
|
|
let proxy_url = build_proxy_url(&options, CHANNEL_ID, &strip_url_scheme(&stub.url));
|
|
let mut item = VideoItem::new(
|
|
stub.id,
|
|
stub.title,
|
|
proxy_url,
|
|
CHANNEL_ID.to_string(),
|
|
stub.thumb,
|
|
stub.duration,
|
|
);
|
|
|
|
if let Ok(detail_html) = self.fetch_html(options, &stub.url).await {
|
|
let (tags, uploader, uploader_url, uploader_id, uploaded_at) =
|
|
self.extract_detail_metadata(&detail_html);
|
|
|
|
if !tags.is_empty() {
|
|
item.tags = Some(tags);
|
|
}
|
|
if let Some(value) = uploader {
|
|
item = item.uploader(value);
|
|
}
|
|
if let Some(value) = uploader_url {
|
|
item = item.uploader_url(value);
|
|
}
|
|
if let Some(value) = uploader_id {
|
|
item.uploaderId = Some(value);
|
|
}
|
|
if let Some(value) = uploaded_at {
|
|
item.uploadedAt = Some(value);
|
|
}
|
|
}
|
|
|
|
item
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl Provider for ClapdatProvider {
|
|
async fn get_videos(
|
|
&self,
|
|
_cache: crate::util::cache::VideoCache,
|
|
_pool: DbPool,
|
|
sort: String,
|
|
query: Option<String>,
|
|
page: String,
|
|
per_page: String,
|
|
options: ServerOptions,
|
|
) -> Vec<VideoItem> {
|
|
let page_num = page.parse::<u16>().unwrap_or(1).max(1);
|
|
let per_page_num = per_page.parse::<usize>().unwrap_or(20).clamp(1, 60);
|
|
let sort_value = if sort.trim().is_empty() {
|
|
options.sort.as_deref().unwrap_or("trending").to_string()
|
|
} else {
|
|
sort
|
|
};
|
|
let query_value = query.unwrap_or_default();
|
|
let target = self.resolve_target(&query_value, &sort_value);
|
|
|
|
let stubs = match &target {
|
|
Target::Search { query } => match self
|
|
.search_videos(&options, query, page_num, per_page_num)
|
|
.await
|
|
{
|
|
Ok(items) => items,
|
|
Err(error) => {
|
|
report_provider_error(CHANNEL_ID, "search_videos", &error.to_string()).await;
|
|
vec![]
|
|
}
|
|
},
|
|
_ => {
|
|
let Some(url) = self.listing_url(&target, page_num) else {
|
|
return vec![];
|
|
};
|
|
match self.fetch_html(&options, &url).await {
|
|
Ok(html) => {
|
|
let parsed = match (&target, page_num) {
|
|
(Target::Trending, 1) => {
|
|
self.parse_home_section_html(&html, "trending-videos")
|
|
}
|
|
(Target::Recent, 1) => {
|
|
self.parse_home_section_html(&html, "recent-videos")
|
|
}
|
|
_ => self.parse_listing_html(&html),
|
|
};
|
|
match parsed {
|
|
Ok(items) => items,
|
|
Err(error) => {
|
|
report_provider_error(
|
|
CHANNEL_ID,
|
|
"parse_listing_html",
|
|
&error.to_string(),
|
|
)
|
|
.await;
|
|
vec![]
|
|
}
|
|
}
|
|
}
|
|
Err(error) => {
|
|
report_provider_error(CHANNEL_ID, "fetch_html", &error.to_string()).await;
|
|
vec![]
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
let mut output = Vec::with_capacity(stubs.len());
|
|
for stub in stubs.into_iter().take(per_page_num) {
|
|
output.push(self.enrich_video(&options, stub).await);
|
|
}
|
|
output
|
|
}
|
|
|
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
|
Some(self.build_channel(clientversion))
|
|
}
|
|
}
|