616 lines
20 KiB
Rust
616 lines
20 KiB
Rust
use crate::DbPool;
|
|
use crate::api::ClientVersion;
|
|
use crate::providers::{Provider, report_provider_error, requester_or_default};
|
|
use crate::status::*;
|
|
use crate::util::cache::VideoCache;
|
|
use crate::util::parse_abbreviated_number;
|
|
use crate::videos::{ServerOptions, VideoItem};
|
|
use async_trait::async_trait;
|
|
use chrono::{Datelike, Local, Months};
|
|
use error_chain::error_chain;
|
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
|
use scraper::{ElementRef, Html, Selector};
|
|
use wreq::Version;
|
|
|
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
|
crate::providers::ProviderChannelMetadata {
|
|
group_id: "mainstream-tube",
|
|
tags: &["tube", "hd", "mixed", "search"],
|
|
};
|
|
|
|
const BASE_URL: &str = "https://www.xvideos.com";
|
|
const CHANNEL_ID: &str = "xvideos";
|
|
const FIREFOX_UA: &str =
|
|
"Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0";
|
|
const HTML_ACCEPT: &str =
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
|
|
|
|
error_chain! {
|
|
foreign_links {
|
|
Io(std::io::Error);
|
|
}
|
|
errors {
|
|
Parse(msg: String) {
|
|
description("parse error")
|
|
display("parse error: {}", msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Slug format is `{SiteName}-{ID}` as returned by /c listing.
|
|
const CATEGORIES: &[(&str, &str)] = &[
|
|
("AI-239", "AI"),
|
|
("Amateur-65", "Amateur"),
|
|
("Anal-12", "Anal"),
|
|
("Arab-159", "Arab"),
|
|
("Asian_Woman-32", "Asian"),
|
|
("ASMR-229", "ASMR"),
|
|
("Ass-14", "Ass"),
|
|
("bbw-51", "BBW"),
|
|
("Bi_Sexual-62", "Bi"),
|
|
("Big_Ass-24", "Big Ass"),
|
|
("Big_Cock-34", "Big Cock"),
|
|
("Big_Tits-23", "Big Tits"),
|
|
("Black_Woman-30", "Black"),
|
|
("Blonde-20", "Blonde"),
|
|
("Blowjob-15", "Blowjob"),
|
|
("Brunette-25", "Brunette"),
|
|
("Cam_Porn-58", "Cam Porn"),
|
|
("Creampie-40", "Creampie"),
|
|
("Cuckold-237", "Cuckold"),
|
|
("Cumshot-18", "Cumshot"),
|
|
("Femdom-235", "Femdom"),
|
|
("Fisting-165", "Fisting"),
|
|
("Fucked_Up_Family-81", "Step Family"),
|
|
("Gangbang-69", "Gangbang"),
|
|
("Gapes-167", "Gapes"),
|
|
("Indian-89", "Indian"),
|
|
("Interracial-27", "Interracial"),
|
|
("Latina-16", "Latina"),
|
|
("Lesbian-26", "Lesbian"),
|
|
("Lingerie-83", "Lingerie"),
|
|
("Mature-38", "Mature"),
|
|
("Milf-19", "MILF"),
|
|
("Oiled-22", "Oiled"),
|
|
("Redhead-31", "Redhead"),
|
|
("Solo_and_Masturbation-33", "Solo"),
|
|
("Squirting-56", "Squirting"),
|
|
("Stockings-28", "Stockings"),
|
|
("Teen-13", "Teen"),
|
|
];
|
|
|
|
#[derive(Debug, Clone)]
|
|
enum Target {
|
|
Latest,
|
|
Best,
|
|
Search(String),
|
|
Archive(String),
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct XvideosProvider;
|
|
|
|
impl XvideosProvider {
|
|
pub fn new() -> Self {
|
|
Self
|
|
}
|
|
|
|
fn build_channel(&self, _cv: ClientVersion) -> Channel {
|
|
let mut cat_options: Vec<FilterOption> = vec![FilterOption {
|
|
id: "all".to_string(),
|
|
title: "All".to_string(),
|
|
}];
|
|
for (slug, label) in CATEGORIES {
|
|
cat_options.push(FilterOption {
|
|
id: slug.to_string(),
|
|
title: label.to_string(),
|
|
});
|
|
}
|
|
|
|
Channel {
|
|
id: CHANNEL_ID.to_string(),
|
|
name: "XVideos".to_string(),
|
|
description:
|
|
"XVideos — one of the world's largest free porn sites with latest, best-of-month, category, tag, and keyword search."
|
|
.to_string(),
|
|
premium: false,
|
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=xvideos.com".to_string(),
|
|
status: "active".to_string(),
|
|
categories: CATEGORIES.iter().map(|(_, label)| label.to_string()).collect(),
|
|
options: vec![
|
|
ChannelOption {
|
|
id: "sort".to_string(),
|
|
title: "Sort".to_string(),
|
|
description: "Browse XVideos ranking feeds.".to_string(),
|
|
systemImage: "list.number".to_string(),
|
|
colorName: "blue".to_string(),
|
|
options: vec![
|
|
FilterOption {
|
|
id: "new".to_string(),
|
|
title: "Latest".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "best".to_string(),
|
|
title: "Best of Month".to_string(),
|
|
},
|
|
],
|
|
multiSelect: false,
|
|
},
|
|
ChannelOption {
|
|
id: "categories".to_string(),
|
|
title: "Categories".to_string(),
|
|
description: "Browse an XVideos category archive.".to_string(),
|
|
systemImage: "square.grid.2x2".to_string(),
|
|
colorName: "orange".to_string(),
|
|
options: cat_options,
|
|
multiSelect: false,
|
|
},
|
|
],
|
|
nsfw: true,
|
|
cacheDuration: Some(1800),
|
|
}
|
|
}
|
|
|
|
fn selector(value: &str) -> Result<Selector> {
|
|
Selector::parse(value)
|
|
.map_err(|e| Error::from(format!("selector `{value}` parse failed: {e}")))
|
|
}
|
|
|
|
fn decode_html(text: &str) -> String {
|
|
decode(text.as_bytes())
|
|
.to_string()
|
|
.unwrap_or_else(|_| text.to_string())
|
|
}
|
|
|
|
fn text_of(el: &ElementRef<'_>) -> String {
|
|
let raw: String = el.text().collect::<Vec<_>>().join(" ");
|
|
Self::decode_html(&raw.split_whitespace().collect::<Vec<_>>().join(" "))
|
|
}
|
|
|
|
fn normalize_key(s: &str) -> String {
|
|
s.trim()
|
|
.trim_start_matches('#')
|
|
.replace(['_', '-'], " ")
|
|
.split_whitespace()
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
.to_ascii_lowercase()
|
|
}
|
|
|
|
fn normalize_url(path: &str) -> String {
|
|
let path = path.trim();
|
|
if path.starts_with("http://") || path.starts_with("https://") {
|
|
return path.to_string();
|
|
}
|
|
if path.starts_with("//") {
|
|
return format!("https:{path}");
|
|
}
|
|
if path.starts_with('/') {
|
|
return format!("{BASE_URL}{path}");
|
|
}
|
|
format!("{BASE_URL}/{path}")
|
|
}
|
|
|
|
fn html_headers(referer: &str) -> Vec<(String, String)> {
|
|
vec![
|
|
("User-Agent".to_string(), FIREFOX_UA.to_string()),
|
|
("Accept".to_string(), HTML_ACCEPT.to_string()),
|
|
("Accept-Language".to_string(), "en-US,en;q=0.5".to_string()),
|
|
("Referer".to_string(), referer.to_string()),
|
|
]
|
|
}
|
|
|
|
// /best always redirects to the previous calendar month's archive.
|
|
fn best_base_url() -> String {
|
|
let now = Local::now();
|
|
let prev = now
|
|
.checked_sub_months(Months::new(1))
|
|
.unwrap_or(now);
|
|
format!("{BASE_URL}/best/{}-{:02}", prev.year(), prev.month())
|
|
}
|
|
|
|
fn target_url(target: &Target, page: u16) -> String {
|
|
match target {
|
|
Target::Latest => {
|
|
if page <= 1 {
|
|
format!("{BASE_URL}/")
|
|
} else {
|
|
// page 2 = /new/1, page 3 = /new/2, ...
|
|
format!("{BASE_URL}/new/{}", page - 1)
|
|
}
|
|
}
|
|
Target::Best => {
|
|
let base = Self::best_base_url();
|
|
if page <= 1 {
|
|
base
|
|
} else {
|
|
format!("{base}/{}", page - 1)
|
|
}
|
|
}
|
|
Target::Search(q) => {
|
|
let encoded: String =
|
|
url::form_urlencoded::byte_serialize(q.trim().as_bytes()).collect();
|
|
if page <= 1 {
|
|
format!("{BASE_URL}/?k={encoded}")
|
|
} else {
|
|
format!("{BASE_URL}/?k={encoded}&p={}", page - 1)
|
|
}
|
|
}
|
|
Target::Archive(base_url) => {
|
|
let base = base_url.trim_end_matches('/');
|
|
if page <= 1 {
|
|
base.to_string()
|
|
} else {
|
|
format!("{base}/{}", page - 1)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn fetch_html(
|
|
requester: &mut crate::util::requester::Requester,
|
|
url: &str,
|
|
) -> Result<String> {
|
|
requester
|
|
.get_with_headers(url, Self::html_headers(url), Some(Version::HTTP_11))
|
|
.await
|
|
.map_err(|e| Error::from(format!("request failed for {url}: {e}")))
|
|
}
|
|
|
|
// Parses "21 min", "1h20min", "2h", "45sec", "MM:SS", "HH:MM:SS"
|
|
fn parse_duration(text: &str) -> u32 {
|
|
let text = text.trim();
|
|
|
|
// Colon-separated formats MM:SS and HH:MM:SS
|
|
let parts: Vec<&str> = text.split(':').collect();
|
|
if parts.len() == 2 {
|
|
let m: u32 = parts[0].trim().parse().unwrap_or(0);
|
|
let s: u32 = parts[1].trim().parse().unwrap_or(0);
|
|
return m * 60 + s;
|
|
}
|
|
if parts.len() == 3 {
|
|
let h: u32 = parts[0].trim().parse().unwrap_or(0);
|
|
let m: u32 = parts[1].trim().parse().unwrap_or(0);
|
|
let s: u32 = parts[2].trim().parse().unwrap_or(0);
|
|
return h * 3600 + m * 60 + s;
|
|
}
|
|
|
|
// Word-based: "1h20min", "30 min", "45sec", etc.
|
|
let low = text.to_ascii_lowercase();
|
|
let h: u32 = low
|
|
.find('h')
|
|
.and_then(|i| low[..i].trim().parse().ok())
|
|
.unwrap_or(0);
|
|
let m: u32 = low.find("min").and_then(|i| {
|
|
let start = low[..i]
|
|
.rfind(|c: char| !c.is_ascii_digit())
|
|
.map(|j| j + 1)
|
|
.unwrap_or(0);
|
|
low[start..i].trim().parse().ok()
|
|
}).unwrap_or(0);
|
|
let s: u32 = low.find("sec").and_then(|i| {
|
|
let start = low[..i]
|
|
.rfind(|c: char| !c.is_ascii_digit())
|
|
.map(|j| j + 1)
|
|
.unwrap_or(0);
|
|
low[start..i].trim().parse().ok()
|
|
}).unwrap_or(0);
|
|
|
|
h * 3600 + m * 60 + s
|
|
}
|
|
|
|
fn parse_views(text: &str) -> Option<u32> {
|
|
// "877.3k Views", "1.2M Views" — strip suffix then parse
|
|
let cleaned = text
|
|
.replace("Views", "")
|
|
.replace("views", "")
|
|
.replace("View", "")
|
|
.replace(',', "");
|
|
parse_abbreviated_number(cleaned.trim())
|
|
}
|
|
|
|
fn parse_listing(html: &str, limit: usize) -> Result<Vec<VideoItem>> {
|
|
let document = Html::parse_document(html);
|
|
|
|
let card_sel = Self::selector("div.thumb-block")?;
|
|
let img_sel = Self::selector("img[data-src]")?;
|
|
let link_sel = Self::selector("a[href]")?;
|
|
let title_sel = Self::selector("p.title a[title], a.title[title]")?;
|
|
let uploader_name_sel = Self::selector("p.metadata a span.name")?;
|
|
let uploader_link_sel = Self::selector("p.metadata a[href]")?;
|
|
let dur_sel = Self::selector(".thumb-under span.duration")?;
|
|
let metadata_sel = Self::selector("p.metadata")?;
|
|
|
|
let mut items = Vec::new();
|
|
|
|
'card: for card in document.select(&card_sel) {
|
|
// Find the anchor whose href contains /video.
|
|
let video_link = card
|
|
.select(&link_sel)
|
|
.find(|el| {
|
|
el.value()
|
|
.attr("href")
|
|
.map(|h| h.contains("/video."))
|
|
.unwrap_or(false)
|
|
});
|
|
let Some(video_link) = video_link else {
|
|
continue;
|
|
};
|
|
|
|
let href = video_link.value().attr("href").unwrap_or_default();
|
|
let page_url = Self::normalize_url(href);
|
|
if page_url.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
// eid: path segment starting with "video." e.g. "video.ohedfck8b21"
|
|
let eid = href
|
|
.split('/')
|
|
.find(|s| s.starts_with("video."))
|
|
.and_then(|s| s.strip_prefix("video."))
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
if eid.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
// Numeric id from data-id attribute; fall back to eid
|
|
let video_id = card
|
|
.value()
|
|
.attr("data-id")
|
|
.filter(|s| !s.is_empty())
|
|
.unwrap_or(&eid)
|
|
.to_string();
|
|
|
|
// Thumbnail (lazy-loaded, stored in data-src)
|
|
let thumb = card
|
|
.select(&img_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("data-src"))
|
|
.map(str::to_string)
|
|
.unwrap_or_default();
|
|
if thumb.is_empty() {
|
|
continue 'card;
|
|
}
|
|
|
|
// Preview video clip (data-pvv on the same img element)
|
|
let preview = card
|
|
.select(&img_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("data-pvv"))
|
|
.map(str::to_string)
|
|
.filter(|s| !s.is_empty());
|
|
|
|
// Title from the title attribute on the link inside p.title
|
|
let title = card
|
|
.select(&title_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("title").map(Self::decode_html))
|
|
.filter(|t| !t.trim().is_empty());
|
|
let Some(title) = title else {
|
|
continue;
|
|
};
|
|
|
|
// Duration from span.duration inside .thumb-under
|
|
let duration = card
|
|
.select(&dur_sel)
|
|
.next()
|
|
.map(|el| Self::parse_duration(&Self::text_of(&el)))
|
|
.unwrap_or(0);
|
|
|
|
// Uploader name and URL
|
|
let uploader_name = card
|
|
.select(&uploader_name_sel)
|
|
.next()
|
|
.map(|el| Self::text_of(&el))
|
|
.filter(|s| !s.is_empty());
|
|
let uploader_url = card
|
|
.select(&uploader_link_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("href").map(Self::normalize_url))
|
|
.filter(|u| !u.is_empty());
|
|
|
|
// Views: scan p.metadata text for "NNN Views"
|
|
let views = card.select(&metadata_sel).next().and_then(|meta| {
|
|
let text = Self::text_of(&meta);
|
|
let low = text.to_ascii_lowercase();
|
|
low.find("views").and_then(|idx| {
|
|
// grab the token immediately before "views"
|
|
text[..idx]
|
|
.split_whitespace()
|
|
.last()
|
|
.and_then(|w| Self::parse_views(w))
|
|
})
|
|
});
|
|
|
|
let mut item = VideoItem::new(
|
|
video_id,
|
|
title.trim().to_string(),
|
|
page_url,
|
|
CHANNEL_ID.to_string(),
|
|
thumb,
|
|
duration,
|
|
);
|
|
if let Some(v) = views {
|
|
item.views = Some(v);
|
|
}
|
|
if let Some(p) = preview {
|
|
item.preview = Some(p);
|
|
}
|
|
if let Some(name) = uploader_name {
|
|
item.uploader = Some(name);
|
|
}
|
|
if let Some(url) = uploader_url {
|
|
let uploader_id = url
|
|
.trim_end_matches('/')
|
|
.rsplit('/')
|
|
.next()
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
if !uploader_id.is_empty() {
|
|
item.uploaderId = Some(format!("{CHANNEL_ID}:{uploader_id}"));
|
|
}
|
|
item.uploaderUrl = Some(url);
|
|
}
|
|
|
|
items.push(item);
|
|
if items.len() >= limit {
|
|
break;
|
|
}
|
|
}
|
|
|
|
Ok(items)
|
|
}
|
|
|
|
fn lookup_category(query: &str) -> Option<String> {
|
|
let normalized = Self::normalize_key(query);
|
|
for (slug, label) in CATEGORIES {
|
|
if Self::normalize_key(label) == normalized || Self::normalize_key(slug) == normalized {
|
|
return Some(format!("{BASE_URL}/c/{slug}"));
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn resolve_query_target(query: &str) -> Target {
|
|
let trimmed = query.trim().trim_start_matches('@');
|
|
|
|
if let Some((kind, value)) = trimmed.split_once(':') {
|
|
let value = value.trim();
|
|
if !value.is_empty() {
|
|
match kind.trim().to_ascii_lowercase().as_str() {
|
|
"tag" => {
|
|
let slug = value.replace(' ', "-").to_ascii_lowercase();
|
|
return Target::Archive(format!("{BASE_URL}/tags/{slug}"));
|
|
}
|
|
"cat" | "category" => {
|
|
if let Some(url) = Self::lookup_category(value) {
|
|
return Target::Archive(url);
|
|
}
|
|
let slug = value.replace(' ', "_");
|
|
return Target::Archive(format!("{BASE_URL}/c/{slug}"));
|
|
}
|
|
"uploader" | "channel" | "profile" => {
|
|
let slug = value.replace(' ', "_").to_ascii_lowercase();
|
|
return Target::Archive(format!("{BASE_URL}/{slug}"));
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Category name lookup
|
|
if let Some(url) = Self::lookup_category(trimmed) {
|
|
return Target::Archive(url);
|
|
}
|
|
|
|
Target::Search(trimmed.to_string())
|
|
}
|
|
|
|
fn resolve_option_target(options: &ServerOptions, sort: &str) -> Target {
|
|
if let Some(cat) = options.categories.as_deref() {
|
|
if cat != "all" && !cat.is_empty() {
|
|
return Target::Archive(format!("{BASE_URL}/c/{cat}"));
|
|
}
|
|
}
|
|
match sort.trim().to_ascii_lowercase().as_str() {
|
|
"best" | "top" => Target::Best,
|
|
_ => Target::Latest,
|
|
}
|
|
}
|
|
|
|
async fn fetch_target(
|
|
&self,
|
|
cache: VideoCache,
|
|
target: Target,
|
|
page: u16,
|
|
per_page: usize,
|
|
options: ServerOptions,
|
|
) -> Result<Vec<VideoItem>> {
|
|
let url = Self::target_url(&target, page);
|
|
let cache_key = format!("{url}#per={per_page}");
|
|
|
|
if let Some((ts, cached)) = cache.get(&cache_key) {
|
|
if ts.elapsed().unwrap_or_default().as_secs() < 300 {
|
|
return Ok(cached.clone());
|
|
}
|
|
}
|
|
|
|
let mut requester = requester_or_default(
|
|
&options,
|
|
CHANNEL_ID,
|
|
"xvideos.fetch_target.missing_requester",
|
|
);
|
|
let html = match Self::fetch_html(&mut requester, &url).await {
|
|
Ok(v) => v,
|
|
Err(e) => {
|
|
report_provider_error(
|
|
CHANNEL_ID,
|
|
"fetch_target.request",
|
|
&format!("url={url}; error={e}"),
|
|
)
|
|
.await;
|
|
return Ok(vec![]);
|
|
}
|
|
};
|
|
|
|
if html.trim().is_empty() {
|
|
report_provider_error(
|
|
CHANNEL_ID,
|
|
"fetch_target.empty",
|
|
&format!("url={url}"),
|
|
)
|
|
.await;
|
|
return Ok(vec![]);
|
|
}
|
|
|
|
let items = Self::parse_listing(&html, per_page)?;
|
|
if !items.is_empty() {
|
|
cache.insert(cache_key, items.clone());
|
|
}
|
|
Ok(items)
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl Provider for XvideosProvider {
|
|
async fn get_videos(
|
|
&self,
|
|
cache: VideoCache,
|
|
_pool: DbPool,
|
|
sort: String,
|
|
query: Option<String>,
|
|
page: String,
|
|
per_page: String,
|
|
options: ServerOptions,
|
|
) -> Vec<VideoItem> {
|
|
let page = page.parse::<u16>().unwrap_or(1).max(1);
|
|
let per_page = per_page.parse::<usize>().unwrap_or(10).clamp(1, 60);
|
|
|
|
let target = match query {
|
|
Some(q) if !q.trim().is_empty() => Self::resolve_query_target(q.trim()),
|
|
_ => Self::resolve_option_target(&options, &sort),
|
|
};
|
|
|
|
match self
|
|
.fetch_target(cache, target, page, per_page, options)
|
|
.await
|
|
{
|
|
Ok(items) => items,
|
|
Err(e) => {
|
|
report_provider_error(
|
|
CHANNEL_ID,
|
|
"get_videos",
|
|
&format!("sort={sort}; page={page}; error={e}"),
|
|
)
|
|
.await;
|
|
vec![]
|
|
}
|
|
}
|
|
}
|
|
|
|
fn get_channel(&self, cv: ClientVersion) -> Option<Channel> {
|
|
Some(self.build_channel(cv))
|
|
}
|
|
}
|