ytdlp command and xgroovy

This commit is contained in:
Simon
2026-07-06 16:07:54 +00:00
parent 55d687c361
commit d1a05de50e
83 changed files with 703 additions and 0 deletions

615
src/providers/xgroovy.rs Normal file
View File

@@ -0,0 +1,615 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error, requester_or_default};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use scraper::{ElementRef, Html, Selector};
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
crate::providers::ProviderChannelMetadata {
group_id: "mainstream-tube",
tags: &["mainstream", "tube", "hd", "general"],
};
const BASE_URL: &str = "https://xgroovy.com";
const CHANNEL_ID: &str = "xgroovy";
const FIREFOX_UA: &str =
"Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0";
const HTML_ACCEPT: &str =
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
error_chain! {
foreign_links {
Io(std::io::Error);
}
errors {
Parse(msg: String) {
description("parse error")
display("parse error: {}", msg)
}
}
}
// Static category list scraped from https://xgroovy.com/tags/ (73 categories)
const CATEGORIES: &[(&str, &str)] = &[
("ai", "AI"),
("amateur", "Amateur"),
("anal", "Anal"),
("arab", "Arab"),
("asian", "Asian"),
("bbc", "BBC"),
("bbw", "BBW"),
("bdsm", "BDSM"),
("beautiful-girl", "Beautiful Girl"),
("big-ass", "Big Ass"),
("big-cock", "Big Cock"),
("big-tits", "Big Tits"),
("bisexual", "Bisexual"),
("blowjob", "Blowjob"),
("brazilian", "Brazilian"),
("british", "British"),
("bukkake", "Bukkake"),
("cartoon", "Cartoon"),
("casting", "Casting"),
("celebrity", "Celebrity"),
("cheating", "Cheating"),
("chinese", "Chinese"),
("compilation", "Compilation"),
("cosplay", "Cosplay"),
("creampie", "Creampie"),
("cuckold", "Cuckold"),
("cumshot", "Cumshot"),
("double-penetration", "Double Penetration"),
("ebony", "Ebony"),
("erotic", "Erotic"),
("family", "Family"),
("femdom", "Femdom"),
("first-time", "First Time"),
("fisting", "Fisting"),
("french", "French"),
("gangbang", "Gangbang"),
("german", "German"),
("groupsex", "Groupsex"),
("hairy", "Hairy"),
("handjob", "Handjob"),
("hentai", "Hentai"),
("indian", "Indian"),
("interracial", "Interracial"),
("italian", "Italian"),
("japanese", "Japanese"),
("latina", "Latina"),
("lesbians", "Lesbians"),
("massage", "Massage"),
("mature", "Mature"),
("milf", "MILF"),
("mom", "Mom"),
("office", "Office"),
("old-young", "Old & Young"),
("orgasm", "Orgasm"),
("outdoor", "Outdoor"),
("petite", "Petite"),
("pov", "POV"),
("public", "Public"),
("reality", "Reality"),
("rough", "Rough"),
("russian", "Russian"),
("school", "School"),
("small-tits", "Small Tits"),
("solo", "Solo"),
("squirt", "Squirt"),
("stockings", "Stockings"),
("teens", "Teens"),
("threesome", "Threesome"),
("toys", "Toys"),
("uniform", "Uniform"),
("vintage", "Vintage"),
("webcam", "Webcam"),
("young", "Young"),
];
#[derive(Debug, Clone)]
enum Target {
New,
Search(String),
Category(String),
Pornstar(String),
Channel(String),
}
#[derive(Debug, Clone)]
pub struct XgroovyProvider;
impl XgroovyProvider {
pub fn new() -> Self {
Self
}
fn build_channel(&self, _cv: ClientVersion) -> Channel {
let mut cat_options = vec![FilterOption {
id: "all".to_string(),
title: "All".to_string(),
}];
for (slug, label) in CATEGORIES {
cat_options.push(FilterOption {
id: slug.to_string(),
title: label.to_string(),
});
}
Channel {
id: CHANNEL_ID.to_string(),
name: "XGroovy".to_string(),
description:
"XGroovy — free porn tube with newest, category, pornstar, channel, and search routing."
.to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=xgroovy.com".to_string(),
status: "active".to_string(),
categories: CATEGORIES
.iter()
.map(|(_, label)| label.to_string())
.collect(),
options: vec![ChannelOption {
id: "categories".to_string(),
title: "Categories".to_string(),
description: "Browse an XGroovy category archive.".to_string(),
systemImage: "square.grid.2x2".to_string(),
colorName: "orange".to_string(),
options: cat_options,
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()),
}
}
fn selector(value: &str) -> Result<Selector> {
Selector::parse(value)
.map_err(|e| Error::from(format!("selector `{value}` parse failed: {e}")))
}
fn decode_html(text: &str) -> String {
decode(text.as_bytes())
.to_string()
.unwrap_or_else(|_| text.to_string())
}
fn text_of(el: &ElementRef<'_>) -> String {
el.text()
.collect::<Vec<_>>()
.join(" ")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
fn normalize_key(s: &str) -> String {
s.trim()
.replace(['-', '_'], " ")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.to_ascii_lowercase()
}
/// xgroovy uses path-segment pagination: page 1 has no trailing segment,
/// page N>=2 appends `/{N}/` to the base path (e.g. `/new/`, `/new/2/`).
fn target_url(target: &Target, page: u16) -> String {
let base = match target {
Target::New => format!("{BASE_URL}/new/"),
Target::Search(q) => {
let encoded = q.trim().replace(' ', "-").to_ascii_lowercase();
format!("{BASE_URL}/search/{encoded}/")
}
Target::Category(slug) => format!("{BASE_URL}/categories/{slug}/"),
Target::Pornstar(slug) => format!("{BASE_URL}/pornstars/{slug}/"),
Target::Channel(slug) => format!("{BASE_URL}/channels/{slug}/"),
};
if page <= 1 {
base
} else {
format!("{base}{page}/")
}
}
/// xgroovy renders duration as free text like "12 min" or "45 sec"
/// rather than a colon-separated timestamp.
fn parse_duration(text: &str) -> Option<u32> {
let text = text.trim().to_ascii_lowercase();
let (num, unit) = text.split_once(char::is_whitespace)?;
let value: u32 = num.parse().ok()?;
match unit {
"min" | "mins" | "minute" | "minutes" => Some(value * 60),
"sec" | "secs" | "second" | "seconds" => Some(value),
"hour" | "hours" | "hr" | "hrs" => Some(value * 3600),
_ => None,
}
}
fn html_headers(referer: &str) -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), FIREFOX_UA.to_string()),
("Accept".to_string(), HTML_ACCEPT.to_string()),
("Referer".to_string(), referer.to_string()),
]
}
fn parse_list_page(html: &str) -> Result<Vec<VideoItem>> {
let document = Html::parse_document(html);
let card_sel = Self::selector("div.item[data-video-id]")?;
let link_sel = Self::selector("a.popito")?;
let img_sel = Self::selector("img.thumb")?;
let title_sel = Self::selector("strong.title")?;
let author_link_sel = Self::selector("div.author-link a")?;
let duration_sel = Self::selector("div.duration")?;
let mut items = Vec::new();
for card in document.select(&card_sel) {
let id = match card.value().attr("data-video-id") {
Some(v) if !v.is_empty() => v.to_string(),
_ => continue,
};
let link = match card.select(&link_sel).next() {
Some(el) => el,
None => continue,
};
let href = link.value().attr("href").unwrap_or_default();
if href.is_empty() {
continue;
}
let page_url = if href.starts_with("https://") {
href.to_string()
} else {
format!("{BASE_URL}{href}")
};
let img = card.select(&img_sel).next();
let thumb = img
.as_ref()
.and_then(|el| {
el.value()
.attr("data-jpg")
.or_else(|| el.value().attr("src"))
})
.unwrap_or_default()
.to_string();
let preview = img
.as_ref()
.and_then(|el| el.value().attr("data-preview"))
.map(str::to_string);
let title = card
.select(&title_sel)
.next()
.map(|el| Self::decode_html(&Self::text_of(&el)))
.filter(|v| !v.is_empty())
.unwrap_or_default();
if title.is_empty() {
continue;
}
let views: Option<u32> = card
.value()
.attr("data-views")
.and_then(|v| v.parse::<u32>().ok());
let rating: Option<f32> = card
.value()
.attr("data-rating")
.and_then(|v| v.parse::<f32>().ok())
.map(|v| v / 100.0);
let duration = card
.select(&duration_sel)
.next()
.map(|el| Self::text_of(&el))
.and_then(|text| Self::parse_duration(&text))
.unwrap_or(0);
let author_el = card.select(&author_link_sel).next();
let uploader = author_el
.as_ref()
.map(|el| Self::decode_html(&Self::text_of(el)))
.filter(|v| !v.is_empty());
let uploader_url = author_el
.and_then(|el| el.value().attr("href"))
.map(|v| {
if v.starts_with("https://") {
v.to_string()
} else {
format!("{BASE_URL}{v}")
}
});
let mut item = VideoItem::new(
id,
title,
page_url,
CHANNEL_ID.to_string(),
thumb,
duration,
);
item.views = views;
item.rating = rating;
item.preview = preview;
item.uploader = uploader;
item.uploaderUrl = uploader_url.clone();
if let Some(url) = &uploader_url {
let slug = url
.trim_end_matches('/')
.rsplit('/')
.next()
.unwrap_or_default();
if !slug.is_empty() {
let kind = if url.contains("/channels/") {
"channel"
} else if url.contains("/pornstars/") {
"pornstar"
} else {
"creator"
};
item.uploaderId = Some(format!("{CHANNEL_ID}:{kind}:{slug}"));
}
}
items.push(item);
}
Ok(items)
}
fn resolve_query_target(&self, query: &str) -> Target {
let trimmed = query.trim();
if let Some((kind, value)) = trimmed.split_once(':') {
let slug = value.trim().replace(' ', "-").to_ascii_lowercase();
if !slug.is_empty() {
match kind.trim().to_ascii_lowercase().as_str() {
"cat" | "category" => return Target::Category(slug),
"pornstar" | "pornstars" | "star" => return Target::Pornstar(slug),
"channel" | "channels" => return Target::Channel(slug),
_ => {}
}
}
}
// Check static category list by label or slug
let normalized = Self::normalize_key(trimmed);
for (slug, label) in CATEGORIES {
if Self::normalize_key(label) == normalized || Self::normalize_key(slug) == normalized
{
return Target::Category(slug.to_string());
}
}
Target::Search(trimmed.to_string())
}
fn resolve_option_target(&self, options: &ServerOptions) -> Target {
if let Some(cat) = options.categories.as_deref() {
if cat != "all" && !cat.is_empty() {
return Target::Category(cat.to_string());
}
}
Target::New
}
async fn fetch_target(
&self,
cache: VideoCache,
target: Target,
page: u16,
per_page: usize,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let url = Self::target_url(&target, page);
let cache_key = format!("{url}#per={per_page}");
if let Some((ts, cached)) = cache.get(&cache_key) {
if ts.elapsed().unwrap_or_default().as_secs() < 300 {
return Ok(cached.clone());
}
}
let mut requester = requester_or_default(&options, CHANNEL_ID, "xgroovy.fetch_target");
let html = requester
.get_with_headers(&url, Self::html_headers(&url), None)
.await
.map_err(|e| Error::from(format!("request failed for {url}: {e}")))?;
if html.trim().is_empty() {
return Err(Error::from(format!("empty response for {url}")));
}
let all = Self::parse_list_page(&html)?;
let items: Vec<VideoItem> = all.into_iter().take(per_page.max(1)).collect();
if !items.is_empty() {
cache.insert(cache_key, items.clone());
}
Ok(items)
}
}
#[async_trait]
impl Provider for XgroovyProvider {
async fn get_videos(
&self,
cache: VideoCache,
_pool: DbPool,
_sort: String,
query: Option<String>,
page: String,
per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let page = page.parse::<u16>().unwrap_or(1).max(1);
let per_page = per_page.parse::<usize>().unwrap_or(10).clamp(1, 60);
let target = match query {
Some(q) if !q.trim().is_empty() => self.resolve_query_target(q.trim()),
_ => self.resolve_option_target(&options),
};
match self
.fetch_target(cache, target, page, per_page, options)
.await
{
Ok(items) => items,
Err(e) => {
report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await;
vec![]
}
}
}
fn get_channel(&self, cv: ClientVersion) -> Option<Channel> {
Some(self.build_channel(cv))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_target_url_pagination() {
assert_eq!(
XgroovyProvider::target_url(&Target::New, 1),
"https://xgroovy.com/new/"
);
assert_eq!(
XgroovyProvider::target_url(&Target::New, 2),
"https://xgroovy.com/new/2/"
);
assert_eq!(
XgroovyProvider::target_url(&Target::Search("hot blonde".to_string()), 1),
"https://xgroovy.com/search/hot-blonde/"
);
assert_eq!(
XgroovyProvider::target_url(&Target::Category("amateur".to_string()), 2),
"https://xgroovy.com/categories/amateur/2/"
);
assert_eq!(
XgroovyProvider::target_url(&Target::Pornstar("abella-danger".to_string()), 1),
"https://xgroovy.com/pornstars/abella-danger/"
);
assert_eq!(
XgroovyProvider::target_url(&Target::Channel("brazzers".to_string()), 3),
"https://xgroovy.com/channels/brazzers/3/"
);
}
#[test]
fn resolves_category_by_label_and_slug() {
let p = XgroovyProvider::new();
assert!(matches!(
p.resolve_query_target("amateur"),
Target::Category(s) if s == "amateur"
));
assert!(matches!(
p.resolve_query_target("Big Ass"),
Target::Category(s) if s == "big-ass"
));
assert!(matches!(
p.resolve_query_target("Old & Young"),
Target::Category(s) if s == "old-young"
));
}
#[test]
fn resolves_explicit_shortcuts() {
let p = XgroovyProvider::new();
assert!(matches!(
p.resolve_query_target("cat:milf"),
Target::Category(s) if s == "milf"
));
assert!(matches!(
p.resolve_query_target("channel:brazzers"),
Target::Channel(s) if s == "brazzers"
));
assert!(matches!(
p.resolve_query_target("pornstar:abella-danger"),
Target::Pornstar(s) if s == "abella-danger"
));
}
#[test]
fn falls_through_to_search() {
let p = XgroovyProvider::new();
assert!(matches!(
p.resolve_query_target("some unknown query"),
Target::Search(_)
));
}
#[test]
fn parses_listing_card() {
let html = r#"
<html><body>
<div class="item " data-video-id="337143" data-history-id="337143" data-views="330079" data-rating="75.942">
<a href="https://xgroovy.com/videos/337143/turkish-18-yo-girl-from-college-gives-a-blowjob-and-gets-fucked-pov/" class="popito">
<div class="img">
<img fetchpriority="high" class="thumb " src="https://i.xgroovy.com/contents/videos_screenshots/337000/337143/608x342/1.jpg" data-jpg="https://i.xgroovy.com/contents/videos_screenshots/337000/337143/640x360/1.jpg" alt="Turkish 18 yo girl" data-cnt="28" data-preview="https://preview.xgroovy.com/videos/337000/337143/337143_pr640.mp4" width="640" height="360"/>
</div>
<strong class="title">
Turkish 18 yo girl from college gives a blowjob and gets fucked POV
</strong>
</a>
<div class="wrap">
<div class="author-link"><a href="https://xgroovy.com/pornstars/assspanker/"><i class="mi star"></i>AssSpanker</a></div>
<div class="views">330k views</div>
<div class="duration">12 min</div>
<div class="rating positive"> <i class="mi thumb_up"></i>75%</div>
</div>
</div>
</body></html>
"#;
let items = XgroovyProvider::parse_list_page(html).expect("parse should succeed");
assert_eq!(items.len(), 1);
let item = &items[0];
assert_eq!(item.id, "337143");
assert_eq!(
item.title,
"Turkish 18 yo girl from college gives a blowjob and gets fucked POV"
);
assert_eq!(
item.url,
"https://xgroovy.com/videos/337143/turkish-18-yo-girl-from-college-gives-a-blowjob-and-gets-fucked-pov/"
);
assert_eq!(
item.thumb,
"https://i.xgroovy.com/contents/videos_screenshots/337000/337143/640x360/1.jpg"
);
assert_eq!(item.duration, 720);
assert_eq!(item.views, Some(330079));
assert_eq!(item.rating, Some(75.942 / 100.0));
assert_eq!(item.uploader.as_deref(), Some("AssSpanker"));
assert_eq!(
item.uploaderUrl.as_deref(),
Some("https://xgroovy.com/pornstars/assspanker/")
);
assert_eq!(
item.uploaderId.as_deref(),
Some("xgroovy:pornstar:assspanker")
);
assert_eq!(
item.preview.as_deref(),
Some("https://preview.xgroovy.com/videos/337000/337143/337143_pr640.mp4")
);
}
}