Add hooporn provider
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
565
src/providers/hooporn.rs
Normal file
565
src/providers/hooporn.rs
Normal file
@@ -0,0 +1,565 @@
|
||||
use crate::DbPool;
|
||||
use crate::api::ClientVersion;
|
||||
use crate::providers::{Provider, report_provider_error, requester_or_default};
|
||||
use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::requester::Requester;
|
||||
use crate::util::time::parse_time_to_seconds;
|
||||
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
||||
use async_trait::async_trait;
|
||||
use error_chain::error_chain;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||
crate::providers::ProviderChannelMetadata {
|
||||
group_id: "studio-network",
|
||||
tags: &["studio", "hd", "pornstars", "free"],
|
||||
};
|
||||
|
||||
const BASE_URL: &str = "https://hooporn.com";
|
||||
const CHANNEL_ID: &str = "hooporn";
|
||||
const ENRICH_CONCURRENCY: usize = 8;
|
||||
|
||||
// Scraped from /categories (`div.channels-card a[href^="/category/"]`).
|
||||
const CATEGORIES: &[(&str, &str)] = &[
|
||||
("anal", "anal"),
|
||||
("ass-licking", "ass licking"),
|
||||
("atm", "ass to mouth"),
|
||||
("babe", "babe"),
|
||||
("bbw", "BBW"),
|
||||
("big-ass", "big ass"),
|
||||
("big-tits", "big tits"),
|
||||
("black", "black"),
|
||||
("blonde", "blonde"),
|
||||
("blowjob", "blowjob"),
|
||||
("bondage", "bondage"),
|
||||
("brunette", "brunette"),
|
||||
("busty", "busty"),
|
||||
("casting", "casting"),
|
||||
("college", "college"),
|
||||
("compilation", "compilation"),
|
||||
("creampie", "creampie"),
|
||||
("cuckold", "cuckold"),
|
||||
("czech", "czech"),
|
||||
("deepthroat", "deepthroat"),
|
||||
("doggystyle", "doggystyle"),
|
||||
("dp", "double penetration"),
|
||||
("ebony", "ebony"),
|
||||
("fantasy", "fantasy"),
|
||||
("fetish", "fetish"),
|
||||
("fingering", "fingering"),
|
||||
("footjob", "footjob"),
|
||||
("foursome", "foursome"),
|
||||
("gangbang", "gangbang"),
|
||||
("gangbang-creampie", "gangbang creampie"),
|
||||
("gaping", "gaping"),
|
||||
("german", "german"),
|
||||
("gloryhole", "gloryhole"),
|
||||
("hairy", "hairy"),
|
||||
("handjob", "handjob"),
|
||||
("hardcore", "hardcore"),
|
||||
("homemade", "homemade"),
|
||||
("interracial", "interracial"),
|
||||
("latina", "latina"),
|
||||
("lesbian", "lesbian"),
|
||||
("lingerie", "lingerie"),
|
||||
("massage", "massage"),
|
||||
("masturbation", "masturbation"),
|
||||
("mature", "mature"),
|
||||
("milf", "MILF"),
|
||||
("orgy", "orgy"),
|
||||
("outdoor", "outdoor"),
|
||||
("petite", "petite"),
|
||||
("pov", "POV"),
|
||||
("public", "public"),
|
||||
("pussy-licking", "pussy licking"),
|
||||
("red-head", "red head"),
|
||||
("riding", "riding"),
|
||||
("russian", "russian"),
|
||||
("skinny", "skinny"),
|
||||
("small-tits", "small tits"),
|
||||
("solo", "solo"),
|
||||
("squirt", "squirt"),
|
||||
("swallow", "swallow"),
|
||||
("teen", "teen"),
|
||||
("threesome", "threesome"),
|
||||
("toys", "toys"),
|
||||
("wife", "wife"),
|
||||
];
|
||||
|
||||
error_chain! {
|
||||
foreign_links {
|
||||
Io(std::io::Error);
|
||||
}
|
||||
errors {
|
||||
Parse(msg: String) {
|
||||
description("parse error")
|
||||
display("parse error: {}", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HoopornProvider;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum Target {
|
||||
Latest,
|
||||
Category(String),
|
||||
Pornstar(String),
|
||||
Studio(String),
|
||||
Search(String),
|
||||
}
|
||||
|
||||
/// Detail-page data: the player iframe plus taxonomy links.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
struct Detail {
|
||||
iframe: Option<String>,
|
||||
tags: Vec<String>,
|
||||
studio: Option<(String, String)>,
|
||||
}
|
||||
|
||||
impl HoopornProvider {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||
let mut category_options = vec![FilterOption {
|
||||
id: "all".to_string(),
|
||||
title: "All".to_string(),
|
||||
}];
|
||||
category_options.extend(CATEGORIES.iter().map(|(slug, title)| FilterOption {
|
||||
id: (*slug).to_string(),
|
||||
title: Self::title_case(title),
|
||||
}));
|
||||
|
||||
Channel {
|
||||
id: CHANNEL_ID.to_string(),
|
||||
name: "HooPorn".to_string(),
|
||||
description: "Free HD studio videos from HooPorn with category, pornstar and studio archives.".to_string(),
|
||||
premium: false,
|
||||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=hooporn.com".to_string(),
|
||||
status: "active".to_string(),
|
||||
categories: vec![],
|
||||
options: vec![
|
||||
ChannelOption {
|
||||
id: "sort".to_string(),
|
||||
title: "Sort".to_string(),
|
||||
description: "Browse HooPorn by newest videos.".to_string(),
|
||||
systemImage: "arrow.up.arrow.down".to_string(),
|
||||
colorName: "blue".to_string(),
|
||||
options: vec![FilterOption {
|
||||
id: "new".to_string(),
|
||||
title: "Latest".to_string(),
|
||||
}],
|
||||
multiSelect: false,
|
||||
},
|
||||
ChannelOption {
|
||||
id: "categories".to_string(),
|
||||
title: "Categories".to_string(),
|
||||
description: "Browse a HooPorn category archive.".to_string(),
|
||||
systemImage: "square.grid.2x2".to_string(),
|
||||
colorName: "orange".to_string(),
|
||||
options: category_options,
|
||||
multiSelect: false,
|
||||
},
|
||||
],
|
||||
nsfw: true,
|
||||
cacheDuration: Some(1800),
|
||||
ytdlpCommand: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn title_case(value: &str) -> String {
|
||||
if value.chars().any(|c| c.is_uppercase()) {
|
||||
return value.to_string();
|
||||
}
|
||||
value
|
||||
.split(' ')
|
||||
.map(|word| {
|
||||
let mut chars = word.chars();
|
||||
match chars.next() {
|
||||
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
|
||||
None => String::new(),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
fn slugify(value: &str) -> String {
|
||||
let mut out = String::new();
|
||||
let mut last_dash = true;
|
||||
for c in value.trim().chars() {
|
||||
if c.is_alphanumeric() {
|
||||
out.extend(c.to_lowercase());
|
||||
last_dash = false;
|
||||
} else if !last_dash {
|
||||
out.push('-');
|
||||
last_dash = true;
|
||||
}
|
||||
}
|
||||
out.trim_matches('-').to_string()
|
||||
}
|
||||
|
||||
fn category_slug(value: &str) -> Option<&'static str> {
|
||||
let slug = Self::slugify(value);
|
||||
if slug.is_empty() {
|
||||
return None;
|
||||
}
|
||||
CATEGORIES
|
||||
.iter()
|
||||
.find_map(|(s, title)| (*s == slug || Self::slugify(title) == slug).then_some(*s))
|
||||
}
|
||||
|
||||
fn pick_target(query: Option<&str>, options: &ServerOptions) -> Target {
|
||||
if let Some(q) = query.map(str::trim).filter(|q| !q.is_empty()) {
|
||||
let lower = q.to_ascii_lowercase();
|
||||
for prefix in ["cat:", "category:"] {
|
||||
if let Some(rest) = lower.strip_prefix(prefix) {
|
||||
let slug = Self::category_slug(rest)
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| Self::slugify(rest));
|
||||
return Target::Category(slug);
|
||||
}
|
||||
}
|
||||
for prefix in ["pornstar:", "star:", "model:"] {
|
||||
if let Some(rest) = lower.strip_prefix(prefix) {
|
||||
return Target::Pornstar(Self::slugify(rest));
|
||||
}
|
||||
}
|
||||
for prefix in ["studio:", "site:"] {
|
||||
if let Some(rest) = lower.strip_prefix(prefix) {
|
||||
return Target::Studio(Self::slugify(rest).replace('-', ""));
|
||||
}
|
||||
}
|
||||
if let Some(slug) = Self::category_slug(q) {
|
||||
return Target::Category(slug.to_string());
|
||||
}
|
||||
return Target::Search(q.to_string());
|
||||
}
|
||||
|
||||
if let Some(cat) = options.categories.as_deref() {
|
||||
let cat = cat.trim();
|
||||
if !cat.is_empty() && !cat.eq_ignore_ascii_case("all") {
|
||||
let slug = Self::category_slug(cat)
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| Self::slugify(cat));
|
||||
return Target::Category(slug);
|
||||
}
|
||||
}
|
||||
Target::Latest
|
||||
}
|
||||
|
||||
fn build_listing_url(target: &Target, page: u32) -> String {
|
||||
match target {
|
||||
Target::Latest => format!("{BASE_URL}/home/{page}"),
|
||||
Target::Category(slug) => format!("{BASE_URL}/category/{slug}/{page}"),
|
||||
Target::Pornstar(slug) => format!("{BASE_URL}/pornstar/{slug}/{page}"),
|
||||
Target::Studio(slug) => format!("{BASE_URL}/studio/{slug}/{page}"),
|
||||
Target::Search(query) => format!(
|
||||
"{BASE_URL}/search?q={}&p={page}",
|
||||
utf8_percent_encode(query, NON_ALPHANUMERIC)
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn selector(value: &str) -> Result<Selector> {
|
||||
Selector::parse(value).map_err(|e| Error::from(format!("selector `{value}` failed: {e}")))
|
||||
}
|
||||
|
||||
fn absolute(url: &str) -> String {
|
||||
if let Some(rest) = url.strip_prefix("//") {
|
||||
format!("https://{rest}")
|
||||
} else if url.starts_with('/') {
|
||||
format!("{BASE_URL}{url}")
|
||||
} else {
|
||||
url.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_listing(html: &str) -> Result<Vec<VideoItem>> {
|
||||
let document = Html::parse_document(html);
|
||||
let card_sel = Self::selector("div.video-card")?;
|
||||
let link_sel = Self::selector("div.video-title a[href^=\"/watch/\"]")?;
|
||||
let img_sel = Self::selector("img")?;
|
||||
let time_sel = Self::selector(".time")?;
|
||||
let created_sel = Self::selector("span.create")?;
|
||||
|
||||
let mut items = Vec::new();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for card in document.select(&card_sel) {
|
||||
let Some(link) = card.select(&link_sel).next() else {
|
||||
continue;
|
||||
};
|
||||
let href = link.value().attr("href").unwrap_or_default();
|
||||
let id = href.trim_start_matches("/watch/").trim_matches('/').to_string();
|
||||
if id.is_empty() || !seen.insert(id.clone()) {
|
||||
continue;
|
||||
}
|
||||
let title = link.text().collect::<String>().trim().to_string();
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let thumb = card
|
||||
.select(&img_sel)
|
||||
.next()
|
||||
.and_then(|i| i.value().attr("data-src"))
|
||||
.map(Self::absolute)
|
||||
.unwrap_or_default();
|
||||
let duration = card
|
||||
.select(&time_sel)
|
||||
.next()
|
||||
.map(|t| t.text().collect::<String>())
|
||||
.and_then(|t| parse_time_to_seconds(t.trim()))
|
||||
.and_then(|s| u32::try_from(s).ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let mut item = VideoItem::new(
|
||||
id.clone(),
|
||||
title,
|
||||
format!("{BASE_URL}/watch/{id}"),
|
||||
CHANNEL_ID.to_string(),
|
||||
thumb,
|
||||
duration,
|
||||
);
|
||||
item.uploadedAt = card
|
||||
.select(&created_sel)
|
||||
.next()
|
||||
.and_then(|c| c.text().collect::<String>().trim().parse::<u64>().ok());
|
||||
item.aspectRatio = Some(16.0 / 9.0);
|
||||
items.push(item);
|
||||
}
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
fn parse_detail(html: &str) -> Result<Detail> {
|
||||
let document = Html::parse_document(html);
|
||||
let iframe_sel = Self::selector("iframe[src*=\"/video/\"]")?;
|
||||
let tag_sel = Self::selector("p.tag-link a[href^=\"/category/\"]")?;
|
||||
let star_sel = Self::selector("a[href^=\"/pornstar/\"]")?;
|
||||
let studio_sel = Self::selector("a[href^=\"/studio/\"]")?;
|
||||
|
||||
let mut detail = Detail::default();
|
||||
detail.iframe = document
|
||||
.select(&iframe_sel)
|
||||
.next()
|
||||
.and_then(|f| f.value().attr("src"))
|
||||
.map(Self::absolute);
|
||||
|
||||
let mut push = |tag: String| {
|
||||
let tag = tag.trim().trim_start_matches('#').trim().to_string();
|
||||
if !tag.is_empty() && !detail.tags.iter().any(|t| t.eq_ignore_ascii_case(&tag)) {
|
||||
detail.tags.push(tag);
|
||||
}
|
||||
};
|
||||
for a in document.select(&tag_sel) {
|
||||
push(a.text().collect::<String>());
|
||||
}
|
||||
for a in document.select(&star_sel) {
|
||||
push(a.text().collect::<String>());
|
||||
}
|
||||
if let Some(a) = document.select(&studio_sel).next() {
|
||||
let name = a.text().collect::<String>().trim().to_string();
|
||||
let slug = a
|
||||
.value()
|
||||
.attr("href")
|
||||
.unwrap_or_default()
|
||||
.trim_start_matches("/studio/")
|
||||
.trim_matches('/')
|
||||
.to_string();
|
||||
if !name.is_empty() && !slug.is_empty() {
|
||||
detail.studio = Some((name, slug));
|
||||
}
|
||||
}
|
||||
Ok(detail)
|
||||
}
|
||||
|
||||
/// The player frame carries `var video = "https://videos…mp4?md5=…&expires=…"`.
|
||||
fn parse_player_video(html: &str) -> Option<String> {
|
||||
let marker = "var video = \"";
|
||||
let start = html.find(marker)? + marker.len();
|
||||
let end = html[start..].find('"')? + start;
|
||||
let url = html[start..end].replace("&", "&");
|
||||
url.starts_with("http").then_some(url)
|
||||
}
|
||||
|
||||
async fn enrich(items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
|
||||
let referer = format!("{BASE_URL}/");
|
||||
stream::iter(items.into_iter().map(|mut item| {
|
||||
let mut requester = requester.clone();
|
||||
let referer = referer.clone();
|
||||
async move {
|
||||
let headers = vec![("Referer".to_string(), referer.clone())];
|
||||
let page = requester
|
||||
.get_with_headers(&item.url, headers.clone(), None)
|
||||
.await
|
||||
.ok()?;
|
||||
let detail = Self::parse_detail(&page).ok()?;
|
||||
let frame = detail.iframe?;
|
||||
let player = requester
|
||||
.get_with_headers(&frame, headers, None)
|
||||
.await
|
||||
.ok()?;
|
||||
let video_url = Self::parse_player_video(&player)?;
|
||||
|
||||
item.formats = Some(vec![
|
||||
VideoFormat::new(video_url, "auto".to_string(), "mp4".to_string())
|
||||
.format_id("mp4".to_string()),
|
||||
]);
|
||||
if !detail.tags.is_empty() {
|
||||
item.tags = Some(detail.tags);
|
||||
}
|
||||
if let Some((name, slug)) = detail.studio {
|
||||
item.uploaderUrl = Some(format!("{BASE_URL}/studio/{slug}"));
|
||||
item.uploaderId = Some(format!("{CHANNEL_ID}:{slug}"));
|
||||
item.uploader = Some(name);
|
||||
}
|
||||
Some(item)
|
||||
}
|
||||
}))
|
||||
.buffered(ENRICH_CONCURRENCY)
|
||||
.filter_map(|item| async move { item })
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn fetch(target: &Target, page: u32, options: &ServerOptions) -> Result<Vec<VideoItem>> {
|
||||
let url = Self::build_listing_url(target, page);
|
||||
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch");
|
||||
let html = requester
|
||||
.get_with_headers(
|
||||
&url,
|
||||
vec![("Referer".to_string(), format!("{BASE_URL}/"))],
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::from(format!("fetch failed for {url}: {e}")))?;
|
||||
let items = Self::parse_listing(&html)?;
|
||||
Ok(Self::enrich(items, requester).await)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for HoopornProvider {
|
||||
async fn get_videos(
|
||||
&self,
|
||||
_cache: VideoCache,
|
||||
_pool: DbPool,
|
||||
_sort: String,
|
||||
query: Option<String>,
|
||||
page: String,
|
||||
_per_page: String,
|
||||
options: ServerOptions,
|
||||
) -> Vec<VideoItem> {
|
||||
let page = page.parse::<u32>().unwrap_or(1).max(1);
|
||||
let target = Self::pick_target(query.as_deref(), &options);
|
||||
|
||||
match Self::fetch(&target, page, &options).await {
|
||||
Ok(items) => items,
|
||||
Err(e) => {
|
||||
report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await;
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn opts() -> ServerOptions {
|
||||
ServerOptions {
|
||||
featured: None,
|
||||
category: None,
|
||||
sites: None,
|
||||
filter: None,
|
||||
language: None,
|
||||
public_url_base: None,
|
||||
requester: None,
|
||||
network: None,
|
||||
stars: None,
|
||||
categories: None,
|
||||
duration: None,
|
||||
sort: None,
|
||||
sexuality: None,
|
||||
client_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_urls() {
|
||||
assert_eq!(HoopornProvider::build_listing_url(&Target::Latest, 2), "https://hooporn.com/home/2");
|
||||
assert_eq!(
|
||||
HoopornProvider::build_listing_url(&Target::Category("milf".into()), 3),
|
||||
"https://hooporn.com/category/milf/3"
|
||||
);
|
||||
assert_eq!(
|
||||
HoopornProvider::build_listing_url(&Target::Search("big tits".into()), 2),
|
||||
"https://hooporn.com/search?q=big%20tits&p=2"
|
||||
);
|
||||
assert_eq!(
|
||||
HoopornProvider::build_listing_url(&Target::Studio("mylf".into()), 1),
|
||||
"https://hooporn.com/studio/mylf/1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routes_queries() {
|
||||
let o = opts();
|
||||
assert_eq!(
|
||||
HoopornProvider::pick_target(Some("Double Penetration"), &o),
|
||||
Target::Category("dp".into())
|
||||
);
|
||||
assert_eq!(
|
||||
HoopornProvider::pick_target(Some("pornstar:Gal Ritchie"), &o),
|
||||
Target::Pornstar("gal-ritchie".into())
|
||||
);
|
||||
assert_eq!(
|
||||
HoopornProvider::pick_target(Some("studio:Family Strokes"), &o),
|
||||
Target::Studio("familystrokes".into())
|
||||
);
|
||||
assert_eq!(
|
||||
HoopornProvider::pick_target(Some("golf"), &o),
|
||||
Target::Search("golf".into())
|
||||
);
|
||||
assert_eq!(HoopornProvider::pick_target(None, &o), Target::Latest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_cards_detail_and_player() {
|
||||
let list = r#"<div class="video-card"><div class="video-card-image">
|
||||
<a class="popout" href="/watch/abc123"><img src="//cdn.hooporn.com/img/blank.gif" data-src="//imgs.hooporn.com/t.jpg?v2" alt="x"></a>
|
||||
<div class="time">37:16</div></div><div class="video-card-body"><div class="video-title">
|
||||
<a class="popout" href="/watch/abc123">A & B - Title</a></div>
|
||||
<div class="video-view"><span class="create">1792152610</span></div></div></div>"#;
|
||||
let items = HoopornProvider::parse_listing(list).unwrap();
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0].title, "A & B - Title");
|
||||
assert_eq!(items[0].thumb, "https://imgs.hooporn.com/t.jpg?v2");
|
||||
assert_eq!(items[0].duration, 37 * 60 + 16);
|
||||
assert_eq!(items[0].uploadedAt, Some(1792152610));
|
||||
|
||||
let detail = r##"<iframe src="//vid.hooporn.com/video/182550"></iframe>
|
||||
<p class="tag-link"><span><a class="popout" href="/category/milf">#milf</a></span></p>
|
||||
<a href="/pornstar/gal-ritchie">gal ritchie</a><a href="/studio/familystrokes">FAMILY STROKES</a>"##;
|
||||
let d = HoopornProvider::parse_detail(detail).unwrap();
|
||||
assert_eq!(d.iframe.as_deref(), Some("https://vid.hooporn.com/video/182550"));
|
||||
assert_eq!(d.tags, vec!["milf".to_string(), "gal ritchie".to_string()]);
|
||||
assert_eq!(d.studio, Some(("FAMILY STROKES".to_string(), "familystrokes".to_string())));
|
||||
|
||||
let player = r#"var video = "https://videos.trendyporn.com/v/a.mp4?md5=X&expires=1";"#;
|
||||
assert_eq!(
|
||||
HoopornProvider::parse_player_video(player).as_deref(),
|
||||
Some("https://videos.trendyporn.com/v/a.mp4?md5=X&expires=1")
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user