use crate::DbPool; use crate::api::ClientVersion; use crate::providers::{ Provider, report_provider_error, report_provider_error_background, requester_or_default, }; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use scraper::{Html, Selector}; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock}; use std::thread; const CHANNEL_ID: &str = "porn4fans"; const BASE_URL: &str = "https://www.porn4fans.com"; /// Bounded concurrency for detail-page enrichment (one fetch per card). const ENRICH_CONCURRENCY: usize = 8; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "onlyfans", tags: &["creator", "premium", "clips"], }; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); } } #[derive(Debug, Clone)] pub struct Porn4fansProvider { url: String, /// Display list exposed via the `categories` status option. categories: Arc>>, /// normalized category title -> site slug, for shortcut routing. category_map: Arc>>, } /// Where a `/api/videos` request should be routed on the site. Every variant /// resolves to the same KVS `get_block` async fragment, just with a different /// path + `block_id` + pagination parameter. #[derive(Debug, Clone)] enum Target { /// Default `/onlyfans-videos/` latest feed. Latest, /// Native keyword search. Search { query: String }, /// `/categories/{slug}/` archive. Category { slug: String }, /// `/tags/{slug}/` archive. Tag { slug: String }, /// `/models/{slug}/` archive (uploader shortcut). Model { slug: String }, } #[derive(Debug, Clone)] struct Porn4fansCard { id: String, title: String, page_url: String, thumb: String, duration: u32, views: Option, preview: Option, uploader: Option, uploader_url: Option, uploader_slug: Option, } impl Porn4fansProvider { pub fn new() -> Self { let provider = Self { url: BASE_URL.to_string(), categories: Arc::new(RwLock::new(vec![FilterOption { id: "all".to_string(), title: "All".to_string(), }])), category_map: Arc::new(RwLock::new(HashMap::new())), }; provider.spawn_initial_load(); provider } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { let categories = self .categories .read() .map(|guard| guard.clone()) .unwrap_or_default(); Channel { id: CHANNEL_ID.to_string(), name: "Porn4Fans".to_string(), description: "OnlyFans creator clips — latest feed, keyword search, plus category, tag and model shortcuts." .to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.porn4fans.com" .to_string(), status: "active".to_string(), categories: categories.iter().map(|value| value.title.clone()).collect(), options: vec![ ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Feed ordering.".to_string(), systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "new".to_string(), title: "Latest".to_string(), }, FilterOption { id: "popular".to_string(), title: "Most Viewed".to_string(), }, FilterOption { id: "rated".to_string(), title: "Top Rated".to_string(), }, FilterOption { id: "longest".to_string(), title: "Longest".to_string(), }, ], multiSelect: false, }, ChannelOption { id: "categories".to_string(), title: "Categories".to_string(), description: "Jump directly to a category archive.".to_string(), systemImage: "square.grid.2x2".to_string(), colorName: "orange".to_string(), options: categories, multiSelect: false, }, ], nsfw: true, cacheDuration: Some(1800), ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()) } } fn sort_by(sort: &str) -> &'static str { match sort { "popular" | "viewed" | "most-viewed" => "video_viewed", "rated" | "rating" | "top-rated" => "rating", "longest" | "duration" => "duration", _ => "post_date", } } /// lowercase, spaces/underscores -> single dash, drop anything else. fn slugify(value: &str) -> String { let mut slug = String::new(); let mut prev_dash = false; for ch in value.trim().to_ascii_lowercase().chars() { if ch.is_ascii_alphanumeric() { slug.push(ch); prev_dash = false; } else if matches!(ch, ' ' | '-' | '_' | '+') { if !prev_dash && !slug.is_empty() { slug.push('-'); prev_dash = true; } } } slug.trim_matches('-').to_string() } /// Normalize a display title for lookup map keys. fn normalize_title(value: &str) -> String { value .trim() .trim_start_matches('#') .replace(['_', '-'], " ") .split_whitespace() .collect::>() .join(" ") .to_ascii_lowercase() } fn resolve_category_slug(&self, value: &str) -> Option { let normalized = Self::normalize_title(value); if normalized.is_empty() || normalized == "all" { return None; } if let Some(slug) = self .category_map .read() .ok() .and_then(|map| map.get(&normalized).cloned()) { return Some(slug); } None } /// Decide where a request is routed: explicit option/prefix first, then a /// bare query that matches a known category, otherwise keyword search. fn target_from_request( &self, query: Option<&str>, category: Option<&str>, ) -> Target { if let Some(category) = category { if let Some(slug) = self.resolve_category_slug(category) { return Target::Category { slug }; } } if let Some(raw) = query { let trimmed = raw.trim(); if !trimmed.is_empty() { for prefix in ["category:", "cat:"] { if let Some(rest) = trimmed.strip_prefix(prefix) { let slug = self .resolve_category_slug(rest) .unwrap_or_else(|| Self::slugify(rest)); if !slug.is_empty() { return Target::Category { slug }; } } } if let Some(rest) = trimmed.strip_prefix("tag:") { let slug = Self::slugify(rest); if !slug.is_empty() { return Target::Tag { slug }; } } for prefix in ["model:", "uploader:", "pornstar:", "star:"] { if let Some(rest) = trimmed.strip_prefix(prefix) { let slug = Self::slugify(rest); if !slug.is_empty() { return Target::Model { slug }; } } } // A bare query that is exactly a known category goes to that // archive (better curated than keyword search). if let Some(slug) = self.resolve_category_slug(trimmed) { return Target::Category { slug }; } return Target::Search { query: trimmed.to_string(), }; } } Target::Latest } /// Build the async `get_block` URL plus the Referer to send with it. fn build_block_url(&self, target: &Target, sort: &str, page: u32) -> (String, String) { let sort_by = Self::sort_by(sort); match target { Target::Latest => { let referer = format!("{}/onlyfans-videos/", self.url); let url = format!( "{referer}?mode=async&function=get_block&block_id=custom_list_videos_latest_videos_list&sort_by={sort_by}&from={page}" ); (url, referer) } Target::Category { slug } => { let referer = format!("{}/categories/{slug}/", self.url); let url = format!( "{referer}?mode=async&function=get_block&block_id=custom_list_videos_common_videos_list&sort_by={sort_by}&from={page}" ); (url, referer) } Target::Tag { slug } => { let referer = format!("{}/tags/{slug}/", self.url); let url = format!( "{referer}?mode=async&function=get_block&block_id=custom_list_videos_common_videos_list&sort_by={sort_by}&from={page}" ); (url, referer) } Target::Model { slug } => { let referer = format!("{}/models/{slug}/", self.url); let url = format!( "{referer}?mode=async&function=get_block&block_id=custom_list_videos_models_videos_list&sort_by={sort_by}&from={page}" ); (url, referer) } Target::Search { query } => { let path = Self::slugify(query); let q = query.split_whitespace().collect::>().join("+"); let referer = format!("{}/search/{path}/", self.url); let url = format!( "{referer}?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search_result&q={q}&category_ids=&sort_by={sort_by}&from_videos={page}" ); (url, referer) } } } fn html_headers(referer: &str) -> Vec<(String, String)> { vec![("Referer".to_string(), referer.to_string())] } async fn get_videos_for_target( &self, cache: VideoCache, target: &Target, sort: &str, page: u32, options: ServerOptions, ) -> Result> { let (video_url, referer) = self.build_block_url(target, sort, page); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { return Ok(items.clone()); } items.clone() } None => vec![], }; let mut requester = requester_or_default(&options, CHANNEL_ID, "get_videos_for_target"); let text = match requester .get_with_headers(&video_url, Self::html_headers(&referer), None) .await { Ok(text) => text, Err(e) => { report_provider_error( CHANNEL_ID, "get_videos_for_target.request", &format!("url={video_url}; error={e}"), ) .await; return Ok(old_items); } }; if text.trim().is_empty() { return Ok(old_items); } let cards = self.parse_video_cards(&text); if cards.is_empty() { return Ok(old_items); } let video_items = self.enrich_cards(cards, &referer, requester).await; if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); return Ok(video_items); } Ok(old_items) } fn decode_html_text(text: &str) -> String { decode(text.as_bytes()) .to_string() .unwrap_or_else(|_| text.to_string()) .split_whitespace() .collect::>() .join(" ") .trim() .to_string() } fn normalize_url(&self, url: &str) -> String { let trimmed = url.trim(); if trimmed.is_empty() { return String::new(); } if trimmed.starts_with("http://") || trimmed.starts_with("https://") { return trimmed.to_string(); } if trimmed.starts_with("//") { return format!("https:{trimmed}"); } if trimmed.starts_with('/') { return format!("{}{}", self.url, trimmed); } format!("{}/{}", self.url, trimmed.trim_start_matches("./")) } /// KVS `get_file` URLs look like `.../10002.mp4/?v-acctoken=...`. Strip the /// slash sitting between the extension and the query so the path ends in a /// real media extension (yt-dlp / health-check media detection keys off it). fn normalize_media_url(raw: &str) -> String { let mut url = raw.trim().to_string(); for ext in [".mp4/", ".m4v/", ".webm/", ".mov/"] { if let Some(idx) = url.find(ext) { let replacement = &ext[..ext.len() - 1]; url.replace_range(idx..idx + ext.len(), replacement); break; } } url } fn id_from_video_url(url: &str) -> Option { let after = url.split("/video/").nth(1)?; let id: String = after.chars().take_while(|c| c.is_ascii_digit()).collect(); (!id.is_empty()).then_some(id) } fn slug_from_model_url(url: &str) -> Option { url.split("/models/") .nth(1)? .split('/') .next() .map(str::to_string) .filter(|slug| !slug.is_empty()) } fn attr(element: &scraper::ElementRef, names: &[&str]) -> Option { for name in names { if let Some(value) = element.value().attr(name) { let trimmed = value.trim(); if !trimmed.is_empty() && !trimmed.starts_with("data:image/") { return Some(trimmed.to_string()); } } } None } fn parse_video_cards(&self, html: &str) -> Vec { let document = Html::parse_document(html); let Ok(item_sel) = Selector::parse("div.item") else { return vec![]; }; let link_sel = Selector::parse("a.img-wrap").ok(); let views_sel = Selector::parse("li.video-item.views span").ok(); let duration_sel = Selector::parse("div.duration").ok(); let img_sel = Selector::parse("img").ok(); let source_sel = Selector::parse("picture source").ok(); let preview_sel = Selector::parse("div.preview-video").ok(); let model_sel = Selector::parse("li.video-item.model a").ok(); let mut cards = Vec::new(); let mut seen = HashSet::new(); for item in document.select(&item_sel) { let Some(link) = link_sel.as_ref().and_then(|sel| item.select(sel).next()) else { continue; }; let Some(href) = link.value().attr("href") else { continue; }; let page_url = self.normalize_url(href); let Some(id) = Self::id_from_video_url(&page_url) else { continue; }; if !seen.insert(id.clone()) { continue; } // Title: prefer the anchor's title attr, fall back to img alt. let title_raw = link .value() .attr("title") .map(str::to_string) .or_else(|| { img_sel .as_ref() .and_then(|sel| item.select(sel).next()) .and_then(|img| img.value().attr("alt").map(str::to_string)) }) .unwrap_or_default(); let title = Self::decode_html_text(&title_raw); let thumb = img_sel .as_ref() .and_then(|sel| item.select(sel).next()) .and_then(|img| Self::attr(&img, &["data-webp", "src", "data-src"])) .or_else(|| { source_sel .as_ref() .and_then(|sel| item.select(sel).next()) .and_then(|src| Self::attr(&src, &["srcset", "data-srcset"])) }) .map(|value| self.normalize_url(value.split_whitespace().next().unwrap_or(&value))) .unwrap_or_default(); let duration = duration_sel .as_ref() .and_then(|sel| item.select(sel).next()) .map(|d| d.text().collect::()) .and_then(|raw| parse_time_to_seconds(raw.trim())) .unwrap_or(0) as u32; let views = views_sel .as_ref() .and_then(|sel| item.select(sel).next()) .map(|v| v.text().collect::()) .and_then(|raw| parse_abbreviated_number(raw.trim())) .filter(|count| *count > 0); let preview = preview_sel .as_ref() .and_then(|sel| item.select(sel).next()) .and_then(|p| Self::attr(&p, &["data-src"])) .map(|raw| self.normalize_url(&raw)); let model = model_sel .as_ref() .and_then(|sel| item.select(sel).next()); let uploader = model .map(|m| Self::decode_html_text(&m.text().collect::())) .filter(|name| !name.is_empty()); let uploader_href = model.and_then(|m| m.value().attr("href").map(str::to_string)); let uploader_url = uploader_href.as_ref().map(|href| self.normalize_url(href)); let uploader_slug = uploader_href .as_deref() .and_then(Self::slug_from_model_url); cards.push(Porn4fansCard { id, title, page_url, thumb, duration, views, preview, uploader, uploader_url, uploader_slug, }); } cards } /// Pull the direct MP4 formats out of the KVS player flashvars. fn extract_formats_from_page(text: &str, referer: &str) -> Vec { let grab = |key: &str| -> Option { let re = Regex::new(&format!(r#"{}\s*:\s*'([^']+)'"#, regex::escape(key))).ok()?; re.captures(text) .and_then(|caps| caps.get(1)) .map(|m| m.as_str().to_string()) .filter(|value| !value.is_empty()) }; let mut formats = Vec::new(); let mut seen = HashSet::new(); // (flashvars url key, quality-label key, fallback label) ordered best-first. let sources = [ ("video_alt_url", "video_alt_url_text", "720p"), ("video_url", "video_url_text", "480p"), ]; for (url_key, label_key, fallback) in sources { let Some(raw_url) = grab(url_key) else { continue; }; let url = Self::normalize_media_url(&raw_url); if !seen.insert(url.clone()) { continue; } let quality = grab(label_key).unwrap_or_else(|| fallback.to_string()); let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string()) .http_header("Referer".to_string(), referer.to_string()); if let Some(height) = quality .trim_end_matches('p') .parse::() .ok() .filter(|h| *h > 0) { format = format.height(height); } formats.push(format); } formats } fn extract_tags_from_page(text: &str) -> Vec { let document = Html::parse_document(text); // The detail page swaps the class names: `.categories-row` holds tag // links and `.tags-row` holds category links. Collect both. let Ok(selector) = Selector::parse(".categories-row a, .tags-row a") else { return vec![]; }; let mut tags = Vec::new(); let mut seen = HashSet::new(); for link in document.select(&selector) { let title = Self::decode_html_text(&link.text().collect::()); if title.is_empty() { continue; } let key = title.to_ascii_lowercase(); if seen.insert(key) { tags.push(title); } } tags } async fn enrich_card( &self, card: Porn4fansCard, listing_referer: String, mut requester: Requester, ) -> VideoItem { let detail = requester .get_with_headers( &card.page_url, Self::html_headers(&listing_referer), None, ) .await .ok(); let (formats, tags) = match detail.as_deref() { Some(text) => ( Self::extract_formats_from_page(text, &self.url), Self::extract_tags_from_page(text), ), None => (vec![], vec![]), }; let mut item = VideoItem::new( card.id, card.title, card.page_url.clone(), CHANNEL_ID.to_string(), card.thumb, card.duration, ); if let Some(views) = card.views { item = item.views(views); } if let Some(uploader) = card.uploader { item = item.uploader(uploader); } if let Some(url) = card.uploader_url { item = item.uploader_url(url); } if let Some(slug) = card.uploader_slug { item.uploaderId = Some(format!("{CHANNEL_ID}:{slug}")); } if !tags.is_empty() { item = item.tags(tags); } if let Some(preview) = card.preview { item = item.preview(preview); } if !formats.is_empty() { item = item.formats(formats); } item } async fn enrich_cards( &self, cards: Vec, listing_referer: &str, requester: Requester, ) -> Vec { stream::iter(cards.into_iter().map(|card| { let requester = requester.clone(); let referer = listing_referer.to_string(); async move { self.enrich_card(card, referer, requester).await } })) .buffered(ENRICH_CONCURRENCY) .collect::>() .await } fn spawn_initial_load(&self) { let url = self.url.clone(); let categories = Arc::clone(&self.categories); let category_map = Arc::clone(&self.category_map); thread::spawn(move || { let runtime = match tokio::runtime::Builder::new_current_thread() .enable_all() .build() { Ok(runtime) => runtime, Err(error) => { report_provider_error_background( CHANNEL_ID, "spawn_initial_load.runtime_build", &error.to_string(), ); return; } }; runtime.block_on(async move { if let Err(error) = Self::load_categories(&url, categories, category_map).await { report_provider_error_background( CHANNEL_ID, "load_categories", &error.to_string(), ); } }); }); } async fn load_categories( url: &str, categories: Arc>>, category_map: Arc>>, ) -> Result<()> { let mut requester = Requester::new(); let category_url = format!("{url}/categories/"); let html = requester .get_with_headers(&category_url, Self::html_headers(url), None) .await .map_err(|error| Error::from(format!("category fetch failed: {error}")))?; let document = Html::parse_document(&html); let selector = Selector::parse("#list_categories_categories_list_items a.item") .map_err(|error| Error::from(format!("category selector failed: {error}")))?; let mut options = vec![FilterOption { id: "all".to_string(), title: "All".to_string(), }]; let mut map = HashMap::new(); for link in document.select(&selector) { let Some(href) = link.value().attr("href") else { continue; }; let Some(slug) = href .split("/categories/") .nth(1) .and_then(|rest| rest.split('/').next()) .filter(|slug| !slug.is_empty()) else { continue; }; let title = link .value() .attr("title") .map(Self::decode_html_text) .filter(|title| !title.is_empty()) .unwrap_or_else(|| Self::decode_html_text(&link.text().collect::())); if title.is_empty() { continue; } let normalized = Self::normalize_title(&title); if normalized.is_empty() || map.contains_key(&normalized) { continue; } options.push(FilterOption { id: normalized.clone(), title, }); map.insert(normalized, slug.to_string()); } if options.len() > 1 { if let Ok(mut guard) = categories.write() { *guard = options; } if let Ok(mut guard) = category_map.write() { *guard = map; } } Ok(()) } } #[async_trait] impl Provider for Porn4fansProvider { async fn get_videos( &self, cache: VideoCache, _pool: DbPool, sort: String, query: Option, page: String, _per_page: String, options: ServerOptions, ) -> Vec { let page = page.parse::().unwrap_or(1).max(1); let target = self.target_from_request(query.as_deref(), options.categories.as_deref()); match self .get_videos_for_target(cache, &target, &sort, page, options) .await { Ok(videos) => videos, Err(e) => { report_provider_error( CHANNEL_ID, "get_videos", &format!("page={page}; target={target:?}; error={e}"), ) .await; vec![] } } } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } } #[cfg(test)] mod tests { use super::*; #[test] fn slugify_collapses_separators() { assert_eq!(Porn4fansProvider::slugify("Big Black Cock"), "big-black-cock"); assert_eq!(Porn4fansProvider::slugify(" big__ass "), "big-ass"); } #[test] fn builds_latest_block_url() { let provider = Porn4fansProvider::new(); let (url, referer) = provider.build_block_url(&Target::Latest, "new", 2); assert_eq!(referer, "https://www.porn4fans.com/onlyfans-videos/"); assert_eq!( url, "https://www.porn4fans.com/onlyfans-videos/?mode=async&function=get_block&block_id=custom_list_videos_latest_videos_list&sort_by=post_date&from=2" ); } #[test] fn builds_search_block_url() { let provider = Porn4fansProvider::new(); let (url, referer) = provider.build_block_url( &Target::Search { query: "big ass".to_string(), }, "popular", 3, ); assert_eq!(referer, "https://www.porn4fans.com/search/big-ass/"); assert_eq!( url, "https://www.porn4fans.com/search/big-ass/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search_result&q=big+ass&category_ids=&sort_by=video_viewed&from_videos=3" ); } #[test] fn category_block_url_uses_common_list() { let provider = Porn4fansProvider::new(); let (url, _) = provider.build_block_url( &Target::Category { slug: "asian".to_string(), }, "new", 1, ); assert_eq!( url, "https://www.porn4fans.com/categories/asian/?mode=async&function=get_block&block_id=custom_list_videos_common_videos_list&sort_by=post_date&from=1" ); } #[test] fn normalizes_kvs_media_url() { assert_eq!( Porn4fansProvider::normalize_media_url( "https://www.porn4fans.com/get_file/4/abc/10000/10002/10002.mp4/?v-acctoken=zzz" ), "https://www.porn4fans.com/get_file/4/abc/10000/10002/10002.mp4?v-acctoken=zzz" ); assert_eq!( Porn4fansProvider::normalize_media_url( "https://www.porn4fans.com/get_file/4/abc/10000/10002/10002_720p.mp4/" ), "https://www.porn4fans.com/get_file/4/abc/10000/10002/10002_720p.mp4" ); } #[test] fn extracts_formats_from_flashvars() { let html = r#" "#; let formats = Porn4fansProvider::extract_formats_from_page(html, "https://www.porn4fans.com/"); assert_eq!(formats.len(), 2); // best-first: 720p before 480p assert!(formats[0].url.ends_with("10002_720p.mp4?v-acctoken=b")); assert!(formats[1].url.ends_with("10002.mp4?v-acctoken=a")); } #[test] fn parses_async_fragment_card() { let provider = Porn4fansProvider::new(); let html = r##" "##; let cards = provider.parse_video_cards(html); assert_eq!(cards.len(), 1); let card = &cards[0]; assert_eq!(card.id, "10002"); assert_eq!( card.page_url, "https://www.porn4fans.com/video/10002/blasianflexcouple-gets-banged-against-the-wall/" ); assert_eq!( card.thumb, "https://www.porn4fans.com/contents/videos_screenshots/10000/10002/800x450/1.jpg" ); assert_eq!(card.duration, 1477); assert_eq!(card.views, Some(103)); assert_eq!(card.uploader.as_deref(), Some("Nina Lee")); assert_eq!(card.uploader_slug.as_deref(), Some("nina-lee")); assert!(card.preview.is_some()); } }