diff --git a/build.rs b/build.rs index 4c15bc8..8d0e856 100644 --- a/build.rs +++ b/build.rs @@ -214,6 +214,11 @@ const PROVIDERS: &[ProviderDef] = &[ module: "freepornvideosxxx", ty: "FreepornvideosxxxProvider", }, + ProviderDef { + id: "freeuseporn", + module: "freeuseporn", + ty: "FreeusepornProvider", + }, ProviderDef { id: "heavyfetish", module: "heavyfetish", diff --git a/src/api.rs b/src/api.rs index c2c0936..27565b1 100644 --- a/src/api.rs +++ b/src/api.rs @@ -554,9 +554,8 @@ async fn uploaders_post( let trace_id = crate::util::flow_debug::next_trace_id("uploaders"); let request = uploader_request.into_inner().normalized(); if !uploader_request_is_valid(&request) { - return Ok(web::HttpResponse::BadRequest().body( - "At least one of uploaderId or uploaderName must be provided", - )); + return Ok(web::HttpResponse::BadRequest() + .body("At least one of uploaderId or uploaderName must be provided")); } let public_url_base = format!( diff --git a/src/providers/freeuseporn.rs b/src/providers/freeuseporn.rs new file mode 100644 index 0000000..84e7c5f --- /dev/null +++ b/src/providers/freeuseporn.rs @@ -0,0 +1,562 @@ +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::{Provider, report_provider_error, requester_or_default}; +use crate::status::*; +use crate::util::cache::VideoCache; +use crate::util::parse_abbreviated_number; +use crate::util::time::parse_time_to_seconds; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; +use async_trait::async_trait; +use error_chain::error_chain; +use htmlentity::entity::{ICodedDataTrait, decode}; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; +use scraper::{Html, Selector}; +use std::collections::HashSet; +use std::vec; + +pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = + crate::providers::ProviderChannelMetadata { + group_id: "fetish-kink", + tags: &["freeuse", "hypno", "mind-control"], + }; + +error_chain! { + foreign_links { + Io(std::io::Error); + HttpRequest(wreq::Error); + } +} + +#[derive(Debug, Clone)] +pub struct FreeusepornProvider { + url: String, +} + +impl FreeusepornProvider { + pub fn new() -> Self { + Self { + url: "https://www.freeuseporn.com".to_string(), + } + } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "freeuseporn".to_string(), + name: "FreeusePorn".to_string(), + description: "FreeusePorn streams freeuse, hypno, mind control, ignored sex, and related fetish videos.".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=freeuseporn.com".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ + ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "recent".to_string(), + title: "Most Recent".to_string(), + }, + FilterOption { + id: "viewed".to_string(), + title: "Most Viewed".to_string(), + }, + FilterOption { + id: "rated".to_string(), + title: "Top Rated".to_string(), + }, + FilterOption { + id: "favorites".to_string(), + title: "Top Favorites".to_string(), + }, + FilterOption { + id: "watched".to_string(), + title: "Being Watched".to_string(), + }, + ], + multiSelect: false, + }, + ChannelOption { + id: "category".to_string(), + title: "Category".to_string(), + description: "Filter by category".to_string(), + systemImage: "square.grid.2x2".to_string(), + colorName: "orange".to_string(), + options: vec![ + FilterOption { + id: "all".to_string(), + title: "All".to_string(), + }, + FilterOption { + id: "mind-control".to_string(), + title: "Mind Control".to_string(), + }, + FilterOption { + id: "general-freeuse".to_string(), + title: "General Freeuse".to_string(), + }, + FilterOption { + id: "free-service".to_string(), + title: "Free Service".to_string(), + }, + FilterOption { + id: "forced".to_string(), + title: "Forced".to_string(), + }, + FilterOption { + id: "japanese".to_string(), + title: "Japanese".to_string(), + }, + FilterOption { + id: "time-stop".to_string(), + title: "Time Stop".to_string(), + }, + FilterOption { + id: "ignored-sex".to_string(), + title: "Ignored Sex".to_string(), + }, + FilterOption { + id: "glory-hole".to_string(), + title: "Glory Hole".to_string(), + }, + ], + multiSelect: false, + }, + ], + nsfw: true, + cacheDuration: Some(1800), + } + } + + fn absolute_url(&self, url: &str) -> String { + if url.starts_with("http://") || url.starts_with("https://") { + url.to_string() + } else if url.starts_with('/') { + format!("{}{}", self.url, url) + } else { + format!("{}/{}", self.url, url.trim_start_matches('/')) + } + } + + fn sort_param(sort: &str) -> &'static str { + match sort { + "viewed" => "mv", + "rated" => "tr", + "favorites" => "tf", + "watched" => "bw", + _ => "mr", + } + } + + fn build_list_url( + &self, + sort: &str, + page: u8, + query: Option<&str>, + category: Option<&str>, + ) -> String { + let path = if let Some(query) = query.map(str::trim).filter(|value| !value.is_empty()) { + format!( + "/search/videos/{}", + utf8_percent_encode(query, NON_ALPHANUMERIC) + ) + } else if let Some(category) = category + .map(str::trim) + .filter(|value| !value.is_empty() && *value != "all") + { + format!("/videos/{}", category) + } else { + "/videos".to_string() + }; + + let mut params = vec![format!("o={}", Self::sort_param(sort))]; + if page > 1 { + params.push(format!("page={page}")); + } + + format!("{}{}?{}", self.url, path, params.join("&")) + } + + fn build_formats(&self, id: &str) -> Vec { + let hd = VideoFormat::new( + format!("{}/media/videos/h264/{}_720p.mp4", self.url, id), + "720p".to_string(), + "video/mp4".to_string(), + ) + .format_id("720p".to_string()) + .format_note("720p".to_string()); + let sd = VideoFormat::new( + format!("{}/media/videos/h264/{}_480p.mp4", self.url, id), + "480p".to_string(), + "video/mp4".to_string(), + ) + .format_id("480p".to_string()) + .format_note("480p".to_string()); + vec![hd, sd] + } + + fn normalized_text(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") + } + + fn decode_text(value: &str) -> String { + decode(value.as_bytes()) + .to_string() + .unwrap_or_else(|_| value.to_string()) + } + + fn parse_views(value: &str) -> Option { + let digits = value + .chars() + .filter(|character| character.is_ascii_digit() || *character == '.' || *character == 'K' || *character == 'M' || *character == 'B' || *character == 'k' || *character == 'm' || *character == 'b') + .collect::(); + if digits.is_empty() { + return None; + } + parse_abbreviated_number(&digits).map(|views| views as u32) + } + + fn parse_rating(value: &str) -> Option { + value + .trim() + .trim_end_matches('%') + .parse::() + .ok() + } + + fn parse_video_item_from_anchor( + &self, + anchor: scraper::ElementRef<'_>, + selectors: &FreeusepornSelectors, + ) -> Option { + let href = anchor.value().attr("href")?; + if !href.contains("/video/") { + return None; + } + + let absolute_url = self.absolute_url(href); + let id = absolute_url.split('/').nth(4)?.to_string(); + if id.is_empty() { + return None; + } + + let title_raw = anchor + .select(&selectors.title) + .next() + .map(|element| Self::normalized_text(&element.text().collect::>().join(" "))) + .filter(|value| !value.is_empty()) + .or_else(|| anchor.value().attr("title").map(Self::normalized_text)) + .or_else(|| { + anchor + .select(&selectors.image) + .next() + .and_then(|element| element.value().attr("alt")) + .map(Self::normalized_text) + })?; + let title = Self::decode_text(&title_raw); + + let thumb = anchor + .select(&selectors.image) + .next() + .and_then(|element| element.value().attr("src")) + .map(|src| self.absolute_url(src)) + .unwrap_or_default(); + + let duration = anchor + .select(&selectors.duration) + .next() + .map(|element| Self::normalized_text(&element.text().collect::>().join(" "))) + .and_then(|value| parse_time_to_seconds(&value)) + .unwrap_or(0) as u32; + + let mut stats = anchor + .select(&selectors.video_stat) + .map(|element| Self::normalized_text(&element.text().collect::>().join(" "))) + .collect::>(); + stats.retain(|value| !value.is_empty()); + let views = stats.first().and_then(|value| Self::parse_views(value)); + let rating = stats.get(1).and_then(|value| Self::parse_rating(value)); + + let mut item = VideoItem::new( + id.clone(), + title, + absolute_url, + "freeuseporn".to_string(), + thumb, + duration, + ) + .views(views.unwrap_or(0)) + .formats(self.build_formats(&id)); + + if views.is_none() { + item.views = None; + } + item.rating = rating; + + Some(item) + } + + fn get_video_items_from_html(&self, html: &str) -> Vec { + if html.trim().is_empty() { + return vec![]; + } + + let document = Html::parse_document(html); + let selectors = FreeusepornSelectors::new(); + let primary_anchors = document + .select(&selectors.list_anchor) + .collect::>(); + let anchors = if primary_anchors.is_empty() { + document + .select(&selectors.fallback_anchor) + .collect::>() + } else { + primary_anchors + }; + + let mut seen = HashSet::new(); + let mut items = Vec::new(); + + for anchor in anchors { + let Some(item) = self.parse_video_item_from_anchor(anchor, &selectors) else { + continue; + }; + if seen.insert(item.id.clone()) { + items.push(item); + } + } + + items + } + + async fn fetch_listing( + &self, + cache: VideoCache, + url: String, + options: ServerOptions, + error_context: &str, + ) -> Result> { + let old_items = match cache.get(&url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } + items.clone() + } + None => vec![], + }; + + let mut requester = requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&url, None).await { + Ok(text) => text, + Err(error) => { + report_provider_error( + "freeuseporn", + error_context, + &format!("url={url}; error={error}"), + ) + .await; + return Ok(old_items); + } + }; + + let items = self.get_video_items_from_html(&text); + if items.is_empty() { + return Ok(old_items); + } + + cache.remove(&url); + cache.insert(url, items.clone()); + Ok(items) + } + + async fn get( + &self, + cache: VideoCache, + page: u8, + sort: &str, + options: ServerOptions, + ) -> Result> { + let url = self.build_list_url(sort, page, None, options.category.as_deref()); + self.fetch_listing(cache, url, options, "get.request").await + } + + async fn query( + &self, + cache: VideoCache, + page: u8, + query: &str, + sort: &str, + options: ServerOptions, + ) -> Result> { + let url = self.build_list_url(sort, page, Some(query), None); + self.fetch_listing(cache, url, options, "query.request").await + } +} + +struct FreeusepornSelectors { + list_anchor: Selector, + fallback_anchor: Selector, + title: Selector, + image: Selector, + duration: Selector, + video_stat: Selector, +} + +impl FreeusepornSelectors { + fn new() -> Self { + Self { + list_anchor: Selector::parse("#videos-list a[href]").expect("valid freeuseporn list selector"), + fallback_anchor: Selector::parse("a[href]").expect("valid freeuseporn fallback selector"), + title: Selector::parse(".v-name").expect("valid freeuseporn title selector"), + image: Selector::parse("img").expect("valid freeuseporn image selector"), + duration: Selector::parse(".duration").expect("valid freeuseporn duration selector"), + video_stat: Selector::parse(".video-stats li").expect("valid freeuseporn stats selector"), + } + } +} + +#[async_trait] +impl Provider for FreeusepornProvider { + async fn get_videos( + &self, + cache: VideoCache, + _pool: DbPool, + sort: String, + query: Option, + page: String, + _per_page: String, + options: ServerOptions, + ) -> Vec { + let page = page.parse::().unwrap_or(1); + let videos = match query { + Some(query) => self.query(cache, page, &query, &sort, options).await, + None => self.get(cache, page, &sort, options).await, + }; + + match videos { + Ok(items) => items, + Err(error) => { + eprintln!("freeuseporn provider error: {error}"); + vec![] + } + } + } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn provider() -> FreeusepornProvider { + FreeusepornProvider::new() + } + + #[test] + fn builds_listing_urls_for_sort_category_and_search() { + let provider = provider(); + + assert_eq!( + provider.build_list_url("recent", 1, None, None), + "https://www.freeuseporn.com/videos?o=mr" + ); + assert_eq!( + provider.build_list_url("viewed", 2, None, Some("mind-control")), + "https://www.freeuseporn.com/videos/mind-control?o=mv&page=2" + ); + assert_eq!( + provider.build_list_url("favorites", 3, Some("mind control"), None), + "https://www.freeuseporn.com/search/videos/mind%20control?o=tf&page=3" + ); + } + + #[test] + fn parses_listing_items_and_builds_formats() { + let provider = provider(); + let html = r#" + + "#; + + let items = provider.get_video_items_from_html(html); + assert_eq!(items.len(), 2); + assert_eq!(items[0].id, "9579"); + assert_eq!(items[0].title, "Nicole Kitt & The Truth"); + assert_eq!( + items[0].url, + "https://www.freeuseporn.com/video/9579/nicole-kitt-shady-slut-keeps-confessing" + ); + assert_eq!( + items[0].thumb, + "https://www.freeuseporn.com/media/videos/tmb/9579/1.jpg" + ); + assert_eq!(items[0].duration, 3549); + assert_eq!(items[0].views, Some(52180)); + assert_eq!(items[0].rating, Some(100.0)); + assert_eq!(items[0].formats.as_ref().map(|formats| formats.len()), Some(2)); + assert_eq!( + items[0] + .formats + .as_ref() + .and_then(|formats| formats.first()) + .map(|format| format.url.as_str()), + Some("https://www.freeuseporn.com/media/videos/h264/9579_720p.mp4") + ); + assert_eq!(items[1].id, "9578"); + assert_eq!(items[1].rating, Some(88.0)); + } +} diff --git a/src/providers/mod.rs b/src/providers/mod.rs index fcbad6c..2c99d47 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -597,7 +597,11 @@ where "provider uploader guard exit provider={} context={} matched={}", provider_name, context, - result.as_ref().ok().and_then(|value| value.as_ref()).is_some() + result + .as_ref() + .ok() + .and_then(|value| value.as_ref()) + .is_some() ); result } @@ -1262,7 +1266,8 @@ mod tests { let now = Instant::now(); let mut counted = 0; for step in 0..VALIDATION_FAILURES_FOR_ERROR { - counted = record_validation_failure(provider_id, now + VALIDATION_COOLDOWN * step as u32); + counted = + record_validation_failure(provider_id, now + VALIDATION_COOLDOWN * step as u32); } assert_eq!(counted, VALIDATION_FAILURES_FOR_ERROR); diff --git a/src/proxies/doodstream.rs b/src/proxies/doodstream.rs index 37cd045..c36bc35 100644 --- a/src/proxies/doodstream.rs +++ b/src/proxies/doodstream.rs @@ -60,9 +60,13 @@ impl DoodstreamProxy { } fn request_headers(detail_url: &str) -> Vec<(String, String)> { - let origin = Self::request_origin(detail_url).unwrap_or_else(|| "https://turboplayers.xyz".to_string()); + let origin = Self::request_origin(detail_url) + .unwrap_or_else(|| "https://turboplayers.xyz".to_string()); vec![ - ("Referer".to_string(), format!("{}/", origin.trim_end_matches('/'))), + ( + "Referer".to_string(), + format!("{}/", origin.trim_end_matches('/')), + ), ("Origin".to_string(), origin), ( "Accept".to_string(), @@ -224,9 +228,11 @@ impl DoodstreamProxy { fn extract_pass_md5_url(text: &str, detail_url: &str) -> Option { let decoded = text.replace("\\/", "/"); - let absolute_regex = - Self::regex(r#"https?://[^\s"'<>]+/pass_md5/[^\s"'<>]+"#)?; - if let Some(url) = absolute_regex.find(&decoded).map(|value| value.as_str().to_string()) { + let absolute_regex = Self::regex(r#"https?://[^\s"'<>]+/pass_md5/[^\s"'<>]+"#)?; + if let Some(url) = absolute_regex + .find(&decoded) + .map(|value| value.as_str().to_string()) + { return Some(url); } @@ -276,7 +282,8 @@ impl DoodstreamProxy { requester: &mut Requester, ) -> Option { let pass_md5_url = Self::extract_pass_md5_url(html, detail_url).or_else(|| { - Self::unpack_packer(html).and_then(|unpacked| Self::extract_pass_md5_url(&unpacked, detail_url)) + Self::unpack_packer(html) + .and_then(|unpacked| Self::extract_pass_md5_url(&unpacked, detail_url)) })?; let headers = vec![ @@ -311,7 +318,9 @@ impl crate::proxies::Proxy for DoodstreamProxy { return url; } - if let Some(url) = Self::resolve_stream_from_pass_md5(&detail_url, &html, &mut requester).await { + if let Some(url) = + Self::resolve_stream_from_pass_md5(&detail_url, &html, &mut requester).await + { return url; } @@ -370,7 +379,8 @@ mod tests { #[test] fn composes_media_url_from_pass_md5_response() { - let pass_md5_url = "https://trailerhg.xyz/pass_md5/abc123/def456?token=t0k3n&expiry=1775000000"; + let pass_md5_url = + "https://trailerhg.xyz/pass_md5/abc123/def456?token=t0k3n&expiry=1775000000"; let body = "https://g4vsrqvtrj.pinebrookproductionlab.shop/1ghkpx2e8jnal/hls3/01/08534/syyzvotfnhaa_l/master.txt"; assert_eq!( DoodstreamProxy::compose_pass_md5_media_url(pass_md5_url, body).as_deref(),