diff --git a/Cargo.toml b/Cargo.toml index 022db2a..d0aee5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ titlecase = "3.6.0" dashmap = "6.1.0" lru = "0.16.3" rand = "0.10.0" +chrono = "0.4.44" [lints.rust] unexpected_cfgs = "allow" diff --git a/src/api.rs b/src/api.rs index 08b0420..0fc3b39 100644 --- a/src/api.rs +++ b/src/api.rs @@ -244,6 +244,7 @@ async fn videos_post( .unwrap_or("") .to_string(); let duration = video_request.duration.as_deref().unwrap_or("").to_string(); + let sexuality = video_request.sexuality.as_deref().unwrap_or("").to_string(); let options = ServerOptions { featured: Some(featured), category: Some(category), @@ -256,6 +257,7 @@ async fn videos_post( categories: Some(categories), duration: Some(duration), sort: Some(sort.clone()), + sexuality: Some(sexuality), }; let mut video_items = run_provider_guarded( &channel, diff --git a/src/providers/xfree.rs b/src/providers/xfree.rs index e886484..b9e0456 100644 --- a/src/providers/xfree.rs +++ b/src/providers/xfree.rs @@ -1,692 +1,283 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{Provider, report_provider_error_background, requester_or_default}; +use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::discord::send_discord_error_report; -use crate::util::parse_abbreviated_number; -use crate::util::time::parse_time_to_seconds; -use crate::videos::{ServerOptions, VideoFormat, VideoItem}; +use crate::util::requester::Requester; +use crate::videos::{ServerOptions, VideoItem}; + use async_trait::async_trait; use error_chain::error_chain; -use futures::stream::{FuturesUnordered, StreamExt}; -use htmlentity::entity::{ICodedDataTrait, decode}; -use regex::Regex; -use std::collections::HashSet; -use std::fmt::Write; +use std::sync::{Arc, RwLock}; use std::vec; -use url::form_urlencoded::{Serializer, parse}; +use wreq::Version; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); + Json(serde_json::Error); + } + errors { + Parse(msg: String) { + description("parse error") + display("parse error: {}", msg) + } } } #[derive(Debug, Clone)] pub struct XfreeProvider { url: String, -} - -#[derive(Debug, Clone)] -struct RawListingItem { - id: String, - title: String, - detail_url: String, - thumb: String, - duration: u32, - views: Option, - uploader: Option, - tags: Vec, + categories: Arc>>, } impl XfreeProvider { pub fn new() -> Self { - Self { + let provider = Self { url: "https://www.xfree.com".to_string(), - } + categories: Arc::new(RwLock::new(vec![])), + }; + provider } - fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; Channel { id: "xfree".to_string(), name: "XFree".to_string(), - description: "Short NSFW clips from xfree.com".to_string(), + description: "Reels & Nudes!".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=xfree.com".to_string(), status: "active".to_string(), - categories: vec![ - "all".to_string(), - "straight".to_string(), - "gay".to_string(), - "trans".to_string(), - ], + categories: self + .categories + .read() + .map(|categories| categories.iter().map(|c| c.title.clone()).collect()) + .unwrap_or_else(|e| { + crate::providers::report_provider_error_background( + "xfree", + "build_channel.categories_read", + &e.to_string(), + ); + vec![] + }), options: vec![ ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort listing preference".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "trending".to_string(), - title: "Trending".to_string(), - }, - FilterOption { - id: "latest".to_string(), - title: "Latest".to_string(), - }, - ], + id: "sexuality".to_string(), + title: "Sexuality".to_string(), + description: "Sexuality of the Videos".to_string(), + systemImage: "heart".to_string(), + colorName: "red".to_string(), multiSelect: false, - }, - ChannelOption { - id: "category".to_string(), - title: "Category".to_string(), - description: "Audience/category feed".to_string(), - systemImage: "line.horizontal.3.decrease.circle".to_string(), - colorName: "green".to_string(), options: vec![ FilterOption { - id: "all".to_string(), - title: "All".to_string(), - }, - FilterOption { - id: "straight".to_string(), + id: "1".to_string(), title: "Straight".to_string(), }, FilterOption { - id: "gay".to_string(), + id: "2".to_string(), title: "Gay".to_string(), }, FilterOption { - id: "trans".to_string(), + id: "3".to_string(), title: "Trans".to_string(), }, ], - multiSelect: false, + }, ], nsfw: true, - cacheDuration: Some(300), + cacheDuration: None, } } - fn normalize_ws(input: &str) -> String { - input.split_whitespace().collect::>().join(" ") - } - - fn decode_html(input: &str) -> String { - decode(input.as_bytes()) - .to_string() - .unwrap_or_else(|_| input.to_string()) - } - - fn clean_media_url(raw: &str) -> String { - let mut out = raw - .trim_matches(|c: char| c == '"' || c == '\'' || c == '\\' || c.is_whitespace()) - .to_string(); - out = out - .replace("\\u0026", "&") - .replace("\\u002F", "/") - .replace("\\/", "/") - .replace("&", "&"); - out = out - .trim_end_matches(|c: char| matches!(c, ',' | ';' | ')' | ']' | '}')) - .to_string(); - if out.starts_with("//") { - return format!("https:{out}"); - } - out - } - - fn is_downloadable_media_url(url: &str) -> bool { - let lower = url.to_ascii_lowercase(); - (lower.starts_with("http://") || lower.starts_with("https://")) - && (lower.contains(".mp4") || lower.contains(".m3u8")) - } - - fn absolute_url(&self, path: &str) -> String { - if path.starts_with("http://") || path.starts_with("https://") { - return path.to_string(); - } - if path.starts_with("//") { - return format!("https:{path}"); - } - if path.starts_with('/') { - return format!("{}{}", self.url, path); - } - format!("{}/{}", self.url, path.trim_start_matches('/')) - } - - fn encode_query_value(value: &str) -> String { - let mut serializer = Serializer::new(String::new()); - serializer.append_pair("q", value); - let encoded = serializer.finish(); - encoded.strip_prefix("q=").unwrap_or(&encoded).to_string() - } - - fn category_value(options: &ServerOptions) -> String { - options - .category - .clone() - .unwrap_or_else(|| "all".to_string()) - .to_ascii_lowercase() - } - - fn sort_value(options: &ServerOptions) -> String { - options - .sort - .clone() - .unwrap_or_else(|| "trending".to_string()) - .to_ascii_lowercase() - } - - fn category_suffix(category: &str) -> Option<&'static str> { - match category { - "gay" => Some("gay"), - "trans" => Some("trans"), - "straight" => Some("straight"), - _ => None, - } - } - - fn with_page(mut url: String, page: u8) -> String { - if page <= 1 { - return url; - } - if url.contains('?') { - url.push_str(&format!("&page={page}")); - } else { - url.push_str(&format!("?page={page}")); - } - url - } - - fn build_listing_urls(&self, page: u8, query: &str, options: &ServerOptions) -> Vec { - let category = Self::category_value(options); - let sort = Self::sort_value(options); - let encoded_query = Self::encode_query_value(query.trim()); - let category_suffix = Self::category_suffix(&category); - let mut urls = Vec::new(); - - if !query.trim().is_empty() { - if let Some(suffix) = category_suffix { - urls.push(Self::with_page( - format!("{}/search-{suffix}?q={encoded_query}", self.url), - page, - )); - } - urls.push(Self::with_page( - format!("{}/search?q={encoded_query}", self.url), - page, - )); - return urls; - } - - let base_category_url = match category_suffix { - Some(suffix) => format!("{}/{}", self.url, suffix), - None => self.url.clone(), - }; - - if sort == "latest" { - urls.push(Self::with_page( - format!("{}/latest", base_category_url), - page, - )); - urls.push(Self::with_page( - format!("{base_category_url}?sort=latest"), - page, - )); - } - urls.push(Self::with_page(base_category_url, page)); - - urls - } - - fn extract_href_param(href: &str, key: &str) -> Option { - let query = href.split('?').nth(1)?; - for (k, v) in parse(query.as_bytes()) { - if k == key { - return Some(v.into_owned()); + fn push_unique(target: &Arc>>, item: FilterOption) { + if let Ok(mut vec) = target.write() { + if !vec.iter().any(|x| x.id == item.id) { + vec.push(item); } } - None } - fn strip_html_tags(text: &str) -> String { - let Ok(tags_re) = Regex::new(r"(?is)<[^>]+>") else { - return text.to_string(); - }; - tags_re.replace_all(text, " ").to_string() - } - - fn extract_duration_seconds(text: &str) -> Option { - let Ok(duration_re) = Regex::new(r"\b(\d{1,2}:\d{2}(?::\d{2})?)\b") else { - return None; - }; - if let Some(caps) = duration_re.captures(text) { - if let Some(raw) = caps.get(1) { - return parse_time_to_seconds(raw.as_str()).map(|v| v as u32); - } - } - - let Ok(iso_re) = Regex::new(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?") else { - return None; - }; - let caps = iso_re.captures(text)?; - let h = caps - .get(1) - .and_then(|m| m.as_str().parse::().ok()) - .unwrap_or(0); - let m = caps - .get(2) - .and_then(|m| m.as_str().parse::().ok()) - .unwrap_or(0); - let s = caps - .get(3) - .and_then(|m| m.as_str().parse::().ok()) - .unwrap_or(0); - let total = h.saturating_mul(3600) + m.saturating_mul(60) + s; - if total > 0 { Some(total) } else { None } - } - - fn extract_views(text: &str) -> Option { - let patterns = [ - r#"(?is)(?:class=["'][^"']*views[^"']*["'][^>]*>|data-views=["']?)([0-9]+(?:\.[0-9]+)?\s*[kmb]?)"#, - r#"(?is)(?:fa-eye|icon-eye|eye[^>]*>)[^0-9]{0,20}([0-9]+(?:\.[0-9]+)?\s*[kmb]?)"#, - r#"(?i)\b([0-9]+(?:\.[0-9]+)?\s*[kmb])\s*(?:views?|view)\b"#, - ]; - - for pattern in patterns { - let Ok(re) = Regex::new(pattern) else { - continue; - }; - let Some(raw) = re - .captures(text) - .and_then(|c| c.get(1)) - .map(|m| m.as_str().to_string()) - else { - continue; - }; - let Some(parsed) = parse_abbreviated_number(&raw) else { - continue; - }; - let has_suffix = raw - .chars() - .any(|c| matches!(c, 'k' | 'K' | 'm' | 'M' | 'b' | 'B')); - if has_suffix || parsed >= 100 { - return Some(parsed); - } - } - - None - } - - fn extract_tags(text: &str) -> Vec { - let Ok(tag_re) = Regex::new(r"#([A-Za-z0-9_]+)") else { - return vec![]; - }; - let mut seen = HashSet::new(); - let mut tags = vec![]; - for caps in tag_re.captures_iter(text) { - let Some(raw) = caps.get(1).map(|m| m.as_str()) else { - continue; - }; - let tag = raw.to_ascii_lowercase(); - if seen.insert(tag.clone()) { - tags.push(tag); - } - } - tags - } - - fn extract_thumb_from_segment(&self, segment: &str) -> String { - let Ok(thumb_re) = Regex::new( - r#"(?is)(https?://[^"' <]*(?:thumbs|peek|prbn)\.xfree\.com[^"' <]*\.(?:jpg|jpeg|png|webp))"#, - ) else { - return String::new(); - }; - if let Some(m) = thumb_re.captures(segment).and_then(|c| c.get(1)) { - return m.as_str().to_string(); - } - - let Ok(img_attr_re) = Regex::new(r#"(?is)(?:src|data-src|data-original)="([^"]+)""#) else { - return String::new(); - }; - if let Some(m) = img_attr_re.captures(segment).and_then(|c| c.get(1)) { - return self.absolute_url(m.as_str()); - } - - String::new() - } - - fn extract_quality_from_url(url: &str) -> String { - let Ok(q_re) = Regex::new(r"(?i)(\d{3,4})p") else { - return "1080".to_string(); - }; - if let Some(q) = q_re.captures(url).and_then(|c| c.get(1)) { - return q.as_str().to_string(); - } - if url.to_ascii_lowercase().contains(".m3u8") { - return "hls".to_string(); - } - "1080".to_string() - } - - fn parse_listing_items(&self, html: &str) -> Vec { - if html.trim().is_empty() { - return vec![]; - } - let Ok(link_re) = Regex::new( - r#"(?is)]+href="(?P/(?:video(?:-[a-z]+)?\?id=\d+[^"]*))"[^>]*>(?P.*?)"#, - ) else { - return vec![]; - }; - let Ok(title_attr_re) = Regex::new(r#"(?is)\btitle="([^"]+)""#) else { - return vec![]; - }; - let Ok(uploader_re) = - Regex::new(r#"(?is)href="/(?:u|user|profile)/[^"]+"[^>]*>\s*([^<]{2,64})\s*<"#) - else { - return vec![]; - }; - - let mut items = vec![]; - let mut seen_ids = HashSet::new(); - - for caps in link_re.captures_iter(html) { - let Some(full) = caps.get(0) else { - continue; - }; - let href = caps.name("href").map(|m| m.as_str()).unwrap_or(""); - let body = caps.name("body").map(|m| m.as_str()).unwrap_or(""); - let Some(id) = Self::extract_href_param(href, "id") else { - continue; - }; - if !seen_ids.insert(id.clone()) { - continue; - } - - let seg_start = full.start().saturating_sub(400); - let seg_end = (full.end() + 700).min(html.len()); - let segment = html.get(seg_start..seg_end).unwrap_or(full.as_str()); - - let title_from_attr = title_attr_re - .captures(full.as_str()) - .and_then(|c| c.get(1)) - .map(|m| m.as_str().to_string()) - .unwrap_or_default(); - let title_from_body = Self::strip_html_tags(body); - let title_from_href = Self::extract_href_param(href, "title") - .map(|s| s.replace('-', " ")) - .unwrap_or_default(); - let title = Self::normalize_ws(&Self::decode_html(if !title_from_attr.is_empty() { - &title_from_attr - } else if !title_from_body.trim().is_empty() { - &title_from_body - } else { - &title_from_href - })); - if title.is_empty() { - continue; - } - - let thumb = self.extract_thumb_from_segment(segment); - let duration = Self::extract_duration_seconds(segment).unwrap_or(0); - let views = Self::extract_views(segment); - let uploader = uploader_re - .captures(segment) - .and_then(|c| c.get(1)) - .map(|m| Self::normalize_ws(m.as_str())) - .filter(|s| !s.is_empty()); - let tags = Self::extract_tags(segment); - - items.push(RawListingItem { - id, - title, - detail_url: self.absolute_url(href), - thumb, - duration, - views, - uploader, - tags, - }); - } - - items - } - - fn extract_media_urls(&self, html: &str) -> Vec { - let mut urls = vec![]; - let mut seen = HashSet::new(); - - let patterns = [ - r#"https?:\\?/\\?/[^"' <>\s]+?\.(?:mp4|m3u8)[^"' <>\s]*"#, - r#"https?://[^"' <>\s]+?\.(?:mp4|m3u8)[^"' <>\s]*"#, - ]; - - for pattern in patterns { - let Ok(re) = Regex::new(pattern) else { - continue; - }; - for m in re.find_iter(html) { - let cleaned = Self::clean_media_url(m.as_str()); - if !Self::is_downloadable_media_url(&cleaned) { - continue; - } - if seen.insert(cleaned.clone()) { - urls.push(cleaned); - } - } - } - - urls - } - - fn extract_detail_tags(html: &str) -> Vec { - let Ok(tag_link_re) = Regex::new(r#"(?is)href="/tag(?:-[a-z]+)?/([^"?#]+)"#) else { - return vec![]; - }; - let mut seen = HashSet::new(); - let mut tags = vec![]; - for caps in tag_link_re.captures_iter(html) { - let Some(raw) = caps.get(1).map(|m| m.as_str()) else { - continue; - }; - let tag = raw - .replace('-', " ") - .replace("%20", " ") - .trim() - .to_ascii_lowercase(); - if tag.is_empty() { - continue; - } - if seen.insert(tag.clone()) { - tags.push(tag); - } - } - tags - } - - fn extract_detail_thumb(&self, html: &str) -> String { - self.extract_thumb_from_segment(html) - } - - async fn fetch_detailed_video_item( - &self, - raw: RawListingItem, - mut requester: crate::util::requester::Requester, - ) -> Option { - let detail_html = match requester.get(&raw.detail_url, None).await { - Ok(t) => t, - Err(e) => { - report_provider_error_background( - "xfree", - "detail.request", - &format!("url={}; error={e}", raw.detail_url), - ); - return None; - } - }; - - let media_urls = self.extract_media_urls(&detail_html); - if media_urls.is_empty() { - report_provider_error_background( - "xfree", - "detail.media", - &format!("no_media_url_found; url={}", raw.detail_url), - ); - return None; - } - - let thumb = if raw.thumb.is_empty() { - self.extract_detail_thumb(&detail_html) - } else { - raw.thumb.clone() - }; - - let duration = if raw.duration > 0 { - raw.duration - } else { - Self::extract_duration_seconds(&detail_html).unwrap_or(0) - }; - - let mut tags = raw.tags.clone(); - for tag in Self::extract_detail_tags(&detail_html) { - if !tags.iter().any(|t| t == &tag) { - tags.push(tag); - } - } - - let mut formats = vec![]; - for media_url in media_urls.iter() { - let format_kind = if media_url.to_ascii_lowercase().contains(".m3u8") { - "m3u8".to_string() - } else { - "mp4".to_string() - }; - let quality = Self::extract_quality_from_url(media_url); - formats.push(VideoFormat::new(media_url.clone(), quality, format_kind)); - } - - let selected_url = media_urls - .iter() - .find(|u| u.to_ascii_lowercase().contains(".mp4")) - .cloned() - .unwrap_or_else(|| media_urls.first().cloned().unwrap_or_default()); - if selected_url.is_empty() { - return None; - } - - let mut item = VideoItem::new( - raw.id, - raw.title, - selected_url, - "xfree".to_string(), - thumb, - duration, - ) - .formats(formats) - .preview( - media_urls - .first() - .cloned() - .unwrap_or_else(|| raw.detail_url.clone()), - ); - - if let Some(views) = raw.views { - item = item.views(views); - } - if let Some(uploader) = raw.uploader { - item = item.uploader(uploader); - } - if !tags.is_empty() { - item = item.tags(tags); - } - - Some(item) - } - - async fn parse_video_items_from_html( - &self, - html: String, - requester: crate::util::requester::Requester, - ) -> Vec { - let listing_items = self.parse_listing_items(&html); - if listing_items.is_empty() { - return vec![]; - } - - let mut in_flight = FuturesUnordered::new(); - let mut items = vec![]; - let mut iter = listing_items.into_iter(); - const MAX_IN_FLIGHT: usize = 5; - - loop { - while in_flight.len() < MAX_IN_FLIGHT { - let Some(raw) = iter.next() else { - break; - }; - in_flight.push(self.fetch_detailed_video_item(raw, requester.clone())); - } - - let Some(result) = in_flight.next().await else { - break; - }; - if let Some(item) = result { - items.push(item); - } - } - - items - } - - async fn fetch( + async fn query( &self, cache: VideoCache, page: u8, query: &str, options: ServerOptions, + pool: DbPool, ) -> Result> { - let urls = self.build_listing_urls(page, query, &options); - let mut requester = requester_or_default(&options, "xfree", "fetch"); - let mut stale_items = vec![]; - - for url in urls { - if let Some((time, items)) = cache.get(&url) { - if time.elapsed().unwrap_or_default().as_secs() < 300 { + let query = if query.is_empty() { "null" } else { query }; + let sexuality = match options.clone().sexuality { + Some(s) if !s.is_empty() => s, + _ => "1".to_string(), + }; + let video_url = format!( + "{}/api/2/search?search={}&lgbt={}&limit=30&offset={}", + self.url, + query.replace(" ", "%20"), + sexuality, + (page as u32 - 1) * 30 + ); + // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 * 24 { + return Ok(items.clone()); + } else { + let _ = cache.check().await; return Ok(items.clone()); } - if stale_items.is_empty() && !items.is_empty() { - stale_items = items.clone(); - } } + None => { + vec![] + } + }; - let html = match requester.get(&url, None).await { - Ok(text) => text, - Err(e) => { - report_provider_error_background( - "xfree", - "listing.request", - &format!("url={url}; error={e}"), - ); - continue; - } - }; - - let items = self - .parse_video_items_from_html(html, requester.clone()) + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + // let _ = requester.get("https://www.xfree.com/", Some(Version::HTTP_2)).await; + let text = match requester.get_with_headers(&video_url, vec![ + ("Apiversion".to_string(), "1.0".to_string()), + ("Accept".to_string(), "application/json text/plain */*".to_string()), + ("Referer".to_string(), "https://www.xfree.com/".to_string()), + ], + Some(Version::HTTP_2), + ).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "xfree", + "query.request", + &format!("url={video_url}; error={e}"), + ) .await; - if !items.is_empty() { - cache.remove(&url); - cache.insert(url, items.clone()); - return Ok(items); + return Ok(old_items); } + }; + let video_items: Vec = self + .get_video_items_from_json(text.clone(), &mut requester, pool) + .await; + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); } + Ok(video_items) + } - Ok(stale_items) + async fn get_video_items_from_json( + &self, + html: String, + _requester: &mut Requester, + _pool: DbPool, + ) -> Vec { + let mut items: Vec = Vec::new(); + let json_result = serde_json::from_str::(&html); + let json = match json_result { + Ok(json) => json, + Err(e) => { + eprintln!("Failed to parse JSON: {e}"); + crate::providers::report_provider_error( + "xfree", + "get_video_items_from_json.parse", + &format!("Failed to parse JSON: {e}"), + ) + .await; + return vec![]; + } + }; + + for post in json.get("body").and_then(|v| v.get("posts")) + .and_then(|p| p.as_array()) + .unwrap_or(&vec![]) + { + let id = post + .get("media") + .and_then(|v| v.get("name")) + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let title = post + .get("title") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let video_url = format!( + "https://cdn.xfree.com/xfree-prod/{}/{}/{}/{}/full.mp4", + id.clone().chars().nth(0).unwrap_or('0'), + id.clone().chars().nth(1).unwrap_or('0'), + id.clone().chars().nth(2).unwrap_or('0'), + id.clone() + ); + + let listsuffix = post + .get("media") + .and_then(|v| v.get("listingSuffix")) + .and_then(|v| v.as_i64()) + .unwrap_or_default(); + let thumb = format!( + "https://thumbs.xfree.com/listing/medium/{}_{}.webp", + id, listsuffix + ); + let views = post.get("viewCount").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + let preview = format!( + "https://cdn.xfree.com/xfree-prod/{}/{}/{}/{}/listing7.mp4", + id.clone().chars().nth(0).unwrap_or('0'), + id.clone().chars().nth(1).unwrap_or('0'), + id.clone().chars().nth(2).unwrap_or('0'), + id.clone() + ); + let duration = post + .get("media") + .and_then(|v| v.get("duration")) + .and_then(|v| v.as_f64()) + .unwrap_or_default() as u32; + let tags = post + .get("tags") + .and_then(|v| v.as_array()) + .unwrap_or(&vec![]) + .iter() + .filter_map(|t| + t.get("tag").and_then(|n| n.as_str()).map(|s| s.to_string())) + .collect::>(); + for tag in tags.iter() { + Self::push_unique(&self.categories, FilterOption { + id: tag.clone(), + title: tag.clone(), + }); + } + let uploader = post + .get("user") + .and_then(|v| v.get("displayName")) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let upload_date = post + .get("publishedDate") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let uploaded_at = chrono::DateTime::parse_from_rfc3339(&upload_date) + .map(|dt| dt.timestamp() as u64) + .unwrap_or(0); + let video_item = VideoItem::new( + id.to_string(), + title, + video_url, + "xfree".to_string(), + thumb, + duration, + ) + .views(views) + .preview(preview) + .tags(tags) + .uploader(uploader) + .uploaded_at(uploaded_at); + items.push(video_item); + } + return items; } } @@ -695,7 +286,7 @@ impl Provider for XfreeProvider { async fn get_videos( &self, cache: VideoCache, - _pool: DbPool, + pool: DbPool, _sort: String, query: Option, page: String, @@ -703,84 +294,16 @@ impl Provider for XfreeProvider { options: ServerOptions, ) -> Vec { let page = page.parse::().unwrap_or(1); - let query = query.unwrap_or_default(); - match self.fetch(cache, page, &query, options).await { - Ok(v) => v, - Err(e) => { - let mut chain_str = String::new(); - for (i, cause) in e.iter().enumerate() { - let _ = writeln!(chain_str, "{}. {}", i + 1, cause); - } - send_discord_error_report( - e.to_string(), - Some(chain_str), - Some("Xfree Provider"), - Some("Failed to fetch videos"), - file!(), - line!(), - module_path!(), - ) - .await; - vec![] - } - } + let res = self.to_owned().query(cache, page, &query.unwrap_or("null".to_string()), options, pool).await; + + res.unwrap_or_else(|e| { + eprintln!("xfree error: {e}"); + vec![] + }) } - fn get_channel(&self, clientversion: ClientVersion) -> Option { - Some(self.build_channel(clientversion)) - } -} - -#[cfg(test)] -mod tests { - use super::XfreeProvider; - - #[test] - fn parses_listing_items_from_html() { - let provider = XfreeProvider::new(); - let html = r#" - - - 1:23 - 12.5K views - - "#; - let items = provider.parse_listing_items(html); - assert_eq!(items.len(), 1); - assert_eq!(items[0].id, "12345"); - assert_eq!(items[0].title, "BBC Anal Test"); - assert_eq!(items[0].duration, 83); - assert_eq!(items[0].views, Some(12_500)); - } - - #[test] - fn extracts_media_urls_from_escaped_html() { - let provider = XfreeProvider::new(); - let html = r#" - - "#; - let urls = provider.extract_media_urls(html); - assert_eq!(urls.len(), 2); - assert!(urls.iter().any(|u| u.contains("clip_720p.mp4"))); - assert!(urls.iter().any(|u| u.contains("master.m3u8"))); - } - - #[test] - fn does_not_take_views_from_plain_title_text() { - let provider = XfreeProvider::new(); - let html = r#" - - - 1:01 - 18 View Example - - "#; - let items = provider.parse_listing_items(html); - assert_eq!(items.len(), 1); - assert_eq!(items[0].views, None); + fn get_channel(&self, v: ClientVersion) -> Option { + Some(self.build_channel(v)) } } diff --git a/src/util/requester.rs b/src/util/requester.rs index fea37d9..183f4f3 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -185,6 +185,15 @@ impl Requester { &mut self, url: &str, _http_version: Option, + ) -> Result { + self.get_with_headers(url, Vec::new(), _http_version).await + } + + pub async fn get_with_headers( + &mut self, + url: &str, + headers: Vec<(String, String)>, + _http_version: Option, ) -> Result { let http_version = match _http_version { Some(v) => v, @@ -192,6 +201,9 @@ impl Requester { }; loop { let mut request = self.client.get(url).version(http_version); + for (key, value) in headers.iter() { + request = request.header(key, value); + } if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); @@ -265,6 +277,9 @@ impl Requester { // Retry the original URL with the updated client & (optional) proxy let mut request = self.client.get(url).version(Version::HTTP_11); + for (key, value) in headers.iter() { + request = request.header(key, value); + } if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); diff --git a/src/videos.rs b/src/videos.rs index b30354f..6df3c24 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -50,7 +50,7 @@ pub struct VideosRequest { pub networks: Option, // pub stars: Option, // pub categories: Option, - pub duration: Option, + pub duration: Option } #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] @@ -66,6 +66,7 @@ pub struct ServerOptions { pub categories: Option, // pub duration: Option, // pub sort: Option, // + pub sexuality: Option, // } #[derive(serde::Serialize, Debug)]