diff --git a/src/providers/paradisehill.rs b/src/providers/paradisehill.rs index 8d27ac7..26c1232 100644 --- a/src/providers/paradisehill.rs +++ b/src/providers/paradisehill.rs @@ -5,11 +5,9 @@ use crate::status::*; use crate::util::cache::VideoCache; use crate::util::requester::Requester; use crate::videos::VideoItem; -use crate::videos::{self, ServerOptions}; +use crate::videos::ServerOptions; use error_chain::error_chain; -use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; -use std::vec; use async_trait::async_trait; error_chain! { @@ -214,77 +212,6 @@ impl ParadisehillProvider { items } - async fn get_video_item(&self, url_str: String, mut requester: Requester) -> Result { - let vid = match requester.get(&url_str, None).await { - Ok(vid) => vid, - Err(e) => { - crate::providers::report_provider_error( - "paradisehill", - "get_video_item.request", - &format!("url={url_str}; error={e}"), - ) - .await; - return Err(Error::from(e.to_string())); - } - }; - let mut title = vid - .split(">().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() - .trim() - .to_string(); - title = decode(title.as_bytes()).to_string().unwrap_or(title); - let thumb = format!( - "{}{}", - self.url, - vid.split(">().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() - .to_string() - ); - - let video_urls = vid.split("var videoList = ").collect::>().get(1).copied().unwrap_or_default() - .split("\"src\":\"") - .collect::>()[1..].to_vec(); - let mut formats = vec![]; - for url in video_urls { - let video_url = url - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() - .replace("\\", "") - .to_string(); - let format = - videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "mp4".to_string()) - // .protocol("https".to_string()) - .format_id(video_url.split("/").last().unwrap_or_default().to_string()) - .format_note(video_url.split("_").last().unwrap_or_default().replace(".mp4", "")) - ; - formats.push(format); - } - - formats.reverse(); - let id = url_str - .split("/") - .collect::>().get(3).copied().unwrap_or_default() - .split("_") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(); - - let video_item = VideoItem::new( - id, - title, - url_str.clone(), - "paradisehill".to_string(), - thumb, - 0, - ) - .aspect_ratio(0.697674419 as f32) - .formats(formats); - - return Ok(video_item); - } } #[async_trait] diff --git a/src/providers/porn00.rs b/src/providers/porn00.rs index f411c34..7c274c7 100644 --- a/src/providers/porn00.rs +++ b/src/providers/porn00.rs @@ -83,7 +83,10 @@ impl Porn00Provider { _ => "list_videos_most_recent_videos", }; - let video_url = format!("{}{}/?mode=async^&function=get_block^&block_id={}^&from={}", self.url, sort_string, list_str, page); + let video_url = format!( + "{}{}/?mode=async&function=get_block&block_id={}&from={}", + self.url, sort_string, list_str, page + ); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { diff --git a/src/providers/xxthots.rs b/src/providers/xxthots.rs index daa97ee..fb4b2c8 100644 --- a/src/providers/xxthots.rs +++ b/src/providers/xxthots.rs @@ -71,20 +71,18 @@ impl XxthotsProvider { sort: &str, options: ServerOptions, ) -> Result> { - let sort_string = match sort { - "popular" => "/most-popular", - "top-rated" => "/top-rated", - _ => "/latest-updates/", - }; - - let list_str = match sort { - "popular" => "list_videos_common_videos_list", - "top-rated" => "list_videos_common_videos_list", - _ => "list_videos_most_recent_videos", + let (sort_path, list_str, sort_by) = match sort { + "popular" => ("/most-popular/", "list_videos_common_videos_list", "video_viewed"), + "top-rated" => ("/top-rated/", "list_videos_common_videos_list", "rating"), + _ => ( + "/latest-updates/", + "list_videos_latest_videos_list", + "post_date", + ), }; let video_url = format!( - "{}{}?mode=async^&function=get_block^&block_id={}^&from={}", - self.url, sort_string, list_str, page + "{}{}?mode=async&function=get_block&block_id={}&sort_by={}&from={}", + self.url, sort_path, list_str, sort_by, page ); let old_items = match cache.get(&video_url) { Some((time, items)) => { @@ -112,6 +110,15 @@ impl XxthotsProvider { return Ok(old_items); } }; + if text.trim().is_empty() { + crate::providers::report_provider_error( + "xxthots", + "get.empty_response", + &format!("url={video_url}"), + ) + .await; + return Ok(old_items); + } let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -161,6 +168,15 @@ impl XxthotsProvider { return Ok(old_items); } }; + if text.trim().is_empty() { + crate::providers::report_provider_error( + "xxthots", + "query.empty_response", + &format!("url={video_url}"), + ) + .await; + return Ok(old_items); + } let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -173,16 +189,15 @@ impl XxthotsProvider { fn get_video_items_from_html(&self, html: String) -> Vec { if html.is_empty() { - println!("HTML is empty"); return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html + let raw_videos: Vec<&str> = html .split("
>().get(0).copied().unwrap_or_default() .split("
") - .collect::>()[1..] - .to_vec(); + .skip(1) + .collect(); for video_segment in &raw_videos { // let vid = video_segment.split("\n").collect::>(); // for (index, line) in vid.iter().enumerate() { @@ -253,11 +268,10 @@ impl Provider for XxthotsProvider { per_page: String, options: ServerOptions, ) -> Vec { - let _ = options; let _ = per_page; let _ = pool; let videos: std::result::Result, Error> = match query { - Some(q) => { + Some(q) if !q.trim().is_empty() => { self.query(cache, page.parse::().unwrap_or(1), &q, options) .await } @@ -265,6 +279,7 @@ impl Provider for XxthotsProvider { self.get(cache, page.parse::().unwrap_or(1), &sort, options) .await } + _ => self.get(cache, page.parse::().unwrap_or(1), &sort, options).await, }; match videos { Ok(v) => v, diff --git a/src/util/proxy.rs b/src/util/proxy.rs index 8ab17c4..9bfdbfb 100644 --- a/src/util/proxy.rs +++ b/src/util/proxy.rs @@ -44,10 +44,6 @@ impl ProxyParseError { } } -pub fn all_proxies_handle() -> Option>>> { - ALL_PROXIES.get().cloned() -} - pub async fn all_proxies_snapshot() -> Option> { let handle = ALL_PROXIES.get()?.clone(); let proxies = handle.read().await; @@ -152,7 +148,7 @@ pub async fn fetch_proxies( added += 1; } } - Ok(Err(err)) => { + Ok(Err(_err)) => { // eprintln!("Proxy verification failed: {err}"); } Err(err) => { diff --git a/src/util/requester.rs b/src/util/requester.rs index c41f317..df7b6ab 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -8,7 +8,6 @@ use wreq::Version; use wreq::header::HeaderValue; use wreq::redirect::Policy; use wreq_util::Emulation; -use rand::seq::SliceRandom; use crate::util::proxy; use crate::util::flaresolverr::FlareSolverrRequest; @@ -23,7 +22,6 @@ pub struct Requester { client: Client, proxy: bool, flaresolverr_session: Option, - use_random_proxy: bool, } impl Requester { @@ -40,7 +38,6 @@ impl Requester { client, proxy: false, flaresolverr_session: None, - use_random_proxy: false, }; proxy::init_all_proxies_background(requester.clone()); @@ -55,10 +52,6 @@ impl Requester { self.proxy = proxy; } - pub fn set_random_proxy(&mut self, random: bool) { - self.use_random_proxy = random; - } - pub async fn get_raw(&mut self, url: &str) -> Result { let client = Client::builder() .cert_verification(false) @@ -79,26 +72,6 @@ impl Requester { request.send().await } - pub async fn get_raw_with_proxy( - &mut self, - url: &str, - proxy: Proxy, - ) -> Result { - let client = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) - .cookie_store(true) - .build() - .expect("Failed to create HTTP client"); - - client - .get(url) - .version(Version::HTTP_11) - .proxy(proxy) - .send() - .await - } - pub async fn get_raw_with_headers( &mut self, url: &str, @@ -218,13 +191,6 @@ pub async fn post_json( request = request.proxy(proxy); } } - // else if self.use_random_proxy { - // let proxies = proxy::all_proxies_snapshot().await.unwrap_or_default(); - // if !proxies.is_empty() { - // let mut random_proxy = proxies.choose_mut(&mut rand::thread_rng()).unwrap().clone(); - // request = request.proxy(random_proxy); - // } - // } let response = request.send().await?; if response.status().is_success() || response.status().as_u16() == 404 { return Ok(response.text().await?);