From d856ade32b76270dfcb335b68a92be9f8471fbec Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 30 Nov 2025 06:53:21 +0000 Subject: [PATCH] adjusted requester to supply http::version itself --- archive/noodlemagazine.rs | 4 +- src/providers/beeg.rs | 10 ++--- src/providers/freshporno.rs | 4 +- src/providers/hanime.rs | 2 +- src/providers/hqporner.rs | 8 ++-- src/providers/missav.rs | 6 +-- src/providers/noodlemagazine.rs | 72 +++++++++++++++++++++++---------- src/providers/omgxxx.rs | 10 ++--- src/providers/paradisehill.rs | 6 +-- src/providers/perverzija.rs | 4 +- src/providers/pmvhaven.rs | 2 +- src/providers/porn00.rs | 4 +- src/providers/pornhat.rs | 4 +- src/providers/pornhub.rs | 4 +- src/providers/pornxp.rs | 4 +- src/providers/pornzog.rs | 2 +- src/providers/redtube.rs | 4 +- src/providers/rule34gen.rs | 4 +- src/providers/rule34video.rs | 4 +- src/providers/sxyprn.rs | 4 +- src/providers/tnaflix.rs | 4 +- src/providers/xxdbx.rs | 4 +- src/providers/xxthots.rs | 4 +- src/providers/youjizz.rs | 4 +- src/proxies/mod.rs | 10 +---- src/proxies/noodlemagazine.rs | 37 ----------------- src/proxies/sxyprn.rs | 2 +- src/proxy.rs | 25 ------------ src/util/requester.rs | 55 ++++++++++++++++++------- src/videos.rs | 9 +++++ 30 files changed, 154 insertions(+), 162 deletions(-) delete mode 100644 src/proxies/noodlemagazine.rs diff --git a/archive/noodlemagazine.rs b/archive/noodlemagazine.rs index c9ae1c3..5141dc3 100644 --- a/archive/noodlemagazine.rs +++ b/archive/noodlemagazine.rs @@ -53,7 +53,7 @@ impl NoodlemagazineProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -90,7 +90,7 @@ impl NoodlemagazineProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/beeg.rs b/src/providers/beeg.rs index d50d0d6..37df40e 100644 --- a/src/providers/beeg.rs +++ b/src/providers/beeg.rs @@ -81,7 +81,7 @@ impl BeegProvider { async fn load_stars(stars: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); let text = requester - .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") + .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None) .await .unwrap(); let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); @@ -103,7 +103,7 @@ impl BeegProvider { async fn load_categories(categories: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); let text = requester - .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") + .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None) .await .unwrap(); let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); @@ -125,7 +125,7 @@ impl BeegProvider { async fn load_sites(sites: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); let text = requester - .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") + .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None) .await .unwrap(); let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); @@ -263,7 +263,7 @@ impl BeegProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); let video_items: Vec = self.get_video_items_from_html(json.clone()); if !video_items.is_empty() { @@ -304,7 +304,7 @@ impl BeegProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); let video_items: Vec = self.get_video_items_from_html(json.clone()); if !video_items.is_empty() { diff --git a/src/providers/freshporno.rs b/src/providers/freshporno.rs index 9f721e5..c0cd2d1 100644 --- a/src/providers/freshporno.rs +++ b/src/providers/freshporno.rs @@ -56,7 +56,7 @@ impl FreshpornoProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -93,7 +93,7 @@ impl FreshpornoProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/hanime.rs b/src/providers/hanime.rs index 4f3f816..7141b7c 100644 --- a/src/providers/hanime.rs +++ b/src/providers/hanime.rs @@ -153,7 +153,7 @@ impl HanimeProvider { let url = format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug); let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url).await.unwrap(); + let text = requester.get(&url, None).await.unwrap(); let urls = text.split("\"servers\"").collect::>()[1]; let mut url_vec = vec![]; diff --git a/src/providers/hqporner.rs b/src/providers/hqporner.rs index 759a49e..608f61c 100644 --- a/src/providers/hqporner.rs +++ b/src/providers/hqporner.rs @@ -64,7 +64,7 @@ impl HqpornerProvider { async fn load_stars(base_url: &str, stars: Arc>>) -> Result<()> { let mut requester = Requester::new(); let text = requester - .get(format!("{}/girls", &base_url).as_str()) + .get(format!("{}/girls", &base_url).as_str(), None) .await .unwrap(); let stars_div = text @@ -96,7 +96,7 @@ impl HqpornerProvider { async fn load_categories(base_url: &str, categories: Arc>>) -> Result<()> { let mut requester = Requester::new(); let text = requester - .get(format!("{}/categories", &base_url).as_str()) + .get(format!("{}/categories", &base_url).as_str(), None) .await .unwrap(); let categories_div = text @@ -176,7 +176,7 @@ impl HqpornerProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self .get_video_items_from_html(text.clone(), &mut requester) .await; @@ -223,7 +223,7 @@ impl HqpornerProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self .get_video_items_from_html(text.clone(), &mut requester) .await; diff --git a/src/providers/missav.rs b/src/providers/missav.rs index 69a7378..3d3bef5 100644 --- a/src/providers/missav.rs +++ b/src/providers/missav.rs @@ -53,7 +53,7 @@ impl MissavProvider { } }; - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); // Pass a reference to options if needed, or reconstruct as needed let video_items: Vec = self.get_video_items_from_html(text.clone(), pool, requester).await; if !video_items.is_empty() { @@ -89,7 +89,7 @@ impl MissavProvider { vec![] } }; - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone(), pool, requester).await; if !video_items.is_empty() { cache.remove(&url_str); @@ -148,7 +148,7 @@ impl MissavProvider { } } drop(conn); - let vid = requester.get(&url_str).await.unwrap(); + let vid = requester.get(&url_str, None).await.unwrap(); let mut title = vid.split(">()[1] .split("\"") .collect::>()[0].trim() diff --git a/src/providers/noodlemagazine.rs b/src/providers/noodlemagazine.rs index 81b8519..a63e265 100644 --- a/src/providers/noodlemagazine.rs +++ b/src/providers/noodlemagazine.rs @@ -3,12 +3,15 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; +use crate::util::requester::Requester; use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; -use crate::videos::{ServerOptions, VideoItem}; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; +use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; +use wreq::Version; use std::{vec}; use titlecase::Titlecase; @@ -76,9 +79,9 @@ impl NoodlemagazineProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); let video_items: Vec = self - .get_video_items_from_html(text.clone()) + .get_video_items_from_html(text.clone(), requester.clone()) .await; if !video_items.is_empty() { cache.remove(&video_url); @@ -116,9 +119,9 @@ impl NoodlemagazineProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); let video_items: Vec = self - .get_video_items_from_html(text.clone()) + .get_video_items_from_html(text.clone(), requester.clone()) .await; if !video_items.is_empty() { cache.remove(&video_url); @@ -132,6 +135,7 @@ impl NoodlemagazineProvider { async fn get_video_items_from_html( &self, html: String, + requester: Requester, ) -> Vec { if html.is_empty() || html.contains("404 Not Found") { return vec![]; @@ -143,24 +147,18 @@ impl NoodlemagazineProvider { .collect::>()[1..] .to_vec(); - let mut video_items: Vec = vec![]; - for raw_video in raw_videos { - if let Ok(video_item) = self.get_video_item(raw_video.to_string()).await { - video_items.push(video_item); - } - } - - // let futures = raw_videos - // .into_iter() - // .map(|el| self.get_video_item(el.to_string())); - // let results: Vec> = join_all(futures).await; - // let video_items: Vec = results.into_iter().filter_map(Result::ok).collect(); + let futures = raw_videos + .into_iter() + .map(|el| self.get_video_item(el.to_string(), requester.clone())); + let results: Vec> = join_all(futures).await; + let video_items: Vec = results.into_iter().filter_map(Result::ok).collect(); return video_items; } async fn get_video_item( &self, video_segment: String, + requester: Requester, ) -> Result { let video_url: String = format!( "{}{}", @@ -208,22 +206,52 @@ impl NoodlemagazineProvider { .collect::>()[0] .trim()).unwrap_or(0); - let proxy_url = format!( - "https://hottub.spacemoehre.de/proxy/noodlemagazine/watch/{}", - video_url.split("/").collect::>()[4] - ); + let formats = match self.extract_media(&video_url, requester.clone()).await{ + Some(f) => f, + None => return Err(Error::from("Failed to extract media formats")), + }; let video_item = VideoItem::new( id, title, - proxy_url, + video_url, "noodlemagazine".to_string(), thumb, duration, ) .views(views) + .formats(formats) ; return Ok(video_item); } + + async fn extract_media(&self, video_url: &String, mut requester: Requester) -> Option> { + let text = requester.get(video_url, Some(Version::HTTP_2)).await.unwrap_or("".to_string()); + if text.is_empty() { + return None; + } + let mut formats = vec![]; + let json_str = text.split("window.playlist = ") + .collect::>()[1] + .split(";") + .collect::>()[0]; + let json: serde_json::Value = serde_json::from_str(json_str).unwrap(); + let sources = json["sources"].as_array().unwrap(); + for source in sources { + let file = source["file"].as_str().unwrap().to_string(); + let label = source["label"].as_str().unwrap_or("unknown").to_string(); + let format = VideoFormat::new( + file, + "video/mp4".to_string(), + label.clone(), + ) + .format_id(label.clone()) + .format_note(label.clone()) + .http_header("Referer".to_string(), video_url.clone()) + ; + formats.push(format); + } + return Some(formats); + } } #[async_trait] diff --git a/src/providers/omgxxx.rs b/src/providers/omgxxx.rs index 61ccb4d..1f62fcd 100644 --- a/src/providers/omgxxx.rs +++ b/src/providers/omgxxx.rs @@ -84,7 +84,7 @@ impl OmgxxxProvider { let mut requester = util::requester::Requester::new(); for page in [1..10].into_iter().flatten() { let text = requester - .get(format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str()) + .get(format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str(), None) .await .unwrap(); if text.contains("404 Not Found") || text.is_empty() { @@ -122,7 +122,7 @@ impl OmgxxxProvider { loop { page += 1; let text = requester - .get(format!("{}/sites/{}/", &base_url, page).as_str()) + .get(format!("{}/sites/{}/", &base_url, page).as_str(), None) .await .unwrap(); if text.contains("404 Not Found") || text.is_empty() { @@ -158,7 +158,7 @@ impl OmgxxxProvider { async fn load_networks(base_url: &str, networks: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); - let text = requester.get(&base_url).await.unwrap(); + let text = requester.get(&base_url, None).await.unwrap(); let networks_div = text.split("class=\"sites__list\"").collect::>()[1] .split("") .collect::>()[0]; @@ -345,7 +345,7 @@ impl OmgxxxProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -393,7 +393,7 @@ impl OmgxxxProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/paradisehill.rs b/src/providers/paradisehill.rs index d89c440..2651f0d 100644 --- a/src/providers/paradisehill.rs +++ b/src/providers/paradisehill.rs @@ -51,7 +51,7 @@ impl ParadisehillProvider { } }; - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); // Pass a reference to options if needed, or reconstruct as needed let video_items: Vec = self .get_video_items_from_html(text.clone(), requester) @@ -93,7 +93,7 @@ impl ParadisehillProvider { vec![] } }; - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); let video_items: Vec = self .get_video_items_from_html(text.clone(), requester) .await; @@ -143,7 +143,7 @@ impl ParadisehillProvider { } async fn get_video_item(&self, url_str: String, mut requester: Requester) -> Result { - let vid = requester.get(&url_str).await.unwrap(); + let vid = requester.get(&url_str, None).await.unwrap(); let mut title = vid .split(">()[1] diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index 82baf82..0fbcd5a 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -71,7 +71,7 @@ impl PerverzijaProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); if !video_items.is_empty() { cache.remove(&url_str); @@ -122,7 +122,7 @@ impl PerverzijaProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); let video_items: Vec = match query_parse { true => { self.get_video_items_from_html_query(text.clone(), pool) diff --git a/src/providers/pmvhaven.rs b/src/providers/pmvhaven.rs index d387773..0023a31 100644 --- a/src/providers/pmvhaven.rs +++ b/src/providers/pmvhaven.rs @@ -175,7 +175,7 @@ impl PmvhavenProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let json = serde_json::from_str::(&text).unwrap_or(serde_json::Value::Null); let video_items: Vec = self .get_video_items_from_json(json) diff --git a/src/providers/porn00.rs b/src/providers/porn00.rs index a70dab5..e6ec322 100644 --- a/src/providers/porn00.rs +++ b/src/providers/porn00.rs @@ -62,7 +62,7 @@ impl Porn00Provider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -99,7 +99,7 @@ impl Porn00Provider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/pornhat.rs b/src/providers/pornhat.rs index 1cf2ce8..1c5fe83 100644 --- a/src/providers/pornhat.rs +++ b/src/providers/pornhat.rs @@ -53,7 +53,7 @@ impl PornhatProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -92,7 +92,7 @@ impl PornhatProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/pornhub.rs b/src/providers/pornhub.rs index bd2bcb2..76d3676 100644 --- a/src/providers/pornhub.rs +++ b/src/providers/pornhub.rs @@ -49,7 +49,7 @@ impl PornhubProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone(),"
    = self.get_video_items_from_html(text.clone(),split_string); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/pornxp.rs b/src/providers/pornxp.rs index 43edd6d..ad5a4ff 100644 --- a/src/providers/pornxp.rs +++ b/src/providers/pornxp.rs @@ -96,7 +96,7 @@ impl PornxpProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -142,7 +142,7 @@ impl PornxpProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/pornzog.rs b/src/providers/pornzog.rs index 95b076a..94298b3 100644 --- a/src/providers/pornzog.rs +++ b/src/providers/pornzog.rs @@ -64,7 +64,7 @@ impl PornzogProvider { let mut requester = options.requester.clone().unwrap(); println!("Fetching URL: {}", video_url); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/redtube.rs b/src/providers/redtube.rs index aeba5e4..e08ca7b 100644 --- a/src/providers/redtube.rs +++ b/src/providers/redtube.rs @@ -50,7 +50,7 @@ impl RedtubeProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -87,7 +87,7 @@ impl RedtubeProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html_query(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/rule34gen.rs b/src/providers/rule34gen.rs index f6bd722..bb93fc1 100644 --- a/src/providers/rule34gen.rs +++ b/src/providers/rule34gen.rs @@ -115,7 +115,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel { }; } let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url).await.unwrap(); + let text = requester.get(&url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&url); @@ -163,7 +163,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url).await.unwrap(); + let text = requester.get(&url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&url); diff --git a/src/providers/rule34video.rs b/src/providers/rule34video.rs index c10d67f..00a4caa 100644 --- a/src/providers/rule34video.rs +++ b/src/providers/rule34video.rs @@ -67,7 +67,7 @@ impl Rule34videoProvider { }; } let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url).await.unwrap(); + let text = requester.get(&url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&url); @@ -115,7 +115,7 @@ impl Rule34videoProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url).await.unwrap(); + let text = requester.get(&url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&url); diff --git a/src/providers/sxyprn.rs b/src/providers/sxyprn.rs index 3295e56..96f93bf 100644 --- a/src/providers/sxyprn.rs +++ b/src/providers/sxyprn.rs @@ -81,7 +81,7 @@ impl SxyprnProvider { } }; - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); // Pass a reference to options if needed, or reconstruct as needed let video_items: Vec = self .get_video_items_from_html(text.clone(), pool, requester) @@ -131,7 +131,7 @@ impl SxyprnProvider { vec![] } }; - let text = requester.get(&url_str).await.unwrap(); + let text = requester.get(&url_str, None).await.unwrap(); let video_items: Vec = self .get_video_items_from_html(text.clone(), pool, requester) .await; diff --git a/src/providers/tnaflix.rs b/src/providers/tnaflix.rs index 3019ea7..60ce1b3 100644 --- a/src/providers/tnaflix.rs +++ b/src/providers/tnaflix.rs @@ -380,7 +380,7 @@ impl TnaflixProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -427,7 +427,7 @@ impl TnaflixProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/xxdbx.rs b/src/providers/xxdbx.rs index fa7d2b5..8dfaedb 100644 --- a/src/providers/xxdbx.rs +++ b/src/providers/xxdbx.rs @@ -106,7 +106,7 @@ impl XxdbxProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -156,7 +156,7 @@ impl XxdbxProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/xxthots.rs b/src/providers/xxthots.rs index 61699d2..24c5471 100644 --- a/src/providers/xxthots.rs +++ b/src/providers/xxthots.rs @@ -62,7 +62,7 @@ impl XxthotsProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -99,7 +99,7 @@ impl XxthotsProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/youjizz.rs b/src/providers/youjizz.rs index 8708528..dfc3582 100644 --- a/src/providers/youjizz.rs +++ b/src/providers/youjizz.rs @@ -59,7 +59,7 @@ impl YoujizzProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -95,7 +95,7 @@ impl YoujizzProvider { let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/proxies/mod.rs b/src/proxies/mod.rs index bf70bf2..561aa24 100644 --- a/src/proxies/mod.rs +++ b/src/proxies/mod.rs @@ -1,15 +1,13 @@ use ntex::web; -use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester, proxies::noodlemagazine::NoodlemagazineProxy}; +use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester}; pub mod sxyprn; pub mod hanimecdn; -pub mod noodlemagazine; #[derive(Debug, Clone)] pub enum AnyProxy { Sxyprn(SxyprnProxy), - Noodlemagazine(NoodlemagazineProxy), } pub trait Proxy { @@ -34,12 +32,6 @@ impl Proxy for AnyProxy { requester, ).await } - AnyProxy::Noodlemagazine(p) => { - p.get_video_url( - url, - requester, - ).await - } } } } \ No newline at end of file diff --git a/src/proxies/noodlemagazine.rs b/src/proxies/noodlemagazine.rs deleted file mode 100644 index 40529f5..0000000 --- a/src/proxies/noodlemagazine.rs +++ /dev/null @@ -1,37 +0,0 @@ -use ntex::web; - -use crate::util::requester::Requester; - - -#[derive(Debug, Clone)] -pub struct NoodlemagazineProxy { -} - -impl NoodlemagazineProxy { - pub fn new() -> Self { - NoodlemagazineProxy { - } - } - - pub async fn get_video_url( - &self, - url: String, - requester: web::types::State, - ) -> String { - let mut requester = requester.get_ref().clone(); - let url = "https://noodlemagazine.com/".to_string() + &url; - let text = requester.get(&url).await.unwrap_or("".to_string()); - if text.is_empty() { - return "".to_string(); - } - let json_str = text.split("window.playlist = ") - .collect::>()[1] - .split(";") - .collect::>()[0]; - let json: serde_json::Value = serde_json::from_str(json_str).unwrap(); - - let sources = json["sources"].as_array().unwrap(); - let video_url = sources[0]["file"].as_str().unwrap().to_string(); - return video_url; - } -} \ No newline at end of file diff --git a/src/proxies/sxyprn.rs b/src/proxies/sxyprn.rs index a4f5e37..db46081 100644 --- a/src/proxies/sxyprn.rs +++ b/src/proxies/sxyprn.rs @@ -40,7 +40,7 @@ impl SxyprnProxy { ) -> String { let mut requester = requester.get_ref().clone(); let url = "https://sxyprn.com/".to_string() + &url; - let text = requester.get(&url).await.unwrap_or("".to_string()); + let text = requester.get(&url, None).await.unwrap_or("".to_string()); if text.is_empty() { return "".to_string(); } diff --git a/src/proxy.rs b/src/proxy.rs index a12b3f0..851aa19 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -1,6 +1,5 @@ use ntex::web::{self, HttpRequest}; -use crate::proxies::noodlemagazine::NoodlemagazineProxy; use crate::proxies::sxyprn::SxyprnProxy; use crate::util::requester::Requester; use crate::proxies::*; @@ -16,16 +15,6 @@ pub fn config(cfg: &mut web::ServiceConfig) { .route(web::post().to(crate::proxies::hanimecdn::get_image)) .route(web::get().to(crate::proxies::hanimecdn::get_image)), ) - .service( - web::resource("/noodlemagazine/{endpoint}*") - .route(web::post().to(noodlemagazine)) - .route(web::get().to(noodlemagazine)), - ) - // .service( - // web::resource("/videos") - // // .route(web::get().to(videos_get)) - // .route(web::post().to(videos_post)), - // ) ; } @@ -43,23 +32,9 @@ async fn sxyprn(req: HttpRequest, .finish()) } -async fn noodlemagazine(req: HttpRequest, - requester: web::types::State,) -> Result { - let proxy = get_proxy(req.uri().to_string().split("/").collect::>()[2]).unwrap(); - let endpoint = req.match_info().query("endpoint").to_string(); - let video_url = match proxy.get_video_url(endpoint, requester).await{ - url if url != "" => url, - _ => "Error".to_string(), - }; - Ok(web::HttpResponse::Found() - .header("Location", format!("{}", video_url)) - .finish()) -} - fn get_proxy(proxy: &str) -> Option { match proxy { "sxyprn" => Some(AnyProxy::Sxyprn(SxyprnProxy::new())), - "noodlemagazine" => Some(AnyProxy::Noodlemagazine(NoodlemagazineProxy::new())), _ => None, } } \ No newline at end of file diff --git a/src/util/requester.rs b/src/util/requester.rs index c99fc2d..cb44bf1 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -83,7 +83,11 @@ impl Requester { request.send().await } - pub async fn get_raw_with_headers(&mut self, url: &str, headers: Vec<(String, String)>) -> Result { + pub async fn get_raw_with_headers( + &mut self, + url: &str, + headers: Vec<(String, String)>, + ) -> Result { let client = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) @@ -106,7 +110,12 @@ impl Requester { request.send().await } - pub async fn post(&mut self, url: &str, data: &S, headers: Vec<(String, String)>) -> Result + pub async fn post( + &mut self, + url: &str, + data: &S, + headers: Vec<(String, String)>, + ) -> Result where S: Serialize + ?Sized, { @@ -123,7 +132,7 @@ impl Requester { for (key, value) in headers.iter() { request = request.header(key, value); } - + if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); @@ -134,21 +143,37 @@ impl Requester { request.send().await } - pub async fn get(&mut self, url: &str) -> Result { - let mut request = self.client.get(url).version(Version::HTTP_11); - - if self.proxy { - if let Ok(proxy_url) = env::var("BURP_URL") { - let proxy = Proxy::all(&proxy_url).unwrap(); - request = request.proxy(proxy); + pub async fn get(&mut self, url: &str, _http_version: Option) -> Result { + let http_version = match _http_version { + Some(v) => v, + None => Version::HTTP_11, + }; + loop { + let mut request = self.client.get(url).version(http_version); + if self.proxy { + if let Ok(proxy_url) = env::var("BURP_URL") { + let proxy = Proxy::all(&proxy_url).unwrap(); + request = request.proxy(proxy); + } + } + let response = request.send().await?; + if response.status().is_success() || response.status().as_u16() == 404 { + return Ok(response.text().await?); + } + if response.status().as_u16() == 429 { + println!("Received 429 Too Many Requests, retrying after delay..."); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } else { + println!( + "Direct request to {} failed with status: {}", + url, + response.status() + ); + break; } } - let response = request.send().await?; - - if response.status().is_success() || response.status().as_u16() == 404 { - return Ok(response.text().await?); - } // If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately, // so no non-Send error value lives across later `.await`s. diff --git a/src/videos.rs b/src/videos.rs index 601486d..bb92195 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -271,6 +271,15 @@ impl VideoFormat { headers.insert(key, value); } } + pub fn http_header(&mut self, key: String, value: String) -> Self { + if self.http_headers.is_none() { + self.http_headers = Some(HashMap::new()); + } + if let Some(headers) = &mut self.http_headers { + headers.insert(key, value); + } + self.to_owned() + } pub fn format_id(mut self, format_id: String) -> Self { self.format_id = Some(format_id); self