diff --git a/Cargo.toml b/Cargo.toml index d13499b..09df2c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ ntex = { version = "2.15.1", features = ["tokio"] } ntex-files = "2.0.0" serde = "1.0.228" serde_json = "1.0.145" -tokio = { version = "1.47.1", features = ["full"] } +tokio = { version = "1.49", features = ["full"] } wreq = { version = "5.3.0", features = ["full", "cookies", "multipart"] } wreq-util = "2" percent-encoding = "2.3.2" diff --git a/docker-compose.yml b/docker-compose.yml index a81a2e3..d883144 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,10 +14,10 @@ services: environment: - RUST_LOG=info - BURP_URL=http://127.0.0.1:8081 # local burpsuite proxy for crawler analysis - - PROXY=1 # 1 for enable, else disabled + - PROXY=0 # 1 for enable, else disabled - DATABASE_URL=hottub.db # sqlite db to store hard to get videos for easy access - FLARE_URL=http://flaresolverr:8191/v1 # flaresolverr to get around cloudflare 403 codes - - DOMAIN=hottub.spacemoehre.de # optional for the 302 forward on "/" + - DOMAIN=hottub.spacemoehre.de # optional for the 302 forward on "/" to restart: unless-stopped working_dir: /app ports: @@ -34,11 +34,13 @@ services: interval: 30s timeout: 5s retries: 3 + start_period: 1s ulimits: nofile: soft: 65536 hard: 65536 +# flaresolverr to bypass cloudflare protections flaresolverr: container_name: flaresolverr ports: @@ -53,9 +55,4 @@ services: max-size: "10m" # Maximum size of each log file (e.g., 10MB) max-file: "3" # Maximum number of log files to keep - restarter: # restarts the flaresolverr so its always ready for work - image: docker:cli - container_name: flaresolverr-restarter - volumes: ["/var/run/docker.sock:/var/run/docker.sock"] - command: ["/bin/sh", "-c", "while true; do sleep 26400; docker restart flaresolverr; done"] - restart: unless-stopped + diff --git a/src/api.rs b/src/api.rs index 6610df9..00d9878 100644 --- a/src/api.rs +++ b/src/api.rs @@ -3,7 +3,7 @@ use ntex::http::header; use ntex::web; use ntex::web::HttpRequest; use std::cmp::Ordering; -use std::fs; +use std::{fs, io}; use tokio::task; use crate::providers::all::AllProvider; @@ -15,6 +15,7 @@ use crate::providers::redtube::RedtubeProvider; use crate::providers::rule34video::Rule34videoProvider; // use crate::providers::spankbang::SpankbangProvider; use crate::util::cache::VideoCache; +use crate::util::discord::send_discord_error_report; use crate::util::requester::Requester; use crate::{DbPool, db, status::*, videos::*}; use cute::c; @@ -104,7 +105,12 @@ pub fn config(cfg: &mut web::ServiceConfig) { web::resource("/videos") // .route(web::get().to(videos_get)) .route(web::post().to(videos_post)), - ); + ) + .service( + web::resource("/test") + .route(web::get().to(test)) + ) + ; } async fn status(req: HttpRequest) -> Result { @@ -345,40 +351,6 @@ async fn status(req: HttpRequest) -> Result { cacheDuration: None, }); - // status.add_channel(Channel { - // id: "spankbang".to_string(), - // name: "SpankBang".to_string(), - // description: "Popular Porn Videos - SpankBang".to_string(), - // premium: false, - // favicon: "https://www.google.com/s2/favicons?sz=64&domain=spankbang.com".to_string(), - // status: "active".to_string(), - // categories: vec![], - // options: vec![ChannelOption { - // id: "sort".to_string(), - // title: "Sort".to_string(), - // description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - // systemImage: "list.number".to_string(), - // colorName: "blue".to_string(), - // options: vec![ - // FilterOption { - // id: "trending_videos".to_string(), - // title: "Trending".to_string(), - // }, - // FilterOption { - // id: "new_videos".to_string(), - // title: "New".to_string(), - // }, - // FilterOption { - // id: "most_popular".to_string(), - // title: "Popular".to_string(), - // }, - // ], - // multiSelect: false, - // }], - // nsfw: true, - //cacheDuration: Some(1800), - // }); - // rule34video status.add_channel(Channel { id: "rule34video".to_string(), @@ -647,41 +619,6 @@ async fn status(req: HttpRequest) -> Result { cacheDuration: Some(1800), }); - // // hentaimoon - // status.add_channel(Channel { - // id: "hentaimoon".to_string(), - // name: "Hentai Moon".to_string(), - // description: "Your Hentai Sputnik".to_string(), - // premium: false, - // favicon: "https://www.google.com/s2/favicons?sz=64&domain=hentai-moon.com".to_string(), - // status: "active".to_string(), - // categories: vec![], - // options: vec![ChannelOption { - // id: "sort".to_string(), - // title: "Sort".to_string(), - // description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - // systemImage: "list.number".to_string(), - // colorName: "blue".to_string(), - // options: vec![ - // FilterOption { - // id: "new".to_string(), - // title: "New".to_string(), - // }, - // FilterOption { - // id: "popular".to_string(), - // title: "Popular".to_string(), - // }, - // FilterOption { - // id: "top-rated".to_string(), - // title: "Top Rated".to_string(), - // }, - // ], - // multiSelect: false, - // }], - // nsfw: true, - // cacheDuration: Some(1800), - // }); - // xxthots status.add_channel(Channel { id: "xxthots".to_string(), @@ -852,20 +789,6 @@ async fn status(req: HttpRequest) -> Result { cacheDuration: None, }); - // noodlemagazine - // status.add_channel(Channel { - // id: "noodlemagazine".to_string(), - // name: "Noodlemagazine".to_string(), - // description: "Discover the Best Adult Videos".to_string(), - // premium: false, - // favicon: "https://www.google.com/s2/favicons?sz=64&domain=noodlemagazine.com".to_string(), - // status: "active".to_string(), - // categories: vec![], - // options: vec![], - // nsfw: true, - // cacheDuration: Some(1800), - // }); - //missav status.add_channel(Channel { id: "missav".to_string(), @@ -1065,7 +988,9 @@ async fn status(req: HttpRequest) -> Result { } for provider in ALL_PROVIDERS.values() { - status.add_channel(provider.get_channel(clientversion.clone())); + if let Some(channel) = provider.get_channel(clientversion.clone()){ + status.add_channel(channel); + } } status.iconUrl = format!("http://{}/favicon.ico", host).to_string(); Ok(web::HttpResponse::Ok().json(&status)) @@ -1261,3 +1186,19 @@ pub fn get_provider(channel: &str) -> Option { x => ALL_PROVIDERS.get(x).cloned(), } } + +pub async fn test() -> Result { + // Simply await the function instead of blocking the thread + let e = io::Error::new(io::ErrorKind::Other, "test error"); + let _ = send_discord_error_report( + e.to_string(), + Some("chain_str".to_string()), + Some("Context"), + Some("xtra info"), + file!(), + line!(), + module_path!(), + ).await; + + Ok(web::HttpResponse::Ok()) +} \ No newline at end of file diff --git a/src/providers/all.rs b/src/providers/all.rs index 6ed2f33..0855532 100644 --- a/src/providers/all.rs +++ b/src/providers/all.rs @@ -94,11 +94,11 @@ impl Provider for AllProvider { return video_items; } - fn get_channel(&self,clientversion:ClientVersion) -> Channel { + fn get_channel(&self,clientversion:ClientVersion) -> Option { println!("Getting channel for placeholder with client version: {:?}",clientversion); let _ = clientversion; - Channel { + Some(Channel { id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None, - } + }) } } diff --git a/src/providers/beeg.rs b/src/providers/beeg.rs index e2716f1..2450b8f 100644 --- a/src/providers/beeg.rs +++ b/src/providers/beeg.rs @@ -3,7 +3,6 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; -use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; use crate::{status::*, util}; use async_trait::async_trait; @@ -311,7 +310,7 @@ impl BeegProvider { }; let id = file.get("id").and_then(|v| v.as_i64()).unwrap_or(0).to_string(); - let title = video + let title = file .get("data") .and_then(|v| v.get(0)) .and_then(|v| v.get("cd_value")) @@ -321,8 +320,7 @@ impl BeegProvider { let duration = file .get("fl_duration") - .and_then(|v| v.as_str()) - .and_then(|s| parse_time_to_seconds(s)) + .and_then(|v| v.as_u64()) .unwrap_or(0); let views = video @@ -378,7 +376,7 @@ impl Provider for BeegProvider { }) } - fn get_channel(&self, clientversion: ClientVersion) -> Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/freshporno.rs b/src/providers/freshporno.rs index e298eb0..539ec41 100644 --- a/src/providers/freshporno.rs +++ b/src/providers/freshporno.rs @@ -192,11 +192,11 @@ impl Provider for FreshpornoProvider { } } - fn get_channel(&self,clientversion:ClientVersion) -> Channel { + fn get_channel(&self,clientversion:ClientVersion) -> Option { println!("Getting channel for placeholder with client version: {:?}",clientversion); let _ = clientversion; - Channel { + Some(Channel { id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None, - } + }) } } diff --git a/src/providers/hqporner.rs b/src/providers/hqporner.rs index 9c898cb..62b8c7f 100644 --- a/src/providers/hqporner.rs +++ b/src/providers/hqporner.rs @@ -458,7 +458,7 @@ impl Provider for HqpornerProvider { } } } - fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/hypnotube.rs b/src/providers/hypnotube.rs index e4d7803..afcf85c 100644 --- a/src/providers/hypnotube.rs +++ b/src/providers/hypnotube.rs @@ -3,14 +3,14 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::discord::send_discord_error_report; +use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; -use htmlentity::entity::{decode, ICodedDataTrait}; +use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; use std::{thread, vec}; use titlecase::Titlecase; @@ -50,7 +50,7 @@ impl HypnotubeProvider { let url = self.url.clone(); let categories = Arc::clone(&self.categories); - thread::spawn(move || { + thread::spawn(async move || { let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -58,14 +58,16 @@ impl HypnotubeProvider { Ok(rt) => rt, Err(e) => { eprintln!("tokio runtime failed: {e}"); - let _ = futures::executor::block_on(send_discord_error_report( - &e, + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), Some("HypnoTube Provider"), Some("Failed to create tokio runtime"), file!(), line!(), module_path!(), - )); + ) + .await; return; } }; @@ -74,13 +76,15 @@ impl HypnotubeProvider { if let Err(e) = Self::load_categories(&url, Arc::clone(&categories)).await { eprintln!("load_categories failed: {e}"); send_discord_error_report( - &e, + e.to_string(), + Some(format_error_chain(&e)), Some("HypnoTube Provider"), Some("Failed to load categories during initial load"), file!(), line!(), module_path!(), - ).await; + ) + .await; } }); }); @@ -89,10 +93,7 @@ impl HypnotubeProvider { async fn load_categories(base: &str, cats: Arc>>) -> Result<()> { let mut requester = Requester::new(); let text = requester - .get( - &format!("{base}/channels/"), - Some(Version::HTTP_11), - ) + .get(&format!("{base}/channels/"), Some(Version::HTTP_11)) .await .map_err(|e| Error::from(format!("{}", e)))?; @@ -123,7 +124,7 @@ impl HypnotubeProvider { } Ok(()) } - + fn build_channel(&self, clientversion: ClientVersion) -> Channel { let _ = clientversion; Channel { @@ -140,8 +141,7 @@ impl HypnotubeProvider { .iter() .map(|c| c.title.clone()) .collect(), - options: vec![ - ChannelOption { + options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Sort the Videos".to_string(), @@ -193,10 +193,7 @@ impl HypnotubeProvider { "longest" => "longest", _ => "videos", }; - let video_url = format!( - "{}/{}/page{}.html", - self.url, sort_string, page - ); + let video_url = format!("{}/{}/page{}.html", self.url, sort_string, page); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { @@ -210,14 +207,15 @@ impl HypnotubeProvider { } }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, Some(Version::HTTP_11)).await.unwrap(); + let text = requester + .get(&video_url, Some(Version::HTTP_11)) + .await + .unwrap(); if text.contains("Sorry, no results were found.") { eprint!("Hypnotube query returned no results for page {}", page); return vec![]; } - let video_items: Vec = self - .get_video_items_from_html(text.clone()) - .await; + let video_items: Vec = self.get_video_items_from_html(text.clone()).await; if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); @@ -242,7 +240,10 @@ impl HypnotubeProvider { }; let video_url = format!( "{}/search/videos/{}/{}/page{}.html", - self.url, query.trim().replace(" ","%20"), sort_string, page + self.url, + query.trim().replace(" ", "%20"), + sort_string, + page ); // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&video_url) { @@ -258,9 +259,19 @@ impl HypnotubeProvider { vec![] } }; - + let mut requester = options.requester.clone().unwrap(); - let text = match requester.post(format!("{}/searchgate.php", self.url).as_str(), format!("q={}&type=videos", query.replace(" ","+")).as_str(), vec![("Content-Type", "application/x-www-form-urlencoded")]).await.unwrap().text().await { + let text = match requester + .post( + format!("{}/searchgate.php", self.url).as_str(), + format!("q={}&type=videos", query.replace(" ", "+")).as_str(), + vec![("Content-Type", "application/x-www-form-urlencoded")], + ) + .await + .unwrap() + .text() + .await + { Ok(t) => t, Err(e) => { eprint!("Hypnotube search POST request failed: {}", e); @@ -273,9 +284,7 @@ impl HypnotubeProvider { eprint!("Hypnotube query returned no results for page {}", page); return vec![]; } - let video_items: Vec = self - .get_video_items_from_html(text.clone()) - .await; + let video_items: Vec = self.get_video_items_from_html(text.clone()).await; if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); @@ -285,10 +294,7 @@ impl HypnotubeProvider { video_items } - async fn get_video_items_from_html( - &self, - html: String - ) -> Vec { + async fn get_video_items_from_html(&self, html: String) -> Vec { if html.is_empty() || html.contains("404 Not Found") { eprint!("Hypnotube returned empty or 404 html"); return vec![]; @@ -302,47 +308,59 @@ impl HypnotubeProvider { Some(b) => b, None => { eprint!("Hypnotube Provider: Failed to get block from html"); - let err = Error::from(ErrorKind::Parse("html".into())); - let _ = futures::executor::block_on(send_discord_error_report( - &err, - Some("Hypnotube Provider"), - Some(&format!("Failed to get block from html:\n```{html}\n```")), - file!(), - line!(), - module_path!(), - )); + let e = Error::from(ErrorKind::Parse("html".into())); + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + Some("Hypnotube Provider"), + Some(&format!("Failed to get block from html:\n```{html}\n```")), + file!(), + line!(), + module_path!(), + ) + .await; return vec![]; - }, + } }; let mut items = vec![]; - for seg in block.split("").skip(1){ + for seg in block.split("").skip(1) { let video_url = match seg .split(" href=\"") .nth(1) - .and_then(|s| s.split('"').next()) { - Some(url) => url.to_string(), - None => { - eprint!("Hypnotube Provider: Failed to parse video url from segment"); - let err = Error::from(ErrorKind::Parse("video url".into())); - let _ = futures::executor::block_on(send_discord_error_report( - &err, - Some("Hypnotube Provider"), - Some(&format!("Failed to parse video url from segment:\n```{seg}\n```")), - file!(), - line!(), - module_path!(), - )); - continue; - } - }; - + .and_then(|s| s.split('"').next()) + { + Some(url) => url.to_string(), + None => { + eprint!("Hypnotube Provider: Failed to parse video url from segment"); + let e = Error::from(ErrorKind::Parse("video url".into())); + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + Some("Hypnotube Provider"), + Some(&format!( + "Failed to parse video url from segment:\n```{seg}\n```" + )), + file!(), + line!(), + module_path!(), + ) + .await; + continue; + } + }; + let mut title = seg .split(" title=\"") .nth(1) .and_then(|s| s.split('"').next()) - .unwrap_or_default().trim().to_string(); + .unwrap_or_default() + .trim() + .to_string(); - title = decode(title.clone().as_bytes()).to_string().unwrap_or(title).titlecase(); + title = decode(title.clone().as_bytes()) + .to_string() + .unwrap_or(title) + .titlecase(); let id = video_url .split('/') .nth(4) @@ -364,7 +382,7 @@ impl HypnotubeProvider { .unwrap_or("") .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - + let views = seg .split("") .nth(1) @@ -411,7 +429,7 @@ impl Provider for HypnotubeProvider { return res; } - fn get_channel(&self, v: ClientVersion) -> Channel { - self.build_channel(v) + fn get_channel(&self, v: ClientVersion) -> Option { + Some(self.build_channel(v)) } } diff --git a/src/providers/javtiful.rs b/src/providers/javtiful.rs index 935d6a6..78aa34a 100644 --- a/src/providers/javtiful.rs +++ b/src/providers/javtiful.rs @@ -3,7 +3,7 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::discord::send_discord_error_report; +use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; @@ -215,15 +215,16 @@ impl JavtifulProvider { Some(b) => b, None => { eprint!("Javtiful Provider: Failed to get block from html"); - let err = Error::from(ErrorKind::Parse("html".into())); - let _ = futures::executor::block_on(send_discord_error_report( - &err, - Some("Javtiful Provider"), - Some(&format!("Failed to get block from html:\n```{html}\n```")), - file!(), - line!(), - module_path!(), - )); + let e = Error::from(ErrorKind::Parse("html".into())); + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + Some("Javtiful Provider"), + Some(&format!("Failed to get block from html:\n```{html}\n```")), + file!(), + line!(), + module_path!(), + ).await; return vec![] }, }; @@ -239,14 +240,22 @@ impl JavtifulProvider { .inspect(|r| { if let Err(e) = r { eprint!("Javtiful Provider: Failed to get video item:{}\n", e); - let _ = futures::executor::block_on(send_discord_error_report( - &e, - Some("Javtiful Provider"), - Some("Failed to get video item"), - file!(), - line!(), - module_path!(), - )); + // Prepare data to move into the background task + let msg = e.to_string(); + let chain = format_error_chain(&e); + + // Spawn the report into the background - NO .await here + tokio::spawn(async move { + let _ = send_discord_error_report( + msg, + Some(chain), + Some("Javtiful Provider"), + Some("Failed to get video item"), + file!(), // Note: these might report the utility line + line!(), // better to hardcode or pass from outside + module_path!(), + ).await; + }); } }) .filter_map(Result::ok) @@ -400,7 +409,7 @@ impl Provider for JavtifulProvider { }) } - fn get_channel(&self, v: ClientVersion) -> Channel { - self.build_channel(v) + fn get_channel(&self, v: ClientVersion) -> Option { + Some(self.build_channel(v)) } } diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 3d98a70..fd855ce 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -82,13 +82,13 @@ pub trait Provider: Send + Sync { options: ServerOptions, ) -> Vec; - fn get_channel(&self, clientversion: ClientVersion) -> Channel { + fn get_channel(&self, clientversion: ClientVersion) -> Option { println!( "Getting channel for placeholder with client version: {:?}", clientversion ); let _ = clientversion; - Channel { + Some(Channel { id: "placeholder".to_string(), name: "PLACEHOLDER".to_string(), description: "PLACEHOLDER FOR PARENT CLASS".to_string(), @@ -99,6 +99,6 @@ pub trait Provider: Send + Sync { options: vec![], nsfw: true, cacheDuration: None, - } + }) } } diff --git a/src/providers/noodlemagazine.rs b/src/providers/noodlemagazine.rs index ef7a7c7..efbb1b0 100644 --- a/src/providers/noodlemagazine.rs +++ b/src/providers/noodlemagazine.rs @@ -302,7 +302,7 @@ impl Provider for NoodlemagazineProvider { }) } - fn get_channel(&self, clientversion: ClientVersion) -> Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/omgxxx.rs b/src/providers/omgxxx.rs index 7ced8b3..30a54f0 100644 --- a/src/providers/omgxxx.rs +++ b/src/providers/omgxxx.rs @@ -621,7 +621,7 @@ impl Provider for OmgxxxProvider { } } } - fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/pimpbunny.rs b/src/providers/pimpbunny.rs index c72c980..4115e9f 100644 --- a/src/providers/pimpbunny.rs +++ b/src/providers/pimpbunny.rs @@ -3,7 +3,7 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::discord::send_discord_error_report; +use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; @@ -101,7 +101,7 @@ impl PimpbunnyProvider { let stars = Arc::clone(&self.stars); let categories = Arc::clone(&self.categories); - thread::spawn(move || { + thread::spawn(async move || { let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -109,14 +109,15 @@ impl PimpbunnyProvider { Ok(rt) => rt, Err(e) => { eprintln!("tokio runtime failed: {e}"); - let _ = futures::executor::block_on(send_discord_error_report( - &e, + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), Some("Pimpbunny Provider"), Some("Failed to create tokio runtime"), file!(), line!(), module_path!(), - )); + ).await; return; } }; @@ -125,7 +126,8 @@ impl PimpbunnyProvider { if let Err(e) = Self::load_stars(&url, Arc::clone(&stars)).await { eprintln!("load_stars failed: {e}"); send_discord_error_report( - &e, + e.to_string(), + Some(format_error_chain(&e)), Some("Pimpbunny Provider"), Some("Failed to load stars during initial load"), file!(), @@ -136,7 +138,8 @@ impl PimpbunnyProvider { if let Err(e) = Self::load_categories(&url, Arc::clone(&categories)).await { eprintln!("load_categories failed: {e}"); send_discord_error_report( - &e, + e.to_string(), + Some(format_error_chain(&e)), Some("Pimpbunny Provider"), Some("Failed to load categories during initial load"), file!(), @@ -529,7 +532,7 @@ impl Provider for PimpbunnyProvider { }) } - fn get_channel(&self, v: ClientVersion) -> Channel { - self.build_channel(v) + fn get_channel(&self, v: ClientVersion) -> Option { + Some(self.build_channel(v)) } } diff --git a/src/providers/pmvhaven.rs b/src/providers/pmvhaven.rs index c1bce8c..f30de9a 100644 --- a/src/providers/pmvhaven.rs +++ b/src/providers/pmvhaven.rs @@ -11,6 +11,7 @@ use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use std::sync::{Arc, RwLock}; use std::vec; +use std::fmt::Write; error_chain! { foreign_links { @@ -277,20 +278,25 @@ impl Provider for PmvhavenProvider { Ok(v) => v, Err(e) => { eprintln!("pmvhaven error: {e}"); - let _ = futures::executor::block_on(send_discord_error_report( - &e, + let mut chain_str = String::new(); + for (i, cause) in e.iter().enumerate() { + let _ = writeln!(chain_str, "{}. {}", i + 1, cause); + } + send_discord_error_report( + e.to_string(), + Some(chain_str), Some("PMVHaven Provider"), Some("Failed to load videos from PMVHaven"), file!(), line!(), module_path!(), - )); + ).await; vec![] } } } - fn get_channel(&self, clientversion: ClientVersion) -> Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/pornxp.rs b/src/providers/pornxp.rs index 8088d5a..ac0f3ba 100644 --- a/src/providers/pornxp.rs +++ b/src/providers/pornxp.rs @@ -265,7 +265,7 @@ impl Provider for PornxpProvider { } } } - fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/rule34gen.rs b/src/providers/rule34gen.rs index bb93fc1..b33d79c 100644 --- a/src/providers/rule34gen.rs +++ b/src/providers/rule34gen.rs @@ -282,7 +282,7 @@ impl Provider for Rule34genProvider { } } - fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/sxyprn.rs b/src/providers/sxyprn.rs index 892fbcf..98de290 100644 --- a/src/providers/sxyprn.rs +++ b/src/providers/sxyprn.rs @@ -1,6 +1,7 @@ use crate::DbPool; use crate::providers::Provider; use crate::util::cache::VideoCache; +use crate::util::discord::format_error_chain; use crate::util::discord::send_discord_error_report; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; @@ -26,14 +27,6 @@ error_chain! { } } -// fn has_blacklisted_class(element: &ElementRef, blacklist: &[&str]) -> bool { -// element -// .value() -// .attr("class") -// .map(|classes| classes.split_whitespace().any(|c| blacklist.contains(&c))) -// .unwrap_or(false) -// } - #[derive(Debug, Clone)] pub struct SxyprnProvider { url: String, @@ -97,6 +90,15 @@ impl SxyprnProvider { Ok(items) => items, Err(e) => { println!("Error parsing video items: {}", e); + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + Some("Sxyprn Provider"), + Some(&format!("URL: {}", url_str)), + file!(), + line!(), + module_path!(), + ).await; return Ok(old_items); } }; @@ -159,14 +161,16 @@ impl SxyprnProvider { { Ok(items) => items, Err(e) => { - println!("Error parsing video items: {}", e); - let _ = futures::executor::block_on(send_discord_error_report( - &e, + println!("Error parsing video items: {}", e);// 1. Convert the error to a string immediately + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), Some("Sxyprn Provider"), - Some(format!("Failed to query videos:\nURL: {}\nQuery: {},", url_str, query).as_str()), + Some(&format!("URL: {}", url_str)), file!(), line!(), - module_path!(),)); + module_path!(), + ).await; return Ok(old_items); } }; @@ -223,7 +227,7 @@ impl SxyprnProvider { let title_parts = video_segment .split("post_text") .nth(1) - .and_then(|s| s.split("style=''>").nth(1)) + .and_then(|s| s.split("style=''>").nth(100000)) .and_then(|s| s.split("").next()) .ok_or_else(|| ErrorKind::Parse("failed to extract title_parts".into()))?; diff --git a/src/providers/tnaflix.rs b/src/providers/tnaflix.rs index 76b12f4..e57456b 100644 --- a/src/providers/tnaflix.rs +++ b/src/providers/tnaflix.rs @@ -554,7 +554,7 @@ impl Provider for TnaflixProvider { } } } - fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/providers/xxdbx.rs b/src/providers/xxdbx.rs index e269db9..71a2c6d 100644 --- a/src/providers/xxdbx.rs +++ b/src/providers/xxdbx.rs @@ -278,7 +278,7 @@ impl Provider for XxdbxProvider { } } } - fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { - self.build_channel(clientversion) + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) } } diff --git a/src/util/discord.rs b/src/util/discord.rs index bc263c2..9712681 100644 --- a/src/util/discord.rs +++ b/src/util/discord.rs @@ -4,35 +4,36 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::json; -/// Send a detailed error report to a Discord webhook -pub async fn send_discord_error_report( - error: &T, - context: Option<&str>, // e.g. provider name, URL, query - extra_info: Option<&str>, // any debug info you want +use crate::util::requester; + +pub fn format_error_chain(err: &dyn Error) -> String { + let mut chain_str = String::new(); + let mut current_err: Option<&dyn Error> = Some(err); + let mut index = 1; + while let Some(e) = current_err { + let _ = writeln!(chain_str, "{}. {}", index, e); + current_err = e.source(); + index += 1; + } + chain_str +} + +pub async fn send_discord_error_report( + error_msg: String, + error_chain: Option, + context: Option<&str>, + extra_info: Option<&str>, file: &str, line: u32, module: &str, ) { + // 1. Get Webhook from Environment let webhook_url = match std::env::var("DISCORD_WEBHOOK") { Ok(url) => url, - Err(_) => return, + Err(_) => return, // Exit silently if no webhook is configured }; - // Discord embed field limits const MAX_FIELD: usize = 1024; - - let mut error_chain = String::new(); - let mut current: &dyn Error = error; - let mut i = 0; - loop { - let _ = writeln!(error_chain, "{}. {}", i + 1, current); - i += 1; - match current.source() { - Some(src) => current = src, - None => break, - } - } - let truncate = |s: &str| { if s.len() > MAX_FIELD { format!("{}…", &s[..MAX_FIELD - 1]) @@ -53,23 +54,23 @@ pub async fn send_discord_error_report( "fields": [ { "name": "Error", - "value": truncate(&error.to_string()), + "value": format!("```{}```", truncate(&error_msg)), "inline": false }, { "name": "Error Chain", - "value": truncate(&error_chain), + "value": truncate(&error_chain.unwrap_or_else(|| "No chain provided".to_string())), "inline": false }, { "name": "Location", "value": format!("`{}`:{}\n`{}`", file, line, module), - "inline": false + "inline": true }, { "name": "Context", "value": truncate(context.unwrap_or("n/a")), - "inline": false + "inline": true }, { "name": "Extra Info", @@ -83,13 +84,7 @@ pub async fn send_discord_error_report( }] }); - // Send (never panic) - if let Err(e) = wreq::Client::new() - .post(webhook_url) - .json(&payload) - .send() - .await - { - eprintln!("Failed to send Discord error report: {e}"); - } -} + let mut requester = requester::Requester::new(); + let _ = requester.post_json(&webhook_url, &payload, vec![]).await; + println!("Error report sent."); +} \ No newline at end of file diff --git a/src/util/requester.rs b/src/util/requester.rs index 3999c0e..c0efdd3 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -47,23 +47,6 @@ impl Requester { self.proxy = proxy; } - // pub fn set_flaresolverr_session(&mut self, session: String) { - // self.flaresolverr_session = Some(session); - // } - - // fn get_url_from_location_header(&self, prev_url: &str, location: &str) -> String { - // if location.starts_with("http://") || location.starts_with("https://") { - // location.to_string() - // } else if location.starts_with("//") { - // format!("{}{}", "https:", location) - // } else if location.starts_with('/') { - // let base_url = prev_url.split('/').take(3).collect::>().join("/"); - // format!("{}{}", base_url, location) - // } else { - // format!("{}/{}", prev_url, location) - // } - // } - pub async fn get_raw(&mut self, url: &str) -> Result { let client = Client::builder() .cert_verification(false)