diff --git a/src/providers/hentaihaven.rs b/src/providers/hentaihaven.rs index 79e0a6b..204885d 100644 --- a/src/providers/hentaihaven.rs +++ b/src/providers/hentaihaven.rs @@ -9,7 +9,7 @@ use crate::{DbPool, db}; use async_trait::async_trait; use error_chain::error_chain; -use futures::future::join_all; +use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; use std::vec; @@ -226,37 +226,39 @@ impl HentaihavenProvider { } }; - let futures = block - .split("id=\"manga-item-") - .skip(1) - .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone())); - join_all(futures) - .await - .into_iter() - .inspect(|r| { - if let Err(e) = r { + let segments: Vec = block.split("id=\"manga-item-").skip(1).map(|el| el.to_string()).collect(); + stream::iter(segments.into_iter().map(|el| { + let pool = pool.clone(); + let requester = requester.clone(); + let provider = self.clone(); + async move { provider.get_video_item(el, pool, requester).await } + })) + .buffer_unordered(4) + .filter_map(|result| async move { + match result { + Ok(item) => Some(item), + Err(e) => { eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e); - // Prepare data to move into the background task let msg = e.to_string(); let chain = format_error_chain(&e); - - // Spawn the report into the background - NO .await here tokio::spawn(async move { let _ = send_discord_error_report( msg, Some(chain), Some("Hentai Haven Provider"), Some("Failed to get video item"), - file!(), // Note: these might report the utility line - line!(), // better to hardcode or pass from outside + file!(), + line!(), module_path!(), ) .await; }); + None } - }) - .filter_map(Result::ok) - .collect() + } + }) + .collect::>() + .await } async fn get_video_items_from_html_search( @@ -292,37 +294,39 @@ impl HentaihavenProvider { } }; - let futures = block - .split("c-tabs-item__content col-6 col-md-12") - .skip(1) - .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone())); - join_all(futures) - .await - .into_iter() - .inspect(|r| { - if let Err(e) = r { + let segments: Vec = block.split("c-tabs-item__content col-6 col-md-12").skip(1).map(|el| el.to_string()).collect(); + stream::iter(segments.into_iter().map(|el| { + let pool = pool.clone(); + let requester = requester.clone(); + let provider = self.clone(); + async move { provider.get_video_item(el, pool, requester).await } + })) + .buffer_unordered(4) + .filter_map(|result| async move { + match result { + Ok(item) => Some(item), + Err(e) => { eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e); - // Prepare data to move into the background task let msg = e.to_string(); let chain = format_error_chain(&e); - - // Spawn the report into the background - NO .await here tokio::spawn(async move { let _ = send_discord_error_report( msg, Some(chain), Some("Hentai Haven Provider"), Some("Failed to get video item"), - file!(), // Note: these might report the utility line - line!(), // better to hardcode or pass from outside + file!(), + line!(), module_path!(), ) .await; }); + None } - }) - .filter_map(Result::ok) - .collect() + } + }) + .collect::>() + .await } async fn get_video_item( @@ -337,45 +341,38 @@ impl HentaihavenProvider { .and_then(|s| s.split('"').next()) .ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))? .to_string(); - let mut conn = match pool.get() { - Ok(conn) => conn, - Err(e) => { - let msg = format!("DB pool error: {}", e); - send_discord_error_report( - msg.clone(), - None, - Some("Hentai Haven Provider"), - Some("get_video_item.pool_get"), - file!(), - line!(), - module_path!(), - ) - .await; - return Err(msg.into()); + + match self.fetch_video_item(&video_url, &mut requester).await { + Ok(video_item) => { + if let Ok(mut conn) = pool.get() { + let _ = db::insert_video( + &mut conn, + &video_url, + &serde_json::to_string(&video_item).unwrap_or_default(), + ); + } + Ok(video_item) } - }; - let db_result = db::get_video(&mut conn, video_url.clone()); - drop(conn); - match db_result { - Ok(Some(video)) => { - let video_item = VideoItem::from(video); - match video_item { - Ok(item) => return Ok(item), - Err(e) => { - eprint!("Failed to convert video from DB result: {}\n", e); + Err(e) => { + if let Ok(mut conn) = pool.get() { + if let Ok(Some(cached)) = db::get_video(&mut conn, video_url.clone()) { + if let Ok(item) = VideoItem::from(cached) { + return Ok(item); + } } } - } - Ok(None) => { - // continue to fetch and parse the video - } - Err(e) => { - eprint!("Database error: {}\n", e); - // continue to fetch and parse the video even if there's a DB error + Err(e) } } + } + + async fn fetch_video_item( + &self, + video_url: &str, + requester: &mut Requester, + ) -> Result { let html = requester - .get(&video_url, Some(Version::HTTP_2)) + .get(video_url, Some(Version::HTTP_2)) .await .map_err(|e| Error::from(format!("Failed to fetch video page: {}", e)))?; @@ -383,7 +380,7 @@ impl HentaihavenProvider { .split("

") .nth(1) .and_then(|s| s.split("

").next()) - .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}").into()))? + .ok_or_else(|| ErrorKind::Parse(format!("video title: {video_url}")))? .trim() .to_string(); title = decode(title.as_bytes()) @@ -394,7 +391,7 @@ impl HentaihavenProvider { .split('/') .nth(4) .and_then(|s| s.split('.').next()) - .ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))? + .ok_or_else(|| ErrorKind::Parse(format!("video id: {video_url}")))? .to_string(); let thumb = html .split("og:image\" content=\"") @@ -495,43 +492,19 @@ impl HentaihavenProvider { formats.push(format); } if formats.is_empty() { - let e = Error::from(format!("No formats found for video URL: {}", video_url)); - return Err(e); + return Err(Error::from(format!("No formats found for video URL: {}", video_url))); } if formats.len() > 1 { title = format!("{} ({} Episodes)", title, formats.len()); } - let video_item = - VideoItem::new(id, title, video_url.clone(), "hentaihaven".into(), thumb, 0) + Ok( + VideoItem::new(id, title, video_url.to_string(), "hentaihaven".into(), thumb, 0) .formats(formats) .tags(tags) .views(views) - .aspect_ratio(0.715); - - match pool.get() { - Ok(mut conn) => { - let _ = db::insert_video( - &mut conn, - &video_url, - &serde_json::to_string(&video_item).unwrap_or_default(), - ); - } - Err(e) => { - send_discord_error_report( - format!("DB pool error: {}", e), - None, - Some("Hentai Haven Provider"), - Some("get_video_item.insert_video.pool_get"), - file!(), - line!(), - module_path!(), - ) - .await; - } - } - - Ok(video_item) + .aspect_ratio(0.715), + ) } }