hentaihaven workflow update

src/providers/hentaihaven.rs

@@ -9,7 +9,7 @@ use crate::{DbPool, db};
 
 use async_trait::async_trait;
 use error_chain::error_chain;
-use futures::future::join_all;
+use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 use std::sync::{Arc, RwLock};
 use std::vec;
@@ -226,37 +226,39 @@ impl HentaihavenProvider {
             }
         };
 
-        let futures = block
+        let segments: Vec<String> = block.split("id=\"manga-item-").skip(1).map(|el| el.to_string()).collect();
-            .split("id=\"manga-item-")
+        stream::iter(segments.into_iter().map(|el| {
-            .skip(1)
+            let pool = pool.clone();
-            .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone()));
+            let requester = requester.clone();
-        join_all(futures)
+            let provider = self.clone();
-            .await
+            async move { provider.get_video_item(el, pool, requester).await }
-            .into_iter()
+        }))
-            .inspect(|r| {
+        .buffer_unordered(4)
-                if let Err(e) = r {
+        .filter_map(|result| async move {
+            match result {
+                Ok(item) => Some(item),
+                Err(e) => {
                     eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e);
-                    // Prepare data to move into the background task
                     let msg = e.to_string();
                     let chain = format_error_chain(&e);
-
-                    // Spawn the report into the background - NO .await here
                     tokio::spawn(async move {
                         let _ = send_discord_error_report(
                             msg,
                             Some(chain),
                             Some("Hentai Haven Provider"),
                             Some("Failed to get video item"),
-                            file!(), // Note: these might report the utility line
+                            file!(),
-                            line!(), // better to hardcode or pass from outside
+                            line!(),
                             module_path!(),
                         )
                         .await;
                     });
+                    None
+                }
             }
         })
-            .filter_map(Result::ok)
+        .collect::<Vec<_>>()
-            .collect()
+        .await
     }
 
     async fn get_video_items_from_html_search(
@@ -292,37 +294,39 @@ impl HentaihavenProvider {
             }
         };
 
-        let futures = block
+        let segments: Vec<String> = block.split("c-tabs-item__content col-6 col-md-12").skip(1).map(|el| el.to_string()).collect();
-            .split("c-tabs-item__content col-6 col-md-12")
+        stream::iter(segments.into_iter().map(|el| {
-            .skip(1)
+            let pool = pool.clone();
-            .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone()));
+            let requester = requester.clone();
-        join_all(futures)
+            let provider = self.clone();
-            .await
+            async move { provider.get_video_item(el, pool, requester).await }
-            .into_iter()
+        }))
-            .inspect(|r| {
+        .buffer_unordered(4)
-                if let Err(e) = r {
+        .filter_map(|result| async move {
+            match result {
+                Ok(item) => Some(item),
+                Err(e) => {
                     eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e);
-                    // Prepare data to move into the background task
                     let msg = e.to_string();
                     let chain = format_error_chain(&e);
-
-                    // Spawn the report into the background - NO .await here
                     tokio::spawn(async move {
                         let _ = send_discord_error_report(
                             msg,
                             Some(chain),
                             Some("Hentai Haven Provider"),
                             Some("Failed to get video item"),
-                            file!(), // Note: these might report the utility line
+                            file!(),
-                            line!(), // better to hardcode or pass from outside
+                            line!(),
                             module_path!(),
                         )
                         .await;
                     });
+                    None
+                }
             }
         })
-            .filter_map(Result::ok)
+        .collect::<Vec<_>>()
-            .collect()
+        .await
     }
 
     async fn get_video_item(
@@ -337,45 +341,38 @@ impl HentaihavenProvider {
             .and_then(|s| s.split('"').next())
             .ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))?
             .to_string();
-        let mut conn = match pool.get() {
+
-            Ok(conn) => conn,
+        match self.fetch_video_item(&video_url, &mut requester).await {
-            Err(e) => {
+            Ok(video_item) => {
-                let msg = format!("DB pool error: {}", e);
+                if let Ok(mut conn) = pool.get() {
-                send_discord_error_report(
+                    let _ = db::insert_video(
-                    msg.clone(),
+                        &mut conn,
-                    None,
+                        &video_url,
-                    Some("Hentai Haven Provider"),
+                        &serde_json::to_string(&video_item).unwrap_or_default(),
-                    Some("get_video_item.pool_get"),
+                    );
-                    file!(),
-                    line!(),
-                    module_path!(),
-                )
-                .await;
-                return Err(msg.into());
                 }
-        };
+                Ok(video_item)
-        let db_result = db::get_video(&mut conn, video_url.clone());
-        drop(conn);
-        match db_result {
-            Ok(Some(video)) => {
-                let video_item = VideoItem::from(video);
-                match video_item {
-                    Ok(item) => return Ok(item),
-                    Err(e) => {
-                        eprint!("Failed to convert video from DB result: {}\n", e);
-                    }
-                }
-            }
-            Ok(None) => {
-                // continue to fetch and parse the video
             }
             Err(e) => {
-                eprint!("Database error: {}\n", e);
+                if let Ok(mut conn) = pool.get() {
-                // continue to fetch and parse the video even if there's a DB error
+                    if let Ok(Some(cached)) = db::get_video(&mut conn, video_url.clone()) {
+                        if let Ok(item) = VideoItem::from(cached) {
+                            return Ok(item);
                         }
                     }
+                }
+                Err(e)
+            }
+        }
+    }
+
+    async fn fetch_video_item(
+        &self,
+        video_url: &str,
+        requester: &mut Requester,
+    ) -> Result<VideoItem> {
         let html = requester
-            .get(&video_url, Some(Version::HTTP_2))
+            .get(video_url, Some(Version::HTTP_2))
             .await
             .map_err(|e| Error::from(format!("Failed to fetch video page: {}", e)))?;
 
@@ -383,7 +380,7 @@ impl HentaihavenProvider {
             .split("<h1>")
             .nth(1)
             .and_then(|s| s.split("</h1>").next())
-            .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}").into()))?
+            .ok_or_else(|| ErrorKind::Parse(format!("video title: {video_url}")))?
             .trim()
             .to_string();
         title = decode(title.as_bytes())
@@ -394,7 +391,7 @@ impl HentaihavenProvider {
             .split('/')
             .nth(4)
             .and_then(|s| s.split('.').next())
-            .ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))?
+            .ok_or_else(|| ErrorKind::Parse(format!("video id: {video_url}")))?
             .to_string();
         let thumb = html
             .split("og:image\" content=\"")
@@ -495,43 +492,19 @@ impl HentaihavenProvider {
             formats.push(format);
         }
         if formats.is_empty() {
-            let e = Error::from(format!("No formats found for video URL: {}", video_url));
+            return Err(Error::from(format!("No formats found for video URL: {}", video_url)));
-            return Err(e);
         }
         if formats.len() > 1 {
             title = format!("{} ({} Episodes)", title, formats.len());
         }
 
-        let video_item =
+        Ok(
-            VideoItem::new(id, title, video_url.clone(), "hentaihaven".into(), thumb, 0)
+            VideoItem::new(id, title, video_url.to_string(), "hentaihaven".into(), thumb, 0)
                 .formats(formats)
                 .tags(tags)
                 .views(views)
-                .aspect_ratio(0.715);
+                .aspect_ratio(0.715),
-
-        match pool.get() {
-            Ok(mut conn) => {
-                let _ = db::insert_video(
-                    &mut conn,
-                    &video_url,
-                    &serde_json::to_string(&video_item).unwrap_or_default(),
-                );
-            }
-            Err(e) => {
-                send_discord_error_report(
-                    format!("DB pool error: {}", e),
-                    None,
-                    Some("Hentai Haven Provider"),
-                    Some("get_video_item.insert_video.pool_get"),
-                    file!(),
-                    line!(),
-                    module_path!(),
         )
-                .await;
-            }
-        }
-
-        Ok(video_item)
     }
 }