updates to sxyprn

src/providers/sxyprn.rs

@@ -1,15 +1,15 @@
-use std::vec;
+use crate::DbPool;
-use error_chain::error_chain;
-use htmlentity::entity::{decode, ICodedDataTrait};
-use scraper::ElementRef;
 use crate::providers::Provider;
 use crate::util::cache::VideoCache;
-use crate::util::time::parse_time_to_seconds;
-use crate::videos::ServerOptions;
-use crate::videos::{VideoItem};
-use crate::DbPool;
 use crate::util::requester::Requester;
+use crate::util::time::parse_time_to_seconds;
+use crate::videos::VideoItem;
+use crate::videos::{self, ServerOptions, VideoFormat};
+use error_chain::error_chain;
+use htmlentity::entity::{ICodedDataTrait, decode};
+use scraper::ElementRef;
 use scraper::{Html, Selector};
+use std::vec;
 
 error_chain! {
     foreign_links {
@@ -19,7 +19,6 @@ error_chain! {
     }
 }
 
-
 fn has_blacklisted_class(element: &ElementRef, blacklist: &[&str]) -> bool {
     element
         .value()
@@ -35,11 +34,17 @@ pub struct SxyprnProvider {
 impl SxyprnProvider {
     pub fn new() -> Self {
         SxyprnProvider {
-            url: "https://sxyprn.com".to_string()
+            url: "https://sxyprn.com".to_string(),
         }
     }
-    async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
+    async fn get(
-
+        &self,
+        cache: VideoCache,
+        pool: DbPool,
+        page: u8,
+        sort: String,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
         let sort_string = match sort.as_str() {
             "views" => "views",
             "rating" => "rating",
@@ -55,14 +60,19 @@ impl SxyprnProvider {
         };
         let mut requester = options.requester.clone().unwrap();
 
-        let url_str = format!("{}/blog/all/{}.html?fl={}&sm={}", self.url, ((page as u32)-1)*20, filter_string, sort_string);
+        let url_str = format!(
-        
+            "{}/blog/all/{}.html?fl={}&sm={}",
+            self.url,
+            ((page as u32) - 1) * 20,
+            filter_string,
+            sort_string
+        );
+
         let old_items = match cache.get(&url_str) {
             Some((time, items)) => {
                 if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
                     return Ok(items.clone());
-                }
+                } else {
-                else{
                     items.clone()
                 }
             }
@@ -73,17 +83,27 @@ impl SxyprnProvider {
 
         let text = requester.get(&url_str).await.unwrap();
         // Pass a reference to options if needed, or reconstruct as needed
-        let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await;
+        let video_items: Vec<VideoItem> = self
+            .get_video_items_from_html(text.clone(), pool, requester)
+            .await;
         if !video_items.is_empty() {
             cache.remove(&url_str);
             cache.insert(url_str.clone(), video_items.clone());
-        } else{
+        } else {
             return Ok(old_items);
         }
         Ok(video_items)
     }
 
-    async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str, sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
+    async fn query(
+        &self,
+        cache: VideoCache,
+        pool: DbPool,
+        page: u8,
+        query: &str,
+        sort: String,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
         let sort_string = match sort.as_str() {
             "views" => "views",
             "rating" => "trending",
@@ -102,52 +122,62 @@ impl SxyprnProvider {
             Some((time, items)) => {
                 if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
                     return Ok(items.clone());
-                }
+                } else {
-                else{
                     let _ = cache.check().await;
-                    return Ok(items.clone())
+                    return Ok(items.clone());
                 }
             }
             None => {
                 vec![]
             }
-        };    
+        };
         let text = requester.get(&url_str).await.unwrap();
-        let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await;
+        let video_items: Vec<VideoItem> = self
+            .get_video_items_from_html(text.clone(), pool, requester)
+            .await;
         if !video_items.is_empty() {
             cache.remove(&url_str);
             cache.insert(url_str.clone(), video_items.clone());
-        } else{
+        } else {
             return Ok(old_items);
         }
         Ok(video_items)
     }
 
-    async fn get_video_items_from_html(&self, html: String, pool: DbPool, requester: Requester) -> Vec<VideoItem> {
+    async fn get_video_items_from_html(
+        &self,
+        html: String,
+        pool: DbPool,
+        requester: Requester,
+    ) -> Vec<VideoItem> {
         if html.is_empty() {
             println!("HTML is empty");
             return vec![];
         }
-        let raw_videos = html
+        let raw_videos = html.split("<script async").collect::<Vec<&str>>()[0]
-            .split("<script async").collect::<Vec<&str>>()[0]
+            .split("post_el_small'")
-            .split("post_el_small'").collect::<Vec<&str>>()[1..]
+            .collect::<Vec<&str>>()[1..]
             .to_vec();
         let mut items: Vec<VideoItem> = Vec::new();
         for video_segment in &raw_videos {
-            let vid = video_segment.split("\n").collect::<Vec<&str>>();
+            // let vid = video_segment.split("\n").collect::<Vec<&str>>();
-            for (index, line) in vid.iter().enumerate() {
+            // for (index, line) in vid.iter().enumerate() {
-                println!("Line {}: {}", index, line.to_string().trim());
+            //     println!("Line {}: {}", index, line.to_string().trim());
-            }
+            // }
-            println!("\n\n\n");
+            // println!("\n\n\n");
-            
-            let video_url = format!("https://hottub.spacemoehre.de/proxy/sxyprn/post/{}",video_segment.split("/post/").collect::<Vec<&str>>()[1]
-                .split("'").collect::<Vec<&str>>()[0]
-                .to_string());
 
-            let title_parts = video_segment.split("post_text").collect::<Vec<&str>>()[1].split("style=''>").collect::<Vec<&str>>()[1]
+            let url = video_segment.split("/post/").collect::<Vec<&str>>()[1]
-                .split("</div>")
+                .split("'")
                 .collect::<Vec<&str>>()[0]
-                ;
+                .to_string();
+
+            let video_url = format!("https://hottub.spacemoehre.de/proxy/sxyprn/post/{}", url);
+
+            let title_parts = video_segment.split("post_text").collect::<Vec<&str>>()[1]
+                .split("style=''>")
+                .collect::<Vec<&str>>()[1]
+                .split("</div>")
+                .collect::<Vec<&str>>()[0];
             let document = Html::parse_document(title_parts);
             let selector = Selector::parse("*").unwrap();
 
@@ -160,62 +190,113 @@ impl SxyprnProvider {
             }
             let mut title = texts[0].clone();
             // html decode
-            title = decode(title.as_bytes()).to_string().unwrap_or(title).replace("  "," ");
+            title = decode(title.as_bytes())
+                .to_string()
+                .unwrap_or(title)
+                .replace("  ", " ");
             title = title.replace(" + ", " ").replace("  ", " ");
 
             // println!("Title: {}", title);
             let id = video_url.split("/").collect::<Vec<&str>>()[6].to_string();
 
-            let thumb = format!("https:{}",video_segment.split("<img class='mini_post_vid_thumb lazyload'").collect::<Vec<&str>>()[1]
+            let thumb = format!(
-                .split("data-src='").collect::<Vec<&str>>()[1]
+                "https:{}",
-                .split("'")
+                video_segment
-                .collect::<Vec<&str>>()[0]
+                    .split("<img class='mini_post_vid_thumb lazyload'")
-                .to_string());
+                    .collect::<Vec<&str>>()[1]
+                    .split("data-src='")
+                    .collect::<Vec<&str>>()[1]
+                    .split("'")
+                    .collect::<Vec<&str>>()[0]
+                    .to_string()
+            );
 
-            let preview = match video_segment.contains("class='hvp_player'"){
+            let preview = match video_segment.contains("class='hvp_player'") {
-                true => Some(format!("https:{}",video_segment
+                true => Some(format!(
-                .split("class='hvp_player'").collect::<Vec<&str>>()[1]
+                    "https:{}",
-                .split(" src='").collect::<Vec<&str>>()[1]
+                    video_segment
-                .split("'")
+                        .split("class='hvp_player'")
-                .collect::<Vec<&str>>()[0]
+                        .collect::<Vec<&str>>()[1]
-                .to_string())),
+                        .split(" src='")
-                false => None
+                        .collect::<Vec<&str>>()[1]
+                        .split("'")
+                        .collect::<Vec<&str>>()[0]
+                        .to_string()
+                )),
+                false => None,
             };
 
-            let views= video_segment
+            let views = video_segment
-                .split("<strong>·</strong> ").collect::<Vec<&str>>()[1]
+                .split("<strong>·</strong> ")
+                .collect::<Vec<&str>>()[1]
                 .split(" ")
                 .collect::<Vec<&str>>()[0]
                 .to_string();
 
-            let raw_duration = video_segment
+            let raw_duration = video_segment.split("duration_small").collect::<Vec<&str>>()[1]
-                .split("duration_small").collect::<Vec<&str>>()[1]
+                .split("title='")
-                .split("title='").collect::<Vec<&str>>()[1]
+                .collect::<Vec<&str>>()[1]
-                .split("'").collect::<Vec<&str>>()[1]
+                .split("'")
-                .split(">").collect::<Vec<&str>>()[1]
+                .collect::<Vec<&str>>()[1]
-                .split("<").collect::<Vec<&str>>()[0]
+                .split(">")
+                .collect::<Vec<&str>>()[1]
+                .split("<")
+                .collect::<Vec<&str>>()[0]
                 .to_string();
-            println!("Duration: {}", raw_duration);
             let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
 
+            let stream_urls = video_segment
+                .split("extlink_icon extlink")
+                .collect::<Vec<&str>>()
+                .iter()
+                .map(|part| {
+                    let url = part
+                        .split("href='")
+                        .collect::<Vec<&str>>()
+                        .last()
+                        .unwrap_or(&"")
+                        .split("'")
+                        .collect::<Vec<&str>>()[0]
+                        .to_string();
+                    url
+                })
+                .filter(|url| url.starts_with("http") && !url.starts_with("https://bigwarp.io/"))
+                .collect::<Vec<String>>();
+
             let mut video_item = VideoItem::new(
                 id,
                 title,
-                video_url.to_string(),
+                url.to_string(),
                 "sxyprn".to_string(),
                 thumb,
                 duration,
             )
-            .views(views.parse::<u32>().unwrap_or(0))
+            .views(views.parse::<u32>().unwrap_or(0));
-            ;
             if let Some(p) = preview {
                 video_item = video_item.preview(p);
             }
+            let mut formats_vec = vec![];
+            formats_vec.push(
+                VideoFormat::new(video_url.clone(), "1080".to_string(), "m3u8".to_string())
+                    .protocol("https".to_string())
+                        .format_id(video_url.clone())
+                        .ext("m3u8".to_string())
+                        .video_ext("m3u8".to_string())
+            );
+            for surl in stream_urls {
+                formats_vec.push(
+                    videos::VideoFormat::new(surl.clone(), "1080".to_string(), "m3u8".to_string())
+                        .protocol("https".to_string())
+                        .format_id(surl)
+                        .ext("m3u8".to_string())
+                        .video_ext("m3u8".to_string())
+                );
+            }
+            video_item = video_item.formats(formats_vec);
             items.push(video_item);
         }
         return items;
     }
-
 }
 
 impl Provider for SxyprnProvider {
@@ -231,8 +312,21 @@ impl Provider for SxyprnProvider {
     ) -> Vec<VideoItem> {
         let _ = per_page;
         let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
-            Some(q) => self.query(cache, pool, page.parse::<u8>().unwrap_or(1), &q, sort, options).await,
+            Some(q) => {
-            None => self.get(cache, pool, page.parse::<u8>().unwrap_or(1), sort, options).await,
+                self.query(
+                    cache,
+                    pool,
+                    page.parse::<u8>().unwrap_or(1),
+                    &q,
+                    sort,
+                    options,
+                )
+                .await
+            }
+            None => {
+                self.get(cache, pool, page.parse::<u8>().unwrap_or(1), sort, options)
+                    .await
+            }
         };
         match videos {
             Ok(v) => v,

src/videos.rs

@@ -229,7 +229,7 @@ impl VideoFormat {
         VideoFormat {
             url,
             quality,
-            format: "mp4".to_string(), // Default format
+            format: format, // Default format
             format_id: Some("mp4-1080".to_string()),
             format_note: None,
             filesize: None,