From e2f3bc2ecb55725c1b0da9b998195e21dc68436d Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 14 Jan 2026 15:41:22 +0000 Subject: [PATCH] bugfixes --- src/providers/hypnotube.rs | 2 - src/providers/javtiful.rs | 3 - src/providers/missav.rs | 351 +++++++++++++++---------------------- 3 files changed, 142 insertions(+), 214 deletions(-) diff --git a/src/providers/hypnotube.rs b/src/providers/hypnotube.rs index 0e27bea..07ada0f 100644 --- a/src/providers/hypnotube.rs +++ b/src/providers/hypnotube.rs @@ -212,7 +212,6 @@ impl HypnotubeProvider { .await .unwrap(); if text.contains("Sorry, no results were found.") { - eprintln!("Hypnotube query returned no results for page {}", page); return vec![]; } let video_items: Vec = self.get_video_items_from_html(text.clone()).await; @@ -281,7 +280,6 @@ impl HypnotubeProvider { // println!("Hypnotube search POST response status: {}", p.text().await.unwrap_or_default()); // let text = requester.get(&video_url, Some(Version::HTTP_11)).await.unwrap(); if text.contains("Sorry, no results were found.") { - eprint!("Hypnotube query returned no results for page {}", page); return vec![]; } let video_items: Vec = self.get_video_items_from_html(text.clone()).await; diff --git a/src/providers/javtiful.rs b/src/providers/javtiful.rs index f059a3a..caf9e81 100644 --- a/src/providers/javtiful.rs +++ b/src/providers/javtiful.rs @@ -133,7 +133,6 @@ impl JavtifulProvider { let mut requester = options.requester.clone().unwrap(); let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); if page > 1 && !text.contains(&format!("
  • {}", page)) { - eprint!("Javtiful query returned no results for page {}", page); return Ok(vec![]); } let video_items: Vec = self @@ -182,7 +181,6 @@ impl JavtifulProvider { let mut requester = options.requester.clone().unwrap(); let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); if page > 1 && !text.contains(&format!("
  • {}", page)) { - eprint!("Javtiful query returned no results for page {}", page); return Ok(vec![]); } let video_items: Vec = self @@ -203,7 +201,6 @@ impl JavtifulProvider { requester: &mut Requester, ) -> Vec { if html.is_empty() || html.contains("404 Not Found") { - eprint!("Javtiful returned empty or 404 html"); return vec![]; } diff --git a/src/providers/missav.rs b/src/providers/missav.rs index 2bd5f74..b8410ce 100644 --- a/src/providers/missav.rs +++ b/src/providers/missav.rs @@ -1,5 +1,6 @@ use std::vec; use async_trait::async_trait; +use diesel::r2d2; use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use futures::future::join_all; @@ -7,17 +8,24 @@ use wreq::Version; use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; +use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::videos::ServerOptions; use crate::videos::{VideoItem}; use crate::DbPool; use crate::util::requester::Requester; - error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); JsonError(serde_json::Error); + Pool(r2d2::Error); // Assuming r2d2 or similar for pool + } + errors { + ParsingError(t: String) { + description("parsing error") + display("Parsing error: '{}'", t) + } } } @@ -25,264 +33,189 @@ error_chain! { pub struct MissavProvider { url: String, } + impl MissavProvider { pub fn new() -> Self { MissavProvider { url: "https://missav.ws".to_string() } } - async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result> { - // Extract needed fields from options at the start - let language = options.language.clone().unwrap(); - let filter = options.filter.clone().unwrap(); - let mut requester = options.requester.clone().unwrap(); - if !sort.is_empty(){ + async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result> { + // Use ok_or to avoid unwrapping options + let language = options.language.as_ref().ok_or("Missing language")?; + let filter = options.filter.as_ref().ok_or("Missing filter")?; + let mut requester = options.requester.clone().ok_or("Missing requester")?; + + if !sort.is_empty() { sort = format!("&sort={}", sort); } let url_str = format!("{}/{}/{}?page={}{}", self.url, language, filter, page, sort); - let old_items = match cache.get(&url_str) { - Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { - return Ok(items.clone()); - } - else{ - items.clone() - } + if let Some((time, items)) = cache.get(&url_str) { + if time.elapsed().unwrap_or_default().as_secs() < 3600 { + return Ok(items.clone()); } - None => { - vec![] - } - }; + } - let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap(); - // Pass a reference to options if needed, or reconstruct as needed - let video_items: Vec = self.get_video_items_from_html(text.clone(), pool, requester).await; + let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { + eprintln!("Error fetching Missav URL {}: {}", url_str, e); + let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); + "".to_string() + }); + let video_items = self.get_video_items_from_html(text, pool, requester).await; + if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); + cache.insert(url_str, video_items.clone()); } Ok(video_items) } - async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str, mut sort: String, options: ServerOptions) -> Result> { - // Extract needed fields from options at the start - let language = options.language.clone().unwrap(); - let mut requester = options.requester.clone().unwrap(); + async fn query(&self, cache: VideoCache, pool: DbPool, page: u8, query: &str, mut sort: String, options: ServerOptions) -> Result> { + let language = options.language.as_ref().ok_or("Missing language")?; + let mut requester = options.requester.clone().ok_or("Missing requester")?; + let search_string = query.replace(" ", "%20"); - if !sort.is_empty(){ + if !sort.is_empty() { sort = format!("&sort={}", sort); } - let url_str = format!( - "{}/{}/search/{}?page={}{}", - self.url, language, search_string, page, sort - ); - // Check our Video Cache. If the result is younger than 1 hour, we return it. - let old_items = match cache.get(&url_str) { - Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { - return Ok(items.clone()); - } - else{ - let _ = cache.check().await; - return Ok(items.clone()) - } + let url_str = format!("{}/{}/search/{}?page={}{}", self.url, language, search_string, page, sort); + + if let Some((time, items)) = cache.get(&url_str) { + if time.elapsed().unwrap_or_default().as_secs() < 3600 { + return Ok(items.clone()); } - None => { - vec![] - } - }; - let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap(); - let video_items: Vec = self.get_video_items_from_html(text.clone(), pool, requester).await; + } + + let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { + eprintln!("Error fetching Missav URL {}: {}", url_str, e); + let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); + "".to_string() + }); + let video_items = self.get_video_items_from_html(text, pool, requester).await; + if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); + cache.insert(url_str, video_items.clone()); } Ok(video_items) } async fn get_video_items_from_html(&self, html: String, pool: DbPool, requester: Requester) -> Vec { - if html.is_empty() { - println!("HTML is empty"); - return vec![]; - } - let raw_videos = html - .split("@mouseenter=\"setPreview(\'") - .collect::>()[1..] - .to_vec(); - let mut urls: Vec = vec![]; - for video_segment in &raw_videos { - // let vid = video_segment.split("\n").collect::>(); - // for (index, line) in vid.iter().enumerate() { - // println!("Line {}: {}", index, line.to_string().trim()); - // } - - let url_str = video_segment.split(">()[1] - .split("\"") - .collect::>()[0] - .to_string(); - urls.push(url_str.clone()); - - } - let futures = urls.into_iter().map(|el| self.get_video_item(el.clone(), pool.clone(), requester.clone())); - let results: Vec> = join_all(futures).await; - let video_items: Vec = results - .into_iter() - .filter_map(Result::ok) - .collect(); + if html.is_empty() { return vec![]; } - return video_items; + let segments: Vec<&str> = html.split("@mouseenter=\"setPreview(\'").collect(); + if segments.len() < 2 { return vec![]; } + + let mut urls = vec![]; + for video_segment in &segments[1..] { + // Safer parsing: find start and end of href + if let Some(start) = video_segment.find(" Result { - let mut conn = pool.get().expect("couldn't get db connection from pool"); - let db_result = db::get_video(&mut conn,url_str.clone()); - match db_result { - Ok(Some(entry)) => { - let video_item: VideoItem = serde_json::from_str(entry.as_str()).unwrap(); - return Ok(video_item) - } - Ok(None) => { - } - Err(e) => { - println!("Error fetching video from database: {}", e); + // 1. Database Check + { + let mut conn = pool.get().map_err(|e| Error::from(format!("Pool error: {}", e)))?; + if let Ok(Some(entry)) = db::get_video(&mut conn, url_str.clone()) { + if let Ok(video_item) = serde_json::from_str::(entry.as_str()) { + return Ok(video_item); + } } } - drop(conn); - let vid = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap(); - let mut title = vid.split(">()[1] - .split("\"") - .collect::>()[0].trim() - .to_string(); + + // 2. Fetch Page + let vid = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { + eprintln!("Error fetching Missav URL {}: {}", url_str, e); + let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); + "".to_string() + }); + + // Helper closure to extract content between two strings + let extract = |html: &str, start_tag: &str, end_tag: &str| -> Option { + let start = html.find(start_tag)? + start_tag.len(); + let rest = &html[start..]; + let end = rest.find(end_tag)?; + Some(rest[..end].to_string()) + }; + + let mut title = extract(&vid, ">()[1] - .split("\"") - .collect::>()[0] - .to_string(); - let raw_duration = vid.split(">()[1] - .split("\"") - .collect::>()[0] - .to_string(); - let duration = raw_duration.parse::().unwrap_or(0); + let thumb = extract(&vid, ">().last().unwrap() - .to_string(); + let duration = extract(&vid, "().ok()) + .unwrap_or(0); + + let id = url_str.split('/').last().ok_or("No ID found")?.to_string(); + + // 3. Extract Tags (Generic approach to avoid repetitive code) let mut tags = vec![]; - if vid.contains("Actress:"){ - for actress_snippet in vid.split("Actress:").collect::>()[1] - .split("").collect::>()[0].split("class=\"text-nord13 font-medium\">"){ - let tag = actress_snippet.split("<").collect::>()[0].trim() - .to_string(); - if !tag.is_empty(){ - tags.push(format!("@actress:{}", tag)); + for (label, prefix) in [("Actress:", "@actress"), ("Actor:", "@actor"), ("Maker:", "@maker"), ("Genre:", "@genre")] { + let marker = format!("{}", label); + if let Some(section) = extract(&vid, &marker, "") { + for part in section.split("class=\"text-nord13 font-medium\">").skip(1) { + if let Some(val) = part.split('<').next() { + let clean = val.trim(); + if !clean.is_empty() { + tags.push(format!("{}:{}", prefix, clean)); + } + } } } } - if vid.contains("Actor:"){ - for actor_snippet in vid.split("Actor:").collect::>()[1] - .split("").collect::>()[0].split("class=\"text-nord13 font-medium\">"){ - let tag = actor_snippet.split("<").collect::>()[0].trim() - .to_string(); - if !tag.is_empty(){ - tags.push(format!("@actor:{}", tag)); - } - } + + // 4. Extract Video URL (The m3u8 logic) + let video_url = (|| { + let parts_str = vid.split("m3u8").nth(1)?.split("https").next()?; + let mut parts: Vec<&str> = parts_str.split('|').collect(); + parts.reverse(); + if parts.len() < 8 { return None; } + Some(format!("https://{}.{}/{}-{}-{}-{}-{}/playlist.m3u8", + parts[1], parts[2], parts[3], parts[4], parts[5], parts[6], parts[7])) + })().ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?; + + let video_item = VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration) + .tags(tags) + .preview(format!("https://fourhoi.com/{}/preview.mp4", url_str.split('/').last().unwrap_or_default())); + + // 5. Cache to DB + if let Ok(mut conn) = pool.get() { + let _ = db::insert_video(&mut conn, &url_str, &serde_json::to_string(&video_item).unwrap_or_default()); } - - if vid.contains("Maker:"){ - for maker_snippet in vid.split("Maker:").collect::>()[1] - .split("").collect::>()[0] - .split("class=\"text-nord13 font-medium\">"){ - let tag = maker_snippet.split("<").collect::>()[0].trim() - .to_string(); - if !tag.is_empty(){ - tags.push(format!("@maker:{}", tag)); - } - } - } - if vid.contains("Genre:"){ - for tag_snippet in vid.split("Genre:").collect::>()[1] - .split("").collect::>()[0].split("class=\"text-nord13 font-medium\">"){ - let tag = tag_snippet.split("<").collect::>()[0].trim() - .to_string(); - if !tag.is_empty(){ - tags.push(format!("@genre:{}", tag)); - } - } - } - - let preview = format!("https://fourhoi.com/{}/preview.mp4",id.clone()); - - let mut video_url_parts = vid.split("m3u8").collect::>()[1] - .split("https").collect::>()[0] - .split("|").collect::>(); - video_url_parts.reverse(); - let video_url = format!("https://{}.{}/{}-{}-{}-{}-{}/playlist.m3u8", - video_url_parts[1], - video_url_parts[2], - video_url_parts[3], - video_url_parts[4], - video_url_parts[5], - video_url_parts[6], - video_url_parts[7] - ); - let video_item = VideoItem::new( - id, - title, - video_url.clone(), - "missav".to_string(), - thumb, - duration, - ) - .tags(tags) - .preview(preview) - ; - - let mut conn = pool.get().expect("couldn't get db connection from pool"); - let insert_result = db::insert_video(&mut conn, &url_str, &serde_json::to_string(&video_item)?); - match insert_result{ - Ok(_) => (), - Err(e) => {println!("{:?}", e); } - } - drop(conn); - - return Ok(video_item); + Ok(video_item) } } #[async_trait] impl Provider for MissavProvider { - async fn get_videos( - &self, - cache: VideoCache, - pool: DbPool, - sort: String, - query: Option, - page: String, - per_page: String, - options: ServerOptions, - ) -> Vec { - let _ = per_page; - let videos: std::result::Result, Error> = match query { - Some(q) => self.query(cache, pool, page.parse::().unwrap_or(1), &q, sort, options).await, - None => self.get(cache, pool, page.parse::().unwrap_or(1), sort, options).await, + async fn get_videos(&self, cache: VideoCache, pool: DbPool, sort: String, query: Option, page: String, _per_page: String, options: ServerOptions) -> Vec { + let page_num = page.parse::().unwrap_or(1); + let result = match query { + Some(q) => self.query(cache, pool, page_num, &q, sort, options).await, + None => self.get(cache, pool, page_num, sort, options).await, }; - match videos { - Ok(v) => v, - Err(e) => { - println!("Error fetching videos: {}", e); - vec![] - } - } + + result.unwrap_or_else(|e| { + eprintln!("Error fetching videos: {}", e); + let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!()); + vec![] + }) } -} +} \ No newline at end of file