diff --git a/Cargo.toml b/Cargo.toml index 157b75f..d13499b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ ntex-files = "2.0.0" serde = "1.0.228" serde_json = "1.0.145" tokio = { version = "1.47.1", features = ["full"] } -wreq = { version = "5.3.0", features = ["full", "cookies"] } +wreq = { version = "5.3.0", features = ["full", "cookies", "multipart"] } wreq-util = "2" percent-encoding = "2.3.2" capitalize = "0.3.4" diff --git a/src/providers/javtiful.rs b/src/providers/javtiful.rs new file mode 100644 index 0000000..7ff2ff4 --- /dev/null +++ b/src/providers/javtiful.rs @@ -0,0 +1,423 @@ +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::Provider; +use crate::status::*; +use crate::util::cache::VideoCache; +use crate::util::discord::send_discord_error_report; +use crate::util::requester::Requester; +use crate::util::time::parse_time_to_seconds; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; + +use async_trait::async_trait; +use error_chain::error_chain; +use futures::future::join_all; +use htmlentity::entity::{decode, ICodedDataTrait}; +use std::sync::{Arc, RwLock}; +use std::{vec}; +use titlecase::Titlecase; +use wreq::Version; + +error_chain! { + foreign_links { + Io(std::io::Error); + HttpRequest(wreq::Error); + Json(serde_json::Error); + } + errors { + Parse(msg: String) { + description("parse error") + display("parse error: {}", msg) + } + } +} + +#[derive(Debug, Clone)] +pub struct JavtifulProvider { + url: String, + categories: Arc>>, +} + +impl JavtifulProvider { + pub fn new() -> Self { + let provider = Self { + url: "https://javtiful.com".to_string(), + categories: Arc::new(RwLock::new(vec![])), + }; + provider + } + + fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; + Channel { + id: "pimpbunny".to_string(), + name: "Javtiful".to_string(), + description: "Watch Porn!".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=javtiful.com".to_string(), + status: "active".to_string(), + categories: self + .categories + .read() + .unwrap() + .iter() + .map(|c| c.title.clone()) + .collect(), + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "newest".into(), + title: "Newest".into(), + }, + FilterOption { + id: "top rated".into(), + title: "Top Rated".into(), + }, + FilterOption { + id: "most viewed".into(), + title: "Most Viewed".into(), + }, + FilterOption { + id: "top favorites".into(), + title: "Top Favorites".into(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: None, + } + } + + fn push_unique(target: &Arc>>, item: FilterOption) { + if let Ok(mut vec) = target.write() { + if !vec.iter().any(|x| x.id == item.id) { + vec.push(item); + } + } + } + + async fn get( + &self, + cache: VideoCache, + page: u8, + sort: &str, + options: ServerOptions, + ) -> Result> { + let sort_string = match sort { + "top rated" => "/sort=top_rated", + "most viewed" => "/sort=most_viewed", + _ => "", + }; + let video_url = format!( + "{}/videos{}?page={}", + self.url, sort_string, page + ); + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } else { + items.clone() + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); + if page > 1 && !text.contains(&format!("
  • {}", page)) { + eprint!("Javtiful query returned no results for page {}", page); + return Ok(vec![]); + } + let video_items: Vec = self + .get_video_items_from_html(text.clone(), &mut requester) + .await; + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + async fn query( + &self, + cache: VideoCache, + page: u8, + query: &str, + options: ServerOptions, + ) -> Result> { + let sort_string = match options.sort.as_deref().unwrap_or("") { + "top rated" => "/sort=top_rated", + "most viewed" => "/sort=most_viewed", + _ => "", + }; + let video_url = format!( + "{}/videos{}?search_query={}&page={}", + self.url, sort_string, query.replace(" ","+"), page + ); + // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } else { + let _ = cache.check().await; + return Ok(items.clone()); + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + + let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); + if page > 1 && !text.contains(&format!("
  • {}", page)) { + eprint!("Javtiful query returned no results for page {}", page); + return Ok(vec![]); + } + let video_items: Vec = self + .get_video_items_from_html(text.clone(), &mut requester) + .await; + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + async fn get_video_items_from_html( + &self, + html: String, + requester: &mut Requester, + ) -> Vec { + if html.is_empty() || html.contains("404 Not Found") { + eprint!("Javtiful returned empty or 404 html"); + return vec![]; + } + + let block = match html + .split("pagination ") + .next() + .and_then(|s| s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4").nth(1)) + { + Some(b) => b, + None => return vec![], + }; + + let futures = block + .split("card ") + .skip(1) + .map(|el| self.get_video_item(el.to_string(), requester.clone())); + + join_all(futures) + .await + .into_iter() + .inspect(|r| { + if let Err(e) = r { + let _ = futures::executor::block_on(send_discord_error_report( + &e, + Some("Javtiful Provider"), + Some("Failed to get video item"), + file!(), + line!(), + module_path!(), + )); + } + }) + .filter_map(Result::ok) + .collect() + } + + async fn get_video_item( + &self, + seg: String, + mut requester: Requester, + ) -> Result { + let video_url = seg + .split(" href=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .ok_or_else(|| ErrorKind::Parse("video url".into()))? + .to_string(); + + let mut title = seg + .split(" alt=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .ok_or_else(|| ErrorKind::Parse("video title".into()))? + .trim() + .to_string(); + + title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase(); + let id = video_url + .split('/') + .nth(5) + .and_then(|s| s.split('.').next()) + .ok_or_else(|| ErrorKind::Parse("video id".into()))? + .to_string(); + let thumb_block = seg + .split("") + .nth(1) + .and_then(|s| s.split('<').next()) + .unwrap_or("") + .to_string(); + let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; + let (tags, formats, views) = + self.extract_media(&video_url, &mut requester).await?; + + if preview.len() == 0 { + preview = format!("https://trailers.jav.si/preview/{id}.mp4"); + } + let video_item = VideoItem::new( + id, + title, + video_url, + "javtiful".into(), + thumb, + duration, + ) + .formats(formats) + .tags(tags) + .preview(preview) + .views(views); + Ok(video_item) + + } + + async fn extract_media( + &self, + url: &str, + requester: &mut Requester, + ) -> Result<(Vec, Vec, u32)> { + let text = requester + .get(url, Some(Version::HTTP_2)) + .await + .map_err(|e| Error::from(format!("{}", e)))?; + let tags = text.split("related-actress").next() + .and_then(|s| s.split("video-comments").next()) + .and_then(|s| s.split(">Tags<").nth(1)) + .map(|tag_block| { + tag_block + .split("') + .nth(1) + .and_then(|s| s.split('<').next()) + .map(|s| decode(s.as_bytes()).to_string().unwrap_or(s.to_string()).titlecase()) + }) + .collect() + }) + .unwrap_or_else(|| vec![]); + for tag in &tags { + Self::push_unique(&self.categories, FilterOption { + id: tag.to_ascii_lowercase().replace(" ","+"), + title: tag.to_string(), + }); + } + let views = text.split(" Views ") + .next() + .and_then(|s| s.split(" ").last()) + .and_then(|s| s.replace(".","") + .parse::().ok()) + .unwrap_or(0); + + let quality="1080p".to_string(); + let video_id = url + .split('/') + .nth(4) + .ok_or_else(|| ErrorKind::Parse("video id for media".into()))? + .to_string(); + + let token = text.split("data-csrf-token=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .ok_or_else(|| ErrorKind::Parse("csrf token".into()))? + .to_string(); + + let form = wreq::multipart::Form::new() + .text("video_id", video_id.clone()) + .text("pid_c", "".to_string()) + .text("token", token.clone()); + let resp = requester + .post_multipart( + "https://javtiful.com/ajax/get_cdn", + form, + vec![("Referer".to_string(), url.to_string())], + Some(Version::HTTP_11), + ) + .await + .map_err(|e| Error::from(format!("{}", e)))?; + let text = resp.text().await?; + let json: serde_json::Value = serde_json::from_str(&text)?; + let video_url = json.get("playlists") + .ok_or_else(|| ErrorKind::Parse("video_url in json".into()))? + .to_string().replace("\"", ""); + Ok(( + tags, + vec![VideoFormat::new(video_url, quality, "video/mp4".into())], + views, + )) + } +} + +#[async_trait] +impl Provider for JavtifulProvider { + async fn get_videos( + &self, + cache: VideoCache, + _pool: DbPool, + sort: String, + query: Option, + page: String, + _per_page: String, + options: ServerOptions, + ) -> Vec { + let page = page.parse::().unwrap_or(1); + + let res = match query { + Some(q) => self.to_owned().query(cache, page, &q, options).await, + None => self.get(cache, page, &sort, options).await, + }; + + res.unwrap_or_else(|e| { + eprintln!("javtiful error: {e}"); + vec![] + }) + } + + fn get_channel(&self, v: ClientVersion) -> Channel { + self.build_channel(v) + } +} diff --git a/src/providers/mod.rs b/src/providers/mod.rs index ba25d5f..85d875e 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -39,6 +39,7 @@ pub mod xxdbx; pub mod hqporner; pub mod noodlemagazine; pub mod pimpbunny; +pub mod javtiful; // convenient alias pub type DynProvider = Arc; @@ -55,6 +56,7 @@ pub static ALL_PROVIDERS: Lazy> = Lazy::new(| m.insert("pmvhaven", Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider); m.insert("noodlemagazine", Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider); m.insert("pimpbunny", Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider); + m.insert("javtiful", Arc::new(javtiful::JavtifulProvider::new()) as DynProvider); // add more here as you migrate them m }); diff --git a/src/util/requester.rs b/src/util/requester.rs index 0cf6bd2..11bc197 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -1,4 +1,5 @@ use serde::Serialize; +use wreq::multipart::Form; use std::env; use wreq::Client; use wreq::Proxy; @@ -119,14 +120,37 @@ impl Requester { where S: Serialize + ?Sized, { - let client = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) - .cookie_store(true) - .build() - .expect("Failed to create HTTP client"); + let mut request = self.client.post(url).version(Version::HTTP_11).json(data); - let mut request = client.post(url).version(Version::HTTP_11).json(data); + // Set custom headers + for (key, value) in headers.iter() { + request = request.header(key, value); + } + + if self.proxy { + if let Ok(proxy_url) = env::var("BURP_URL") { + let proxy = Proxy::all(&proxy_url).unwrap(); + request = request.proxy(proxy); + } + } + + request.send().await + } + + pub async fn post_multipart( + &mut self, + url: &str, + form: Form, + headers: Vec<(String, String)>, + _http_version: Option, + ) -> Result + { + let http_version = match _http_version { + Some(v) => v, + None => Version::HTTP_11, + }; + + let mut request = self.client.post(url).multipart(form).version(http_version); // Set custom headers for (key, value) in headers.iter() {