diff --git a/src/db.rs b/src/db.rs index 31d4f97..b558b89 100644 --- a/src/db.rs +++ b/src/db.rs @@ -22,4 +22,8 @@ pub fn insert_video(conn: &mut SqliteConnection, new_id: &str, new_url: &str) -> }).execute(conn) } +pub fn delete_video(conn: &mut SqliteConnection, video_id: String) -> Result { + use crate::schema::videos::dsl::*; + diesel::delete(videos.filter(id.eq(video_id))).execute(conn) +} diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index 14ca541..8a9894e 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -3,10 +3,14 @@ use std::env; use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use futures::future::join_all; +use serde::Serialize; use wreq::Client; use wreq_util::Emulation; +use serde::Deserialize; use crate::db; +use crate::providers::perverzija; use crate::providers::Provider; +use crate::schema::videos::url; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; @@ -18,10 +22,17 @@ error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); + JsonError(serde_json::Error); } } +#[derive(Debug, Deserialize, Serialize)] +struct PerverzijaDbEntry { + url_string: String, + tags_strings: Vec, +} + #[derive(Debug, Clone)] pub struct PerverzijaProvider { url: String, @@ -38,15 +49,15 @@ impl PerverzijaProvider { if featured == "featured" { prefix_uri = "featured-scenes/".to_string(); } - let mut url = format!("{}{}page/{}/", self.url, prefix_uri, page); + let mut url_str = format!("{}{}page/{}/", self.url, prefix_uri, page); if page == 1 { - url = format!("{}{}", self.url, prefix_uri); + url_str = format!("{}{}", self.url, prefix_uri); } - let old_items = match cache.get(&url) { + let old_items = match cache.get(&url_str) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { - println!("Cache hit for URL: {}", url); + println!("Cache hit for URL: {}", url_str); return Ok(items.clone()); } else{ @@ -63,14 +74,14 @@ impl PerverzijaProvider { .emulation(Emulation::Firefox136) .build()?; - let response = client.get(url.clone()).send().await?; + let response = client.get(url_str.clone()).send().await?; // print!("Response: {:?}\n", response); if response.status().is_success() { let text = response.text().await?; let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); if !video_items.is_empty() { - cache.remove(&url); - cache.insert(url.clone(), video_items.clone()); + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); } else{ return Ok(old_items); } @@ -81,7 +92,7 @@ impl PerverzijaProvider { let result = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), - url: url.clone(), + url: url_str.clone(), maxTimeout: 60000, }) .await; @@ -96,8 +107,8 @@ impl PerverzijaProvider { } }; if !video_items.is_empty() { - cache.remove(&url); - cache.insert(url.clone(), video_items.clone()); + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); } else { return Ok(old_items); } @@ -107,27 +118,26 @@ impl PerverzijaProvider { async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result> { let mut query_parse = true; let search_string = query.replace(" ", "+"); - let mut url = format!( + let mut url_str = format!( "{}page/{}/?s={}", self.url, page, search_string ); if page == 1 { - url = format!("{}?s={}", self.url, search_string); + url_str = format!("{}?s={}", self.url, search_string); } if query.starts_with("@studio:") { let studio_name = query.replace("@studio:", ""); - url = format!("{}studio/{}/page/{}/", self.url, studio_name, page); + url_str = format!("{}studio/{}/page/{}/", self.url, studio_name, page); query_parse = false; } else if query.starts_with("@stars:") { let stars_name = query.replace("@stars:", ""); - url = format!("{}stars/{}/page/{}/", self.url, stars_name, page); + url_str = format!("{}stars/{}/page/{}/", self.url, stars_name, page); query_parse = false; } - url = url.replace("page/1/", ""); - println!("Query URL: {}", url); + url_str = url_str.replace("page/1/", ""); // Check our Video Cache. If the result is younger than 1 hour, we return it. - let old_items = match cache.get(&url) { + let old_items = match cache.get(&url_str) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { return Ok(items.clone()); @@ -145,18 +155,16 @@ impl PerverzijaProvider { .emulation(Emulation::Firefox136) .build()?; - let response = client.get(url.clone()).send().await?; + let response = client.get(url_str.clone()).send().await?; if response.status().is_success() { let text = response.text().await?; let video_items: Vec = match query_parse{ - true => { println!("query"); - self.get_video_items_from_html_query(text.clone(), pool).await}, - false => { println!("no query"); - self.get_video_items_from_html(text.clone(), pool)} + true => {self.get_video_items_from_html_query(text.clone(), pool).await}, + false => {self.get_video_items_from_html(text.clone(), pool)} }; if !video_items.is_empty() { - cache.remove(&url); - cache.insert(url.clone(), video_items.clone()); + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); } else{ return Ok(old_items); } @@ -167,7 +175,7 @@ impl PerverzijaProvider { let result = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), - url: url.clone(), + url: url_str.clone(), maxTimeout: 60000, }) .await; @@ -181,8 +189,8 @@ impl PerverzijaProvider { } }; if !video_items.is_empty() { - cache.remove(&url); - cache.insert(url.clone(), video_items.clone()); + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); } else{ return Ok(old_items); } @@ -215,11 +223,11 @@ impl PerverzijaProvider { .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let url = vid[1].split("iframe src="").collect::>()[1] + let url_str = vid[1].split("iframe src="").collect::>()[1] .split(""") .collect::>()[0] .to_string().replace("index.php", "xs1.php"); - let id = url.split("data=").collect::>()[1] + let id = url_str.split("data=").collect::>()[1] .split("&") .collect::>()[0] .to_string(); @@ -258,10 +266,10 @@ impl PerverzijaProvider { .to_string(); let mut conn = pool.get().expect("couldn't get db connection from pool"); - let _ = db::insert_video(&mut conn, &id_url, &url); + let _ = db::insert_video(&mut conn, &id_url, &url_str); drop(conn); let referer_url = "https://xtremestream.xyz/".to_string(); - let embed = VideoEmbed::new(embed_html, url.clone()); + let embed = VideoEmbed::new(embed_html, url_str.clone()); let mut tags: Vec = Vec::new(); // Placeholder for tags, adjust as needed @@ -305,7 +313,7 @@ impl PerverzijaProvider { ).tags(tags); // .embed(embed.clone()); let mut format = - videos::VideoFormat::new(url.clone(), "1080".to_string(), "m3u8".to_string()); + videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); @@ -368,32 +376,42 @@ impl PerverzijaProvider { let mut conn = pool.get().expect("couldn't get db connection from pool"); let db_result = db::get_video(&mut conn,lookup_url.clone()); match db_result { - Ok(Some(url)) => { - if url.starts_with("!"){ - return Err("Video was removed".into()); + Ok(Some(entry)) => { + if entry.starts_with("{"){ // replace old urls with new json objects + let entry = serde_json::from_str::(entry.as_str())?; + let url_str = entry.url_string; + let tags = entry.tags_strings; + if url_str.starts_with("!"){ + return Err("Video was removed".into()); + } + let mut id = url_str.split("data=").collect::>()[1] + .to_string(); + if id.contains("&"){ + id = id.split("&").collect::>()[0].to_string() + } + let mut video_item = VideoItem::new( + id, + title, + url_str.clone(), + "perverzija".to_string(), + thumb, + duration, + ) + .tags(tags) + ; + let mut format = + videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); + format.add_http_header("Referer".to_string(), referer_url.clone()); + if let Some(formats) = video_item.formats.as_mut() { + formats.push(format); + } else { + video_item.formats = Some(vec![format]); + } + return Ok(video_item) } - let mut id = url.split("data=").collect::>()[1] - .to_string(); - if id.contains("&"){ - id = id.split("&").collect::>()[0].to_string() - } - let mut video_item = VideoItem::new( - id, - title, - url.clone(), - "perverzija".to_string(), - thumb, - duration, - ); - let mut format = - videos::VideoFormat::new(url.clone(), "1080".to_string(), "m3u8".to_string()); - format.add_http_header("Referer".to_string(), referer_url.clone()); - if let Some(formats) = video_item.formats.as_mut() { - formats.push(format); - } else { - video_item.formats = Some(vec![format]); - } - return Ok(video_item); + else{ + let _ = db::delete_video(&mut conn,lookup_url.clone()); + }; } Ok(None) => { }, @@ -419,25 +437,63 @@ impl PerverzijaProvider { }; - let mut url = text.split("