diff --git a/src/api.rs b/src/api.rs index 96dc516..7cc3081 100644 --- a/src/api.rs +++ b/src/api.rs @@ -3,6 +3,7 @@ use ntex::web; use ntex::web::HttpRequest; use crate::providers::perverzija::PerverzijaProvider; +use crate::util::cache::VideoCache; use crate::{providers::*, status::*, videos::*}; pub fn config(cfg: &mut web::ServiceConfig) { @@ -163,6 +164,7 @@ async fn status(req: HttpRequest) -> Result { async fn videos_post( video_request: web::types::Json, + cache: web::types::State ) -> Result { let mut videos = Videos { pageInfo: PageInfo { @@ -198,7 +200,7 @@ async fn videos_post( let featured = video_request.featured.as_deref().unwrap_or("all").to_string(); let provider = PerverzijaProvider::new(); let video_items = provider - .get_videos(channel, sort, query, page.to_string(), perPage.to_string(), featured) + .get_videos(cache.get_ref().clone(), channel, sort, query, page.to_string(), perPage.to_string(), featured) .await; videos.items = video_items.clone(); Ok(web::HttpResponse::Ok().json(&videos)) diff --git a/src/main.rs b/src/main.rs index 81ab6c1..9485837 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ #![allow(non_snake_case)] use ntex_files as fs; - use ntex::web; mod api; mod status; @@ -14,9 +13,11 @@ async fn main() -> std::io::Result<()> { std::env::set_var("RUST_BACKTRACE", "1"); env_logger::init(); // You need this to actually see logs + let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new(); - web::HttpServer::new(|| { + web::HttpServer::new(move || { web::App::new() + .state(cache.clone()) .wrap(web::middleware::Logger::default()) .service(web::scope("/api").configure(api::config)) .service(fs::Files::new("/", "static")) diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 9a69418..ceb6320 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -1,6 +1,6 @@ -use crate::videos::{Video_Item}; +use crate::{util::cache::VideoCache, videos::Video_Item}; pub mod perverzija; pub trait Provider{ - async fn get_videos(&self, channel: String, sort: String, query: Option, page: String, per_page: String, featured: String) -> Vec; + async fn get_videos(&self, cache: VideoCache ,channel: String, sort: String, query: Option, page: String, per_page: String, featured: String) -> Vec; } \ No newline at end of file diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index 1a6b689..af4e72f 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -5,6 +5,7 @@ use htmlentity::entity::{decode, ICodedDataTrait}; use reqwest::{Proxy}; use crate::providers::Provider; +use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; use crate::videos::{self, Video_Embed, Video_Item}; // Make sure Provider trait is imported @@ -25,7 +26,7 @@ impl PerverzijaProvider { url: "https://tube.perverzija.com/".to_string(), } } - async fn get(&self, page: &u8, featured: String) -> Result> { + async fn get(&self, cache:VideoCache ,page: &u8, featured: String) -> Result> { println!("get"); //TODO @@ -43,6 +44,21 @@ impl PerverzijaProvider { if page == &1 { url = format!("{}{}", self.url, prefix_uri); } + + let old_items = match cache.get(&url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { + println!("Cache hit for URL: {}", url); + return Ok(items.clone()); + } + else{ + items.clone() + } + } + None => { + vec![] + } + }; let client = match env::var("BURP_URL").as_deref() { @@ -63,6 +79,12 @@ impl PerverzijaProvider { if response.status().is_success() { let text = response.text().await?; let video_items: Vec = self.get_video_items_from_html(text.clone()); + if !video_items.is_empty() { + cache.remove(&url); + cache.insert(url.clone(), video_items.clone()); + } else{ + return Ok(old_items); + } Ok(video_items) } else { let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); @@ -85,10 +107,16 @@ impl PerverzijaProvider { return Err("Failed to solve FlareSolverr".into()); } }; + if !video_items.is_empty() { + cache.remove(&url); + cache.insert(url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } Ok(video_items) } } - async fn query(&self, page: &u8, query: &str) -> Result> { + async fn query(&self, cache: VideoCache, page: &u8, query: &str) -> Result> { println!("query: {}", query); let search_string = query.replace(" ", "+"); let mut url = format!( @@ -99,7 +127,21 @@ impl PerverzijaProvider { url = format!("{}advanced-search/?_sf_s={}", self.url, search_string); } - + // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { + println!("Cache hit for URL: {}", url); + return Ok(items.clone()); + } + else{ + items.clone() + } + } + None => { + vec![] + } + }; let client = match env::var("BURP_URL").as_deref() { Ok(burp_url) => reqwest::Client::builder() @@ -118,6 +160,12 @@ impl PerverzijaProvider { if response.status().is_success() { let text = response.text().await?; let video_items: Vec = self.get_video_items_from_html_query(text.clone()); + if !video_items.is_empty() { + cache.remove(&url); + cache.insert(url.clone(), video_items.clone()); + } else{ + return Ok(old_items); + } Ok(video_items) } else { let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); @@ -140,6 +188,12 @@ impl PerverzijaProvider { return Err("Failed to solve FlareSolverr".into()); } }; + if !video_items.is_empty() { + cache.remove(&url); + cache.insert(url.clone(), video_items.clone()); + } else{ + return Ok(old_items); + } Ok(video_items) } } @@ -353,6 +407,7 @@ impl PerverzijaProvider { impl Provider for PerverzijaProvider { async fn get_videos( &self, + cache: VideoCache, _channel: String, sort: String, query: Option, @@ -363,8 +418,8 @@ impl Provider for PerverzijaProvider { let _ = per_page; let _ = sort; let videos: std::result::Result, Error> = match query { - Some(q) => self.query(&page.parse::().unwrap_or(1), &q).await, - None => self.get(&page.parse::().unwrap_or(1), featured).await, + Some(q) => self.query(cache, &page.parse::().unwrap_or(1), &q).await, + None => self.get(cache, &page.parse::().unwrap_or(1), featured).await, }; match videos { Ok(v) => v, diff --git a/src/util/cache.rs b/src/util/cache.rs new file mode 100644 index 0000000..028e24b --- /dev/null +++ b/src/util/cache.rs @@ -0,0 +1,34 @@ +use std::time::SystemTime; + +use std::sync::{Arc, Mutex}; +use crate::videos::Video_Item; + +#[derive(Clone)] +pub struct VideoCache{ + cache: Arc)>>>, // url -> time+Items +} +impl VideoCache { + pub fn new() -> Self { + VideoCache { + cache: Arc::new(Mutex::new(std::collections::HashMap::new())), + } + } + + pub fn get(&self, key: &str) -> Option<(SystemTime, Vec)> { + let cache = self.cache.lock().ok()?; + cache.get(key).cloned() + } + + pub fn insert(&self, key: String, value: Vec) { + if let Ok(mut cache) = self.cache.lock() { + cache.insert(key.clone(), (SystemTime::now(), value.clone())); + } + } + + pub fn remove(&self, key: &str) { + if let Ok(mut cache) = self.cache.lock() { + cache.remove(key); + } + } + +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index 41aed28..dce15b4 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,2 +1,3 @@ pub mod time; -pub mod flaresolverr; \ No newline at end of file +pub mod flaresolverr; +pub mod cache; \ No newline at end of file diff --git a/src/videos.rs b/src/videos.rs index d3dc66e..744ed99 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct Videos_Request { //"versionInstallDate":"2025-06-03T18:20:20Z","languageCode":"en","appInstallDate":"2025-06-03T18:20:20Z","server":"spacemoehre","sexu - pub clientHash: String, // "a07b23c9b07813c65050e2a4041ca777", + pub clientHash: Option, // "a07b23c9b07813c65050e2a4041ca777", pub blockedKeywords: Option, // "kittens", pub countryCode: Option, // "DE", pub clientVersion: Option, // "2.1.4-22b",