From 60a07269f61b234f821b3863d275281f563aaba4 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 6 Jun 2025 07:48:21 +0000 Subject: [PATCH] clean cache, handled warnings etc --- Cargo.toml | 1 + migrations/create_videos/up.sql | 2 +- src/api.rs | 3 +-- src/main.rs | 1 - src/providers/hanime.rs | 28 ++++++++++----------- src/providers/mod.rs | 6 ++--- src/providers/perverzija.rs | 44 ++++++++++++++++----------------- src/status.rs | 8 ++++-- src/util/cache.rs | 38 +++++++++++++++++++++++----- src/videos.rs | 32 +++++++++++++----------- 10 files changed, 96 insertions(+), 67 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 272a3f6..f701937 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,4 +18,5 @@ once_cell = "1.21.3" reqwest = { version = "0.12.18", features = ["blocking", "json", "rustls-tls"] } serde = "1.0.219" serde_json = "1.0.140" +tokio = { version = "1.45.1", features = ["full"] } diff --git a/migrations/create_videos/up.sql b/migrations/create_videos/up.sql index 781ca1b..67059f5 100644 --- a/migrations/create_videos/up.sql +++ b/migrations/create_videos/up.sql @@ -1,7 +1,7 @@ -- Your SQL goes here CREATE TABLE videos ( id TEXT NOT NULL PRIMARY KEY, -- like url parts to uniquely identify a video - url TEXT NOT NULL--, + url TEXT NOT NULL UNIQUE--, --views INTEGER, --rating INTEGER, --uploader TEXT diff --git a/src/api.rs b/src/api.rs index a95578b..15cd01c 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,4 +1,3 @@ -use futures::channel; use ntex::http::header; use ntex::web; use ntex::web::HttpRequest; @@ -237,7 +236,7 @@ async fn status(req: HttpRequest) -> Result { } async fn videos_post( - video_request: web::types::Json, + video_request: web::types::Json, cache: web::types::State, pool: web::types::State, ) -> Result { diff --git a/src/main.rs b/src/main.rs index 5d68c6c..403686d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,4 @@ #![allow(non_snake_case)] -#[macro_use] extern crate diesel; use diesel::{r2d2::{self, ConnectionManager}, SqliteConnection}; diff --git a/src/providers/hanime.rs b/src/providers/hanime.rs index a913b98..855d7f2 100644 --- a/src/providers/hanime.rs +++ b/src/providers/hanime.rs @@ -1,17 +1,13 @@ -use std::time::Duration; use std::vec; use std::env; use error_chain::error_chain; -use htmlentity::entity::{decode, ICodedDataTrait}; use reqwest::{Proxy}; use futures::future::join_all; use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; -use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; -use crate::util::time::parse_time_to_seconds; -use crate::videos::{self, Video_Embed, Video_Item}; +use crate::videos::{self, VideoItem}; use crate::DbPool; // Make sure Provider trait is imported error_chain! { @@ -33,6 +29,7 @@ struct HanimeSearchRequest{ page: u8 } +#[allow(dead_code)] impl HanimeSearchRequest { pub fn new() -> Self { HanimeSearchRequest { @@ -116,6 +113,7 @@ struct HanimeSearchResult{ } +#[allow(dead_code)] pub struct HanimeProvider { url: String, } @@ -127,7 +125,7 @@ impl HanimeProvider { } } - async fn get_video_item(&self, hit: HanimeSearchResult, pool: DbPool) -> Result { + async fn get_video_item(&self, hit: HanimeSearchResult, pool: DbPool) -> Result { let mut conn = pool.get().expect("couldn't get db connection from pool"); let db_result = db::get_video(&mut conn,format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug.clone())); drop(conn); @@ -138,12 +136,12 @@ impl HanimeProvider { let channel = "hanime".to_string(); // Placeholder, adjust as needed match db_result { Ok(Some(video_url)) => { - return Ok(Video_Item::new(id, title, video_url.clone(), channel, thumb, duration) + return Ok(VideoItem::new(id, title, video_url.clone(), channel, thumb, duration) .tags(hit.tags) .uploader(hit.brand) .views(hit.views as u32) .rating((hit.likes as f32 / (hit.likes + hit.dislikes)as f32) * 100 as f32) - .formats(vec![videos::Video_Format::new(video_url.clone(), "1080".to_string(), "m3u8".to_string())])); + .formats(vec![videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "m3u8".to_string())])); } Ok(None) => (), Err(e) => { @@ -189,16 +187,16 @@ impl HanimeProvider { let mut conn = pool.get().expect("couldn't get db connection from pool"); let _ = db::insert_video(&mut conn, &format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug.clone()), &url_vec[0].clone()); drop(conn); - Ok(Video_Item::new(id, title, url_vec[0].clone(), channel, thumb, duration) + Ok(VideoItem::new(id, title, url_vec[0].clone(), channel, thumb, duration) .tags(hit.tags) .uploader(hit.brand) .views(hit.views as u32) .rating((hit.likes as f32 / (hit.likes + hit.dislikes)as f32) * 100 as f32) - .formats(vec![videos::Video_Format::new(url_vec[0].clone(), "1080".to_string(), "m3u8".to_string())])) + .formats(vec![videos::VideoFormat::new(url_vec[0].clone(), "1080".to_string(), "m3u8".to_string())])) } - async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, query: String, sort:String) -> Result> { + async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, query: String, sort:String) -> Result> { let index = format!("{}:{}:{}", query, page, sort); let order_by = sort.split(".").collect::>()[0].to_string(); let ordering = sort.split(".").collect::>()[1].to_string(); @@ -251,8 +249,8 @@ impl HanimeProvider { .map_err(|e| format!("Failed to parse hits JSON: {}", e))?; // let timeout_duration = Duration::from_secs(120); let futures = hits_json.into_iter().map(|el| self.get_video_item(el.clone(), pool.clone())); - let results: Vec> = join_all(futures).await; - let video_items: Vec = results + let results: Vec> = join_all(futures).await; + let video_items: Vec = results .into_iter() .filter_map(Result::ok) .collect(); @@ -278,12 +276,12 @@ impl Provider for HanimeProvider { page: String, per_page: String, featured: String, - ) -> Vec { + ) -> Vec { let _ = featured; let _ = per_page; let _ = sort; println!("Sort: {:?}", sort); - let videos: std::result::Result, Error> = match query { + let videos: std::result::Result, Error> = match query { Some(q) => self.get(cache, pool, page.parse::().unwrap_or(1), q, sort).await, None => self.get(cache, pool, page.parse::().unwrap_or(1), "".to_string(), sort).await, }; diff --git a/src/providers/mod.rs b/src/providers/mod.rs index b8c9df7..6091a8d 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -1,10 +1,10 @@ -use crate::{providers::{hanime::HanimeProvider, perverzija::PerverzijaProvider}, util::cache::VideoCache, videos::Video_Item, DbPool}; +use crate::{providers::{hanime::HanimeProvider, perverzija::PerverzijaProvider}, util::cache::VideoCache, videos::VideoItem, DbPool}; pub mod perverzija; pub mod hanime; pub trait Provider{ - async fn get_videos(&self, cache: VideoCache, pool: DbPool, channel: String, sort: String, query: Option, page: String, per_page: String, featured: String) -> Vec; + async fn get_videos(&self, cache: VideoCache, pool: DbPool, channel: String, sort: String, query: Option, page: String, per_page: String, featured: String) -> Vec; } pub enum AnyProvider { @@ -12,7 +12,7 @@ pub enum AnyProvider { Hanime(HanimeProvider), } impl Provider for AnyProvider { - async fn get_videos(&self, cache: VideoCache, pool:DbPool, channel: String, sort: String, query: Option, page: String, per_page: String, featured: String) -> Vec { + async fn get_videos(&self, cache: VideoCache, pool:DbPool, channel: String, sort: String, query: Option, page: String, per_page: String, featured: String) -> Vec { match self { AnyProvider::Perverzija(p) => p.get_videos(cache, pool, channel, sort, query, page, per_page, featured).await, AnyProvider::Hanime(p) => p.get_videos(cache, pool, channel, sort, query, page, per_page, featured).await, diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index edb9f01..d827f83 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -4,14 +4,13 @@ use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use reqwest::{Proxy}; use futures::future::join_all; -use ntex::web; use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; -use crate::videos::{self, Video_Embed, Video_Item}; +use crate::videos::{self, VideoEmbed, VideoItem}; use crate::DbPool; // Make sure Provider trait is imported error_chain! { @@ -30,7 +29,7 @@ impl PerverzijaProvider { url: "https://tube.perverzija.com/".to_string(), } } - async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result> { + async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result> { //TODO // let mut url = Url::parse("https://example.net")?; // url.query_pairs_mut().append_pair("foo", "bar"); @@ -80,7 +79,7 @@ impl PerverzijaProvider { // print!("Response: {:?}\n", response); if response.status().is_success() { let text = response.text().await?; - let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); + let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); @@ -117,7 +116,7 @@ impl PerverzijaProvider { Ok(video_items) } } - async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result> { + async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result> { let search_string = query.replace(" ", "+"); let mut url = format!( "{}page/{}/?s={}", @@ -134,7 +133,8 @@ impl PerverzijaProvider { return Ok(items.clone()); } else{ - items.clone() + let _ = cache.check().await; + return Ok(items.clone()) } } None => { @@ -157,7 +157,7 @@ impl PerverzijaProvider { let response = client.get(url.clone()).send().await?; if response.status().is_success() { let text = response.text().await?; - let video_items: Vec = self.get_video_items_from_html_query(text.clone(), pool).await; + let video_items: Vec = self.get_video_items_from_html_query(text.clone(), pool).await; if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); @@ -194,12 +194,12 @@ impl PerverzijaProvider { } } - fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec { + fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec { if html.is_empty() { println!("HTML is empty"); return vec![]; } - let mut items: Vec = Vec::new(); + let mut items: Vec = Vec::new(); let video_listing_content = html.split("video-listing-content").collect::>()[1]; let raw_videos = video_listing_content .split("video-item post") @@ -261,7 +261,7 @@ impl PerverzijaProvider { let _ = db::insert_video(&mut conn, &id_url, &url); drop(conn); let referer_url = "https://xtremestream.xyz/".to_string(); - let embed = Video_Embed::new(embed_html, url.clone()); + let embed = VideoEmbed::new(embed_html, url.clone()); let mut tags: Vec = Vec::new(); // Placeholder for tags, adjust as needed for tag in vid[0].split(" ").collect::>(){ @@ -273,7 +273,7 @@ impl PerverzijaProvider { } } } - let mut video_item = Video_Item::new( + let mut video_item = VideoItem::new( id, title, embed.source.clone(), @@ -283,7 +283,7 @@ impl PerverzijaProvider { ).tags(tags); // .embed(embed.clone()); let mut format = - videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string()); + videos::VideoFormat::new(url.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); @@ -296,21 +296,21 @@ impl PerverzijaProvider { return items; } - async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec { + async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec { let raw_videos = html .split("video-item post") .collect::>()[1..] .to_vec(); let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone())); - let results: Vec> = join_all(futures).await; - let items: Vec = results + let results: Vec> = join_all(futures).await; + let items: Vec = results .into_iter() .filter_map(Result::ok) .collect(); return items; } - async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result { + async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result { let vid = snippet.split("\n").collect::>(); if vid.len() > 30 { return Err("Unexpected video snippet length".into()); @@ -355,7 +355,7 @@ impl PerverzijaProvider { if id.contains("&"){ id = id.split("&").collect::>()[0].to_string() } - let mut video_item = Video_Item::new( + let mut video_item = VideoItem::new( id, title, url.clone(), @@ -364,7 +364,7 @@ impl PerverzijaProvider { duration, ); let mut format = - videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string()); + videos::VideoFormat::new(url.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); @@ -438,7 +438,7 @@ impl PerverzijaProvider { } } - let mut video_item = Video_Item::new( + let mut video_item = VideoItem::new( id, title, url.clone(), @@ -449,7 +449,7 @@ impl PerverzijaProvider { .tags(tags); // .embed(embed.clone()); let mut format = - videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string()); + videos::VideoFormat::new(url.clone(), "1080".to_string(), "m3u8".to_string()); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); @@ -471,10 +471,10 @@ impl Provider for PerverzijaProvider { page: String, per_page: String, featured: String, - ) -> Vec { + ) -> Vec { let _ = per_page; let _ = sort; - let videos: std::result::Result, Error> = match query { + let videos: std::result::Result, Error> = match query { Some(q) => self.query(cache, pool, page.parse::().unwrap_or(1), &q).await, None => self.get(cache, pool, page.parse::().unwrap_or(1), featured).await, }; diff --git a/src/status.rs b/src/status.rs index e3cc71c..8782933 100644 --- a/src/status.rs +++ b/src/status.rs @@ -44,11 +44,11 @@ pub struct Options { pub description: String, //"Sort the videos by new or old.", pub systemImage: String, //"sort.image", pub colorName: String, //"blue", - pub options: Vec, + pub options: Vec, } #[derive(serde::Serialize)] -pub struct Option_Value { +pub struct OptionValue { pub id: String, //"new", pub title: String, //"New", pub description: Option, //"Sort the videos by new or old." @@ -107,15 +107,19 @@ impl Status { .to_string(), } } + #[allow(dead_code)] pub fn add_notice(&mut self, notice: Notice) { self.notices.push(notice); } + #[allow(dead_code)] pub fn add_channel(&mut self, channel: Channel) { self.channels.push(channel); } + #[allow(dead_code)] pub fn add_option(&mut self, option: Options) { self.options.push(option); } + #[allow(dead_code)] pub fn add_category(&mut self, category: String) { self.categories.push(category); } diff --git a/src/util/cache.rs b/src/util/cache.rs index 028e24b..a93f656 100644 --- a/src/util/cache.rs +++ b/src/util/cache.rs @@ -1,11 +1,14 @@ -use std::time::SystemTime; +use std::time::{SystemTime}; use std::sync::{Arc, Mutex}; -use crate::videos::Video_Item; +use std::time::Duration; + + +use crate::videos::VideoItem; #[derive(Clone)] pub struct VideoCache{ - cache: Arc)>>>, // url -> time+Items + cache: Arc)>>>, // url -> time+Items } impl VideoCache { pub fn new() -> Self { @@ -14,21 +17,44 @@ impl VideoCache { } } - pub fn get(&self, key: &str) -> Option<(SystemTime, Vec)> { + pub fn get(&self, key: &str) -> Option<(SystemTime, Vec)> { let cache = self.cache.lock().ok()?; cache.get(key).cloned() } - pub fn insert(&self, key: String, value: Vec) { + pub fn insert(&self, key: String, value: Vec) { if let Ok(mut cache) = self.cache.lock() { cache.insert(key.clone(), (SystemTime::now(), value.clone())); } } - pub fn remove(&self, key: &str) { if let Ok(mut cache) = self.cache.lock() { cache.remove(key); } } + pub fn entries(&self) -> Option))>> { + if let Ok(cache) = self.cache.lock() { + // Return a cloned vector of the cache entries + return Some(cache.iter().map(|(k, v)| (k.clone(), v.clone())).collect()); + } + None + } + + pub async fn check(&self) -> Result<(), Box>{ + let iter = match self.entries(){ + Some(iter) => iter, + None => return Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Could not get entries"))) + }; + + for (key, (time, _items)) in iter { + if let Ok(elapsed) = time.elapsed() { + if elapsed > Duration::from_secs(60*60){ + println!("Key: {}, elapsed: {:?}", key, elapsed); + self.remove(&key); + } + } + } + Ok(()) + } } \ No newline at end of file diff --git a/src/videos.rs b/src/videos.rs index d4befc4..ee9c893 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; #[derive(serde::Serialize, serde::Deserialize, Debug)] -pub struct Videos_Request { +pub struct VideosRequest { //"versionInstallDate":"2025-06-03T18:20:20Z","languageCode":"en","appInstallDate":"2025-06-03T18:20:20Z","server":"spacemoehre","sexu pub clientHash: Option, // "a07b23c9b07813c65050e2a4041ca777", pub blockedKeywords: Option, // "kittens", @@ -35,20 +35,20 @@ pub struct PageInfo { #[derive(serde::Serialize, Debug, Clone)] -pub struct Video_Embed{ +pub struct VideoEmbed{ pub html: String, pub source: String, } -impl Video_Embed { +impl VideoEmbed { pub fn new(html: String, source: String) -> Self { - Video_Embed { + VideoEmbed { html, source, } } } #[derive(serde::Serialize, Debug, Clone)] -pub struct Video_Item { +pub struct VideoItem { pub duration: u32, // 110, pub views: Option, // 14622653, pub rating: Option, // 0.0, @@ -62,10 +62,11 @@ pub struct Video_Item { pub verified: Option, // false, pub tags: Option>, // [], pub uploadedAt: Option, // 1741142954 - pub formats: Option>, // Additional HTTP headers if needed - pub embed: Option, // Optional embed information + pub formats: Option>, // Additional HTTP headers if needed + pub embed: Option, // Optional embed information } -impl Video_Item { +#[allow(dead_code)] +impl VideoItem { pub fn new( id: String, title: String, @@ -74,7 +75,7 @@ impl Video_Item { thumb: String, duration: u32, ) -> Self { - Video_Item { + VideoItem { duration: duration, // Placeholder, adjust as needed views: None, // Placeholder, adjust as needed rating: None, // Placeholder, adjust as needed @@ -120,18 +121,18 @@ impl Video_Item { self.uploadedAt = Some(uploaded_at); self } - pub fn formats(mut self, formats: Vec) -> Self { + pub fn formats(mut self, formats: Vec) -> Self { self.formats = Some(formats); self } - pub fn embed(mut self, embed: Video_Embed) -> Self { + pub fn embed(mut self, embed: VideoEmbed) -> Self { self.embed = Some(embed); self } } #[derive(serde::Serialize, Debug, Clone)] -pub struct Video_Format { +pub struct VideoFormat { url: String, quality: String, format: String, @@ -159,9 +160,10 @@ pub struct Video_Format { http_headers: Option>, } -impl Video_Format { +impl VideoFormat { pub fn new(url: String, quality: String, format: String) -> Self { - Video_Format { + let _ = format; + VideoFormat { url, quality, format: "mp4".to_string(), // Default format @@ -201,5 +203,5 @@ impl Video_Format { #[derive(serde::Serialize, Debug)] pub struct Videos { pub pageInfo: PageInfo, - pub items: Vec, + pub items: Vec, }