diff --git a/Cargo.toml b/Cargo.toml index 5276f01..e169a20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,9 @@ capitalize = "0.3.4" url = "2.5.4" base64 = "0.22.1" scraper = "0.24.0" +once_cell = "1.21.3" +rustc-hash = "2.1.1" +async-trait = "0.1" [lints.rust] unexpected_cfgs = "allow" diff --git a/src/api.rs b/src/api.rs index 5604e1a..fdaa56c 100644 --- a/src/api.rs +++ b/src/api.rs @@ -17,11 +17,13 @@ use crate::providers::rule34video::Rule34videoProvider; // use crate::providers::spankbang::SpankbangProvider; use crate::util::cache::VideoCache; use crate::util::requester::Requester; -use crate::{DbPool, db, providers::*, status::*, videos::*}; +use crate::{DbPool, db, status::*, videos::*}; use cute::c; +use std::sync::Arc; +use crate::providers::{Provider, DynProvider, ALL_PROVIDERS}; -#[derive(Debug)] -struct ClientVersion { +#[derive(Debug, Clone)] +pub struct ClientVersion { version: u32, subversion: u32, name: String, @@ -923,7 +925,7 @@ async fn status(req: HttpRequest) -> Result { cacheDuration: None, }); - // porn00 + // noodlemagazine // status.add_channel(Channel { // id: "noodlemagazine".to_string(), // name: "Noodlemagazine".to_string(), @@ -1070,42 +1072,6 @@ async fn status(req: HttpRequest) -> Result { cacheDuration: None, }); - status.add_channel(Channel{ - id: "omgxxx".to_string(), - name: "OMG XXX".to_string(), - description: "Free Porn Site".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.omg.xxx".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ - ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "latest-updates".to_string(), - title: "Latest".to_string(), - }, - FilterOption { - id: "most-popular".to_string(), - title: "Most Viewed".to_string(), - }, - FilterOption { - id: "top-rated".to_string(), - title: "Top Rated".to_string(), - }, - ], - multiSelect: false, - } - ], - nsfw: true, - cacheDuration: None, - }); - if clientversion >= ClientVersion::new(22, 105, "22i".to_string()) { //sxyprn status.add_channel(Channel { @@ -1170,6 +1136,14 @@ async fn status(req: HttpRequest) -> Result { cacheDuration: Some(1800), }); } + + for provider in ALL_PROVIDERS.values() { + println!( + "Loaded provider (dyn): {:?}", + std::any::type_name::<&dyn Provider>() + ); + status.add_channel(provider.get_channel(clientversion.clone())); + } status.iconUrl = format!("http://{}/favicon.ico", host).to_string(); Ok(web::HttpResponse::Ok().json(&status)) } @@ -1296,54 +1270,35 @@ async fn videos_post( Ok(web::HttpResponse::Ok().json(&videos)) } -pub fn get_provider(channel: &str) -> Option { +pub fn get_provider(channel: &str) -> Option { match channel { - "all" => Some(AnyProvider::All(AllProvider::new())), - "perverzija" => Some(AnyProvider::Perverzija(PerverzijaProvider::new())), - "hanime" => Some(AnyProvider::Hanime(HanimeProvider::new())), - // "spankbang" => Some(AnyProvider::Spankbang(SpankbangProvider::new())), - "pornhub" => Some(AnyProvider::Pornhub(PornhubProvider::new())), - "pmvhaven" => Some(AnyProvider::Pmvhaven(PmvhavenProvider::new())), - "rule34video" => Some(AnyProvider::Rule34video(Rule34videoProvider::new())), - "redtube" => Some(AnyProvider::Redtube(RedtubeProvider::new())), - "okporn" => Some(AnyProvider::Okporn(OkpornProvider::new())), - "pornhat" => Some(AnyProvider::Pornhat( - crate::providers::pornhat::PornhatProvider::new(), - )), - "perfectgirls" => Some(AnyProvider::Perfectgirls( + "all" => Some(Arc::new(AllProvider::new())), + "perverzija" => Some(Arc::new(PerverzijaProvider::new())), + "hanime" => Some(Arc::new(HanimeProvider::new())), + "pornhub" => Some(Arc::new(PornhubProvider::new())), + "pmvhaven" => Some(Arc::new(PmvhavenProvider::new())), + "rule34video" => Some(Arc::new(Rule34videoProvider::new())), + "redtube" => Some(Arc::new(RedtubeProvider::new())), + "okporn" => Some(Arc::new(OkpornProvider::new())), + "pornhat" => Some(Arc::new(crate::providers::pornhat::PornhatProvider::new())), + "perfectgirls" => Some(Arc::new( crate::providers::perfectgirls::PerfectgirlsProvider::new(), )), - "okxxx" => Some(AnyProvider::Okxxx( - crate::providers::okxxx::OkxxxProvider::new(), - )), - "homoxxx" => Some(AnyProvider::Homoxxx( - crate::providers::homoxxx::HomoxxxProvider::new(), - )), - // "hentaimoon" => Some(AnyProvider::Hentaimoon(crate::providers::hentaimoon::HentaimoonProvider::new())), - "missav" => Some(AnyProvider::Missav( - crate::providers::missav::MissavProvider::new(), - )), - "xxthots" => Some(AnyProvider::Xxthots( - crate::providers::xxthots::XxthotsProvider::new(), - )), - "sxyprn" => Some(AnyProvider::Sxyprn( - crate::providers::sxyprn::SxyprnProvider::new(), - )), - "porn00" => Some(AnyProvider::Porn00( - crate::providers::porn00::Porn00Provider::new(), - )), - // "noodlemagazine" => Some(AnyProvider::Noodlemagazine(crate::providers::noodlemagazine::NoodlemagazineProvider::new())), - "freshporno" => Some(AnyProvider::Freshporno( + "okxxx" => Some(Arc::new(crate::providers::okxxx::OkxxxProvider::new())), + "homoxxx" => Some(Arc::new(crate::providers::homoxxx::HomoxxxProvider::new())), + "missav" => Some(Arc::new(crate::providers::missav::MissavProvider::new())), + "xxthots" => Some(Arc::new(crate::providers::xxthots::XxthotsProvider::new())), + "sxyprn" => Some(Arc::new(crate::providers::sxyprn::SxyprnProvider::new())), + "porn00" => Some(Arc::new(crate::providers::porn00::Porn00Provider::new())), + "freshporno" => Some(Arc::new( crate::providers::freshporno::FreshpornoProvider::new(), )), - "youjizz" => Some(AnyProvider::Youjizz( - crate::providers::youjizz::YoujizzProvider::new(), - )), - "paradisehill" => Some(AnyProvider::Paradisehill( + "youjizz" => Some(Arc::new(crate::providers::youjizz::YoujizzProvider::new())), + "paradisehill" => Some(Arc::new( crate::providers::paradisehill::ParadisehillProvider::new(), )), - "pornzog" => Some(AnyProvider::Pornzog(crate::providers::pornzog::PornzogProvider::new())), - "omgxxx" => Some(AnyProvider::Omgxxx(crate::providers::omgxxx::OmgxxxProvider::new())), - _ => Some(AnyProvider::Perverzija(PerverzijaProvider::new())), + "pornzog" => Some(Arc::new(crate::providers::pornzog::PornzogProvider::new())), + // fallback to the dynamic registry + x => ALL_PROVIDERS.get(x).cloned(), } } diff --git a/src/db.rs b/src/db.rs index 6763518..c833356 100644 --- a/src/db.rs +++ b/src/db.rs @@ -46,8 +46,8 @@ pub fn has_table( use diesel::sql_types::Text; #[derive(QueryableByName)] struct TableName { - #[sql_type = "Text"] - #[column_name = "name"] + #[diesel(sql_type = Text)] + #[diesel(column_name = name)] name: String, } diff --git a/src/providers/all.rs b/src/providers/all.rs index 09bddd4..abbd0d4 100644 --- a/src/providers/all.rs +++ b/src/providers/all.rs @@ -1,8 +1,10 @@ use std::fs; +use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; -use crate::api::get_provider; -use crate::providers::{AnyProvider, Provider}; +use crate::api::{get_provider, ClientVersion}; +use crate::providers::{DynProvider, Provider}; +use crate::status::Channel; use crate::util::cache::VideoCache; use crate::util::interleave; use crate::videos::{ServerOptions, VideoItem}; @@ -27,6 +29,7 @@ impl AllProvider { } } +#[async_trait] impl Provider for AllProvider { async fn get_videos( &self, @@ -48,11 +51,11 @@ impl Provider for AllProvider { .collect::>(); sites_str = providers.join(","); } - let sites = sites_str + let providers: Vec = sites_str .split(',') - .map(|s| s.to_string()) // or s.to_owned() - .collect::>(); - let providers = sites.iter().map(|el| get_provider(el.as_str()).unwrap()).collect::>(); + .filter(|s| !s.is_empty()) + .filter_map(|s| get_provider(s)) // assumes get_provider -> Option + .collect(); let futures = providers.iter().map(|provider| { provider.get_videos( @@ -71,4 +74,12 @@ impl Provider for AllProvider { return video_items; } + + fn get_channel(&self,clientversion:ClientVersion) -> Channel { + println!("Getting channel for placeholder with client version: {:?}",clientversion); + let _ = clientversion; + Channel { + id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None, + } + } } diff --git a/src/providers/freshporno.rs b/src/providers/freshporno.rs index 8f45035..9f721e5 100644 --- a/src/providers/freshporno.rs +++ b/src/providers/freshporno.rs @@ -1,7 +1,10 @@ +use crate::api::ClientVersion; +use crate::status::Channel; use crate::DbPool; use crate::providers::Provider; use crate::util::cache::VideoCache; use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::vec; @@ -17,6 +20,7 @@ error_chain! { pub struct FreshpornoProvider { url: String, } + impl FreshpornoProvider { pub fn new() -> Self { FreshpornoProvider { @@ -156,6 +160,7 @@ impl FreshpornoProvider { } +#[async_trait] impl Provider for FreshpornoProvider { async fn get_videos( &self, @@ -187,4 +192,12 @@ impl Provider for FreshpornoProvider { } } } + + fn get_channel(&self,clientversion:ClientVersion) -> Channel { + println!("Getting channel for placeholder with client version: {:?}",clientversion); + let _ = clientversion; + Channel { + id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None, + } + } } diff --git a/src/providers/hanime.rs b/src/providers/hanime.rs index 5634603..51b5090 100644 --- a/src/providers/hanime.rs +++ b/src/providers/hanime.rs @@ -1,4 +1,5 @@ use std::vec; +use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; use wreq::Client; @@ -253,6 +254,7 @@ impl HanimeProvider { } } +#[async_trait] impl Provider for HanimeProvider { async fn get_videos( &self, diff --git a/src/providers/homoxxx.rs b/src/providers/homoxxx.rs index b5d67bb..2c22c89 100644 --- a/src/providers/homoxxx.rs +++ b/src/providers/homoxxx.rs @@ -4,6 +4,7 @@ use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::env; @@ -243,6 +244,7 @@ impl HomoxxxProvider { } +#[async_trait] impl Provider for HomoxxxProvider { async fn get_videos( &self, diff --git a/src/providers/missav.rs b/src/providers/missav.rs index c46049a..69a7378 100644 --- a/src/providers/missav.rs +++ b/src/providers/missav.rs @@ -1,4 +1,5 @@ use std::vec; +use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use futures::future::join_all; @@ -253,6 +254,7 @@ impl MissavProvider { } } +#[async_trait] impl Provider for MissavProvider { async fn get_videos( &self, diff --git a/src/providers/mod.rs b/src/providers/mod.rs index bc7210c..a4ac45d 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -1,7 +1,15 @@ +use async_trait::async_trait; +use once_cell::sync::Lazy; +use rustc_hash::FxHashMap as HashMap; +use std::sync::Arc; + use crate::{ - providers::{ - all::AllProvider, hanime::HanimeProvider, okporn::OkpornProvider, perverzija::PerverzijaProvider, pmvhaven::PmvhavenProvider, pornhub::PornhubProvider, redtube::RedtubeProvider, rule34video::Rule34videoProvider - }, util::cache::VideoCache, videos::{ServerOptions, VideoItem}, DbPool + DbPool, + api::ClientVersion, + providers::omgxxx::OmgxxxProvider, + status::Channel, + util::cache::VideoCache, + videos::{ServerOptions, VideoItem}, }; pub mod all; @@ -10,27 +18,37 @@ pub mod perverzija; pub mod pmvhaven; pub mod pornhub; // pub mod spankbang; -pub mod rule34video; -pub mod redtube; -pub mod okporn; -pub mod pornhat; -pub mod perfectgirls; -pub mod okxxx; pub mod homoxxx; +pub mod okporn; +pub mod okxxx; +pub mod perfectgirls; +pub mod pornhat; +pub mod redtube; +pub mod rule34video; // pub mod hentaimoon; pub mod missav; -pub mod xxthots; -pub mod sxyprn; pub mod porn00; +pub mod sxyprn; +pub mod xxthots; // pub mod noodlemagazine; pub mod freshporno; -pub mod youjizz; +pub mod omgxxx; pub mod paradisehill; pub mod pornzog; -pub mod omgxxx; +pub mod youjizz; +// convenient alias +pub type DynProvider = Arc; -pub trait Provider { +pub static ALL_PROVIDERS: Lazy> = Lazy::new(|| { + let mut m = HashMap::default(); + m.insert("omgxxx", Arc::new(OmgxxxProvider::new()) as DynProvider); + // add more here as you migrate them + m +}); + +#[async_trait] +pub trait Provider: Send + Sync { async fn get_videos( &self, cache: VideoCache, @@ -41,156 +59,24 @@ pub trait Provider { per_page: String, options: ServerOptions, ) -> Vec; -} -#[derive(Debug, Clone)] -pub enum AnyProvider { - All(AllProvider), - Perverzija(PerverzijaProvider), - Hanime(HanimeProvider), - // Spankbang(SpankbangProvider), - Pornhub(PornhubProvider), - Pmvhaven(PmvhavenProvider), - Rule34video(Rule34videoProvider), - Redtube(RedtubeProvider), - Okporn(OkpornProvider), - Pornhat(crate::providers::pornhat::PornhatProvider), - Perfectgirls(crate::providers::perfectgirls::PerfectgirlsProvider), - Okxxx(crate::providers::okxxx::OkxxxProvider), - Homoxxx(crate::providers::homoxxx::HomoxxxProvider), - // Hentaimoon(crate::providers::hentaimoon::HentaimoonProvider), - Missav(crate::providers::missav::MissavProvider), - Xxthots(crate::providers::xxthots::XxthotsProvider), - Sxyprn(crate::providers::sxyprn::SxyprnProvider), - Porn00(crate::providers::porn00::Porn00Provider), - // Noodlemagazine(crate::providers::noodlemagazine::NoodlemagazineProvider), - Freshporno(crate::providers::freshporno::FreshpornoProvider), - Youjizz(crate::providers::youjizz::YoujizzProvider), - Paradisehill(crate::providers::paradisehill::ParadisehillProvider), - Pornzog(crate::providers::pornzog::PornzogProvider), - Omgxxx(crate::providers::omgxxx::OmgxxxProvider), -} - -impl Provider for AnyProvider { - async fn get_videos( - &self, - cache: VideoCache, - pool: DbPool, - sort: String, - query: Option, - page: String, - per_page: String, - options: ServerOptions - ) -> Vec { + fn get_channel(&self, clientversion: ClientVersion) -> Channel { println!( - "/api/videos: sort={:?}, query={:?}, page={:?}, provider={:?}", - sort, query, page, self + "Getting channel for placeholder with client version: {:?}", + clientversion ); - match self { - AnyProvider::Perverzija(p) => { - p.get_videos( - cache.clone(), - pool.clone(), - sort.clone(), - query.clone(), - page.clone(), - per_page.clone(), - options, - ) - .await - } - AnyProvider::Hanime(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - // AnyProvider::Spankbang(p) => { - // p.get_videos(cache, pool, sort, query, page, per_page, options,) - // .await - // } - AnyProvider::Pornhub(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Pmvhaven(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Rule34video(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Redtube(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::All(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Okporn(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Pornhat(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Perfectgirls(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Okxxx(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Homoxxx(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - // AnyProvider::Hentaimoon(p) => { - // p.get_videos(cache, pool, sort, query, page, per_page, options,) - // .await - // } - AnyProvider::Missav(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Xxthots(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Sxyprn(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Porn00(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - // AnyProvider::Noodlemagazine(p) => { - // p.get_videos(cache, pool, sort, query, page, per_page, options,) - // .await - // } - AnyProvider::Freshporno(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Youjizz(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - }, - AnyProvider::Paradisehill(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Pornzog(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } - AnyProvider::Omgxxx(p) => { - p.get_videos(cache, pool, sort, query, page, per_page, options,) - .await - } + let _ = clientversion; + Channel { + id: "placeholder".to_string(), + name: "PLACEHOLDER".to_string(), + description: "PLACEHOLDER FOR PARENT CLASS".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![], + nsfw: true, + cacheDuration: None, } } } diff --git a/src/providers/okporn.rs b/src/providers/okporn.rs index 6835a0c..134b38f 100644 --- a/src/providers/okporn.rs +++ b/src/providers/okporn.rs @@ -10,6 +10,7 @@ use std::env; use std::vec; use wreq::{Client}; use wreq_util::Emulation; +use async_trait::async_trait; error_chain! { foreign_links { @@ -233,6 +234,7 @@ impl OkpornProvider { } +#[async_trait] impl Provider for OkpornProvider { async fn get_videos( &self, diff --git a/src/providers/okxxx.rs b/src/providers/okxxx.rs index 31da329..5244c6c 100644 --- a/src/providers/okxxx.rs +++ b/src/providers/okxxx.rs @@ -11,6 +11,7 @@ use std::env; use std::vec; use wreq::{Client}; use wreq_util::Emulation; +use async_trait::async_trait; error_chain! { foreign_links { @@ -278,6 +279,7 @@ impl OkxxxProvider { } +#[async_trait] impl Provider for OkxxxProvider { async fn get_videos( &self, diff --git a/src/providers/omgxxx.rs b/src/providers/omgxxx.rs index 93ed577..f5c1012 100644 --- a/src/providers/omgxxx.rs +++ b/src/providers/omgxxx.rs @@ -1,3 +1,4 @@ +use crate::api::ClientVersion; use crate::status::*; use crate::util::parse_abbreviated_number; use crate::DbPool; @@ -8,6 +9,7 @@ use crate::videos::{ServerOptions, VideoItem}; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::vec; +use async_trait::async_trait; error_chain! { foreign_links { @@ -19,15 +21,21 @@ error_chain! { #[derive(Debug, Clone)] pub struct OmgxxxProvider { url: String, + sites: Vec, + networks: Vec, } impl OmgxxxProvider { pub fn new() -> Self { OmgxxxProvider { url: "https://www.omg.xxx".to_string(), + sites: vec![], + networks: vec![], } } - async fn get_channel(&self) -> crate::status::Channel { + fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; + let channel: crate::status::Channel = Channel{ id: "omgxxx".to_string(), name: "OMG XXX".to_string(), @@ -58,11 +66,31 @@ impl OmgxxxProvider { }, ], multiSelect: false, + }, + ChannelOption { + id: "sites".to_string(), + title: "Sites".to_string(), + description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), + systemImage: "list.bullet.indent".to_string(), + colorName: "green".to_string(), + options: self.sites.clone(), + multiSelect: false, + }, + ChannelOption { + id: "networks".to_string(), + title: "Networks".to_string(), + description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), + systemImage: "list.dash".to_string(), + colorName: "purple".to_string(), + options: self.networks.clone(), + multiSelect: false, } + ], nsfw: true, cacheDuration: None, }; + return channel; } @@ -94,7 +122,6 @@ impl OmgxxxProvider { }; let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url).await.unwrap(); let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { @@ -158,10 +185,10 @@ impl OmgxxxProvider { .split("class=\"item\"").collect::>()[1..] .to_vec(); for video_segment in &raw_videos { - let vid = video_segment.split("\n").collect::>(); - for (index, line) in vid.iter().enumerate() { - println!("Line {}: {}", index, line); - } + // let vid = video_segment.split("\n").collect::>(); + // for (index, line) in vid.iter().enumerate() { + // println!("Line {}: {}", index, line); + // } let video_url: String = video_segment.split(">()[1] .split("\"") .collect::>()[0].to_string(); @@ -226,6 +253,7 @@ impl OmgxxxProvider { } +#[async_trait] impl Provider for OmgxxxProvider { async fn get_videos( &self, @@ -257,4 +285,8 @@ impl Provider for OmgxxxProvider { } } } + fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { + println!("Getting channel for omgxxx with client version: {:?}", clientversion); + self.build_channel(clientversion) + } } diff --git a/src/providers/paradisehill.rs b/src/providers/paradisehill.rs index 8a603e6..d89c440 100644 --- a/src/providers/paradisehill.rs +++ b/src/providers/paradisehill.rs @@ -8,6 +8,7 @@ use error_chain::error_chain; use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; use std::vec; +use async_trait::async_trait; error_chain! { foreign_links { @@ -173,7 +174,7 @@ impl ParadisehillProvider { .to_string(); let format = videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "mp4".to_string()) - .protocol("https".to_string()) + // .protocol("https".to_string()) .format_id(video_url.split("/").last().unwrap().to_string()) .format_note(format!("{}", video_url.split("_").last().unwrap().replace(".mp4", "").to_string())) ; @@ -203,6 +204,7 @@ impl ParadisehillProvider { } } +#[async_trait] impl Provider for ParadisehillProvider { async fn get_videos( &self, diff --git a/src/providers/perfectgirls.rs b/src/providers/perfectgirls.rs index a442944..5d22f8d 100644 --- a/src/providers/perfectgirls.rs +++ b/src/providers/perfectgirls.rs @@ -11,6 +11,7 @@ use std::env; use std::vec; use wreq::{Client}; use wreq_util::Emulation; +use async_trait::async_trait; error_chain! { foreign_links { @@ -281,6 +282,7 @@ impl PerfectgirlsProvider { } +#[async_trait] impl Provider for PerfectgirlsProvider { async fn get_videos( &self, diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index 0f35e18..82baf82 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -1,21 +1,19 @@ -use std::vec; -use std::env; -use error_chain::error_chain; -use htmlentity::entity::{decode, ICodedDataTrait}; -use futures::future::join_all; -use serde::Serialize; -use wreq::Client; -use wreq_util::Emulation; -use serde::Deserialize; +use crate::DbPool; use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; -use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; use crate::videos::ServerOptions; use crate::videos::{self, VideoEmbed, VideoItem}; -use crate::DbPool; - +use async_trait::async_trait; +use error_chain::error_chain; +use futures::future::join_all; +use htmlentity::entity::{ICodedDataTrait, decode}; +use serde::Deserialize; +use serde::Serialize; +use std::vec; +use wreq::Client; +use wreq_util::Emulation; error_chain! { foreign_links { @@ -25,7 +23,6 @@ error_chain! { } } - #[derive(Debug, Deserialize, Serialize)] struct PerverzijaDbEntry { url_string: String, @@ -42,8 +39,14 @@ impl PerverzijaProvider { url: "https://tube.perverzija.com/".to_string(), } } - async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result> { - + async fn get( + &self, + cache: VideoCache, + pool: DbPool, + page: u8, + options: ServerOptions, + ) -> Result> { + let featured = options.featured.unwrap_or("".to_string()); let mut prefix_uri = "".to_string(); if featured == "featured" { prefix_uri = "featured-scenes/".to_string(); @@ -52,14 +55,13 @@ impl PerverzijaProvider { if page == 1 { url_str = format!("{}{}", self.url, prefix_uri); } - + let old_items = match cache.get(&url_str) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { println!("Cache hit for URL: {}", url_str); return Ok(items.clone()); - } - else{ + } else { items.clone() } } @@ -68,59 +70,28 @@ impl PerverzijaProvider { } }; - - let client = Client::builder() - .emulation(Emulation::Firefox136) - .build()?; - - let response = client.get(url_str.clone()).send().await?; - // print!("Response: {:?}\n", response); - if response.status().is_success() { - let text = response.text().await?; - let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); - } - Ok(video_items) + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&url_str).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); + if !video_items.is_empty() { + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); - let flare = Flaresolverr::new(flare_url); - let result = flare - .solve(FlareSolverrRequest { - cmd: "request.get".to_string(), - url: url_str.clone(), - maxTimeout: 60000, - }) - .await; - let video_items = match result { - Ok(res) => { - // println!("FlareSolverr response: {}", res); - self.get_video_items_from_html(res.solution.response, pool) - } - Err(e) => { - println!("Error solving FlareSolverr: {}", e); - return Err("Failed to solve FlareSolverr".into()); - } - }; - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + return Ok(old_items); } + Ok(video_items) } - async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result> { + async fn query( + &self, + cache: VideoCache, + pool: DbPool, + page: u8, + query: &str, + options: ServerOptions, + ) -> Result> { let mut query_parse = true; let search_string = query.replace(" ", "+"); - let mut url_str = format!( - "{}page/{}/?s={}", - self.url, page, search_string - ); + let mut url_str = format!("{}page/{}/?s={}", self.url, page, search_string); if page == 1 { url_str = format!("{}?s={}", self.url, search_string); } @@ -140,61 +111,32 @@ impl PerverzijaProvider { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { return Ok(items.clone()); - } - else{ + } else { let _ = cache.check().await; - return Ok(items.clone()) + return Ok(items.clone()); } } None => { vec![] } - }; - let client = Client::builder() - .emulation(Emulation::Firefox136) - .build()?; - - let response = client.get(url_str.clone()).send().await?; - if response.status().is_success() { - let text = response.text().await?; - let video_items: Vec = match query_parse{ - true => {self.get_video_items_from_html_query(text.clone(), pool).await}, - false => {self.get_video_items_from_html(text.clone(), pool)} - }; - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); + }; + + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&url_str).await.unwrap(); + let video_items: Vec = match query_parse { + true => { + self.get_video_items_from_html_query(text.clone(), pool) + .await } - Ok(video_items) + false => self.get_video_items_from_html(text.clone(), pool), + }; + if !video_items.is_empty() { + cache.remove(&url_str); + cache.insert(url_str.clone(), video_items.clone()); } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); - let flare = Flaresolverr::new(flare_url); - let result = flare - .solve(FlareSolverrRequest { - cmd: "request.get".to_string(), - url: url_str.clone(), - maxTimeout: 60000, - }) - .await; - let video_items = match result { - Ok(res) => { - self.get_video_items_from_html_query(res.solution.response, pool).await - } - Err(e) => { - println!("Error solving FlareSolverr: {}", e); - return Err("Failed to solve FlareSolverr".into()); - } - }; - if !video_items.is_empty() { - cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); - } else{ - return Ok(old_items); - } - Ok(video_items) + return Ok(old_items); } + Ok(video_items) } fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec { @@ -228,8 +170,9 @@ impl PerverzijaProvider { let url_str = vid[1].split("iframe src="").collect::>()[1] .split(""") .collect::>()[0] - .to_string().replace("index.php", "xs1.php"); - if url_str.starts_with("https://streamtape.com/"){ + .to_string() + .replace("index.php", "xs1.php"); + if url_str.starts_with("https://streamtape.com/") { continue; // Skip Streamtape links } let id = url_str.split("data=").collect::>()[1] @@ -245,16 +188,18 @@ impl PerverzijaProvider { }; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - if !vid[4].contains("srcset=") && vid[4].split("src=\"").collect::>().len() == 1{ - for (index, line) in vid.iter().enumerate(){ + if !vid[4].contains("srcset=") + && vid[4].split("src=\"").collect::>().len() == 1 + { + for (index, line) in vid.iter().enumerate() { println!("Line {}: {}\n\n", index, line); } } let mut thumb = "".to_string(); - for v in vid.clone(){ + for v in vid.clone() { let line = v.trim(); - if line.starts_with(">()[1] .split("\"") .collect::>()[0] @@ -269,7 +214,7 @@ impl PerverzijaProvider { .split("'") .collect::>()[0] .to_string(); - + let mut conn = pool.get().expect("couldn't get db connection from pool"); let _ = db::insert_video(&mut conn, &id_url, &url_str); drop(conn); @@ -280,18 +225,20 @@ impl PerverzijaProvider { let studios_parts = vid[7].split("a href=\"").collect::>(); for studio in studios_parts.iter().skip(1) { - if studio.starts_with("https://tube.perverzija.com/studio/"){ + if studio.starts_with("https://tube.perverzija.com/studio/") { tags.push( studio.split("/\"").collect::>()[0] - .replace("https://tube.perverzija.com/studio/", "@studio:") + .replace("https://tube.perverzija.com/studio/", "@studio:") .to_string(), ); - } + } } - for tag in vid[0].split(" ").collect::>(){ + for tag in vid[0].split(" ").collect::>() { if tag.starts_with("stars-") { - let tag_name = tag.split("stars-").collect::>()[1].split("\"").collect::>()[0] + let tag_name = tag.split("stars-").collect::>()[1] + .split("\"") + .collect::>()[0] .to_string(); if !tag_name.is_empty() { tags.push(format!("@stars:{}", tag_name)); @@ -299,10 +246,9 @@ impl PerverzijaProvider { } } - for tag in vid[0].split(" ").collect::>(){ + for tag in vid[0].split(" ").collect::>() { if tag.starts_with("tag-") { - let tag_name = tag.split("tag-").collect::>()[1] - .to_string(); + let tag_name = tag.split("tag-").collect::>()[1].to_string(); if !tag_name.is_empty() { tags.push(tag_name.replace("-", " ").to_string()); } @@ -315,7 +261,8 @@ impl PerverzijaProvider { "perverzija".to_string(), thumb, duration, - ).tags(tags); + ) + .tags(tags); // .embed(embed.clone()); let mut format = videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); @@ -331,19 +278,15 @@ impl PerverzijaProvider { return items; } - async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec { - let raw_videos = html - .split("video-item post") - .collect::>()[1..] - .to_vec(); - let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone())); - let results: Vec> = join_all(futures).await; - let items: Vec = results + async fn get_video_items_from_html_query(&self, html: String, pool: DbPool) -> Vec { + let raw_videos = html.split("video-item post").collect::>()[1..].to_vec(); + let futures = raw_videos .into_iter() - .filter_map(Result::ok) - .collect(); + .map(|el| self.get_video_item(el, pool.clone())); + let results: Vec> = join_all(futures).await; + let items: Vec = results.into_iter().filter_map(Result::ok).collect(); - return items; + return items; } async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result { @@ -358,9 +301,9 @@ impl PerverzijaProvider { .to_string(); title = decode(title.as_bytes()).to_string().unwrap_or(title); - let thumb = match vid[6].split(" src=\"").collect::>().len(){ - 1=>{ - for (index,line) in vid.iter().enumerate() { + let thumb = match vid[6].split(" src=\"").collect::>().len() { + 1 => { + for (index, line) in vid.iter().enumerate() { println!("Line {}: {}", index, line.to_string().trim()); } return Err("Failed to parse thumbnail URL".into()); @@ -379,19 +322,19 @@ impl PerverzijaProvider { let referer_url = "https://xtremestream.xyz/".to_string(); let mut conn = pool.get().expect("couldn't get db connection from pool"); - let db_result = db::get_video(&mut conn,lookup_url.clone()); + let db_result = db::get_video(&mut conn, lookup_url.clone()); match db_result { Ok(Some(entry)) => { - if entry.starts_with("{"){ // replace old urls with new json objects + if entry.starts_with("{") { + // replace old urls with new json objects let entry = serde_json::from_str::(entry.as_str())?; let url_str = entry.url_string; let tags = entry.tags_strings; - if url_str.starts_with("!"){ + if url_str.starts_with("!") { return Err("Video was removed".into()); } - let mut id = url_str.split("data=").collect::>()[1] - .to_string(); - if id.contains("&"){ + let mut id = url_str.split("data=").collect::>()[1].to_string(); + if id.contains("&") { id = id.split("&").collect::>()[0].to_string() } let mut video_item = VideoItem::new( @@ -402,38 +345,35 @@ impl PerverzijaProvider { thumb, duration, ) - .tags(tags) - ; - let mut format = - videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); + .tags(tags); + let mut format = videos::VideoFormat::new( + url_str.clone(), + "1080".to_string(), + "m3u8".to_string(), + ); format.add_http_header("Referer".to_string(), referer_url.clone()); if let Some(formats) = video_item.formats.as_mut() { formats.push(format); } else { video_item.formats = Some(vec![format]); } - return Ok(video_item) - } - else{ - let _ = db::delete_video(&mut conn,lookup_url.clone()); + return Ok(video_item); + } else { + let _ = db::delete_video(&mut conn, lookup_url.clone()); }; } - Ok(None) => { - }, + Ok(None) => {} Err(e) => { println!("Error fetching video from database: {}", e); // return Err(format!("Error fetching video from database: {}", e).into()); } } drop(conn); - - let client = Client::builder() - .emulation(Emulation::Firefox136) - .build()?; - + let client = Client::builder().emulation(Emulation::Firefox136).build()?; + let response = client.get(lookup_url.clone()).send().await?; - let text = match response.status().is_success(){ + let text = match response.status().is_success() { true => response.text().await?, false => { println!("Failed to fetch video details"); @@ -441,49 +381,65 @@ impl PerverzijaProvider { } }; - let mut url_str = text.split("