simon/hottub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

overhault to fix warnings etc

Browse Source
This commit is contained in:
Simon
2025-10-04 14:28:29 +00:00
parent d84cc715a8
commit 28a4c57616
29 changed files with 889 additions and 1338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.toml
View File

@@ -23,6 +23,9 @@ capitalize = "0.3.4"
url = "2.5.4" url = "2.5.4"
base64 = "0.22.1" base64 = "0.22.1"
scraper = "0.24.0" scraper = "0.24.0"
once_cell = "1.21.3"
rustc-hash = "2.1.1"
async-trait = "0.1"
[lints.rust] [lints.rust]
unexpected_cfgs = "allow" unexpected_cfgs = "allow"

119
src/api.rs
View File

@@ -17,11 +17,13 @@ use crate::providers::rule34video::Rule34videoProvider;
// use crate::providers::spankbang::SpankbangProvider; // use crate::providers::spankbang::SpankbangProvider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::requester::Requester; use crate::util::requester::Requester;
use crate::{DbPool, db, providers::*, status::*, videos::*}; use crate::{DbPool, db, status::*, videos::*};
use cute::c; use cute::c;
use std::sync::Arc;
use crate::providers::{Provider, DynProvider, ALL_PROVIDERS};
#[derive(Debug)] #[derive(Debug, Clone)]
struct ClientVersion { pub struct ClientVersion {
version: u32, version: u32,
subversion: u32, subversion: u32,
name: String, name: String,
@@ -923,7 +925,7 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
cacheDuration: None, cacheDuration: None,
}); });
// porn00 // noodlemagazine
// status.add_channel(Channel { // status.add_channel(Channel {
// id: "noodlemagazine".to_string(), // id: "noodlemagazine".to_string(),
// name: "Noodlemagazine".to_string(), // name: "Noodlemagazine".to_string(),
@@ -1070,42 +1072,6 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
cacheDuration: None, cacheDuration: None,
}); });
status.add_channel(Channel{
id: "omgxxx".to_string(),
name: "OMG XXX".to_string(),
description: "Free Porn Site".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.omg.xxx".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![
ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "latest-updates".to_string(),
title: "Latest".to_string(),
},
FilterOption {
id: "most-popular".to_string(),
title: "Most Viewed".to_string(),
},
FilterOption {
id: "top-rated".to_string(),
title: "Top Rated".to_string(),
},
],
multiSelect: false,
}
],
nsfw: true,
cacheDuration: None,
});
if clientversion >= ClientVersion::new(22, 105, "22i".to_string()) { if clientversion >= ClientVersion::new(22, 105, "22i".to_string()) {
//sxyprn //sxyprn
status.add_channel(Channel { status.add_channel(Channel {
@@ -1170,6 +1136,14 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
cacheDuration: Some(1800), cacheDuration: Some(1800),
}); });
} }
for provider in ALL_PROVIDERS.values() {
println!(
"Loaded provider (dyn): {:?}",
std::any::type_name::<&dyn Provider>()
);
status.add_channel(provider.get_channel(clientversion.clone()));
}
status.iconUrl = format!("http://{}/favicon.ico", host).to_string(); status.iconUrl = format!("http://{}/favicon.ico", host).to_string();
Ok(web::HttpResponse::Ok().json(&status)) Ok(web::HttpResponse::Ok().json(&status))
} }
@@ -1296,54 +1270,35 @@ async fn videos_post(
Ok(web::HttpResponse::Ok().json(&videos)) Ok(web::HttpResponse::Ok().json(&videos))
} }
pub fn get_provider(channel: &str) -> Option<AnyProvider> { pub fn get_provider(channel: &str) -> Option<DynProvider> {
match channel { match channel {
"all" => Some(AnyProvider::All(AllProvider::new())), "all" => Some(Arc::new(AllProvider::new())),
"perverzija" => Some(AnyProvider::Perverzija(PerverzijaProvider::new())), "perverzija" => Some(Arc::new(PerverzijaProvider::new())),
"hanime" => Some(AnyProvider::Hanime(HanimeProvider::new())), "hanime" => Some(Arc::new(HanimeProvider::new())),
// "spankbang" => Some(AnyProvider::Spankbang(SpankbangProvider::new())), "pornhub" => Some(Arc::new(PornhubProvider::new())),
"pornhub" => Some(AnyProvider::Pornhub(PornhubProvider::new())), "pmvhaven" => Some(Arc::new(PmvhavenProvider::new())),
"pmvhaven" => Some(AnyProvider::Pmvhaven(PmvhavenProvider::new())), "rule34video" => Some(Arc::new(Rule34videoProvider::new())),
"rule34video" => Some(AnyProvider::Rule34video(Rule34videoProvider::new())), "redtube" => Some(Arc::new(RedtubeProvider::new())),
"redtube" => Some(AnyProvider::Redtube(RedtubeProvider::new())), "okporn" => Some(Arc::new(OkpornProvider::new())),
"okporn" => Some(AnyProvider::Okporn(OkpornProvider::new())), "pornhat" => Some(Arc::new(crate::providers::pornhat::PornhatProvider::new())),
"pornhat" => Some(AnyProvider::Pornhat( "perfectgirls" => Some(Arc::new(
crate::providers::pornhat::PornhatProvider::new(),
)),
"perfectgirls" => Some(AnyProvider::Perfectgirls(
crate::providers::perfectgirls::PerfectgirlsProvider::new(), crate::providers::perfectgirls::PerfectgirlsProvider::new(),
)), )),
"okxxx" => Some(AnyProvider::Okxxx( "okxxx" => Some(Arc::new(crate::providers::okxxx::OkxxxProvider::new())),
crate::providers::okxxx::OkxxxProvider::new(), "homoxxx" => Some(Arc::new(crate::providers::homoxxx::HomoxxxProvider::new())),
)), "missav" => Some(Arc::new(crate::providers::missav::MissavProvider::new())),
"homoxxx" => Some(AnyProvider::Homoxxx( "xxthots" => Some(Arc::new(crate::providers::xxthots::XxthotsProvider::new())),
crate::providers::homoxxx::HomoxxxProvider::new(), "sxyprn" => Some(Arc::new(crate::providers::sxyprn::SxyprnProvider::new())),
)), "porn00" => Some(Arc::new(crate::providers::porn00::Porn00Provider::new())),
// "hentaimoon" => Some(AnyProvider::Hentaimoon(crate::providers::hentaimoon::HentaimoonProvider::new())), "freshporno" => Some(Arc::new(
"missav" => Some(AnyProvider::Missav(
crate::providers::missav::MissavProvider::new(),
)),
"xxthots" => Some(AnyProvider::Xxthots(
crate::providers::xxthots::XxthotsProvider::new(),
)),
"sxyprn" => Some(AnyProvider::Sxyprn(
crate::providers::sxyprn::SxyprnProvider::new(),
)),
"porn00" => Some(AnyProvider::Porn00(
crate::providers::porn00::Porn00Provider::new(),
)),
// "noodlemagazine" => Some(AnyProvider::Noodlemagazine(crate::providers::noodlemagazine::NoodlemagazineProvider::new())),
"freshporno" => Some(AnyProvider::Freshporno(
crate::providers::freshporno::FreshpornoProvider::new(), crate::providers::freshporno::FreshpornoProvider::new(),
)), )),
"youjizz" => Some(AnyProvider::Youjizz( "youjizz" => Some(Arc::new(crate::providers::youjizz::YoujizzProvider::new())),
crate::providers::youjizz::YoujizzProvider::new(), "paradisehill" => Some(Arc::new(
)),
"paradisehill" => Some(AnyProvider::Paradisehill(
crate::providers::paradisehill::ParadisehillProvider::new(), crate::providers::paradisehill::ParadisehillProvider::new(),
)), )),
"pornzog" => Some(AnyProvider::Pornzog(crate::providers::pornzog::PornzogProvider::new())), "pornzog" => Some(Arc::new(crate::providers::pornzog::PornzogProvider::new())),
"omgxxx" => Some(AnyProvider::Omgxxx(crate::providers::omgxxx::OmgxxxProvider::new())), // fallback to the dynamic registry
_ => Some(AnyProvider::Perverzija(PerverzijaProvider::new())), x => ALL_PROVIDERS.get(x).cloned(),
} }
} }

4
src/db.rs
View File

@@ -46,8 +46,8 @@ pub fn has_table(
use diesel::sql_types::Text; use diesel::sql_types::Text;
#[derive(QueryableByName)] #[derive(QueryableByName)]
struct TableName { struct TableName {
#[sql_type = "Text"] #[diesel(sql_type = Text)]
#[column_name = "name"] #[diesel(column_name = name)]
name: String, name: String,
} }

23
src/providers/all.rs
View File

@@ -1,8 +1,10 @@
use std::fs; use std::fs;
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use crate::api::get_provider; use crate::api::{get_provider, ClientVersion};
use crate::providers::{AnyProvider, Provider}; use crate::providers::{DynProvider, Provider};
use crate::status::Channel;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::interleave; use crate::util::interleave;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
@@ -27,6 +29,7 @@ impl AllProvider {
} }
} }
#[async_trait]
impl Provider for AllProvider { impl Provider for AllProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -48,11 +51,11 @@ impl Provider for AllProvider {
.collect::<Vec<String>>(); .collect::<Vec<String>>();
sites_str = providers.join(","); sites_str = providers.join(",");
} }
let sites = sites_str let providers: Vec<DynProvider> = sites_str
.split(',') .split(',')
.map(|s| s.to_string()) // or s.to_owned() .filter(|s| !s.is_empty())
.collect::<Vec<String>>(); .filter_map(|s| get_provider(s)) // assumes get_provider -> Option<DynProvider>
let providers = sites.iter().map(|el| get_provider(el.as_str()).unwrap()).collect::<Vec<AnyProvider>>(); .collect();
let futures = providers.iter().map(|provider| { let futures = providers.iter().map(|provider| {
provider.get_videos( provider.get_videos(
@@ -71,4 +74,12 @@ impl Provider for AllProvider {
return video_items; return video_items;
} }
fn get_channel(&self,clientversion:ClientVersion) -> Channel {
println!("Getting channel for placeholder with client version: {:?}",clientversion);
let _ = clientversion;
Channel {
id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None,
}
}
} }

13
src/providers/freshporno.rs
View File

@@ -1,7 +1,10 @@
use crate::api::ClientVersion;
use crate::status::Channel;
use crate::DbPool; use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
@@ -17,6 +20,7 @@ error_chain! {
pub struct FreshpornoProvider { pub struct FreshpornoProvider {
url: String, url: String,
} }
impl FreshpornoProvider { impl FreshpornoProvider {
pub fn new() -> Self { pub fn new() -> Self {
FreshpornoProvider { FreshpornoProvider {
@@ -156,6 +160,7 @@ impl FreshpornoProvider {
} }
#[async_trait]
impl Provider for FreshpornoProvider { impl Provider for FreshpornoProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -187,4 +192,12 @@ impl Provider for FreshpornoProvider {
} }
} }
} }
fn get_channel(&self,clientversion:ClientVersion) -> Channel {
println!("Getting channel for placeholder with client version: {:?}",clientversion);
let _ = clientversion;
Channel {
id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None,
}
}
} }

2
src/providers/hanime.rs
View File

@@ -1,4 +1,5 @@
use std::vec; use std::vec;
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use wreq::Client; use wreq::Client;
@@ -253,6 +254,7 @@ impl HanimeProvider {
} }
} }
#[async_trait]
impl Provider for HanimeProvider { impl Provider for HanimeProvider {
async fn get_videos( async fn get_videos(
&self, &self,

2
src/providers/homoxxx.rs
View File

@@ -4,6 +4,7 @@ use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env; use std::env;
@@ -243,6 +244,7 @@ impl HomoxxxProvider {
} }
#[async_trait]
impl Provider for HomoxxxProvider { impl Provider for HomoxxxProvider {
async fn get_videos( async fn get_videos(
&self, &self,

2
src/providers/missav.rs
View File

@@ -1,4 +1,5 @@
use std::vec; use std::vec;
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait}; use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all; use futures::future::join_all;
@@ -253,6 +254,7 @@ impl MissavProvider {
} }
} }
#[async_trait]
impl Provider for MissavProvider { impl Provider for MissavProvider {
async fn get_videos( async fn get_videos(
&self, &self,

208
src/providers/mod.rs
View File

@@ -1,7 +1,15 @@
use async_trait::async_trait;
use once_cell::sync::Lazy;
use rustc_hash::FxHashMap as HashMap;
use std::sync::Arc;
use crate::{ use crate::{
providers::{ DbPool,
all::AllProvider, hanime::HanimeProvider, okporn::OkpornProvider, perverzija::PerverzijaProvider, pmvhaven::PmvhavenProvider, pornhub::PornhubProvider, redtube::RedtubeProvider, rule34video::Rule34videoProvider api::ClientVersion,
}, util::cache::VideoCache, videos::{ServerOptions, VideoItem}, DbPool providers::omgxxx::OmgxxxProvider,
status::Channel,
util::cache::VideoCache,
videos::{ServerOptions, VideoItem},
}; };
pub mod all; pub mod all;
@@ -10,27 +18,37 @@ pub mod perverzija;
pub mod pmvhaven; pub mod pmvhaven;
pub mod pornhub; pub mod pornhub;
// pub mod spankbang; // pub mod spankbang;
pub mod rule34video;
pub mod redtube;
pub mod okporn;
pub mod pornhat;
pub mod perfectgirls;
pub mod okxxx;
pub mod homoxxx; pub mod homoxxx;
pub mod okporn;
pub mod okxxx;
pub mod perfectgirls;
pub mod pornhat;
pub mod redtube;
pub mod rule34video;
// pub mod hentaimoon; // pub mod hentaimoon;
pub mod missav; pub mod missav;
pub mod xxthots;
pub mod sxyprn;
pub mod porn00; pub mod porn00;
pub mod sxyprn;
pub mod xxthots;
// pub mod noodlemagazine; // pub mod noodlemagazine;
pub mod freshporno; pub mod freshporno;
pub mod youjizz; pub mod omgxxx;
pub mod paradisehill; pub mod paradisehill;
pub mod pornzog; pub mod pornzog;
pub mod omgxxx; pub mod youjizz;
// convenient alias
pub type DynProvider = Arc<dyn Provider>;
pub trait Provider { pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| {
let mut m = HashMap::default();
m.insert("omgxxx", Arc::new(OmgxxxProvider::new()) as DynProvider);
// add more here as you migrate them
m
});
#[async_trait]
pub trait Provider: Send + Sync {
async fn get_videos( async fn get_videos(
&self, &self,
cache: VideoCache, cache: VideoCache,
@@ -41,156 +59,24 @@ pub trait Provider {
per_page: String, per_page: String,
options: ServerOptions, options: ServerOptions,
) -> Vec<VideoItem>; ) -> Vec<VideoItem>;
}
#[derive(Debug, Clone)] fn get_channel(&self, clientversion: ClientVersion) -> Channel {
pub enum AnyProvider {
All(AllProvider),
Perverzija(PerverzijaProvider),
Hanime(HanimeProvider),
// Spankbang(SpankbangProvider),
Pornhub(PornhubProvider),
Pmvhaven(PmvhavenProvider),
Rule34video(Rule34videoProvider),
Redtube(RedtubeProvider),
Okporn(OkpornProvider),
Pornhat(crate::providers::pornhat::PornhatProvider),
Perfectgirls(crate::providers::perfectgirls::PerfectgirlsProvider),
Okxxx(crate::providers::okxxx::OkxxxProvider),
Homoxxx(crate::providers::homoxxx::HomoxxxProvider),
// Hentaimoon(crate::providers::hentaimoon::HentaimoonProvider),
Missav(crate::providers::missav::MissavProvider),
Xxthots(crate::providers::xxthots::XxthotsProvider),
Sxyprn(crate::providers::sxyprn::SxyprnProvider),
Porn00(crate::providers::porn00::Porn00Provider),
// Noodlemagazine(crate::providers::noodlemagazine::NoodlemagazineProvider),
Freshporno(crate::providers::freshporno::FreshpornoProvider),
Youjizz(crate::providers::youjizz::YoujizzProvider),
Paradisehill(crate::providers::paradisehill::ParadisehillProvider),
Pornzog(crate::providers::pornzog::PornzogProvider),
Omgxxx(crate::providers::omgxxx::OmgxxxProvider),
}
impl Provider for AnyProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
sort: String,
query: Option<String>,
page: String,
per_page: String,
options: ServerOptions
) -> Vec<VideoItem> {
println!( println!(
"/api/videos: sort={:?}, query={:?}, page={:?}, provider={:?}", "Getting channel for placeholder with client version: {:?}",
sort, query, page, self clientversion
); );
match self { let _ = clientversion;
AnyProvider::Perverzija(p) => { Channel {
p.get_videos( id: "placeholder".to_string(),
cache.clone(), name: "PLACEHOLDER".to_string(),
pool.clone(), description: "PLACEHOLDER FOR PARENT CLASS".to_string(),
sort.clone(), premium: false,
query.clone(), favicon: "https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),
page.clone(), status: "active".to_string(),
per_page.clone(), categories: vec![],
options, options: vec![],
) nsfw: true,
.await cacheDuration: None,
}
AnyProvider::Hanime(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
// AnyProvider::Spankbang(p) => {
// p.get_videos(cache, pool, sort, query, page, per_page, options,)
// .await
// }
AnyProvider::Pornhub(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Pmvhaven(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Rule34video(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Redtube(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::All(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Okporn(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Pornhat(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Perfectgirls(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Okxxx(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Homoxxx(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
// AnyProvider::Hentaimoon(p) => {
// p.get_videos(cache, pool, sort, query, page, per_page, options,)
// .await
// }
AnyProvider::Missav(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Xxthots(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Sxyprn(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Porn00(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
// AnyProvider::Noodlemagazine(p) => {
// p.get_videos(cache, pool, sort, query, page, per_page, options,)
// .await
// }
AnyProvider::Freshporno(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Youjizz(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
},
AnyProvider::Paradisehill(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Pornzog(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Omgxxx(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
} }
} }
} }

2
src/providers/okporn.rs
View File

@@ -10,6 +10,7 @@ use std::env;
use std::vec; use std::vec;
use wreq::{Client}; use wreq::{Client};
use wreq_util::Emulation; use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -233,6 +234,7 @@ impl OkpornProvider {
} }
#[async_trait]
impl Provider for OkpornProvider { impl Provider for OkpornProvider {
async fn get_videos( async fn get_videos(
&self, &self,

2
src/providers/okxxx.rs
View File

@@ -11,6 +11,7 @@ use std::env;
use std::vec; use std::vec;
use wreq::{Client}; use wreq::{Client};
use wreq_util::Emulation; use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -278,6 +279,7 @@ impl OkxxxProvider {
} }
#[async_trait]
impl Provider for OkxxxProvider { impl Provider for OkxxxProvider {
async fn get_videos( async fn get_videos(
&self, &self,

44
src/providers/omgxxx.rs
View File

@@ -1,3 +1,4 @@
use crate::api::ClientVersion;
use crate::status::*; use crate::status::*;
use crate::util::parse_abbreviated_number; use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
@@ -8,6 +9,7 @@ use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -19,15 +21,21 @@ error_chain! {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct OmgxxxProvider { pub struct OmgxxxProvider {
url: String, url: String,
sites: Vec<FilterOption>,
networks: Vec<FilterOption>,
} }
impl OmgxxxProvider { impl OmgxxxProvider {
pub fn new() -> Self { pub fn new() -> Self {
OmgxxxProvider { OmgxxxProvider {
url: "https://www.omg.xxx".to_string(), url: "https://www.omg.xxx".to_string(),
sites: vec![],
networks: vec![],
} }
} }
async fn get_channel(&self) -> crate::status::Channel { fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion;
let channel: crate::status::Channel = Channel{ let channel: crate::status::Channel = Channel{
id: "omgxxx".to_string(), id: "omgxxx".to_string(),
name: "OMG XXX".to_string(), name: "OMG XXX".to_string(),
@@ -58,11 +66,31 @@ impl OmgxxxProvider {
}, },
], ],
multiSelect: false, multiSelect: false,
},
ChannelOption {
id: "sites".to_string(),
title: "Sites".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.bullet.indent".to_string(),
colorName: "green".to_string(),
options: self.sites.clone(),
multiSelect: false,
},
ChannelOption {
id: "networks".to_string(),
title: "Networks".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.dash".to_string(),
colorName: "purple".to_string(),
options: self.networks.clone(),
multiSelect: false,
} }
], ],
nsfw: true, nsfw: true,
cacheDuration: None, cacheDuration: None,
}; };
return channel; return channel;
} }
@@ -94,7 +122,6 @@ impl OmgxxxProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
@@ -158,10 +185,10 @@ impl OmgxxxProvider {
.split("class=\"item\"").collect::<Vec<&str>>()[1..] .split("class=\"item\"").collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
for video_segment in &raw_videos { for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>(); // let vid = video_segment.split("\n").collect::<Vec<&str>>();
for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
} // }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1] let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0].to_string(); .collect::<Vec<&str>>()[0].to_string();
@@ -226,6 +253,7 @@ impl OmgxxxProvider {
} }
#[async_trait]
impl Provider for OmgxxxProvider { impl Provider for OmgxxxProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -257,4 +285,8 @@ impl Provider for OmgxxxProvider {
} }
} }
} }
fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel {
println!("Getting channel for omgxxx with client version: {:?}", clientversion);
self.build_channel(clientversion)
}
} }

4
src/providers/paradisehill.rs
View File

@@ -8,6 +8,7 @@ use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -173,7 +174,7 @@ impl ParadisehillProvider {
.to_string(); .to_string();
let format = let format =
videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "mp4".to_string()) videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "mp4".to_string())
.protocol("https".to_string()) // .protocol("https".to_string())
.format_id(video_url.split("/").last().unwrap().to_string()) .format_id(video_url.split("/").last().unwrap().to_string())
.format_note(format!("{}", video_url.split("_").last().unwrap().replace(".mp4", "").to_string())) .format_note(format!("{}", video_url.split("_").last().unwrap().replace(".mp4", "").to_string()))
; ;
@@ -203,6 +204,7 @@ impl ParadisehillProvider {
} }
} }
#[async_trait]
impl Provider for ParadisehillProvider { impl Provider for ParadisehillProvider {
async fn get_videos( async fn get_videos(
&self, &self,

2
src/providers/perfectgirls.rs
View File

@@ -11,6 +11,7 @@ use std::env;
use std::vec; use std::vec;
use wreq::{Client}; use wreq::{Client};
use wreq_util::Emulation; use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -281,6 +282,7 @@ impl PerfectgirlsProvider {
} }
#[async_trait]
impl Provider for PerfectgirlsProvider { impl Provider for PerfectgirlsProvider {
async fn get_videos( async fn get_videos(
&self, &self,

351
src/providers/perverzija.rs
View File

@@ -1,21 +1,19 @@
use std::vec; use crate::DbPool;
use std::env;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all;
use serde::Serialize;
use wreq::Client;
use wreq_util::Emulation;
use serde::Deserialize;
use crate::db; use crate::db;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::ServerOptions; use crate::videos::ServerOptions;
use crate::videos::{self, VideoEmbed, VideoItem}; use crate::videos::{self, VideoEmbed, VideoItem};
use crate::DbPool; use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use serde::Deserialize;
use serde::Serialize;
use std::vec;
use wreq::Client;
use wreq_util::Emulation;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -25,7 +23,6 @@ error_chain! {
} }
} }
#[derive(Debug, Deserialize, Serialize)] #[derive(Debug, Deserialize, Serialize)]
struct PerverzijaDbEntry { struct PerverzijaDbEntry {
url_string: String, url_string: String,
@@ -42,8 +39,14 @@ impl PerverzijaProvider {
url: "https://tube.perverzija.com/".to_string(), url: "https://tube.perverzija.com/".to_string(),
} }
} }
async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result<Vec<VideoItem>> { async fn get(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let featured = options.featured.unwrap_or("".to_string());
let mut prefix_uri = "".to_string(); let mut prefix_uri = "".to_string();
if featured == "featured" { if featured == "featured" {
prefix_uri = "featured-scenes/".to_string(); prefix_uri = "featured-scenes/".to_string();
@@ -52,14 +55,13 @@ impl PerverzijaProvider {
if page == 1 { if page == 1 {
url_str = format!("{}{}", self.url, prefix_uri); url_str = format!("{}{}", self.url, prefix_uri);
} }
let old_items = match cache.get(&url_str) { let old_items = match cache.get(&url_str) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
println!("Cache hit for URL: {}", url_str); println!("Cache hit for URL: {}", url_str);
return Ok(items.clone()); return Ok(items.clone());
} } else {
else{
items.clone() items.clone()
} }
} }
@@ -68,59 +70,28 @@ impl PerverzijaProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap();
let client = Client::builder() let text = requester.get(&url_str).await.unwrap();
.emulation(Emulation::Firefox136) let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool);
.build()?; if !video_items.is_empty() {
cache.remove(&url_str);
let response = client.get(url_str.clone()).send().await?; cache.insert(url_str.clone(), video_items.clone());
// print!("Response: {:?}\n", response);
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else{
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url_str.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response, pool)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result<Vec<VideoItem>> { async fn query(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let mut query_parse = true; let mut query_parse = true;
let search_string = query.replace(" ", "+"); let search_string = query.replace(" ", "+");
let mut url_str = format!( let mut url_str = format!("{}page/{}/?s={}", self.url, page, search_string);
"{}page/{}/?s={}",
self.url, page, search_string
);
if page == 1 { if page == 1 {
url_str = format!("{}?s={}", self.url, search_string); url_str = format!("{}?s={}", self.url, search_string);
} }
@@ -140,61 +111,32 @@ impl PerverzijaProvider {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
return Ok(items.clone()); return Ok(items.clone());
} } else {
else{
let _ = cache.check().await; let _ = cache.check().await;
return Ok(items.clone()) return Ok(items.clone());
} }
} }
None => { None => {
vec![] vec![]
} }
}; };
let client = Client::builder()
.emulation(Emulation::Firefox136) let mut requester = options.requester.clone().unwrap();
.build()?; let text = requester.get(&url_str).await.unwrap();
let video_items: Vec<VideoItem> = match query_parse {
let response = client.get(url_str.clone()).send().await?; true => {
if response.status().is_success() { self.get_video_items_from_html_query(text.clone(), pool)
let text = response.text().await?; .await
let video_items: Vec<VideoItem> = match query_parse{
true => {self.get_video_items_from_html_query(text.clone(), pool).await},
false => {self.get_video_items_from_html(text.clone(), pool)}
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else{
return Ok(old_items);
} }
Ok(video_items) false => self.get_video_items_from_html(text.clone(), pool),
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url_str.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
self.get_video_items_from_html_query(res.solution.response, pool).await
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else{
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<VideoItem> { fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
@@ -228,8 +170,9 @@ impl PerverzijaProvider {
let url_str = vid[1].split("iframe src=&quot;").collect::<Vec<&str>>()[1] let url_str = vid[1].split("iframe src=&quot;").collect::<Vec<&str>>()[1]
.split("&quot;") .split("&quot;")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string().replace("index.php", "xs1.php"); .to_string()
if url_str.starts_with("https://streamtape.com/"){ .replace("index.php", "xs1.php");
if url_str.starts_with("https://streamtape.com/") {
continue; // Skip Streamtape links continue; // Skip Streamtape links
} }
let id = url_str.split("data=").collect::<Vec<&str>>()[1] let id = url_str.split("data=").collect::<Vec<&str>>()[1]
@@ -245,16 +188,18 @@ impl PerverzijaProvider {
}; };
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
if !vid[4].contains("srcset=") && vid[4].split("src=\"").collect::<Vec<&str>>().len() == 1{ if !vid[4].contains("srcset=")
for (index, line) in vid.iter().enumerate(){ && vid[4].split("src=\"").collect::<Vec<&str>>().len() == 1
{
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}\n\n", index, line); println!("Line {}: {}\n\n", index, line);
} }
} }
let mut thumb = "".to_string(); let mut thumb = "".to_string();
for v in vid.clone(){ for v in vid.clone() {
let line = v.trim(); let line = v.trim();
if line.starts_with("<img "){ if line.starts_with("<img ") {
thumb = line.split(" src=\"").collect::<Vec<&str>>()[1] thumb = line.split(" src=\"").collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
@@ -269,7 +214,7 @@ impl PerverzijaProvider {
.split("'") .split("'")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string(); .to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool"); let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(&mut conn, &id_url, &url_str); let _ = db::insert_video(&mut conn, &id_url, &url_str);
drop(conn); drop(conn);
@@ -280,18 +225,20 @@ impl PerverzijaProvider {
let studios_parts = vid[7].split("a href=\"").collect::<Vec<&str>>(); let studios_parts = vid[7].split("a href=\"").collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) { for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/"){ if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push( tags.push(
studio.split("/\"").collect::<Vec<&str>>()[0] studio.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/studio/", "@studio:") .replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(), .to_string(),
); );
} }
} }
for tag in vid[0].split(" ").collect::<Vec<&str>>(){ for tag in vid[0].split(" ").collect::<Vec<&str>>() {
if tag.starts_with("stars-") { if tag.starts_with("stars-") {
let tag_name = tag.split("stars-").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0] let tag_name = tag.split("stars-").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string(); .to_string();
if !tag_name.is_empty() { if !tag_name.is_empty() {
tags.push(format!("@stars:{}", tag_name)); tags.push(format!("@stars:{}", tag_name));
@@ -299,10 +246,9 @@ impl PerverzijaProvider {
} }
} }
for tag in vid[0].split(" ").collect::<Vec<&str>>(){ for tag in vid[0].split(" ").collect::<Vec<&str>>() {
if tag.starts_with("tag-") { if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1] let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1].to_string();
.to_string();
if !tag_name.is_empty() { if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string()); tags.push(tag_name.replace("-", " ").to_string());
} }
@@ -315,7 +261,8 @@ impl PerverzijaProvider {
"perverzija".to_string(), "perverzija".to_string(),
thumb, thumb,
duration, duration,
).tags(tags); )
.tags(tags);
// .embed(embed.clone()); // .embed(embed.clone());
let mut format = let mut format =
videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string());
@@ -331,19 +278,15 @@ impl PerverzijaProvider {
return items; return items;
} }
async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec<VideoItem> { async fn get_video_items_from_html_query(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
let raw_videos = html let raw_videos = html.split("video-item post").collect::<Vec<&str>>()[1..].to_vec();
.split("video-item post") let futures = raw_videos
.collect::<Vec<&str>>()[1..]
.to_vec();
let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let items: Vec<VideoItem> = results
.into_iter() .into_iter()
.filter_map(Result::ok) .map(|el| self.get_video_item(el, pool.clone()));
.collect(); let results: Vec<Result<VideoItem>> = join_all(futures).await;
let items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return items; return items;
} }
async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<VideoItem> { async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<VideoItem> {
@@ -358,9 +301,9 @@ impl PerverzijaProvider {
.to_string(); .to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = match vid[6].split(" src=\"").collect::<Vec<&str>>().len(){ let thumb = match vid[6].split(" src=\"").collect::<Vec<&str>>().len() {
1=>{ 1 => {
for (index,line) in vid.iter().enumerate() { for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line.to_string().trim()); println!("Line {}: {}", index, line.to_string().trim());
} }
return Err("Failed to parse thumbnail URL".into()); return Err("Failed to parse thumbnail URL".into());
@@ -379,19 +322,19 @@ impl PerverzijaProvider {
let referer_url = "https://xtremestream.xyz/".to_string(); let referer_url = "https://xtremestream.xyz/".to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool"); let mut conn = pool.get().expect("couldn't get db connection from pool");
let db_result = db::get_video(&mut conn,lookup_url.clone()); let db_result = db::get_video(&mut conn, lookup_url.clone());
match db_result { match db_result {
Ok(Some(entry)) => { Ok(Some(entry)) => {
if entry.starts_with("{"){ // replace old urls with new json objects if entry.starts_with("{") {
// replace old urls with new json objects
let entry = serde_json::from_str::<PerverzijaDbEntry>(entry.as_str())?; let entry = serde_json::from_str::<PerverzijaDbEntry>(entry.as_str())?;
let url_str = entry.url_string; let url_str = entry.url_string;
let tags = entry.tags_strings; let tags = entry.tags_strings;
if url_str.starts_with("!"){ if url_str.starts_with("!") {
return Err("Video was removed".into()); return Err("Video was removed".into());
} }
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1] let mut id = url_str.split("data=").collect::<Vec<&str>>()[1].to_string();
.to_string(); if id.contains("&") {
if id.contains("&"){
id = id.split("&").collect::<Vec<&str>>()[0].to_string() id = id.split("&").collect::<Vec<&str>>()[0].to_string()
} }
let mut video_item = VideoItem::new( let mut video_item = VideoItem::new(
@@ -402,38 +345,35 @@ impl PerverzijaProvider {
thumb, thumb,
duration, duration,
) )
.tags(tags) .tags(tags);
; let mut format = videos::VideoFormat::new(
let mut format = url_str.clone(),
videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string()); "1080".to_string(),
"m3u8".to_string(),
);
format.add_http_header("Referer".to_string(), referer_url.clone()); format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() { if let Some(formats) = video_item.formats.as_mut() {
formats.push(format); formats.push(format);
} else { } else {
video_item.formats = Some(vec![format]); video_item.formats = Some(vec![format]);
} }
return Ok(video_item) return Ok(video_item);
} } else {
else{ let _ = db::delete_video(&mut conn, lookup_url.clone());
let _ = db::delete_video(&mut conn,lookup_url.clone());
}; };
} }
Ok(None) => { Ok(None) => {}
},
Err(e) => { Err(e) => {
println!("Error fetching video from database: {}", e); println!("Error fetching video from database: {}", e);
// return Err(format!("Error fetching video from database: {}", e).into()); // return Err(format!("Error fetching video from database: {}", e).into());
} }
} }
drop(conn); drop(conn);
let client = Client::builder() let client = Client::builder().emulation(Emulation::Firefox136).build()?;
.emulation(Emulation::Firefox136)
.build()?;
let response = client.get(lookup_url.clone()).send().await?; let response = client.get(lookup_url.clone()).send().await?;
let text = match response.status().is_success(){ let text = match response.status().is_success() {
true => response.text().await?, true => response.text().await?,
false => { false => {
println!("Failed to fetch video details"); println!("Failed to fetch video details");
@@ -441,49 +381,65 @@ impl PerverzijaProvider {
} }
}; };
let mut url_str = text.split("<iframe src=\"").collect::<Vec<&str>>()[1] let mut url_str = text.split("<iframe src=\"").collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string().replace("index.php","xs1.php"); .to_string()
if !url_str.contains("xtremestream.xyz"){ .replace("index.php", "xs1.php");
if !url_str.contains("xtremestream.xyz") {
url_str = "!".to_string() url_str = "!".to_string()
} }
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
let studios_parts = text.split("<strong>Studio: </strong>").collect::<Vec<&str>>()[1].split("</div>").collect::<Vec<&str>>()[0].split("<a href=\"").collect::<Vec<&str>>(); let studios_parts = text
.split("<strong>Studio: </strong>")
.collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0]
.split("<a href=\"")
.collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) { for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/"){ if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push( tags.push(
studio.split("/\"").collect::<Vec<&str>>()[0] studio.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/studio/", "@studio:") .replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(), .to_string(),
); );
} }
} }
if text.contains("<strong>Stars: </strong>"){ if text.contains("<strong>Stars: </strong>") {
let stars_parts: Vec<&str> = text.split("<strong>Stars: </strong>").collect::<Vec<&str>>()[1].split("</div>").collect::<Vec<&str>>()[0].split("<a href=\"").collect::<Vec<&str>>(); let stars_parts: Vec<&str> = text
.split("<strong>Stars: </strong>")
.collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0]
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in stars_parts.iter().skip(1) { for star in stars_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/"){ if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push( tags.push(
star.split("/\"").collect::<Vec<&str>>()[0] star.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/stars/", "@stars:") .replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(), .to_string(),
); );
} }
} }
} }
let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>()[1].split("</div>").collect::<Vec<&str>>()[0].split("<a href=\"").collect::<Vec<&str>>(); let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0]
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in tags_parts.iter().skip(1) { for star in tags_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/"){ if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push( tags.push(
star.split("/\"").collect::<Vec<&str>>()[0] star.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/stars/", "@stars:") .replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(), .to_string(),
); );
} }
} }
let perverzija_db_entry = PerverzijaDbEntry { let perverzija_db_entry = PerverzijaDbEntry {
@@ -491,18 +447,23 @@ impl PerverzijaProvider {
tags_strings: tags.clone(), tags_strings: tags.clone(),
}; };
let mut conn = pool.get().expect("couldn't get db connection from pool"); let mut conn = pool.get().expect("couldn't get db connection from pool");
let insert_result = db::insert_video(&mut conn, &lookup_url, &serde_json::to_string(&perverzija_db_entry)?); let insert_result = db::insert_video(
match insert_result{ &mut conn,
&lookup_url,
&serde_json::to_string(&perverzija_db_entry)?,
);
match insert_result {
Ok(_) => (), Ok(_) => (),
Err(e) => {println!("{:?}", e); } Err(e) => {
println!("{:?}", e);
}
} }
drop(conn); drop(conn);
if !url_str.contains("xtremestream.xyz"){ if !url_str.contains("xtremestream.xyz") {
return Err("Video URL does not contain xtremestream.xyz".into()); return Err("Video URL does not contain xtremestream.xyz".into());
} }
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1] let mut id = url_str.split("data=").collect::<Vec<&str>>()[1].to_string();
.to_string(); if id.contains("&") {
if id.contains("&"){
id = id.split("&").collect::<Vec<&str>>()[0].to_string() id = id.split("&").collect::<Vec<&str>>()[0].to_string()
} }
// if !vid[6].contains(" src=\""){ // if !vid[6].contains(" src=\""){
@@ -513,9 +474,6 @@ impl PerverzijaProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim()); // println!("Line {}: {}", index, line.to_string().trim());
// } // }
let mut video_item = VideoItem::new( let mut video_item = VideoItem::new(
id, id,
@@ -539,6 +497,7 @@ impl PerverzijaProvider {
} }
} }
#[async_trait]
impl Provider for PerverzijaProvider { impl Provider for PerverzijaProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -553,8 +512,14 @@ impl Provider for PerverzijaProvider {
let _ = per_page; let _ = per_page;
let _ = sort; let _ = sort;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => self.query(cache, pool, page.parse::<u8>().unwrap_or(1), &q).await, Some(q) => {
None => self.get(cache, pool, page.parse::<u8>().unwrap_or(1), options.featured.unwrap()).await, self.query(cache, pool, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(cache, pool, page.parse::<u8>().unwrap_or(1), options)
.await
}
}; };
match videos { match videos {
Ok(v) => v, Ok(v) => v,

264
src/providers/pmvhaven.rs
View File

@@ -2,15 +2,11 @@ use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use cute::c; use cute::c;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::ICodedDataTrait; // use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
use std::vec; use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
#[macro_use(c)]
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -157,50 +153,55 @@ impl PmvhavenSearch {
#[derive(serde::Deserialize)] #[derive(serde::Deserialize)]
struct PmvhavenVideo { struct PmvhavenVideo {
title: String, //JAV Addiction Therapy", title: String, //JAV Addiction Therapy",
uploader: Option<String>, //itonlygetsworse", _uploader: Option<String>, //itonlygetsworse",
duration: f32, //259.093333, duration: f32, //259.093333,
width: Option<String>, //3840", _width: Option<String>, //3840",
height: Option<String>, //2160", _height: Option<String>, //2160",
ratio: Option<u32>, //50, _ratio: Option<u32>, //50,
thumbnails: Vec<Option<String>>, //[ thumbnails: Vec<Option<String>>, //[
// "placeholder", // "placeholder",
// "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.png", // "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.png",
// "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/webp320_686f24e96f7124f3dfbe90ab.webp" // "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/webp320_686f24e96f7124f3dfbe90ab.webp"
// ], // ],
views: u32, //1971, views: u32, //1971,
url: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.mp4", _url: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.mp4",
previewUrlCompressed: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/videoPreview/comus_686f24e96f7124f3dfbe90ab.mp4", previewUrlCompressed: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/videoPreview/comus_686f24e96f7124f3dfbe90ab.mp4",
seizureWarning: Option<bool>, //false, _seizureWarning: Option<bool>, //false,
isoDate: Option<String>, //2025-07-10T02:52:26.000Z", _isoDate: Option<String>, //2025-07-10T02:52:26.000Z",
gayContent: Option<bool>, //false, _gayContent: Option<bool>, //false,
transContent: Option<bool>, //false, _transContent: Option<bool>, //false,
creator: Option<String>, //itonlygetsworse", creator: Option<String>, //itonlygetsworse",
_id: String, //686f2aeade2062f93d72931f", _id: String, //686f2aeade2062f93d72931f",
totalRaters: Option<u32>, //42, _totalRaters: Option<u32>, //42,
rating: Option<u32>, //164 _rating: Option<u32>, //164
} }
impl PmvhavenVideo { impl PmvhavenVideo {
fn to_videoitem(self) -> VideoItem { fn to_videoitem(self) -> VideoItem {
let encoded_title = percent_encode_emojis(&self.title); // let encoded_title = percent_encode_emojis(&self.title);
let thumbnail = self.thumbnails[self.thumbnails.len()-1].clone().unwrap_or("".to_string()); let thumbnail = self.thumbnails[self.thumbnails.len() - 1]
let video_id = thumbnail.split("_").collect::<Vec<&str>>().last().unwrap_or(&"").to_string().split('.').next().unwrap_or("").to_string(); .clone()
.unwrap_or("".to_string());
// let video_id = thumbnail.split("_").collect::<Vec<&str>>().last().unwrap_or(&"").to_string().split('.').next().unwrap_or("").to_string();
let mut item = VideoItem::new( let mut item = VideoItem::new(
self._id.clone(), self._id.clone(),
self.title.clone(), self.title.clone(),
format!("https://pmvhaven.com/video/{}_{}", self.title.replace(" ","-"), self._id), format!(
"https://pmvhaven.com/video/{}_{}",
self.title.replace(" ", "-"),
self._id
),
"pmvhaven".to_string(), "pmvhaven".to_string(),
thumbnail, thumbnail,
self.duration as u32, self.duration as u32,
) )
.views(self.views); .views(self.views);
item = match self.creator{ item = match self.creator {
Some(c) => item.uploader(c), Some(c) => item.uploader(c),
_ => item, _ => item,
}; };
item = match self.previewUrlCompressed{ item = match self.previewUrlCompressed {
Some(u) => item.preview(u), Some(u) => item.preview(u),
_ => item, _ => item,
}; };
@@ -210,18 +211,33 @@ impl PmvhavenVideo {
} }
// Define a percent-encoding set that encodes all non-ASCII characters // Define a percent-encoding set that encodes all non-ASCII characters
const EMOJI_ENCODE_SET: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'#').add(b'%').add(b'<').add(b'>').add(b'?').add(b'[').add(b'\\').add(b']').add(b'^').add(b'`').add(b'{').add(b'|').add(b'}'); // const EMOJI_ENCODE_SET: &AsciiSet = &CONTROLS
// .add(b' ')
// .add(b'"')
// .add(b'#')
// .add(b'%')
// .add(b'<')
// .add(b'>')
// .add(b'?')
// .add(b'[')
// .add(b'\\')
// .add(b']')
// .add(b'^')
// .add(b'`')
// .add(b'{')
// .add(b'|')
// .add(b'}');
//
// Helper function to percent-encode emojis and other non-ASCII chars // Helper function to percent-encode emojis and other non-ASCII chars
fn percent_encode_emojis(s: &str) -> String { // fn percent_encode_emojis(s: &str) -> String {
utf8_percent_encode(s, EMOJI_ENCODE_SET).to_string() // utf8_percent_encode(s, EMOJI_ENCODE_SET).to_string()
} // }
#[derive(serde::Deserialize)] #[derive(serde::Deserialize)]
struct PmvhavenResponse { struct PmvhavenResponse {
httpStatusCode: Option<u32>, _httpStatusCode: Option<u32>,
data: Vec<PmvhavenVideo>, data: Vec<PmvhavenVideo>,
count: Option<u32>, _count: Option<u32>,
} }
impl PmvhavenResponse { impl PmvhavenResponse {
@@ -240,22 +256,46 @@ impl PmvhavenProvider {
url: "https://pmvhaven.com".to_string(), url: "https://pmvhaven.com".to_string(),
} }
} }
async fn get(&self, cache: VideoCache, page: u8, category: String, sort:String) -> Result<Vec<VideoItem>> { async fn get(
&self,
cache: VideoCache,
page: u8,
sort: String,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let category = options.category.unwrap_or("".to_string());
let index = format!("pmvhaven:{}:{}", page, category); let index = format!("pmvhaven:{}:{}", page, category);
let url = format!("{}/api/getmorevideos", self.url); let url = format!("{}/api/getmorevideos", self.url);
let mut request = PmvhavenRequest::new(page as u32); let mut request = PmvhavenRequest::new(page as u32);
request.activeView = sort; request.activeView = sort;
println!("Category: {}", category);
request = match category.as_str() { request = match category.as_str() {
"hypno" => { request.hypno(); request }, "hypno" => {
"pmv" => { request.pmv(); request }, request.hypno();
"hmv" => { request.hmv(); request }, request
"tiktok" => { request.tiktok(); request }, }
"koreanbj" => { request.koreanbj(); request }, "pmv" => {
"other" => { request.other(); request }, request.pmv();
request
}
"hmv" => {
request.hmv();
request
}
"tiktok" => {
request.tiktok();
request
}
"koreanbj" => {
request.koreanbj();
request
}
"other" => {
request.other();
request
}
_ => request, _ => request,
}; };
let old_items = match cache.get(&index) { let old_items = match cache.get(&index) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -270,42 +310,35 @@ impl PmvhavenProvider {
} }
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let mut requester = options.requester.clone().unwrap();
let client = Client::builder() let response = requester.post(&url, &request).await.unwrap();
.cert_verification(false) let videos = match response.json::<PmvhavenResponse>().await {
.emulation(Emulation::Firefox136) Ok(resp) => resp,
.build()?; Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
let response = client
.post(url.clone())
// .proxy(proxy.clone())
.json(&request)
.header("Content-Type", "text/plain;charset=UTF-8")
.send()
.await?;
if response.status().is_success() {
let videos = match response.json::<PmvhavenResponse>().await {
Ok(resp) => resp,
Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items); return Ok(old_items);
} }
return Ok(video_items); };
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
} }
Err("Failed to get Videos".into()) return Ok(video_items);
} }
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let index = format!("pmvhaven:{}:{}", query, page); let index = format!("pmvhaven:{}:{}", query, page);
let url = format!("{}/api/v2/search", self.url); let url = format!("{}/api/v2/search", self.url);
let request = PmvhavenSearch::new(query.to_string(),page as u32); let request = PmvhavenSearch::new(query.to_string(), page as u32);
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&index) { let old_items = match cache.get(&index) {
Some((time, items)) => { Some((time, items)) => {
@@ -321,66 +354,27 @@ impl PmvhavenProvider {
} }
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let mut requester = options.requester.clone().unwrap();
let client = Client::builder() let response = requester.post(&url, &request).await.unwrap();
.cert_verification(false) let videos = match response.json::<PmvhavenResponse>().await {
.emulation(Emulation::Firefox136) Ok(resp) => resp,
.build()?; Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
let response = client
.post(url.clone())
// .proxy(proxy.clone())
.json(&request)
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.send()
.await?;
if response.status().is_success() {
let videos = match response.json::<PmvhavenResponse>().await {
Ok(resp) => resp,
Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items); return Ok(old_items);
} }
return Ok(video_items); };
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
} }
// else { return Ok(video_items);
// let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
// let flare = Flaresolverr::new(flare_url);
// let result = flare
// .solve(FlareSolverrRequest {
// cmd: "request.get".to_string(),
// url: url.clone(),
// maxTimeout: 60000,
// })
// .await;
// let video_items = match result {
// Ok(res) => self.get_video_items_from_html(res.solution.response),
// Err(e) => {
// println!("Error solving FlareSolverr: {}", e);
// return Err("Failed to solve FlareSolverr".into());
// }
// };
// if !video_items.is_empty() {
// cache.remove(&url);
// cache.insert(url.clone(), video_items.clone());
// } else {
// return Ok(old_items);
// }
// Ok(video_items)
// }
Err("Failed to query Videos".into())
} }
} }
#[async_trait]
impl Provider for PmvhavenProvider { impl Provider for PmvhavenProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -395,11 +389,19 @@ impl Provider for PmvhavenProvider {
let _ = per_page; let _ = per_page;
let _ = pool; // Ignored in this implementation let _ = pool; // Ignored in this implementation
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await, Some(q) => {
None => { self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
self.get(cache, page.parse::<u8>().unwrap_or(1), options.category.unwrap(), sort)
.await .await
} }
None => {
self.get(
cache,
page.parse::<u8>().unwrap_or(1),
sort,
options,
)
.await
}
}; };
match videos { match videos {
Ok(v) => v, Ok(v) => v,

2
src/providers/porn00.rs
View File

@@ -7,6 +7,7 @@ use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -170,6 +171,7 @@ impl Porn00Provider {
} }
#[async_trait]
impl Provider for Porn00Provider { impl Provider for Porn00Provider {
async fn get_videos( async fn get_videos(
&self, &self,

114
src/providers/pornhat.rs
View File

@@ -2,15 +2,12 @@ use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec; use std::vec;
use wreq::{Client, Proxy}; use async_trait::async_trait;
use wreq_util::Emulation;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -34,6 +31,7 @@ impl PornhatProvider {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
sort: &str, sort: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let sort_string = match sort { let sort_string = match sort {
"trending" => "/trending", "trending" => "/trending",
@@ -54,63 +52,23 @@ impl PornhatProvider {
vec![] vec![]
} }
}; };
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&video_url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
let mut response = client.get(video_url.clone()) cache.remove(&video_url);
// .proxy(proxy.clone()) cache.insert(video_url.clone(), video_items.clone());
.send().await?;
if response.status().is_redirection(){
println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap());
response = client.get(response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
async fn query( async fn query(
&self, &self,
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
query: &str, query: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
@@ -133,23 +91,8 @@ impl PornhatProvider {
vec![] vec![]
} }
}; };
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&video_url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -158,31 +101,6 @@ impl PornhatProvider {
return Ok(old_items); return Ok(old_items);
} }
Ok(video_items) Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
} }
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> { fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -278,6 +196,7 @@ impl PornhatProvider {
} }
#[async_trait]
impl Provider for PornhatProvider { impl Provider for PornhatProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -289,16 +208,15 @@ impl Provider for PornhatProvider {
per_page: String, per_page: String,
options: ServerOptions, options: ServerOptions,
) -> Vec<VideoItem> { ) -> Vec<VideoItem> {
let _ = options;
let _ = per_page; let _ = per_page;
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,) self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await .await
} }
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await .await
} }
}; };

146
src/providers/pornhub.rs
View File

@@ -2,15 +2,12 @@ use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec; use std::vec;
use wreq::{Client, Proxy}; use async_trait::async_trait;
use wreq_util::Emulation;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -34,6 +31,7 @@ impl PornhubProvider {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
sort: &str, sort: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let video_url = format!("{}/video?o={}&page={}", self.url, sort, page); let video_url = format!("{}/video?o={}&page={}", self.url, sort, page);
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
@@ -50,56 +48,16 @@ impl PornhubProvider {
} }
}; };
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let mut requester = options.requester.clone().unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),"<ul id=\"video");
let mut response = client.get(video_url.clone()) if !video_items.is_empty() {
// .proxy(proxy.clone()) cache.remove(&video_url);
.send().await?; cache.insert(video_url.clone(), video_items.clone());
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),"<ul id=\"video");
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response,"<ul id=\"video")
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
async fn query( async fn query(
@@ -108,6 +66,7 @@ impl PornhubProvider {
page: u8, page: u8,
query: &str, query: &str,
sort: &str, sort: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let mut split_string = "<ul id=\"video"; let mut split_string = "<ul id=\"video";
let search_string = query.to_lowercase().trim().replace(" ", "+"); let search_string = query.to_lowercase().trim().replace(" ", "+");
@@ -156,55 +115,17 @@ impl PornhubProvider {
} }
}; };
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let mut requester = options.requester.clone().unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),split_string);
let mut response = client.get(video_url.clone()) if !video_items.is_empty() {
//.proxy(proxy.clone()) cache.remove(&video_url);
.send().await?; cache.insert(video_url.clone(), video_items.clone());
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),split_string);
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response,split_string),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
fn get_video_items_from_html(&self, html: String, split_string: &str) -> Vec<VideoItem> { fn get_video_items_from_html(&self, html: String, split_string: &str) -> Vec<VideoItem> {
@@ -226,7 +147,7 @@ impl PornhubProvider {
if video_segment.contains("wrapVideoBlock"){ if video_segment.contains("wrapVideoBlock"){
continue; // Skip if the segment is a wrapVideoBlock continue; // Skip if the segment is a wrapVideoBlock
} }
let mut video_url: String = String::new(); let video_url: String;
if !video_segment.contains("<a href=\"") { if !video_segment.contains("<a href=\"") {
let url_part = video_segment.split("data-video-vkey=\"").collect::<Vec<&str>>()[1] let url_part = video_segment.split("data-video-vkey=\"").collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
@@ -242,7 +163,7 @@ impl PornhubProvider {
} }
video_url = format!("{}{}", self.url, url_part); video_url = format!("{}{}", self.url, url_part);
} }
if video_url == "https://www.pornhub.comjavascript:void(0)".to_string() { if video_url.starts_with("https://www.pornhub.comjavascript:void(0)") {
continue; continue;
} }
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1] let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
@@ -275,10 +196,8 @@ impl PornhubProvider {
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string(); .to_string();
let uploaderBlock;
let uploader_href;
let mut uploaderBlock = String::new();
let mut uploader_href = vec![];
let mut tag = String::new(); let mut tag = String::new();
if video_segment.contains("videoUploaderBlock") { if video_segment.contains("videoUploaderBlock") {
@@ -291,6 +210,9 @@ impl PornhubProvider {
tag = format!("@{}:{}", uploader_href[1], uploader_href[2].replace("-", " ")); tag = format!("@{}:{}", uploader_href[1], uploader_href[2].replace("-", " "));
} }
else{
uploader_href = vec![];
}
let mut video_item = VideoItem::new( let mut video_item = VideoItem::new(
@@ -309,15 +231,6 @@ impl PornhubProvider {
video_item = video_item.tags(vec![tag]) video_item = video_item.tags(vec![tag])
.uploader(uploader_href[2].to_string()); .uploader(uploader_href[2].to_string());
} }
// if video_segment.contains("data-mediabook=\"") {
// let preview = video_segment.split("data-mediabook=\"").collect::<Vec<&str>>()[1]
// .split("\"")
// .collect::<Vec<&str>>()[0]
// .to_string();
// video_item = video_item.preview(preview);
// }
items.push(video_item); items.push(video_item);
} }
return items; return items;
@@ -326,6 +239,7 @@ impl PornhubProvider {
} }
#[async_trait]
impl Provider for PornhubProvider { impl Provider for PornhubProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -346,11 +260,11 @@ impl Provider for PornhubProvider {
} }
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort) self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort, options)
.await .await
} }
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await .await
} }
}; };

3
src/providers/pornzog.rs
View File

@@ -1,12 +1,12 @@
use crate::DbPool; use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -156,6 +156,7 @@ impl PornzogProvider {
} }
} }
#[async_trait]
impl Provider for PornzogProvider { impl Provider for PornzogProvider {
async fn get_videos( async fn get_videos(
&self, &self,

214
src/providers/redtube.rs
View File

@@ -1,17 +1,14 @@
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use serde_json::Value; use serde_json::Value;
use std::env;
use std::vec; use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -35,8 +32,10 @@ impl RedtubeProvider {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
sort: &str, sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let video_url = format!("{}/mostviewed", self.url); let _ = sort;
let video_url = format!("{}/mostviewed?page={}", self.url, page);
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -50,57 +49,16 @@ impl RedtubeProvider {
vec![] vec![]
} }
}; };
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&video_url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
let mut response = client.get(video_url.clone()) cache.remove(&video_url);
// .proxy(proxy.clone()) cache.insert(video_url.clone(), video_items.clone());
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
async fn query( async fn query(
&self, &self,
@@ -108,7 +66,9 @@ impl RedtubeProvider {
page: u8, page: u8,
query: &str, query: &str,
sort: &str, sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let _ = sort; //TODO
let search_string = query.to_lowercase().trim().replace(" ", "+"); let search_string = query.to_lowercase().trim().replace(" ", "+");
let video_url = format!("{}/?search={}&page={}", self.url, search_string, page); let video_url = format!("{}/?search={}&page={}", self.url, search_string, page);
@@ -126,56 +86,16 @@ impl RedtubeProvider {
vec![] vec![]
} }
}; };
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&video_url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone());
if !video_items.is_empty() {
let mut response = client.get(video_url.clone()) cache.remove(&video_url);
//.proxy(proxy.clone()) cache.insert(video_url.clone(), video_items.clone());
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html_query(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> { fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -184,7 +104,11 @@ impl RedtubeProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("<script type=\"application/ld+json\">").collect::<Vec<&str>>()[1].split("</script>").collect::<Vec<&str>>()[0]; let video_listing_content = html
.split("<script type=\"application/ld+json\">")
.collect::<Vec<&str>>()[1]
.split("</script>")
.collect::<Vec<&str>>()[0];
let mut videos: Value = serde_json::from_str(video_listing_content).unwrap(); let mut videos: Value = serde_json::from_str(video_listing_content).unwrap();
for vid in videos.as_array_mut().unwrap() { for vid in videos.as_array_mut().unwrap() {
let video_url: String = vid["embedUrl"].as_str().unwrap_or("").to_string(); let video_url: String = vid["embedUrl"].as_str().unwrap_or("").to_string();
@@ -193,7 +117,11 @@ impl RedtubeProvider {
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("=").collect::<Vec<&str>>()[1].to_string(); let id = video_url.split("=").collect::<Vec<&str>>()[1].to_string();
let raw_duration = vid["duration"].as_str().unwrap_or("0"); let raw_duration = vid["duration"].as_str().unwrap_or("0");
let duration = raw_duration.replace("PT", "").replace("S","").parse::<u32>().unwrap(); let duration = raw_duration
.replace("PT", "")
.replace("S", "")
.parse::<u32>()
.unwrap();
let views: u64 = vid["interactionCount"].as_u64().unwrap_or(0); let views: u64 = vid["interactionCount"].as_u64().unwrap_or(0);
let thumb = vid["thumbnailUrl"].as_str().unwrap_or("").to_string(); let thumb = vid["thumbnailUrl"].as_str().unwrap_or("").to_string();
@@ -205,8 +133,7 @@ impl RedtubeProvider {
thumb, thumb,
duration, duration,
) )
.views(views as u32) .views(views as u32);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
@@ -219,39 +146,64 @@ impl RedtubeProvider {
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("videos_grid").collect::<Vec<&str>>()[1]; let video_listing_content = html.split("videos_grid").collect::<Vec<&str>>()[1];
let videos = video_listing_content.split("<li id=\"tags_videos_").collect::<Vec<&str>>()[1..].to_vec(); let videos = video_listing_content
.split("<li id=\"tags_videos_")
.collect::<Vec<&str>>()[1..]
.to_vec();
for vid in videos { for vid in videos {
// for (i, c) in vid.split("\n").enumerate() { // for (i, c) in vid.split("\n").enumerate() {
// println!("{}: {}", i, c); // println!("{}: {}", i, c);
// } // }
let id = vid.split("data-video-id=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].to_string(); let id = vid.split("data-video-id=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let video_url = format!("{}/{}", self.url, id); let video_url = format!("{}/{}", self.url, id);
let title = vid.split(" <a title=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].trim().to_string(); let title = vid.split(" <a title=\"").collect::<Vec<&str>>()[1]
let thumb = vid.split("<img").collect::<Vec<&str>>()[1].split(" data-src=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].to_string(); .split("\"")
let raw_duration = vid.split("<span class=\"video-properties tm_video_duration\">").collect::<Vec<&str>>()[1].split("</span>").collect::<Vec<&str>>()[0].trim().to_string(); .collect::<Vec<&str>>()[0]
.trim()
.to_string();
let thumb = vid.split("<img").collect::<Vec<&str>>()[1]
.split(" data-src=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let raw_duration = vid
.split("<span class=\"video-properties tm_video_duration\">")
.collect::<Vec<&str>>()[1]
.split("</span>")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views_str = vid.split("<span class='info-views'>").collect::<Vec<&str>>()[1].split("</span>").collect::<Vec<&str>>()[0].trim().to_string(); let views_str = vid
.split("<span class='info-views'>")
.collect::<Vec<&str>>()[1]
.split("</span>")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
let views = parse_abbreviated_number(&views_str).unwrap_or(0) as u32; let views = parse_abbreviated_number(&views_str).unwrap_or(0) as u32;
let preview = vid.split("<img").collect::<Vec<&str>>()[1].split(" data-mediabook=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].to_string(); let preview = vid.split("<img").collect::<Vec<&str>>()[1]
.split(" data-mediabook=\"")
let video_item = VideoItem::new( .collect::<Vec<&str>>()[1]
id, .split("\"")
title, .collect::<Vec<&str>>()[0]
video_url, .to_string();
"redtube".to_string(),
thumb, let video_item =
duration, VideoItem::new(id, title, video_url, "redtube".to_string(), thumb, duration)
) .views(views)
.views(views) .preview(preview);
.preview(preview)
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait]
impl Provider for RedtubeProvider { impl Provider for RedtubeProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -265,18 +217,18 @@ impl Provider for RedtubeProvider {
) -> Vec<VideoItem> { ) -> Vec<VideoItem> {
let _ = options; let _ = options;
let _ = per_page; let _ = per_page;
let _ = pool; let _ = pool;
let mut sort = sort.to_lowercase(); let mut sort = sort.to_lowercase();
if sort.contains("date"){ if sort.contains("date") {
sort = "mr".to_string(); sort = "mr".to_string();
} }
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort) self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort, options)
.await .await
} }
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await .await
} }
}; };

120
src/providers/rule34video.rs
View File

@@ -2,16 +2,13 @@ use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec; use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -35,6 +32,7 @@ impl Rule34videoProvider {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
sort: &str, sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let now = SystemTime::now() let now = SystemTime::now()
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
@@ -68,56 +66,16 @@ impl Rule34videoProvider {
} }
}; };
} }
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
let mut response = client.get(url.clone()) cache.remove(&url);
// .proxy(proxy.clone()) cache.insert(url.clone(), video_items.clone());
.send().await?;
while response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
async fn query( async fn query(
&self, &self,
@@ -125,6 +83,7 @@ impl Rule34videoProvider {
page: u8, page: u8,
query: &str, query: &str,
sort: &str, sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let now = SystemTime::now() let now = SystemTime::now()
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
@@ -155,23 +114,8 @@ impl Rule34videoProvider {
vec![] vec![]
} }
}; };
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url); cache.remove(&url);
@@ -180,31 +124,6 @@ impl Rule34videoProvider {
return Ok(old_items); return Ok(old_items);
} }
Ok(video_items) Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
} }
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> { fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -254,10 +173,10 @@ impl Rule34videoProvider {
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string(); .to_string();
let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>()[1] // let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>()[1]
.split("\"") // .split("\"")
.collect::<Vec<&str>>()[0] // .collect::<Vec<&str>>()[0]
.to_string(); // .to_string();
let video_item = VideoItem::new( let video_item = VideoItem::new(
@@ -281,6 +200,7 @@ impl Rule34videoProvider {
} }
#[async_trait]
impl Provider for Rule34videoProvider { impl Provider for Rule34videoProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -297,11 +217,11 @@ impl Provider for Rule34videoProvider {
let _ = pool; // Ignored in this implementation let _ = pool; // Ignored in this implementation
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort) self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort, options)
.await .await
} }
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await .await
} }
}; };

21
src/providers/sxyprn.rs
View File

@@ -4,12 +4,12 @@ use crate::util::cache::VideoCache;
use crate::util::requester::Requester; use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::VideoItem; use crate::videos::VideoItem;
use crate::videos::{self, ServerOptions, VideoFormat}; use crate::videos::ServerOptions;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use scraper::ElementRef;
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -19,13 +19,13 @@ error_chain! {
} }
} }
fn has_blacklisted_class(element: &ElementRef, blacklist: &[&str]) -> bool { // fn has_blacklisted_class(element: &ElementRef, blacklist: &[&str]) -> bool {
element // element
.value() // .value()
.attr("class") // .attr("class")
.map(|classes| classes.split_whitespace().any(|c| blacklist.contains(&c))) // .map(|classes| classes.split_whitespace().any(|c| blacklist.contains(&c)))
.unwrap_or(false) // .unwrap_or(false)
} // }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct SxyprnProvider { pub struct SxyprnProvider {
@@ -150,6 +150,8 @@ impl SxyprnProvider {
pool: DbPool, pool: DbPool,
requester: Requester, requester: Requester,
) -> Vec<VideoItem> { ) -> Vec<VideoItem> {
let _ = requester;
let _ = pool;
if html.is_empty() { if html.is_empty() {
println!("HTML is empty"); println!("HTML is empty");
return vec![]; return vec![];
@@ -284,6 +286,7 @@ impl SxyprnProvider {
} }
} }
#[async_trait]
impl Provider for SxyprnProvider { impl Provider for SxyprnProvider {
async fn get_videos( async fn get_videos(
&self, &self,

168
src/providers/xxthots.rs
View File

@@ -1,16 +1,13 @@
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec; use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -34,6 +31,7 @@ impl XxthotsProvider {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
sort: &str, sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let sort_string = match sort { let sort_string = match sort {
"popular" => "/most-popular", "popular" => "/most-popular",
@@ -46,7 +44,10 @@ impl XxthotsProvider {
"top-rated" => "list_videos_common_videos_list", "top-rated" => "list_videos_common_videos_list",
_ => "list_videos_most_recent_videos", _ => "list_videos_most_recent_videos",
}; };
let video_url = format!("{}{}?mode=async^&function=get_block^&block_id={}^&from={}", self.url, sort_string, list_str, page); let video_url = format!(
"{}{}?mode=async^&function=get_block^&block_id={}^&from={}",
self.url, sort_string, list_str, page
);
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -60,65 +61,29 @@ impl XxthotsProvider {
vec![] vec![]
} }
}; };
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&video_url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
let mut response = client.get(video_url.clone()) cache.remove(&video_url);
// .proxy(proxy.clone()) cache.insert(video_url.clone(), video_items.clone());
.send().await?;
if response.status().is_redirection(){
response = client.get(response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
async fn query( async fn query(
&self, &self,
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
query: &str, query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let video_url = format!("{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&category_ids=&sort_by=&from_videos={}&from_albums={}&", self.url, search_string, page, page); let video_url = format!(
"{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&category_ids=&sort_by=&from_videos={}&from_albums={}&",
self.url, search_string, page, page
);
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
@@ -133,56 +98,16 @@ impl XxthotsProvider {
vec![] vec![]
} }
}; };
let mut requester = options.requester.clone().unwrap();
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let text = requester.get(&video_url).await.unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
let mut response = client.get(video_url.clone()) cache.remove(&video_url);
// .proxy(proxy.clone()) cache.insert(video_url.clone(), video_items.clone());
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} else { } else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); return Ok(old_items);
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
} }
Ok(video_items)
} }
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> { fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -191,7 +116,9 @@ impl XxthotsProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>()[0] let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()[0]
.split("<div class=\"thumb thumb_rel item \">") .split("<div class=\"thumb thumb_rel item \">")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -201,8 +128,9 @@ impl XxthotsProvider {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1] let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0].to_string(); .collect::<Vec<&str>>()[0]
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1] let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
@@ -210,20 +138,28 @@ impl XxthotsProvider {
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string(); let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>()[1] let raw_duration = video_segment
.split("<div class=\"time\">")
.collect::<Vec<&str>>()[1]
.split("<") .split("<")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"lazy-load").collect::<Vec<&str>>()[1] let thumb = video_segment
.split("data-original=\"").collect::<Vec<&str>>()[1] .split("<img class=\"lazy-load")
.collect::<Vec<&str>>()[1]
.split("data-original=\"")
.collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string(); .to_string();
let views_part = video_segment.split("svg-icon icon-eye").collect::<Vec<&str>>()[1] let views_part = video_segment
.split("</i>").collect::<Vec<&str>>()[1] .split("svg-icon icon-eye")
.collect::<Vec<&str>>()[1]
.split("</i>")
.collect::<Vec<&str>>()[1]
.split("<") .split("<")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
.to_string(); .to_string();
@@ -237,16 +173,14 @@ impl XxthotsProvider {
thumb, thumb,
duration, duration,
) )
.views(views) .views(views);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait]
impl Provider for XxthotsProvider { impl Provider for XxthotsProvider {
async fn get_videos( async fn get_videos(
&self, &self,
@@ -263,11 +197,11 @@ impl Provider for XxthotsProvider {
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,) self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await .await
} }
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await .await
} }
}; };

2
src/providers/youjizz.rs
View File

@@ -7,6 +7,7 @@ use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -169,6 +170,7 @@ impl YoujizzProvider {
} }
#[async_trait]
impl Provider for YoujizzProvider { impl Provider for YoujizzProvider {
async fn get_videos( async fn get_videos(
&self, &self,

2
src/status.rs
View File

@@ -33,7 +33,7 @@ pub struct ChannelOption {
pub multiSelect: bool, //true pub multiSelect: bool, //true
} }
#[derive(serde::Serialize)] #[derive(serde::Serialize, Debug, Clone)]
pub struct FilterOption{ pub struct FilterOption{
pub id: String, //"sort", pub id: String, //"sort",
pub title: String, //"Sort", pub title: String, //"Sort",

9
src/util/mod.rs
View File

@@ -1,5 +1,3 @@
use crate::providers::AnyProvider;
pub mod time; pub mod time;
pub mod flaresolverr; pub mod flaresolverr;
pub mod cache; pub mod cache;
@@ -43,11 +41,4 @@ pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {
} }
result result
}
pub fn get_all_providers() -> Vec<AnyProvider>{
return vec![];
} }

221
src/util/requester.rs
View File

@@ -1,15 +1,19 @@
use wreq::header::HeaderValue; use serde::Serialize;
use wreq::redirect::Policy; use std::env;
use wreq::Client; use wreq::Client;
use wreq::Proxy; use wreq::Proxy;
use wreq::Response; use wreq::Response;
use wreq::Version; use wreq::Version;
use wreq::header::HeaderValue;
use wreq::redirect::Policy;
use wreq_util::Emulation; use wreq_util::Emulation;
use std::env;
use crate::util::flaresolverr::FlareSolverrRequest; use crate::util::flaresolverr::FlareSolverrRequest;
use crate::util::flaresolverr::Flaresolverr; use crate::util::flaresolverr::Flaresolverr;
// A Send + Sync error type for all async paths
type AnyErr = Box<dyn std::error::Error + Send + Sync + 'static>;
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
pub struct Requester { pub struct Requester {
#[serde(skip)] #[serde(skip)]
@@ -21,11 +25,11 @@ pub struct Requester {
impl Requester { impl Requester {
pub fn new() -> Self { pub fn new() -> Self {
let client = Client::builder() let client = Client::builder()
.cert_verification(false) .cert_verification(false)
.emulation(Emulation::Firefox136) .emulation(Emulation::Firefox136)
.cookie_store(true) .cookie_store(true)
.redirect(Policy::default()) .redirect(Policy::default())
.build() .build()
.expect("Failed to create HTTP client"); .expect("Failed to create HTTP client");
Requester { Requester {
@@ -34,121 +38,150 @@ impl Requester {
flaresolverr_session: None, flaresolverr_session: None,
} }
} }
pub fn set_proxy(&mut self, proxy: bool) { pub fn set_proxy(&mut self, proxy: bool) {
if proxy{ if proxy {
println!("Proxy enabled"); println!("Proxy enabled");
} }
self.proxy = proxy; self.proxy = proxy;
} }
pub fn set_flaresolverr_session(&mut self, session: String) { // pub fn set_flaresolverr_session(&mut self, session: String) {
self.flaresolverr_session = Some(session); // self.flaresolverr_session = Some(session);
} // }
fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String { // fn get_url_from_location_header(&self, prev_url: &str, location: &str) -> String {
if location.starts_with("http://") || location.starts_with("https://") { // if location.starts_with("http://") || location.starts_with("https://") {
location.to_string() // location.to_string()
} else if location.starts_with("//") { // } else if location.starts_with("//") {
format!("{}{}", "https:", location) // Replace with your base URL // format!("{}{}", "https:", location)
} else if location.starts_with("/") { // } else if location.starts_with('/') {
let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/"); // let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/");
format!("{}{}", base_url, location) // format!("{}{}", base_url, location)
} else { // } else {
format!("{}/{}", prev_url, location) // format!("{}/{}", prev_url, location)
} // }
} // }
pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> { pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
let client = Client::builder() let client = Client::builder()
.cert_verification(false) .cert_verification(false)
.emulation(Emulation::Firefox136) .emulation(Emulation::Firefox136)
.cookie_store(true) .cookie_store(true)
.build() .build()
.expect("Failed to create HTTP client"); .expect("Failed to create HTTP client");
let mut request = client.get(url).version(Version::HTTP_11); let mut request = client.get(url).version(Version::HTTP_11);
let proxy;
if self.proxy { if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") { if let Ok(proxy_url) = env::var("BURP_URL") {
proxy = Proxy::all(&proxy_url).unwrap(); let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy.clone()); request = request.proxy(proxy);
} }
} }
// Directly propagate the error from send()
request.send().await request.send().await
} }
pub async fn get(&mut self, url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> { pub async fn post<S>(&mut self, url: &str, data: &S) -> Result<Response, wreq::Error>
let mut request = self.client.get(url).version(Version::HTTP_11); where
let mut proxy; S: Serialize + ?Sized,
{
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.build()
.expect("Failed to create HTTP client");
let mut request = client.post(url).version(Version::HTTP_11).json(data);
if self.proxy { if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") { if let Ok(proxy_url) = env::var("BURP_URL") {
proxy = Proxy::all(&proxy_url).unwrap(); let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy.clone()); request = request.proxy(proxy);
} }
} }
let mut response = request.send().await?; request.send().await
}
pub async fn get(&mut self, url: &str) -> Result<String, AnyErr> {
let mut request = self.client.get(url).version(Version::HTTP_11);
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy);
}
}
let response = request.send().await?;
if response.status().is_success() { if response.status().is_success() {
return Ok(response.text().await?); return Ok(response.text().await?);
} else { }
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let mut flare = Flaresolverr::new(flare_url);
if self.proxy && env::var("BURP_URL").is_ok() {
flare.set_proxy(true);
}
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url.to_string(),
maxTimeout: 60000,
})
.await;
match result {
Ok(res) => {
let cookie_url = url.split("/").collect::<Vec<&str>>()[..3].join("/");
self.client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.redirect(Policy::default())
.build()
.expect("Failed to create HTTP client");
let useragent = res.solution.userAgent;
self.client.update()
.headers(|headers| {
headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
})
.apply()
.unwrap();
for cookie in res.solution.cookies {
let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
// Parse the domain string into a Url
if let Ok(url) = url::Url::parse(cookie_url.as_str()) {
self.client.set_cookie(&url, header);
}
}
request = self.client.get(url).version(Version::HTTP_11);
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy.clone());
}
}
response = request.send().await?; // If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately,
if response.status().is_success() { // so no non-Send error value lives across later `.await`s.
return Ok(response.text().await?); let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
} let mut flare = Flaresolverr::new(flare_url);
if self.proxy && env::var("BURP_URL").is_ok() {
flare.set_proxy(true);
}
Ok(res.solution.response) let res = flare
} .solve(FlareSolverrRequest {
Err(e) => { cmd: "request.get".to_string(),
return Err(format!("Failed to solve FlareSolverr: {e}").into()); url: url.to_string(),
} maxTimeout: 60000,
})
.await
.map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?;
// Rebuild client and apply UA/cookies from FlareSolverr
let cookie_origin = url.split('/').take(3).collect::<Vec<&str>>().join("/");
self.client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.redirect(Policy::default())
.build()
.expect("Failed to create HTTP client");
let useragent = res.solution.userAgent;
self.client
.update()
.headers(|headers| {
headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
})
.apply()
.unwrap();
if let Ok(origin) = url::Url::parse(&cookie_origin) {
for cookie in res.solution.cookies {
let header =
HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
self.client.set_cookie(&origin, header);
} }
} }
// Retry the original URL with the updated client & (optional) proxy
let mut request = self.client.get(url).version(Version::HTTP_11);
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy);
}
}
let response = request.send().await?;
if response.status().is_success() {
return Ok(response.text().await?);
}
// Fall back to FlareSolverr-provided body
Ok(res.solution.response)
} }
} }

160
src/videos.rs
View File

@@ -270,86 +270,86 @@ impl VideoFormat {
self.format_note = Some(format_note); self.format_note = Some(format_note);
self self
} }
pub fn filesize(mut self, filesize: u32) -> Self { // pub fn filesize(mut self, filesize: u32) -> Self {
self.filesize = Some(filesize); // self.filesize = Some(filesize);
self // self
} // }
pub fn asr(mut self, asr: u32) -> Self { // pub fn asr(mut self, asr: u32) -> Self {
self.asr = Some(asr); // self.asr = Some(asr);
self // self
} // }
pub fn fps(mut self, fps: u32) -> Self { // pub fn fps(mut self, fps: u32) -> Self {
self.fps = Some(fps); // self.fps = Some(fps);
self // self
} // }
pub fn width(mut self, width: u32) -> Self { // pub fn width(mut self, width: u32) -> Self {
self.width = Some(width); // self.width = Some(width);
self // self
} // }
pub fn height(mut self, height: u32) -> Self { // pub fn height(mut self, height: u32) -> Self {
self.height = Some(height); // self.height = Some(height);
self // self
} // }
pub fn tbr(mut self, tbr: u32) -> Self { // pub fn tbr(mut self, tbr: u32) -> Self {
self.tbr = Some(tbr); // self.tbr = Some(tbr);
self // self
} // }
pub fn language(mut self, language: String) -> Self { // pub fn language(mut self, language: String) -> Self {
self.language = Some(language); // self.language = Some(language);
self // self
} // }
pub fn language_preference(mut self, language_preference: u32) -> Self { // pub fn language_preference(mut self, language_preference: u32) -> Self {
self.language_preference = Some(language_preference); // self.language_preference = Some(language_preference);
self // self
} // }
pub fn ext(mut self, ext: String) -> Self { // pub fn ext(mut self, ext: String) -> Self {
self.ext = Some(ext); // self.ext = Some(ext);
self // self
} // }
pub fn vcodec(mut self, vcodec: String) -> Self { // pub fn vcodec(mut self, vcodec: String) -> Self {
self.vcodec = Some(vcodec); // self.vcodec = Some(vcodec);
self // self
} // }
pub fn acodec(mut self, acodec: String) -> Self { // pub fn acodec(mut self, acodec: String) -> Self {
self.acodec = Some(acodec); // self.acodec = Some(acodec);
self // self
} // }
pub fn dynamic_range(mut self, dynamic_range: String) -> Self { // pub fn dynamic_range(mut self, dynamic_range: String) -> Self {
self.dynamic_range = Some(dynamic_range); // self.dynamic_range = Some(dynamic_range);
self // self
} // }
pub fn abr(mut self, abr: u32) -> Self { // pub fn abr(mut self, abr: u32) -> Self {
self.abr = Some(abr); // self.abr = Some(abr);
self // self
} // }
pub fn vbr(mut self, vbr: u32) -> Self { // pub fn vbr(mut self, vbr: u32) -> Self {
self.vbr = Some(vbr); // self.vbr = Some(vbr);
self // self
} // }
pub fn container(mut self, container: String) -> Self { // pub fn container(mut self, container: String) -> Self {
self.container = Some(container); // self.container = Some(container);
self // self
} // }
pub fn protocol(mut self, protocol: String) -> Self { // pub fn protocol(mut self, protocol: String) -> Self {
self.protocol = Some(protocol); // self.protocol = Some(protocol);
self // self
} // }
pub fn audio_ext(mut self, audio_ext: String) -> Self { // pub fn audio_ext(mut self, audio_ext: String) -> Self {
self.audio_ext = Some(audio_ext); // self.audio_ext = Some(audio_ext);
self // self
} // }
pub fn video_ext(mut self, video_ext: String) -> Self { // pub fn video_ext(mut self, video_ext: String) -> Self {
self.video_ext = Some(video_ext); // self.video_ext = Some(video_ext);
self // self
} // }
pub fn resolution(mut self, resolution: String) -> Self { // pub fn resolution(mut self, resolution: String) -> Self {
self.resolution = Some(resolution); // self.resolution = Some(resolution);
self // self
} // }
pub fn http_headers(mut self, http_headers: HashMap<String, String>) -> Self { // pub fn http_headers(mut self, http_headers: HashMap<String, String>) -> Self {
self.http_headers = Some(http_headers); // self.http_headers = Some(http_headers);
self // self
} // }
} }
#[derive(serde::Serialize, Debug)] #[derive(serde::Serialize, Debug)]
pub struct Videos { pub struct Videos {