simon/hottub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

overhault to fix warnings etc

Browse Source
This commit is contained in:
Simon
2025-10-04 14:28:29 +00:00
parent d84cc715a8
commit 28a4c57616
29 changed files with 889 additions and 1338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.toml
View File

@@ -23,6 +23,9 @@ capitalize = "0.3.4"
url = "2.5.4"
base64 = "0.22.1"
scraper = "0.24.0"
once_cell = "1.21.3"
rustc-hash = "2.1.1"
async-trait = "0.1"
[lints.rust]
unexpected_cfgs = "allow"

119
src/api.rs
View File

@@ -17,11 +17,13 @@ use crate::providers::rule34video::Rule34videoProvider;
// use crate::providers::spankbang::SpankbangProvider;
use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::{DbPool, db, providers::*, status::*, videos::*};
use crate::{DbPool, db, status::*, videos::*};
use cute::c;
use std::sync::Arc;
use crate::providers::{Provider, DynProvider, ALL_PROVIDERS};
#[derive(Debug)]
struct ClientVersion {
#[derive(Debug, Clone)]
pub struct ClientVersion {
version: u32,
subversion: u32,
name: String,
@@ -923,7 +925,7 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
cacheDuration: None,
});
// porn00
// noodlemagazine
// status.add_channel(Channel {
// id: "noodlemagazine".to_string(),
// name: "Noodlemagazine".to_string(),
@@ -1070,42 +1072,6 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
cacheDuration: None,
});
status.add_channel(Channel{
id: "omgxxx".to_string(),
name: "OMG XXX".to_string(),
description: "Free Porn Site".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.omg.xxx".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![
ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "latest-updates".to_string(),
title: "Latest".to_string(),
},
FilterOption {
id: "most-popular".to_string(),
title: "Most Viewed".to_string(),
},
FilterOption {
id: "top-rated".to_string(),
title: "Top Rated".to_string(),
},
],
multiSelect: false,
}
],
nsfw: true,
cacheDuration: None,
});
if clientversion >= ClientVersion::new(22, 105, "22i".to_string()) {
//sxyprn
status.add_channel(Channel {
@@ -1170,6 +1136,14 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
cacheDuration: Some(1800),
});
}
for provider in ALL_PROVIDERS.values() {
println!(
"Loaded provider (dyn): {:?}",
std::any::type_name::<&dyn Provider>()
);
status.add_channel(provider.get_channel(clientversion.clone()));
}
status.iconUrl = format!("http://{}/favicon.ico", host).to_string();
Ok(web::HttpResponse::Ok().json(&status))
}
@@ -1296,54 +1270,35 @@ async fn videos_post(
Ok(web::HttpResponse::Ok().json(&videos))
}
pub fn get_provider(channel: &str) -> Option<AnyProvider> {
pub fn get_provider(channel: &str) -> Option<DynProvider> {
match channel {
"all" => Some(AnyProvider::All(AllProvider::new())),
"perverzija" => Some(AnyProvider::Perverzija(PerverzijaProvider::new())),
"hanime" => Some(AnyProvider::Hanime(HanimeProvider::new())),
// "spankbang" => Some(AnyProvider::Spankbang(SpankbangProvider::new())),
"pornhub" => Some(AnyProvider::Pornhub(PornhubProvider::new())),
"pmvhaven" => Some(AnyProvider::Pmvhaven(PmvhavenProvider::new())),
"rule34video" => Some(AnyProvider::Rule34video(Rule34videoProvider::new())),
"redtube" => Some(AnyProvider::Redtube(RedtubeProvider::new())),
"okporn" => Some(AnyProvider::Okporn(OkpornProvider::new())),
"pornhat" => Some(AnyProvider::Pornhat(
crate::providers::pornhat::PornhatProvider::new(),
)),
"perfectgirls" => Some(AnyProvider::Perfectgirls(
"all" => Some(Arc::new(AllProvider::new())),
"perverzija" => Some(Arc::new(PerverzijaProvider::new())),
"hanime" => Some(Arc::new(HanimeProvider::new())),
"pornhub" => Some(Arc::new(PornhubProvider::new())),
"pmvhaven" => Some(Arc::new(PmvhavenProvider::new())),
"rule34video" => Some(Arc::new(Rule34videoProvider::new())),
"redtube" => Some(Arc::new(RedtubeProvider::new())),
"okporn" => Some(Arc::new(OkpornProvider::new())),
"pornhat" => Some(Arc::new(crate::providers::pornhat::PornhatProvider::new())),
"perfectgirls" => Some(Arc::new(
crate::providers::perfectgirls::PerfectgirlsProvider::new(),
)),
"okxxx" => Some(AnyProvider::Okxxx(
crate::providers::okxxx::OkxxxProvider::new(),
)),
"homoxxx" => Some(AnyProvider::Homoxxx(
crate::providers::homoxxx::HomoxxxProvider::new(),
)),
// "hentaimoon" => Some(AnyProvider::Hentaimoon(crate::providers::hentaimoon::HentaimoonProvider::new())),
"missav" => Some(AnyProvider::Missav(
crate::providers::missav::MissavProvider::new(),
)),
"xxthots" => Some(AnyProvider::Xxthots(
crate::providers::xxthots::XxthotsProvider::new(),
)),
"sxyprn" => Some(AnyProvider::Sxyprn(
crate::providers::sxyprn::SxyprnProvider::new(),
)),
"porn00" => Some(AnyProvider::Porn00(
crate::providers::porn00::Porn00Provider::new(),
)),
// "noodlemagazine" => Some(AnyProvider::Noodlemagazine(crate::providers::noodlemagazine::NoodlemagazineProvider::new())),
"freshporno" => Some(AnyProvider::Freshporno(
"okxxx" => Some(Arc::new(crate::providers::okxxx::OkxxxProvider::new())),
"homoxxx" => Some(Arc::new(crate::providers::homoxxx::HomoxxxProvider::new())),
"missav" => Some(Arc::new(crate::providers::missav::MissavProvider::new())),
"xxthots" => Some(Arc::new(crate::providers::xxthots::XxthotsProvider::new())),
"sxyprn" => Some(Arc::new(crate::providers::sxyprn::SxyprnProvider::new())),
"porn00" => Some(Arc::new(crate::providers::porn00::Porn00Provider::new())),
"freshporno" => Some(Arc::new(
crate::providers::freshporno::FreshpornoProvider::new(),
)),
"youjizz" => Some(AnyProvider::Youjizz(
crate::providers::youjizz::YoujizzProvider::new(),
)),
"paradisehill" => Some(AnyProvider::Paradisehill(
"youjizz" => Some(Arc::new(crate::providers::youjizz::YoujizzProvider::new())),
"paradisehill" => Some(Arc::new(
crate::providers::paradisehill::ParadisehillProvider::new(),
)),
"pornzog" => Some(AnyProvider::Pornzog(crate::providers::pornzog::PornzogProvider::new())),
"omgxxx" => Some(AnyProvider::Omgxxx(crate::providers::omgxxx::OmgxxxProvider::new())),
_ => Some(AnyProvider::Perverzija(PerverzijaProvider::new())),
"pornzog" => Some(Arc::new(crate::providers::pornzog::PornzogProvider::new())),
// fallback to the dynamic registry
x => ALL_PROVIDERS.get(x).cloned(),
}
}

4
src/db.rs
View File

@@ -46,8 +46,8 @@ pub fn has_table(
use diesel::sql_types::Text;
#[derive(QueryableByName)]
struct TableName {
#[sql_type = "Text"]
#[column_name = "name"]
#[diesel(sql_type = Text)]
#[diesel(column_name = name)]
name: String,
}

23
src/providers/all.rs
View File

@@ -1,8 +1,10 @@
use std::fs;
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use crate::api::get_provider;
use crate::providers::{AnyProvider, Provider};
use crate::api::{get_provider, ClientVersion};
use crate::providers::{DynProvider, Provider};
use crate::status::Channel;
use crate::util::cache::VideoCache;
use crate::util::interleave;
use crate::videos::{ServerOptions, VideoItem};
@@ -27,6 +29,7 @@ impl AllProvider {
}
}
#[async_trait]
impl Provider for AllProvider {
async fn get_videos(
&self,
@@ -48,11 +51,11 @@ impl Provider for AllProvider {
.collect::<Vec<String>>();
sites_str = providers.join(",");
}
let sites = sites_str
let providers: Vec<DynProvider> = sites_str
.split(',')
.map(|s| s.to_string()) // or s.to_owned()
.collect::<Vec<String>>();
let providers = sites.iter().map(|el| get_provider(el.as_str()).unwrap()).collect::<Vec<AnyProvider>>();
.filter(|s| !s.is_empty())
.filter_map(|s| get_provider(s)) // assumes get_provider -> Option<DynProvider>
.collect();
let futures = providers.iter().map(|provider| {
provider.get_videos(
@@ -71,4 +74,12 @@ impl Provider for AllProvider {
return video_items;
}
fn get_channel(&self,clientversion:ClientVersion) -> Channel {
println!("Getting channel for placeholder with client version: {:?}",clientversion);
let _ = clientversion;
Channel {
id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None,
}
}
}

13
src/providers/freshporno.rs
View File

@@ -1,7 +1,10 @@
use crate::api::ClientVersion;
use crate::status::Channel;
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
@@ -17,6 +20,7 @@ error_chain! {
pub struct FreshpornoProvider {
url: String,
}
impl FreshpornoProvider {
pub fn new() -> Self {
FreshpornoProvider {
@@ -156,6 +160,7 @@ impl FreshpornoProvider {
}
#[async_trait]
impl Provider for FreshpornoProvider {
async fn get_videos(
&self,
@@ -187,4 +192,12 @@ impl Provider for FreshpornoProvider {
}
}
}
fn get_channel(&self,clientversion:ClientVersion) -> Channel {
println!("Getting channel for placeholder with client version: {:?}",clientversion);
let _ = clientversion;
Channel {
id:"placeholder".to_string(),name:"PLACEHOLDER".to_string(),description:"PLACEHOLDER FOR PARENT CLASS".to_string(),premium:false,favicon:"https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),status:"active".to_string(),categories:vec![],options:vec![],nsfw:true,cacheDuration:None,
}
}
}

2
src/providers/hanime.rs
View File

@@ -1,4 +1,5 @@
use std::vec;
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use wreq::Client;
@@ -253,6 +254,7 @@ impl HanimeProvider {
}
}
#[async_trait]
impl Provider for HanimeProvider {
async fn get_videos(
&self,

2
src/providers/homoxxx.rs
View File

@@ -4,6 +4,7 @@ use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
@@ -243,6 +244,7 @@ impl HomoxxxProvider {
}
#[async_trait]
impl Provider for HomoxxxProvider {
async fn get_videos(
&self,

2
src/providers/missav.rs
View File

@@ -1,4 +1,5 @@
use std::vec;
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all;
@@ -253,6 +254,7 @@ impl MissavProvider {
}
}
#[async_trait]
impl Provider for MissavProvider {
async fn get_videos(
&self,

208
src/providers/mod.rs
View File

@@ -1,7 +1,15 @@
use async_trait::async_trait;
use once_cell::sync::Lazy;
use rustc_hash::FxHashMap as HashMap;
use std::sync::Arc;
use crate::{
providers::{
all::AllProvider, hanime::HanimeProvider, okporn::OkpornProvider, perverzija::PerverzijaProvider, pmvhaven::PmvhavenProvider, pornhub::PornhubProvider, redtube::RedtubeProvider, rule34video::Rule34videoProvider
}, util::cache::VideoCache, videos::{ServerOptions, VideoItem}, DbPool
DbPool,
api::ClientVersion,
providers::omgxxx::OmgxxxProvider,
status::Channel,
util::cache::VideoCache,
videos::{ServerOptions, VideoItem},
};
pub mod all;
@@ -10,27 +18,37 @@ pub mod perverzija;
pub mod pmvhaven;
pub mod pornhub;
// pub mod spankbang;
pub mod rule34video;
pub mod redtube;
pub mod okporn;
pub mod pornhat;
pub mod perfectgirls;
pub mod okxxx;
pub mod homoxxx;
pub mod okporn;
pub mod okxxx;
pub mod perfectgirls;
pub mod pornhat;
pub mod redtube;
pub mod rule34video;
// pub mod hentaimoon;
pub mod missav;
pub mod xxthots;
pub mod sxyprn;
pub mod porn00;
pub mod sxyprn;
pub mod xxthots;
// pub mod noodlemagazine;
pub mod freshporno;
pub mod youjizz;
pub mod omgxxx;
pub mod paradisehill;
pub mod pornzog;
pub mod omgxxx;
pub mod youjizz;
// convenient alias
pub type DynProvider = Arc<dyn Provider>;
pub trait Provider {
pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| {
let mut m = HashMap::default();
m.insert("omgxxx", Arc::new(OmgxxxProvider::new()) as DynProvider);
// add more here as you migrate them
m
});
#[async_trait]
pub trait Provider: Send + Sync {
async fn get_videos(
&self,
cache: VideoCache,
@@ -41,156 +59,24 @@ pub trait Provider {
per_page: String,
options: ServerOptions,
) -> Vec<VideoItem>;
}
#[derive(Debug, Clone)]
pub enum AnyProvider {
All(AllProvider),
Perverzija(PerverzijaProvider),
Hanime(HanimeProvider),
// Spankbang(SpankbangProvider),
Pornhub(PornhubProvider),
Pmvhaven(PmvhavenProvider),
Rule34video(Rule34videoProvider),
Redtube(RedtubeProvider),
Okporn(OkpornProvider),
Pornhat(crate::providers::pornhat::PornhatProvider),
Perfectgirls(crate::providers::perfectgirls::PerfectgirlsProvider),
Okxxx(crate::providers::okxxx::OkxxxProvider),
Homoxxx(crate::providers::homoxxx::HomoxxxProvider),
// Hentaimoon(crate::providers::hentaimoon::HentaimoonProvider),
Missav(crate::providers::missav::MissavProvider),
Xxthots(crate::providers::xxthots::XxthotsProvider),
Sxyprn(crate::providers::sxyprn::SxyprnProvider),
Porn00(crate::providers::porn00::Porn00Provider),
// Noodlemagazine(crate::providers::noodlemagazine::NoodlemagazineProvider),
Freshporno(crate::providers::freshporno::FreshpornoProvider),
Youjizz(crate::providers::youjizz::YoujizzProvider),
Paradisehill(crate::providers::paradisehill::ParadisehillProvider),
Pornzog(crate::providers::pornzog::PornzogProvider),
Omgxxx(crate::providers::omgxxx::OmgxxxProvider),
}
impl Provider for AnyProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
sort: String,
query: Option<String>,
page: String,
per_page: String,
options: ServerOptions
) -> Vec<VideoItem> {
fn get_channel(&self, clientversion: ClientVersion) -> Channel {
println!(
"/api/videos: sort={:?}, query={:?}, page={:?}, provider={:?}",
sort, query, page, self
"Getting channel for placeholder with client version: {:?}",
clientversion
);
match self {
AnyProvider::Perverzija(p) => {
p.get_videos(
cache.clone(),
pool.clone(),
sort.clone(),
query.clone(),
page.clone(),
per_page.clone(),
options,
)
.await
}
AnyProvider::Hanime(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
// AnyProvider::Spankbang(p) => {
// p.get_videos(cache, pool, sort, query, page, per_page, options,)
// .await
// }
AnyProvider::Pornhub(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Pmvhaven(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Rule34video(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Redtube(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::All(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Okporn(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Pornhat(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Perfectgirls(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Okxxx(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Homoxxx(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
// AnyProvider::Hentaimoon(p) => {
// p.get_videos(cache, pool, sort, query, page, per_page, options,)
// .await
// }
AnyProvider::Missav(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Xxthots(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Sxyprn(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Porn00(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
// AnyProvider::Noodlemagazine(p) => {
// p.get_videos(cache, pool, sort, query, page, per_page, options,)
// .await
// }
AnyProvider::Freshporno(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Youjizz(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
},
AnyProvider::Paradisehill(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Pornzog(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
AnyProvider::Omgxxx(p) => {
p.get_videos(cache, pool, sort, query, page, per_page, options,)
.await
}
let _ = clientversion;
Channel {
id: "placeholder".to_string(),
name: "PLACEHOLDER".to_string(),
description: "PLACEHOLDER FOR PARENT CLASS".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![],
nsfw: true,
cacheDuration: None,
}
}
}

2
src/providers/okporn.rs
View File

@@ -10,6 +10,7 @@ use std::env;
use std::vec;
use wreq::{Client};
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -233,6 +234,7 @@ impl OkpornProvider {
}
#[async_trait]
impl Provider for OkpornProvider {
async fn get_videos(
&self,

2
src/providers/okxxx.rs
View File

@@ -11,6 +11,7 @@ use std::env;
use std::vec;
use wreq::{Client};
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -278,6 +279,7 @@ impl OkxxxProvider {
}
#[async_trait]
impl Provider for OkxxxProvider {
async fn get_videos(
&self,

44
src/providers/omgxxx.rs
View File

@@ -1,3 +1,4 @@
use crate::api::ClientVersion;
use crate::status::*;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
@@ -8,6 +9,7 @@ use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -19,15 +21,21 @@ error_chain! {
#[derive(Debug, Clone)]
pub struct OmgxxxProvider {
url: String,
sites: Vec<FilterOption>,
networks: Vec<FilterOption>,
}
impl OmgxxxProvider {
pub fn new() -> Self {
OmgxxxProvider {
url: "https://www.omg.xxx".to_string(),
sites: vec![],
networks: vec![],
}
}
async fn get_channel(&self) -> crate::status::Channel {
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion;
let channel: crate::status::Channel = Channel{
id: "omgxxx".to_string(),
name: "OMG XXX".to_string(),
@@ -58,11 +66,31 @@ impl OmgxxxProvider {
},
],
multiSelect: false,
},
ChannelOption {
id: "sites".to_string(),
title: "Sites".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.bullet.indent".to_string(),
colorName: "green".to_string(),
options: self.sites.clone(),
multiSelect: false,
},
ChannelOption {
id: "networks".to_string(),
title: "Networks".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.dash".to_string(),
colorName: "purple".to_string(),
options: self.networks.clone(),
multiSelect: false,
}
],
nsfw: true,
cacheDuration: None,
};
return channel;
}
@@ -94,7 +122,6 @@ impl OmgxxxProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
@@ -158,10 +185,10 @@ impl OmgxxxProvider {
.split("class=\"item\"").collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>();
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line);
}
// let vid = video_segment.split("\n").collect::<Vec<&str>>();
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0].to_string();
@@ -226,6 +253,7 @@ impl OmgxxxProvider {
}
#[async_trait]
impl Provider for OmgxxxProvider {
async fn get_videos(
&self,
@@ -257,4 +285,8 @@ impl Provider for OmgxxxProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel {
println!("Getting channel for omgxxx with client version: {:?}", clientversion);
self.build_channel(clientversion)
}
}

4
src/providers/paradisehill.rs
View File

@@ -8,6 +8,7 @@ use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -173,7 +174,7 @@ impl ParadisehillProvider {
.to_string();
let format =
videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "mp4".to_string())
.protocol("https".to_string())
// .protocol("https".to_string())
.format_id(video_url.split("/").last().unwrap().to_string())
.format_note(format!("{}", video_url.split("_").last().unwrap().replace(".mp4", "").to_string()))
;
@@ -203,6 +204,7 @@ impl ParadisehillProvider {
}
}
#[async_trait]
impl Provider for ParadisehillProvider {
async fn get_videos(
&self,

2
src/providers/perfectgirls.rs
View File

@@ -11,6 +11,7 @@ use std::env;
use std::vec;
use wreq::{Client};
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -281,6 +282,7 @@ impl PerfectgirlsProvider {
}
#[async_trait]
impl Provider for PerfectgirlsProvider {
async fn get_videos(
&self,

333
src/providers/perverzija.rs
View File

@@ -1,21 +1,19 @@
use std::vec;
use std::env;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all;
use serde::Serialize;
use wreq::Client;
use wreq_util::Emulation;
use serde::Deserialize;
use crate::DbPool;
use crate::db;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::ServerOptions;
use crate::videos::{self, VideoEmbed, VideoItem};
use crate::DbPool;
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use serde::Deserialize;
use serde::Serialize;
use std::vec;
use wreq::Client;
use wreq_util::Emulation;
error_chain! {
foreign_links {
@@ -25,7 +23,6 @@ error_chain! {
}
}
#[derive(Debug, Deserialize, Serialize)]
struct PerverzijaDbEntry {
url_string: String,
@@ -42,8 +39,14 @@ impl PerverzijaProvider {
url: "https://tube.perverzija.com/".to_string(),
}
}
async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result<Vec<VideoItem>> {
async fn get(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let featured = options.featured.unwrap_or("".to_string());
let mut prefix_uri = "".to_string();
if featured == "featured" {
prefix_uri = "featured-scenes/".to_string();
@@ -58,8 +61,7 @@ impl PerverzijaProvider {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
println!("Cache hit for URL: {}", url_str);
return Ok(items.clone());
}
else{
} else {
items.clone()
}
}
@@ -68,59 +70,28 @@ impl PerverzijaProvider {
}
};
let client = Client::builder()
.emulation(Emulation::Firefox136)
.build()?;
let response = client.get(url_str.clone()).send().await?;
// print!("Response: {:?}\n", response);
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else{
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url_str.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response, pool)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result<Vec<VideoItem>> {
async fn query(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let mut query_parse = true;
let search_string = query.replace(" ", "+");
let mut url_str = format!(
"{}page/{}/?s={}",
self.url, page, search_string
);
let mut url_str = format!("{}page/{}/?s={}", self.url, page, search_string);
if page == 1 {
url_str = format!("{}?s={}", self.url, search_string);
}
@@ -140,61 +111,32 @@ impl PerverzijaProvider {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
return Ok(items.clone());
}
else{
} else {
let _ = cache.check().await;
return Ok(items.clone())
return Ok(items.clone());
}
}
None => {
vec![]
}
};
let client = Client::builder()
.emulation(Emulation::Firefox136)
.build()?;
let response = client.get(url_str.clone()).send().await?;
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = match query_parse{
true => {self.get_video_items_from_html_query(text.clone(), pool).await},
false => {self.get_video_items_from_html(text.clone(), pool)}
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else{
return Ok(old_items);
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str).await.unwrap();
let video_items: Vec<VideoItem> = match query_parse {
true => {
self.get_video_items_from_html_query(text.clone(), pool)
.await
}
Ok(video_items)
false => self.get_video_items_from_html(text.clone(), pool),
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url_str.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
self.get_video_items_from_html_query(res.solution.response, pool).await
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url_str);
cache.insert(url_str.clone(), video_items.clone());
} else{
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
@@ -228,8 +170,9 @@ impl PerverzijaProvider {
let url_str = vid[1].split("iframe src=&quot;").collect::<Vec<&str>>()[1]
.split("&quot;")
.collect::<Vec<&str>>()[0]
.to_string().replace("index.php", "xs1.php");
if url_str.starts_with("https://streamtape.com/"){
.to_string()
.replace("index.php", "xs1.php");
if url_str.starts_with("https://streamtape.com/") {
continue; // Skip Streamtape links
}
let id = url_str.split("data=").collect::<Vec<&str>>()[1]
@@ -245,16 +188,18 @@ impl PerverzijaProvider {
};
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
if !vid[4].contains("srcset=") && vid[4].split("src=\"").collect::<Vec<&str>>().len() == 1{
for (index, line) in vid.iter().enumerate(){
if !vid[4].contains("srcset=")
&& vid[4].split("src=\"").collect::<Vec<&str>>().len() == 1
{
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}\n\n", index, line);
}
}
let mut thumb = "".to_string();
for v in vid.clone(){
for v in vid.clone() {
let line = v.trim();
if line.starts_with("<img "){
if line.starts_with("<img ") {
thumb = line.split(" src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
@@ -280,18 +225,20 @@ impl PerverzijaProvider {
let studios_parts = vid[7].split("a href=\"").collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/"){
if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push(
studio.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/studio/", "@studio:")
.replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(),
);
}
}
for tag in vid[0].split(" ").collect::<Vec<&str>>(){
for tag in vid[0].split(" ").collect::<Vec<&str>>() {
if tag.starts_with("stars-") {
let tag_name = tag.split("stars-").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0]
let tag_name = tag.split("stars-").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
if !tag_name.is_empty() {
tags.push(format!("@stars:{}", tag_name));
@@ -299,10 +246,9 @@ impl PerverzijaProvider {
}
}
for tag in vid[0].split(" ").collect::<Vec<&str>>(){
for tag in vid[0].split(" ").collect::<Vec<&str>>() {
if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1]
.to_string();
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1].to_string();
if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string());
}
@@ -315,7 +261,8 @@ impl PerverzijaProvider {
"perverzija".to_string(),
thumb,
duration,
).tags(tags);
)
.tags(tags);
// .embed(embed.clone());
let mut format =
videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string());
@@ -331,19 +278,15 @@ impl PerverzijaProvider {
return items;
}
async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec<VideoItem> {
let raw_videos = html
.split("video-item post")
.collect::<Vec<&str>>()[1..]
.to_vec();
let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let items: Vec<VideoItem> = results
async fn get_video_items_from_html_query(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
let raw_videos = html.split("video-item post").collect::<Vec<&str>>()[1..].to_vec();
let futures = raw_videos
.into_iter()
.filter_map(Result::ok)
.collect();
.map(|el| self.get_video_item(el, pool.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return items;
return items;
}
async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<VideoItem> {
@@ -358,9 +301,9 @@ impl PerverzijaProvider {
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = match vid[6].split(" src=\"").collect::<Vec<&str>>().len(){
1=>{
for (index,line) in vid.iter().enumerate() {
let thumb = match vid[6].split(" src=\"").collect::<Vec<&str>>().len() {
1 => {
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line.to_string().trim());
}
return Err("Failed to parse thumbnail URL".into());
@@ -379,19 +322,19 @@ impl PerverzijaProvider {
let referer_url = "https://xtremestream.xyz/".to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let db_result = db::get_video(&mut conn,lookup_url.clone());
let db_result = db::get_video(&mut conn, lookup_url.clone());
match db_result {
Ok(Some(entry)) => {
if entry.starts_with("{"){ // replace old urls with new json objects
if entry.starts_with("{") {
// replace old urls with new json objects
let entry = serde_json::from_str::<PerverzijaDbEntry>(entry.as_str())?;
let url_str = entry.url_string;
let tags = entry.tags_strings;
if url_str.starts_with("!"){
if url_str.starts_with("!") {
return Err("Video was removed".into());
}
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1]
.to_string();
if id.contains("&"){
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1].to_string();
if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
}
let mut video_item = VideoItem::new(
@@ -402,24 +345,24 @@ impl PerverzijaProvider {
thumb,
duration,
)
.tags(tags)
;
let mut format =
videos::VideoFormat::new(url_str.clone(), "1080".to_string(), "m3u8".to_string());
.tags(tags);
let mut format = videos::VideoFormat::new(
url_str.clone(),
"1080".to_string(),
"m3u8".to_string(),
);
format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() {
formats.push(format);
} else {
video_item.formats = Some(vec![format]);
}
return Ok(video_item)
}
else{
let _ = db::delete_video(&mut conn,lookup_url.clone());
return Ok(video_item);
} else {
let _ = db::delete_video(&mut conn, lookup_url.clone());
};
}
Ok(None) => {
},
Ok(None) => {}
Err(e) => {
println!("Error fetching video from database: {}", e);
// return Err(format!("Error fetching video from database: {}", e).into());
@@ -427,13 +370,10 @@ impl PerverzijaProvider {
}
drop(conn);
let client = Client::builder()
.emulation(Emulation::Firefox136)
.build()?;
let client = Client::builder().emulation(Emulation::Firefox136).build()?;
let response = client.get(lookup_url.clone()).send().await?;
let text = match response.status().is_success(){
let text = match response.status().is_success() {
true => response.text().await?,
false => {
println!("Failed to fetch video details");
@@ -441,46 +381,62 @@ impl PerverzijaProvider {
}
};
let mut url_str = text.split("<iframe src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string().replace("index.php","xs1.php");
if !url_str.contains("xtremestream.xyz"){
.to_string()
.replace("index.php", "xs1.php");
if !url_str.contains("xtremestream.xyz") {
url_str = "!".to_string()
}
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
let studios_parts = text.split("<strong>Studio: </strong>").collect::<Vec<&str>>()[1].split("</div>").collect::<Vec<&str>>()[0].split("<a href=\"").collect::<Vec<&str>>();
let studios_parts = text
.split("<strong>Studio: </strong>")
.collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0]
.split("<a href=\"")
.collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/"){
if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push(
studio.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/studio/", "@studio:")
.replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(),
);
}
}
if text.contains("<strong>Stars: </strong>"){
let stars_parts: Vec<&str> = text.split("<strong>Stars: </strong>").collect::<Vec<&str>>()[1].split("</div>").collect::<Vec<&str>>()[0].split("<a href=\"").collect::<Vec<&str>>();
if text.contains("<strong>Stars: </strong>") {
let stars_parts: Vec<&str> = text
.split("<strong>Stars: </strong>")
.collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0]
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in stars_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/"){
if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push(
star.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/stars/", "@stars:")
.replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(),
);
}
}
}
}
let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>()[1].split("</div>").collect::<Vec<&str>>()[0].split("<a href=\"").collect::<Vec<&str>>();
let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0]
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in tags_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/"){
if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push(
star.split("/\"").collect::<Vec<&str>>()[0]
.replace("https://tube.perverzija.com/stars/", "@stars:")
.replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(),
);
}
@@ -491,18 +447,23 @@ impl PerverzijaProvider {
tags_strings: tags.clone(),
};
let mut conn = pool.get().expect("couldn't get db connection from pool");
let insert_result = db::insert_video(&mut conn, &lookup_url, &serde_json::to_string(&perverzija_db_entry)?);
match insert_result{
let insert_result = db::insert_video(
&mut conn,
&lookup_url,
&serde_json::to_string(&perverzija_db_entry)?,
);
match insert_result {
Ok(_) => (),
Err(e) => {println!("{:?}", e); }
Err(e) => {
println!("{:?}", e);
}
}
drop(conn);
if !url_str.contains("xtremestream.xyz"){
if !url_str.contains("xtremestream.xyz") {
return Err("Video URL does not contain xtremestream.xyz".into());
}
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1]
.to_string();
if id.contains("&"){
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1].to_string();
if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
}
// if !vid[6].contains(" src=\""){
@@ -514,9 +475,6 @@ impl PerverzijaProvider {
// println!("Line {}: {}", index, line.to_string().trim());
// }
let mut video_item = VideoItem::new(
id,
title,
@@ -539,6 +497,7 @@ impl PerverzijaProvider {
}
}
#[async_trait]
impl Provider for PerverzijaProvider {
async fn get_videos(
&self,
@@ -553,8 +512,14 @@ impl Provider for PerverzijaProvider {
let _ = per_page;
let _ = sort;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => self.query(cache, pool, page.parse::<u8>().unwrap_or(1), &q).await,
None => self.get(cache, pool, page.parse::<u8>().unwrap_or(1), options.featured.unwrap()).await,
Some(q) => {
self.query(cache, pool, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(cache, pool, page.parse::<u8>().unwrap_or(1), options)
.await
}
};
match videos {
Ok(v) => v,

262
src/providers/pmvhaven.rs
View File

@@ -2,15 +2,11 @@ use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use cute::c;
use error_chain::error_chain;
use htmlentity::entity::ICodedDataTrait;
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
// use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
#[macro_use(c)]
error_chain! {
foreign_links {
@@ -157,50 +153,55 @@ impl PmvhavenSearch {
#[derive(serde::Deserialize)]
struct PmvhavenVideo {
title: String, //JAV Addiction Therapy",
uploader: Option<String>, //itonlygetsworse",
duration: f32, //259.093333,
width: Option<String>, //3840",
height: Option<String>, //2160",
ratio: Option<u32>, //50,
title: String, //JAV Addiction Therapy",
_uploader: Option<String>, //itonlygetsworse",
duration: f32, //259.093333,
_width: Option<String>, //3840",
_height: Option<String>, //2160",
_ratio: Option<u32>, //50,
thumbnails: Vec<Option<String>>, //[
// "placeholder",
// "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.png",
// "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/webp320_686f24e96f7124f3dfbe90ab.webp"
// ],
views: u32, //1971,
url: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.mp4",
views: u32, //1971,
_url: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.mp4",
previewUrlCompressed: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/videoPreview/comus_686f24e96f7124f3dfbe90ab.mp4",
seizureWarning: Option<bool>, //false,
isoDate: Option<String>, //2025-07-10T02:52:26.000Z",
gayContent: Option<bool>, //false,
transContent: Option<bool>, //false,
_seizureWarning: Option<bool>, //false,
_isoDate: Option<String>, //2025-07-10T02:52:26.000Z",
_gayContent: Option<bool>, //false,
_transContent: Option<bool>, //false,
creator: Option<String>, //itonlygetsworse",
_id: String, //686f2aeade2062f93d72931f",
totalRaters: Option<u32>, //42,
rating: Option<u32>, //164
_id: String, //686f2aeade2062f93d72931f",
_totalRaters: Option<u32>, //42,
_rating: Option<u32>, //164
}
impl PmvhavenVideo {
fn to_videoitem(self) -> VideoItem {
let encoded_title = percent_encode_emojis(&self.title);
let thumbnail = self.thumbnails[self.thumbnails.len()-1].clone().unwrap_or("".to_string());
let video_id = thumbnail.split("_").collect::<Vec<&str>>().last().unwrap_or(&"").to_string().split('.').next().unwrap_or("").to_string();
// let encoded_title = percent_encode_emojis(&self.title);
let thumbnail = self.thumbnails[self.thumbnails.len() - 1]
.clone()
.unwrap_or("".to_string());
// let video_id = thumbnail.split("_").collect::<Vec<&str>>().last().unwrap_or(&"").to_string().split('.').next().unwrap_or("").to_string();
let mut item = VideoItem::new(
self._id.clone(),
self.title.clone(),
format!("https://pmvhaven.com/video/{}_{}", self.title.replace(" ","-"), self._id),
format!(
"https://pmvhaven.com/video/{}_{}",
self.title.replace(" ", "-"),
self._id
),
"pmvhaven".to_string(),
thumbnail,
self.duration as u32,
)
.views(self.views);
item = match self.creator{
item = match self.creator {
Some(c) => item.uploader(c),
_ => item,
};
item = match self.previewUrlCompressed{
item = match self.previewUrlCompressed {
Some(u) => item.preview(u),
_ => item,
};
@@ -210,18 +211,33 @@ impl PmvhavenVideo {
}
// Define a percent-encoding set that encodes all non-ASCII characters
const EMOJI_ENCODE_SET: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'#').add(b'%').add(b'<').add(b'>').add(b'?').add(b'[').add(b'\\').add(b']').add(b'^').add(b'`').add(b'{').add(b'|').add(b'}');
// const EMOJI_ENCODE_SET: &AsciiSet = &CONTROLS
// .add(b' ')
// .add(b'"')
// .add(b'#')
// .add(b'%')
// .add(b'<')
// .add(b'>')
// .add(b'?')
// .add(b'[')
// .add(b'\\')
// .add(b']')
// .add(b'^')
// .add(b'`')
// .add(b'{')
// .add(b'|')
// .add(b'}');
//
// Helper function to percent-encode emojis and other non-ASCII chars
fn percent_encode_emojis(s: &str) -> String {
utf8_percent_encode(s, EMOJI_ENCODE_SET).to_string()
}
// fn percent_encode_emojis(s: &str) -> String {
// utf8_percent_encode(s, EMOJI_ENCODE_SET).to_string()
// }
#[derive(serde::Deserialize)]
struct PmvhavenResponse {
httpStatusCode: Option<u32>,
_httpStatusCode: Option<u32>,
data: Vec<PmvhavenVideo>,
count: Option<u32>,
_count: Option<u32>,
}
impl PmvhavenResponse {
@@ -240,19 +256,43 @@ impl PmvhavenProvider {
url: "https://pmvhaven.com".to_string(),
}
}
async fn get(&self, cache: VideoCache, page: u8, category: String, sort:String) -> Result<Vec<VideoItem>> {
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: String,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let category = options.category.unwrap_or("".to_string());
let index = format!("pmvhaven:{}:{}", page, category);
let url = format!("{}/api/getmorevideos", self.url);
let mut request = PmvhavenRequest::new(page as u32);
request.activeView = sort;
println!("Category: {}", category);
request = match category.as_str() {
"hypno" => { request.hypno(); request },
"pmv" => { request.pmv(); request },
"hmv" => { request.hmv(); request },
"tiktok" => { request.tiktok(); request },
"koreanbj" => { request.koreanbj(); request },
"other" => { request.other(); request },
"hypno" => {
request.hypno();
request
}
"pmv" => {
request.pmv();
request
}
"hmv" => {
request.hmv();
request
}
"tiktok" => {
request.tiktok();
request
}
"koreanbj" => {
request.koreanbj();
request
}
"other" => {
request.other();
request
}
_ => request,
};
@@ -270,42 +310,35 @@ impl PmvhavenProvider {
}
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let response = client
.post(url.clone())
// .proxy(proxy.clone())
.json(&request)
.header("Content-Type", "text/plain;charset=UTF-8")
.send()
.await?;
if response.status().is_success() {
let videos = match response.json::<PmvhavenResponse>().await {
Ok(resp) => resp,
Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
let mut requester = options.requester.clone().unwrap();
let response = requester.post(&url, &request).await.unwrap();
let videos = match response.json::<PmvhavenResponse>().await {
Ok(resp) => resp,
Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
return Ok(old_items);
}
return Ok(video_items);
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Err("Failed to get Videos".into())
return Ok(video_items);
}
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let index = format!("pmvhaven:{}:{}", query, page);
let url = format!("{}/api/v2/search", self.url);
let request = PmvhavenSearch::new(query.to_string(),page as u32);
let request = PmvhavenSearch::new(query.to_string(), page as u32);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&index) {
Some((time, items)) => {
@@ -321,66 +354,27 @@ impl PmvhavenProvider {
}
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let response = client
.post(url.clone())
// .proxy(proxy.clone())
.json(&request)
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.send()
.await?;
if response.status().is_success() {
let videos = match response.json::<PmvhavenResponse>().await {
Ok(resp) => resp,
Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
let mut requester = options.requester.clone().unwrap();
let response = requester.post(&url, &request).await.unwrap();
let videos = match response.json::<PmvhavenResponse>().await {
Ok(resp) => resp,
Err(e) => {
println!("Failed to parse PmvhavenResponse: {}", e);
return Ok(old_items);
}
return Ok(video_items);
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
// else {
// let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
// let flare = Flaresolverr::new(flare_url);
// let result = flare
// .solve(FlareSolverrRequest {
// cmd: "request.get".to_string(),
// url: url.clone(),
// maxTimeout: 60000,
// })
// .await;
// let video_items = match result {
// Ok(res) => self.get_video_items_from_html(res.solution.response),
// Err(e) => {
// println!("Error solving FlareSolverr: {}", e);
// return Err("Failed to solve FlareSolverr".into());
// }
// };
// if !video_items.is_empty() {
// cache.remove(&url);
// cache.insert(url.clone(), video_items.clone());
// } else {
// return Ok(old_items);
// }
// Ok(video_items)
// }
Err("Failed to query Videos".into())
return Ok(video_items);
}
}
#[async_trait]
impl Provider for PmvhavenProvider {
async fn get_videos(
&self,
@@ -395,11 +389,19 @@ impl Provider for PmvhavenProvider {
let _ = per_page;
let _ = pool; // Ignored in this implementation
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), options.category.unwrap(), sort)
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(
cache,
page.parse::<u8>().unwrap_or(1),
sort,
options,
)
.await
}
};
match videos {
Ok(v) => v,

2
src/providers/porn00.rs
View File

@@ -7,6 +7,7 @@ use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -170,6 +171,7 @@ impl Porn00Provider {
}
#[async_trait]
impl Provider for Porn00Provider {
async fn get_videos(
&self,

114
src/providers/pornhat.rs
View File

@@ -2,15 +2,12 @@ use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -34,6 +31,7 @@ impl PornhatProvider {
cache: VideoCache,
page: u8,
sort: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"trending" => "/trending",
@@ -54,63 +52,23 @@ impl PornhatProvider {
vec![]
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap());
response = client.get(response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
@@ -133,23 +91,8 @@ impl PornhatProvider {
vec![]
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -158,31 +101,6 @@ impl PornhatProvider {
return Ok(old_items);
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -278,6 +196,7 @@ impl PornhatProvider {
}
#[async_trait]
impl Provider for PornhatProvider {
async fn get_videos(
&self,
@@ -289,16 +208,15 @@ impl Provider for PornhatProvider {
per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let _ = options;
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};

146
src/providers/pornhub.rs
View File

@@ -2,15 +2,12 @@ use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -34,6 +31,7 @@ impl PornhubProvider {
cache: VideoCache,
page: u8,
sort: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> {
let video_url = format!("{}/video?o={}&page={}", self.url, sort, page);
let old_items = match cache.get(&video_url) {
@@ -50,56 +48,16 @@ impl PornhubProvider {
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),"<ul id=\"video");
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),"<ul id=\"video");
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response,"<ul id=\"video")
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
@@ -108,6 +66,7 @@ impl PornhubProvider {
page: u8,
query: &str,
sort: &str,
options:ServerOptions
) -> Result<Vec<VideoItem>> {
let mut split_string = "<ul id=\"video";
let search_string = query.to_lowercase().trim().replace(" ", "+");
@@ -156,55 +115,17 @@ impl PornhubProvider {
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
//.proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),split_string);
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),split_string);
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response,split_string),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String, split_string: &str) -> Vec<VideoItem> {
@@ -226,7 +147,7 @@ impl PornhubProvider {
if video_segment.contains("wrapVideoBlock"){
continue; // Skip if the segment is a wrapVideoBlock
}
let mut video_url: String = String::new();
let video_url: String;
if !video_segment.contains("<a href=\"") {
let url_part = video_segment.split("data-video-vkey=\"").collect::<Vec<&str>>()[1]
.split("\"")
@@ -242,7 +163,7 @@ impl PornhubProvider {
}
video_url = format!("{}{}", self.url, url_part);
}
if video_url == "https://www.pornhub.comjavascript:void(0)".to_string() {
if video_url.starts_with("https://www.pornhub.comjavascript:void(0)") {
continue;
}
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
@@ -275,10 +196,8 @@ impl PornhubProvider {
.collect::<Vec<&str>>()[0]
.to_string();
let mut uploaderBlock = String::new();
let mut uploader_href = vec![];
let uploaderBlock;
let uploader_href;
let mut tag = String::new();
if video_segment.contains("videoUploaderBlock") {
@@ -291,6 +210,9 @@ impl PornhubProvider {
tag = format!("@{}:{}", uploader_href[1], uploader_href[2].replace("-", " "));
}
else{
uploader_href = vec![];
}
let mut video_item = VideoItem::new(
@@ -309,15 +231,6 @@ impl PornhubProvider {
video_item = video_item.tags(vec![tag])
.uploader(uploader_href[2].to_string());
}
// if video_segment.contains("data-mediabook=\"") {
// let preview = video_segment.split("data-mediabook=\"").collect::<Vec<&str>>()[1]
// .split("\"")
// .collect::<Vec<&str>>()[0]
// .to_string();
// video_item = video_item.preview(preview);
// }
items.push(video_item);
}
return items;
@@ -326,6 +239,7 @@ impl PornhubProvider {
}
#[async_trait]
impl Provider for PornhubProvider {
async fn get_videos(
&self,
@@ -346,11 +260,11 @@ impl Provider for PornhubProvider {
}
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort)
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};

3
src/providers/pornzog.rs
View File

@@ -1,12 +1,12 @@
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -156,6 +156,7 @@ impl PornzogProvider {
}
}
#[async_trait]
impl Provider for PornzogProvider {
async fn get_videos(
&self,

210
src/providers/redtube.rs
View File

@@ -1,17 +1,14 @@
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use serde_json::Value;
use std::env;
use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
error_chain! {
foreign_links {
@@ -35,8 +32,10 @@ impl RedtubeProvider {
cache: VideoCache,
page: u8,
sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let video_url = format!("{}/mostviewed", self.url);
let _ = sort;
let video_url = format!("{}/mostviewed?page={}", self.url, page);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -50,57 +49,16 @@ impl RedtubeProvider {
vec![]
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
&self,
@@ -108,7 +66,9 @@ impl RedtubeProvider {
page: u8,
query: &str,
sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> {
let _ = sort; //TODO
let search_string = query.to_lowercase().trim().replace(" ", "+");
let video_url = format!("{}/?search={}&page={}", self.url, search_string, page);
@@ -126,56 +86,16 @@ impl RedtubeProvider {
vec![]
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
//.proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html_query(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -184,7 +104,11 @@ impl RedtubeProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("<script type=\"application/ld+json\">").collect::<Vec<&str>>()[1].split("</script>").collect::<Vec<&str>>()[0];
let video_listing_content = html
.split("<script type=\"application/ld+json\">")
.collect::<Vec<&str>>()[1]
.split("</script>")
.collect::<Vec<&str>>()[0];
let mut videos: Value = serde_json::from_str(video_listing_content).unwrap();
for vid in videos.as_array_mut().unwrap() {
let video_url: String = vid["embedUrl"].as_str().unwrap_or("").to_string();
@@ -193,7 +117,11 @@ impl RedtubeProvider {
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("=").collect::<Vec<&str>>()[1].to_string();
let raw_duration = vid["duration"].as_str().unwrap_or("0");
let duration = raw_duration.replace("PT", "").replace("S","").parse::<u32>().unwrap();
let duration = raw_duration
.replace("PT", "")
.replace("S", "")
.parse::<u32>()
.unwrap();
let views: u64 = vid["interactionCount"].as_u64().unwrap_or(0);
let thumb = vid["thumbnailUrl"].as_str().unwrap_or("").to_string();
@@ -205,8 +133,7 @@ impl RedtubeProvider {
thumb,
duration,
)
.views(views as u32)
;
.views(views as u32);
items.push(video_item);
}
return items;
@@ -219,39 +146,64 @@ impl RedtubeProvider {
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("videos_grid").collect::<Vec<&str>>()[1];
let videos = video_listing_content.split("<li id=\"tags_videos_").collect::<Vec<&str>>()[1..].to_vec();
let videos = video_listing_content
.split("<li id=\"tags_videos_")
.collect::<Vec<&str>>()[1..]
.to_vec();
for vid in videos {
// for (i, c) in vid.split("\n").enumerate() {
// println!("{}: {}", i, c);
// }
let id = vid.split("data-video-id=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].to_string();
let id = vid.split("data-video-id=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let video_url = format!("{}/{}", self.url, id);
let title = vid.split(" <a title=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].trim().to_string();
let thumb = vid.split("<img").collect::<Vec<&str>>()[1].split(" data-src=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].to_string();
let raw_duration = vid.split("<span class=\"video-properties tm_video_duration\">").collect::<Vec<&str>>()[1].split("</span>").collect::<Vec<&str>>()[0].trim().to_string();
let title = vid.split(" <a title=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
let thumb = vid.split("<img").collect::<Vec<&str>>()[1]
.split(" data-src=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let raw_duration = vid
.split("<span class=\"video-properties tm_video_duration\">")
.collect::<Vec<&str>>()[1]
.split("</span>")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views_str = vid.split("<span class='info-views'>").collect::<Vec<&str>>()[1].split("</span>").collect::<Vec<&str>>()[0].trim().to_string();
let views_str = vid
.split("<span class='info-views'>")
.collect::<Vec<&str>>()[1]
.split("</span>")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
let views = parse_abbreviated_number(&views_str).unwrap_or(0) as u32;
let preview = vid.split("<img").collect::<Vec<&str>>()[1].split(" data-mediabook=\"").collect::<Vec<&str>>()[1].split("\"").collect::<Vec<&str>>()[0].to_string();
let preview = vid.split("<img").collect::<Vec<&str>>()[1]
.split(" data-mediabook=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let video_item = VideoItem::new(
id,
title,
video_url,
"redtube".to_string(),
thumb,
duration,
)
.views(views)
.preview(preview)
;
let video_item =
VideoItem::new(id, title, video_url, "redtube".to_string(), thumb, duration)
.views(views)
.preview(preview);
items.push(video_item);
}
return items;
}
}
#[async_trait]
impl Provider for RedtubeProvider {
async fn get_videos(
&self,
@@ -267,16 +219,16 @@ impl Provider for RedtubeProvider {
let _ = per_page;
let _ = pool;
let mut sort = sort.to_lowercase();
if sort.contains("date"){
if sort.contains("date") {
sort = "mr".to_string();
}
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort)
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};

120
src/providers/rule34video.rs
View File

@@ -2,16 +2,13 @@ use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
use std::time::{SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -35,6 +32,7 @@ impl Rule34videoProvider {
cache: VideoCache,
page: u8,
sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -68,56 +66,16 @@ impl Rule34videoProvider {
}
};
}
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(url.clone())
// .proxy(proxy.clone())
.send().await?;
while response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
&self,
@@ -125,6 +83,7 @@ impl Rule34videoProvider {
page: u8,
query: &str,
sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -155,23 +114,8 @@ impl Rule34videoProvider {
vec![]
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
@@ -180,31 +124,6 @@ impl Rule34videoProvider {
return Ok(old_items);
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -254,10 +173,10 @@ impl Rule34videoProvider {
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
// let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>()[1]
// .split("\"")
// .collect::<Vec<&str>>()[0]
// .to_string();
let video_item = VideoItem::new(
@@ -281,6 +200,7 @@ impl Rule34videoProvider {
}
#[async_trait]
impl Provider for Rule34videoProvider {
async fn get_videos(
&self,
@@ -297,11 +217,11 @@ impl Provider for Rule34videoProvider {
let _ = pool; // Ignored in this implementation
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort)
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, &sort, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};

21
src/providers/sxyprn.rs
View File

@@ -4,12 +4,12 @@ use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds;
use crate::videos::VideoItem;
use crate::videos::{self, ServerOptions, VideoFormat};
use crate::videos::ServerOptions;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use scraper::ElementRef;
use scraper::{Html, Selector};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -19,13 +19,13 @@ error_chain! {
}
}
fn has_blacklisted_class(element: &ElementRef, blacklist: &[&str]) -> bool {
element
.value()
.attr("class")
.map(|classes| classes.split_whitespace().any(|c| blacklist.contains(&c)))
.unwrap_or(false)
}
// fn has_blacklisted_class(element: &ElementRef, blacklist: &[&str]) -> bool {
// element
// .value()
// .attr("class")
// .map(|classes| classes.split_whitespace().any(|c| blacklist.contains(&c)))
// .unwrap_or(false)
// }
#[derive(Debug, Clone)]
pub struct SxyprnProvider {
@@ -150,6 +150,8 @@ impl SxyprnProvider {
pool: DbPool,
requester: Requester,
) -> Vec<VideoItem> {
let _ = requester;
let _ = pool;
if html.is_empty() {
println!("HTML is empty");
return vec![];
@@ -284,6 +286,7 @@ impl SxyprnProvider {
}
}
#[async_trait]
impl Provider for SxyprnProvider {
async fn get_videos(
&self,

168
src/providers/xxthots.rs
View File

@@ -1,16 +1,13 @@
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
error_chain! {
foreign_links {
@@ -34,6 +31,7 @@ impl XxthotsProvider {
cache: VideoCache,
page: u8,
sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"popular" => "/most-popular",
@@ -46,7 +44,10 @@ impl XxthotsProvider {
"top-rated" => "list_videos_common_videos_list",
_ => "list_videos_most_recent_videos",
};
let video_url = format!("{}{}?mode=async^&function=get_block^&block_id={}^&from={}", self.url, sort_string, list_str, page);
let video_url = format!(
"{}{}?mode=async^&function=get_block^&block_id={}^&from={}",
self.url, sort_string, list_str, page
);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -60,65 +61,29 @@ impl XxthotsProvider {
vec![]
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let video_url = format!("{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&category_ids=&sort_by=&from_videos={}&from_albums={}&", self.url, search_string, page, page);
let video_url = format!(
"{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&category_ids=&sort_by=&from_videos={}&from_albums={}&",
self.url, search_string, page, page
);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
@@ -133,56 +98,16 @@ impl XxthotsProvider {
vec![]
}
};
let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
}
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: video_url.clone(),
maxTimeout: 60000,
})
.await;
let video_items = match result {
Ok(res) => self.get_video_items_from_html(res.solution.response),
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
return Err("Failed to solve FlareSolverr".into());
}
};
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -191,7 +116,9 @@ impl XxthotsProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>()[0]
let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()[0]
.split("<div class=\"thumb thumb_rel item \">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -201,8 +128,9 @@ impl XxthotsProvider {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0].to_string();
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
@@ -210,20 +138,28 @@ impl XxthotsProvider {
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>()[1]
let raw_duration = video_segment
.split("<div class=\"time\">")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"lazy-load").collect::<Vec<&str>>()[1]
.split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = video_segment
.split("<img class=\"lazy-load")
.collect::<Vec<&str>>()[1]
.split("data-original=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let views_part = video_segment.split("svg-icon icon-eye").collect::<Vec<&str>>()[1]
.split("</i>").collect::<Vec<&str>>()[1]
let views_part = video_segment
.split("svg-icon icon-eye")
.collect::<Vec<&str>>()[1]
.split("</i>")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
@@ -237,16 +173,14 @@ impl XxthotsProvider {
thumb,
duration,
)
.views(views)
;
.views(views);
items.push(video_item);
}
return items;
}
}
#[async_trait]
impl Provider for XxthotsProvider {
async fn get_videos(
&self,
@@ -263,11 +197,11 @@ impl Provider for XxthotsProvider {
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};

2
src/providers/youjizz.rs
View File

@@ -7,6 +7,7 @@ use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -169,6 +170,7 @@ impl YoujizzProvider {
}
#[async_trait]
impl Provider for YoujizzProvider {
async fn get_videos(
&self,

2
src/status.rs
View File

@@ -33,7 +33,7 @@ pub struct ChannelOption {
pub multiSelect: bool, //true
}
#[derive(serde::Serialize)]
#[derive(serde::Serialize, Debug, Clone)]
pub struct FilterOption{
pub id: String, //"sort",
pub title: String, //"Sort",

9
src/util/mod.rs
View File

@@ -1,5 +1,3 @@
use crate::providers::AnyProvider;
pub mod time;
pub mod flaresolverr;
pub mod cache;
@@ -44,10 +42,3 @@ pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {
result
}
pub fn get_all_providers() -> Vec<AnyProvider>{
return vec![];
}

217
src/util/requester.rs
View File

@@ -1,15 +1,19 @@
use wreq::header::HeaderValue;
use wreq::redirect::Policy;
use serde::Serialize;
use std::env;
use wreq::Client;
use wreq::Proxy;
use wreq::Response;
use wreq::Version;
use wreq::header::HeaderValue;
use wreq::redirect::Policy;
use wreq_util::Emulation;
use std::env;
use crate::util::flaresolverr::FlareSolverrRequest;
use crate::util::flaresolverr::Flaresolverr;
// A Send + Sync error type for all async paths
type AnyErr = Box<dyn std::error::Error + Send + Sync + 'static>;
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
pub struct Requester {
#[serde(skip)]
@@ -21,11 +25,11 @@ pub struct Requester {
impl Requester {
pub fn new() -> Self {
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.redirect(Policy::default())
.build()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.redirect(Policy::default())
.build()
.expect("Failed to create HTTP client");
Requester {
@@ -34,121 +38,150 @@ impl Requester {
flaresolverr_session: None,
}
}
pub fn set_proxy(&mut self, proxy: bool) {
if proxy{
if proxy {
println!("Proxy enabled");
}
self.proxy = proxy;
}
pub fn set_flaresolverr_session(&mut self, session: String) {
self.flaresolverr_session = Some(session);
}
// pub fn set_flaresolverr_session(&mut self, session: String) {
// self.flaresolverr_session = Some(session);
// }
fn get_url_from_location_header(&self, prev_url: &str,location: &str) -> String {
if location.starts_with("http://") || location.starts_with("https://") {
location.to_string()
} else if location.starts_with("//") {
format!("{}{}", "https:", location) // Replace with your base URL
} else if location.starts_with("/") {
let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/");
format!("{}{}", base_url, location)
} else {
format!("{}/{}", prev_url, location)
}
}
// fn get_url_from_location_header(&self, prev_url: &str, location: &str) -> String {
// if location.starts_with("http://") || location.starts_with("https://") {
// location.to_string()
// } else if location.starts_with("//") {
// format!("{}{}", "https:", location)
// } else if location.starts_with('/') {
// let base_url = prev_url.split('/').take(3).collect::<Vec<&str>>().join("/");
// format!("{}{}", base_url, location)
// } else {
// format!("{}/{}", prev_url, location)
// }
// }
pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.build()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.build()
.expect("Failed to create HTTP client");
let mut request = client.get(url).version(Version::HTTP_11);
let proxy;
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy.clone());
let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy);
}
}
// Directly propagate the error from send()
request.send().await
}
pub async fn get(&mut self, url: &str) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let mut request = self.client.get(url).version(Version::HTTP_11);
let mut proxy;
pub async fn post<S>(&mut self, url: &str, data: &S) -> Result<Response, wreq::Error>
where
S: Serialize + ?Sized,
{
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.build()
.expect("Failed to create HTTP client");
let mut request = client.post(url).version(Version::HTTP_11).json(data);
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy.clone());
let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy);
}
}
let mut response = request.send().await?;
request.send().await
}
pub async fn get(&mut self, url: &str) -> Result<String, AnyErr> {
let mut request = self.client.get(url).version(Version::HTTP_11);
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy);
}
}
let response = request.send().await?;
if response.status().is_success() {
return Ok(response.text().await?);
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let mut flare = Flaresolverr::new(flare_url);
if self.proxy && env::var("BURP_URL").is_ok() {
flare.set_proxy(true);
}
let result = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url.to_string(),
maxTimeout: 60000,
})
.await;
match result {
Ok(res) => {
let cookie_url = url.split("/").collect::<Vec<&str>>()[..3].join("/");
self.client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.redirect(Policy::default())
.build()
.expect("Failed to create HTTP client");
let useragent = res.solution.userAgent;
self.client.update()
.headers(|headers| {
headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
})
.apply()
.unwrap();
for cookie in res.solution.cookies {
let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
// Parse the domain string into a Url
if let Ok(url) = url::Url::parse(cookie_url.as_str()) {
self.client.set_cookie(&url, header);
}
}
request = self.client.get(url).version(Version::HTTP_11);
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy.clone());
}
}
}
response = request.send().await?;
if response.status().is_success() {
return Ok(response.text().await?);
}
// If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately,
// so no non-Send error value lives across later `.await`s.
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let mut flare = Flaresolverr::new(flare_url);
if self.proxy && env::var("BURP_URL").is_ok() {
flare.set_proxy(true);
}
Ok(res.solution.response)
}
Err(e) => {
return Err(format!("Failed to solve FlareSolverr: {e}").into());
}
let res = flare
.solve(FlareSolverrRequest {
cmd: "request.get".to_string(),
url: url.to_string(),
maxTimeout: 60000,
})
.await
.map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?;
// Rebuild client and apply UA/cookies from FlareSolverr
let cookie_origin = url.split('/').take(3).collect::<Vec<&str>>().join("/");
self.client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.cookie_store(true)
.redirect(Policy::default())
.build()
.expect("Failed to create HTTP client");
let useragent = res.solution.userAgent;
self.client
.update()
.headers(|headers| {
headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
})
.apply()
.unwrap();
if let Ok(origin) = url::Url::parse(&cookie_origin) {
for cookie in res.solution.cookies {
let header =
HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
self.client.set_cookie(&origin, header);
}
}
// Retry the original URL with the updated client & (optional) proxy
let mut request = self.client.get(url).version(Version::HTTP_11);
if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") {
let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy);
}
}
let response = request.send().await?;
if response.status().is_success() {
return Ok(response.text().await?);
}
// Fall back to FlareSolverr-provided body
Ok(res.solution.response)
}
}

160
src/videos.rs
View File

@@ -270,86 +270,86 @@ impl VideoFormat {
self.format_note = Some(format_note);
self
}
pub fn filesize(mut self, filesize: u32) -> Self {
self.filesize = Some(filesize);
self
}
pub fn asr(mut self, asr: u32) -> Self {
self.asr = Some(asr);
self
}
pub fn fps(mut self, fps: u32) -> Self {
self.fps = Some(fps);
self
}
pub fn width(mut self, width: u32) -> Self {
self.width = Some(width);
self
}
pub fn height(mut self, height: u32) -> Self {
self.height = Some(height);
self
}
pub fn tbr(mut self, tbr: u32) -> Self {
self.tbr = Some(tbr);
self
}
pub fn language(mut self, language: String) -> Self {
self.language = Some(language);
self
}
pub fn language_preference(mut self, language_preference: u32) -> Self {
self.language_preference = Some(language_preference);
self
}
pub fn ext(mut self, ext: String) -> Self {
self.ext = Some(ext);
self
}
pub fn vcodec(mut self, vcodec: String) -> Self {
self.vcodec = Some(vcodec);
self
}
pub fn acodec(mut self, acodec: String) -> Self {
self.acodec = Some(acodec);
self
}
pub fn dynamic_range(mut self, dynamic_range: String) -> Self {
self.dynamic_range = Some(dynamic_range);
self
}
pub fn abr(mut self, abr: u32) -> Self {
self.abr = Some(abr);
self
}
pub fn vbr(mut self, vbr: u32) -> Self {
self.vbr = Some(vbr);
self
}
pub fn container(mut self, container: String) -> Self {
self.container = Some(container);
self
}
pub fn protocol(mut self, protocol: String) -> Self {
self.protocol = Some(protocol);
self
}
pub fn audio_ext(mut self, audio_ext: String) -> Self {
self.audio_ext = Some(audio_ext);
self
}
pub fn video_ext(mut self, video_ext: String) -> Self {
self.video_ext = Some(video_ext);
self
}
pub fn resolution(mut self, resolution: String) -> Self {
self.resolution = Some(resolution);
self
}
pub fn http_headers(mut self, http_headers: HashMap<String, String>) -> Self {
self.http_headers = Some(http_headers);
self
}
// pub fn filesize(mut self, filesize: u32) -> Self {
// self.filesize = Some(filesize);
// self
// }
// pub fn asr(mut self, asr: u32) -> Self {
// self.asr = Some(asr);
// self
// }
// pub fn fps(mut self, fps: u32) -> Self {
// self.fps = Some(fps);
// self
// }
// pub fn width(mut self, width: u32) -> Self {
// self.width = Some(width);
// self
// }
// pub fn height(mut self, height: u32) -> Self {
// self.height = Some(height);
// self
// }
// pub fn tbr(mut self, tbr: u32) -> Self {
// self.tbr = Some(tbr);
// self
// }
// pub fn language(mut self, language: String) -> Self {
// self.language = Some(language);
// self
// }
// pub fn language_preference(mut self, language_preference: u32) -> Self {
// self.language_preference = Some(language_preference);
// self
// }
// pub fn ext(mut self, ext: String) -> Self {
// self.ext = Some(ext);
// self
// }
// pub fn vcodec(mut self, vcodec: String) -> Self {
// self.vcodec = Some(vcodec);
// self
// }
// pub fn acodec(mut self, acodec: String) -> Self {
// self.acodec = Some(acodec);
// self
// }
// pub fn dynamic_range(mut self, dynamic_range: String) -> Self {
// self.dynamic_range = Some(dynamic_range);
// self
// }
// pub fn abr(mut self, abr: u32) -> Self {
// self.abr = Some(abr);
// self
// }
// pub fn vbr(mut self, vbr: u32) -> Self {
// self.vbr = Some(vbr);
// self
// }
// pub fn container(mut self, container: String) -> Self {
// self.container = Some(container);
// self
// }
// pub fn protocol(mut self, protocol: String) -> Self {
// self.protocol = Some(protocol);
// self
// }
// pub fn audio_ext(mut self, audio_ext: String) -> Self {
// self.audio_ext = Some(audio_ext);
// self
// }
// pub fn video_ext(mut self, video_ext: String) -> Self {
// self.video_ext = Some(video_ext);
// self
// }
// pub fn resolution(mut self, resolution: String) -> Self {
// self.resolution = Some(resolution);
// self
// }
// pub fn http_headers(mut self, http_headers: HashMap<String, String>) -> Self {
// self.http_headers = Some(http_headers);
// self
// }
}
#[derive(serde::Serialize, Debug)]
pub struct Videos {