simon/hottub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: simon:53ac33f856a49bceadb235dbbf9d200672bb1404
Branches Tags
simon:main simon:master
...
compare: simon:main
Branches Tags
simon:main simon:master

24 Commits
53ac33f856 ... main

Author SHA1 Message Date
Simon
b986faa1d4 healthcheck, logging and ulimit adjustment 2025-12-05 09:11:59 +00:00
Simon
7124b388fa cleanup 2025-12-05 09:09:47 +00:00
Simon
632931f515 search bugfix 2025-12-04 20:12:57 +00:00
Simon
9739560c03 removed unimportant prints 2025-12-04 13:51:34 +00:00
Simon
80d874a004 query bug fix 2025-12-04 13:37:24 +00:00
Simon
64dc7455ee http version 2 2025-12-04 13:27:16 +00:00
Simon
9e30eedc77 run init load in its own thread 2025-12-04 13:11:46 +00:00
Simon
75e28608bd missav bugfixes 2025-12-04 11:54:31 +00:00
Simon
e22a3f2d6d prevent empty tags/formats 2025-12-01 16:07:45 +00:00
Simon
07b812be64 pimpbunny 2025-11-30 14:15:09 +00:00
Simon
61e38caed5 fixed wrong order of format/quality 2025-11-30 07:05:49 +00:00
Simon
e5a6c8decc reverse formats order so high quality is selected first 2025-11-30 07:03:42 +00:00
Simon
d856ade32b adjusted requester to supply http::version itself 2025-11-30 06:53:21 +00:00
Simon
2de6a7d42b testing found 2025-11-29 20:14:59 +00:00
Simon
39e38249b7 noodlemagazine 2025-11-29 20:08:46 +00:00
Simon
e924c89573 undo 2025-11-29 18:52:48 +00:00
Simon
3f57569511 htmlencode videourl 2025-11-29 18:46:28 +00:00
Simon
23190ee05c bugfix 2025-11-29 17:22:41 +00:00
Simon
12053ce6db removed debug print 2025-11-29 17:21:04 +00:00
Simon
5522f2e37d pmvhaven backend fix 2025-11-29 17:16:21 +00:00
Simon
8f885c79d4 send categories in channel info 2025-11-29 15:56:22 +00:00
Simon
d7e7f70bd2 bugfixes 2025-11-29 14:20:36 +00:00
Simon
0e02a1b821 tags upgrade 2025-11-29 13:55:56 +00:00
Simon
cafb990fd4 removed debug prints 2025-11-29 08:24:14 +00:00
40 changed files with 1444 additions and 743 deletions
Expand all files Collapse all files

1
Cargo.toml
View File

@@ -27,6 +27,7 @@ once_cell = "1.21.3"
rustc-hash = "2.1.1" rustc-hash = "2.1.1"
async-trait = "0.1" async-trait = "0.1"
regex = "1.12.2" regex = "1.12.2"
titlecase = "3.6.0"
[lints.rust] [lints.rust]
unexpected_cfgs = "allow" unexpected_cfgs = "allow"

1
archive/hentaimoon.rs
View File

@@ -51,7 +51,6 @@ impl HentaimoonProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()

197
archive/noodlemagazine.rs
View File

@@ -1,197 +0,0 @@
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client, Proxy};
use wreq_util::Emulation;
error_chain! {
foreign_links {
Io(std::io::Error);
HttpRequest(wreq::Error);
}
}
#[derive(Debug, Clone)]
pub struct NoodlemagazineProvider {
url: String,
}
impl NoodlemagazineProvider {
pub fn new() -> Self {
NoodlemagazineProvider {
url: "https://noodlemagazine.com".to_string(),
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let video_url = format!("{}/popular/recent?p={}", self.url, page-1);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone());
} else {
items.clone()
}
}
None => {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.replace(" ", "%20");
let video_url = format!("{}/video/{}?p={}", self.url, search_string, page);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone());
} else {
let _ = cache.check().await;
return Ok(items.clone());
}
}
None => {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
if html.is_empty() {
println!("HTML is empty");
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("- Made with <svg ").collect::<Vec<&str>>()[0]
.split("<div class=\"item\">")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
// let vid = video_segment.split("\n").collect::<Vec<&str>>();
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}",self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0].to_string());
let mut title = video_segment.split("\"title\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("#clock-o").collect::<Vec<&str>>()[1]
.split("</svg>").collect::<Vec<&str>>()[1]
.split("<").collect::<Vec<&str>>()[0]
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img ").collect::<Vec<&str>>()[1]
.split("data-src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let views_part = video_segment.split("#eye").collect::<Vec<&str>>()[1]
.split("</svg>").collect::<Vec<&str>>()[1]
.split("<").collect::<Vec<&str>>()[0]
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
let video_item = VideoItem::new(
id,
title,
video_url.to_string(),
"noodlemagazine".to_string(),
thumb,
duration,
)
.views(views)
;
items.push(video_item);
}
return items;
}
}
impl Provider for NoodlemagazineProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
sort: String,
query: Option<String>,
page: String,
per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), options)
.await
}
};
match videos {
Ok(v) => v,
Err(e) => {
println!("Error fetching videos: {}", e);
vec![]
}
}
}
}

16
docker-compose.yml
View File

@@ -23,7 +23,21 @@ services:
ports: ports:
- 80:18080 - 80:18080
- 6901:6901 # vnc port to access burpsuite - 6901:6901 # vnc port to access burpsuite
- 8081:8080 # burpsuite port of http proxy - 8081:8080 # burpsuite port of http(s) proxy
logging:
driver: "json-file"
options:
max-size: "10m" # Maximum size of each log file (e.g., 10MB)
max-file: "3" # Maximum number of log files to keep
healthcheck:
test: ["CMD-SHELL", "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:18080/api/status | grep -q 200"]
interval: 30s
timeout: 5s
retries: 3
ulimits:
nofile:
soft: 65536
hard: 65536
flaresolverr: flaresolverr:
container_name: flaresolverr container_name: flaresolverr

79
src/api.rs
View File

@@ -10,7 +10,6 @@ use crate::providers::all::AllProvider;
use crate::providers::hanime::HanimeProvider; use crate::providers::hanime::HanimeProvider;
use crate::providers::okporn::OkpornProvider; use crate::providers::okporn::OkpornProvider;
use crate::providers::perverzija::PerverzijaProvider; use crate::providers::perverzija::PerverzijaProvider;
use crate::providers::pmvhaven::PmvhavenProvider;
use crate::providers::pornhub::PornhubProvider; use crate::providers::pornhub::PornhubProvider;
use crate::providers::redtube::RedtubeProvider; use crate::providers::redtube::RedtubeProvider;
use crate::providers::rule34video::Rule34videoProvider; use crate::providers::rule34video::Rule34videoProvider;
@@ -164,82 +163,6 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
nsfw: true, nsfw: true,
cacheDuration: Some(1800), cacheDuration: Some(1800),
}); });
if clientversion >= ClientVersion::new(22, 101, "22e".to_string()) {
// pmvhaven
status.add_channel(Channel {
id: "pmvhaven".to_string(),
name: "Pmvhaven".to_string(),
description: "Explore a curated collection of captivating PMV".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=pmvhaven.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![
ChannelOption {
id: "category".to_string(),
title: "Category".to_string(),
description: "Category of PMV Video get".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "folder".to_string(),
colorName: "yellow".to_string(),
options: vec![
FilterOption {
id: "all".to_string(),
title: "All".to_string(),
},
FilterOption {
id: "pmv".to_string(),
title: "PMV".to_string(),
},
FilterOption {
id: "hmv".to_string(),
title: "HMV".to_string(),
},
FilterOption {
id: "tiktok".to_string(),
title: "Tiktok".to_string(),
},
FilterOption {
id: "koreanbj".to_string(),
title: "KoreanBJ".to_string(),
},
FilterOption {
id: "hypno".to_string(),
title: "Hypno".to_string(),
},
FilterOption {
id: "other".to_string(),
title: "Other".to_string(),
},
],
multiSelect: false,
},
ChannelOption {
id: "sort".to_string(),
title: "Filter".to_string(),
description: "Filter PMV Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "Newest".to_string(),
title: "Newest".to_string(),
},
FilterOption {
id: "Top Rated".to_string(),
title: "Top Rated".to_string(),
},
FilterOption {
id: "Most Viewed".to_string(),
title: "Most Viewed".to_string(),
},
],
multiSelect: false,
},
],
nsfw: true,
cacheDuration: Some(1800),
});
}
if clientversion >= ClientVersion::new(22, 97, "22a".to_string()) { if clientversion >= ClientVersion::new(22, 97, "22a".to_string()) {
// perverzija // perverzija
status.add_channel(Channel { status.add_channel(Channel {
@@ -1241,6 +1164,7 @@ async fn videos_post(
stars: Some(stars), stars: Some(stars),
categories: Some(categories), categories: Some(categories),
duration: Some(duration), duration: Some(duration),
sort: Some(sort.clone())
}; };
let video_items = provider let video_items = provider
.get_videos( .get_videos(
@@ -1308,7 +1232,6 @@ pub fn get_provider(channel: &str) -> Option<DynProvider> {
"perverzija" => Some(Arc::new(PerverzijaProvider::new())), "perverzija" => Some(Arc::new(PerverzijaProvider::new())),
"hanime" => Some(Arc::new(HanimeProvider::new())), "hanime" => Some(Arc::new(HanimeProvider::new())),
"pornhub" => Some(Arc::new(PornhubProvider::new())), "pornhub" => Some(Arc::new(PornhubProvider::new())),
"pmvhaven" => Some(Arc::new(PmvhavenProvider::new())),
"rule34video" => Some(Arc::new(Rule34videoProvider::new())), "rule34video" => Some(Arc::new(Rule34videoProvider::new())),
"redtube" => Some(Arc::new(RedtubeProvider::new())), "redtube" => Some(Arc::new(RedtubeProvider::new())),
"okporn" => Some(Arc::new(OkpornProvider::new())), "okporn" => Some(Arc::new(OkpornProvider::new())),

12
src/main.rs
View File

@@ -2,7 +2,7 @@
#![allow(non_snake_case)] #![allow(non_snake_case)]
use std::env; use std::{env, thread};
use diesel::{r2d2::{self, ConnectionManager}, SqliteConnection}; use diesel::{r2d2::{self, ConnectionManager}, SqliteConnection};
use dotenvy::dotenv; use dotenvy::dotenv;
@@ -51,7 +51,17 @@ async fn main() -> std::io::Result<()> {
let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new(); let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new();
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
rt.block_on(async move {
providers::init_providers_now(); providers::init_providers_now();
});
});
web::HttpServer::new(move || { web::HttpServer::new(move || {
web::App::new() web::App::new()

11
src/providers/beeg.rs
View File

@@ -81,7 +81,7 @@ impl BeegProvider {
async fn load_stars(stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> { async fn load_stars(stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new(); let mut requester = util::requester::Requester::new();
let text = requester let text = requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None)
.await .await
.unwrap(); .unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap(); let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
@@ -103,7 +103,7 @@ impl BeegProvider {
async fn load_categories(categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> { async fn load_categories(categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new(); let mut requester = util::requester::Requester::new();
let text = requester let text = requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None)
.await .await
.unwrap(); .unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap(); let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
@@ -125,7 +125,7 @@ impl BeegProvider {
async fn load_sites(sites: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> { async fn load_sites(sites: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new(); let mut requester = util::requester::Requester::new();
let text = requester let text = requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None)
.await .await
.unwrap(); .unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap(); let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
@@ -252,7 +252,6 @@ impl BeegProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -263,7 +262,7 @@ impl BeegProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap(); let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
@@ -304,7 +303,7 @@ impl BeegProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap(); let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
if !video_items.is_empty() { if !video_items.is_empty() {

5
src/providers/freshporno.rs
View File

@@ -43,7 +43,6 @@ impl FreshpornoProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -56,7 +55,7 @@ impl FreshpornoProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -93,7 +92,7 @@ impl FreshpornoProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

2
src/providers/hanime.rs
View File

@@ -153,7 +153,7 @@ impl HanimeProvider {
let url = format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug); let url = format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug);
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap(); let text = requester.get(&url, None).await.unwrap();
let urls = text.split("\"servers\"").collect::<Vec<&str>>()[1]; let urls = text.split("\"servers\"").collect::<Vec<&str>>()[1];
let mut url_vec = vec![]; let mut url_vec = vec![];

1
src/providers/homoxxx.rs
View File

@@ -44,7 +44,6 @@ impl HomoxxxProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()

291
src/providers/hqporner.rs
View File

@@ -1,16 +1,18 @@
use crate::util::requester::Requester; use crate::DbPool;
use crate::{DbPool};
use crate::api::ClientVersion; use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::sync::{Arc, RwLock};
use std::{thread, vec};
use titlecase::Titlecase;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -22,19 +24,107 @@ error_chain! {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct HqpornerProvider { pub struct HqpornerProvider {
url: String, url: String,
// stars: Arc<RwLock<Vec<String>>>, stars: Arc<RwLock<Vec<FilterOption>>>,
// categories: Arc<RwLock<Vec<String>>>, categories: Arc<RwLock<Vec<FilterOption>>>,
} }
impl HqpornerProvider { impl HqpornerProvider {
pub fn new() -> Self { pub fn new() -> Self {
let provider = HqpornerProvider { let provider = HqpornerProvider {
url: "https://hqporner.com".to_string(), url: "https://hqporner.com".to_string(),
// stars: Arc::new(RwLock::new(vec![])), stars: Arc::new(RwLock::new(vec![])),
// categories: Arc::new(RwLock::new(vec![])), categories: Arc::new(RwLock::new(vec![])),
}; };
provider.spawn_initial_load();
provider provider
} }
fn spawn_initial_load(&self) {
let url = self.url.clone();
let stars = Arc::clone(&self.stars);
let categories = Arc::clone(&self.categories);
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
rt.block_on(async move {
if let Err(e) = Self::load_stars(&url, stars).await {
eprintln!("load_stars failed: {e}");
}
if let Err(e) = Self::load_categories(&url, categories).await {
eprintln!("load_categories failed: {e}");
}
});
});
}
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(format!("{}/girls", &base_url).as_str(), None)
.await
.unwrap();
let stars_div = text
.split("<span>Girls</span>")
.collect::<Vec<&str>>().last().unwrap()
.split("</ul>")
.collect::<Vec<&str>>()[0];
for stars_element in stars_div.split("<li ").collect::<Vec<&str>>()[1..].to_vec() {
let star_id = stars_element.split("href=\"/actress/").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let star_name = stars_element.split("<a ").collect::<Vec<&str>>()[1]
.split(">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
Self::push_unique(
&stars,
FilterOption {
id: star_id,
title: star_name,
},
);
}
return Ok(());
}
async fn load_categories(base_url: &str, categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(format!("{}/categories", &base_url).as_str(), None)
.await
.unwrap();
let categories_div = text
.split("<span>Categories</span>")
.collect::<Vec<&str>>().last().unwrap()
.split("</ul>")
.collect::<Vec<&str>>()[0];
for categories_element in categories_div.split("<li ").collect::<Vec<&str>>()[1..].to_vec() {
let category_id = categories_element.split("href=\"/category/").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let category_name = categories_element.split("<a ").collect::<Vec<&str>>()[1]
.split(">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.titlecase();
Self::push_unique(
&categories,
FilterOption {
id: category_id,
title: category_name,
},
);
}
return Ok(());
}
fn build_channel(&self, clientversion: ClientVersion) -> Channel { fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion; let _ = clientversion;
Channel { Channel {
@@ -44,13 +134,24 @@ impl HqpornerProvider {
premium: false, premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=hqporner.com".to_string(), favicon: "https://www.google.com/s2/favicons?sz=64&domain=hqporner.com".to_string(),
status: "active".to_string(), status: "active".to_string(),
categories: vec![], categories: self.categories.read().unwrap().iter().map(|c| c.title.clone()).collect(),
options: vec![], options: vec![],
nsfw: true, nsfw: true,
cacheDuration: None, cacheDuration: None,
} }
} }
// Push one item with minimal lock time and dedup by id
fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
if let Ok(mut vec) = target.write() {
if !vec.iter().any(|x| x.id == item.id) {
vec.push(item);
// Optional: keep it sorted for nicer UX
// vec.sort_by(|a,b| a.title.cmp(&b.title));
}
}
}
async fn get( async fn get(
&self, &self,
cache: VideoCache, cache: VideoCache,
@@ -63,7 +164,6 @@ impl HqpornerProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -75,8 +175,10 @@ impl HqpornerProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), &mut requester).await; let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone()); cache.insert(video_url.clone(), video_items.clone());
@@ -95,7 +197,14 @@ impl HqpornerProvider {
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let search_string = query.trim().to_string(); let search_string = query.trim().to_string();
let video_url = format!("{}/?q={}&p={}", self.url, search_string, page); let mut video_url = format!("{}/?q={}&p={}", self.url, search_string, page);
if let Some(star) = self.stars.read().unwrap().iter().find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
video_url = format!("{}/actress/{}/{}", self.url, star.id, page);
}
if let Some(cat) = self.categories.read().unwrap().iter().find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
video_url = format!("{}/category/{}/{}", self.url, cat.id, page);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
@@ -113,8 +222,10 @@ impl HqpornerProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), &mut requester).await; let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone()); cache.insert(video_url.clone(), video_items.clone());
@@ -124,7 +235,11 @@ impl HqpornerProvider {
Ok(video_items) Ok(video_items)
} }
async fn get_video_items_from_html(&self, html: String, requester: &mut Requester) -> Vec<VideoItem> { async fn get_video_items_from_html(
&self,
html: String,
requester: &mut Requester,
) -> Vec<VideoItem> {
if html.is_empty() || html.contains("404 Not Found") { if html.is_empty() || html.contains("404 Not Found") {
return vec![]; return vec![];
} }
@@ -135,34 +250,41 @@ impl HqpornerProvider {
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
let futures = raw_videos.into_iter().map(|el| self.get_video_item(el.to_string(), requester.clone())); let futures = raw_videos
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results
.into_iter() .into_iter()
.filter_map(Result::ok) .map(|el| self.get_video_item(el.to_string(), requester.clone()));
.collect(); let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return video_items; return video_items;
} }
async fn get_video_item(&self, video_segment: String, mut requester: Requester) -> Result<VideoItem> { async fn get_video_item(
let vid = video_segment.split("\n").collect::<Vec<&str>>(); &self,
for (index, line) in vid.iter().enumerate() { video_segment: String,
println!("Line {}: {}\n", index, line); mut requester: Requester,
} ) -> Result<VideoItem> {
let video_url: String = format!( let video_url: String = format!(
"{}{}", "{}{}",
self.url, self.url,
video_segment.split("<a href=\"").collect::<Vec<&str>>()[1] video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"").collect::<Vec<&str>>()[0] .split("\"")
.collect::<Vec<&str>>()[0]
.to_string() .to_string()
); );
let mut title = video_segment let mut title = video_segment
.split("<h3 class=\"meta-data-title\">").collect::<Vec<&str>>()[1] .split("<h3 class=\"meta-data-title\">")
.split(">").collect::<Vec<&str>>()[1] .collect::<Vec<&str>>()[1]
.split("<").collect::<Vec<&str>>()[0] .split(">")
.trim().to_string(); .collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes())
.to_string()
.unwrap_or(title)
.titlecase();
let id = video_url.split("/").collect::<Vec<&str>>()[4] let id = video_url.split("/").collect::<Vec<&str>>()[4]
.split(".") .split(".")
.collect::<Vec<&str>>()[0] .collect::<Vec<&str>>()[0]
@@ -186,7 +308,10 @@ impl HqpornerProvider {
.to_string(); .to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32; let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let formats = get_media_formats(&video_url, &mut requester).await; let (tags, formats) = match self.extract_media(&video_url, &mut requester).await{
Ok((t, f)) => (t, f),
Err(_) => return Err(Error::from("Video media extraction failed")),
};
if formats.is_empty() { if formats.is_empty() {
return Err(Error::from("No formats found for video")); return Err(Error::from("No formats found for video"));
} }
@@ -199,42 +324,106 @@ impl HqpornerProvider {
duration, duration,
) )
.formats(formats) .formats(formats)
; .tags(tags);
return Ok(video_item); return Ok(video_item);
} }
async fn extract_media(
&self,
video_page_url: &str,
requester: &mut Requester,
) -> Result<(Vec<String>, Vec<VideoFormat>)> {
let mut formats = vec![];
let mut tags = vec![];
let text = requester.get_raw_with_headers(&video_page_url, vec![("Referer".to_string(), "https://hqporner.com".to_string())])
.await.unwrap().text().await.unwrap();
if text.contains("Why do I see it?") {
return Ok((tags, formats));
}
let stars_elements = text.split("icon fa-star-o").collect::<Vec<&str>>()[1]
.split("</li>")
.collect::<Vec<&str>>()[0]
.split("href=\"/actress/")
.collect::<Vec<&str>>()[1..]
.to_vec();
for star_el in stars_elements {
let star_id = star_el.split("\"").collect::<Vec<&str>>()[0].to_string();
let star_name = star_el.split("\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
tags.push(star_name.clone());
Self::push_unique(&self.stars, FilterOption {
id: star_id,
title: star_name.clone(),
});
}
let categories_elements = text.split("This video belongs to the following categories").collect::<Vec<&str>>()[1]
.split("</p>")
.collect::<Vec<&str>>()[0]
.split("href=\"/category/")
.collect::<Vec<&str>>()[1..]
.to_vec();
for categories_el in categories_elements {
let category_id = categories_el.split("\"").collect::<Vec<&str>>()[0].to_string();
let category_name = categories_el.split("\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0].titlecase();
tags.push(category_name.clone());
Self::push_unique(&self.categories, FilterOption {
id: category_id,
title: category_name.clone(),
});
} }
async fn get_media_formats(video_page_url: &str, requester: &mut Requester) -> Vec<VideoFormat> { let video_url = format!(
let mut formats = vec![]; "https:{}",
let text = requester.get(&video_page_url).await.unwrap(); text.split("url: '/blocks/altplayer.php?i=")
if text.contains("Why do I see it?"){
return formats;
}
for (index, line) in text.split("\n").enumerate() {
println!("Line {}: {}\n", index, line);
}
let video_url = format!("https:{}", text
.split("url: '/blocks/altplayer.php?i=")
.collect::<Vec<&str>>()[1] .collect::<Vec<&str>>()[1]
.split("'") .split("'")
.collect::<Vec<&str>>()[0]); .collect::<Vec<&str>>()[0]
);
let text2 = requester.get_raw_with_headers(&video_url, vec![("Referer".to_string(), "https://hqporner.com/".to_string())]).await.unwrap().text().await.unwrap(); let text2 = requester
.get_raw_with_headers(
&video_url,
vec![("Referer".to_string(), "https://hqporner.com/".to_string())],
)
.await
.unwrap()
.text()
.await
.unwrap();
match text2.split("<video ").collect::<Vec<&str>>().len() > 2 {
false => return Err(Error::from("No video element found")),
true => (),
}
let video_element = text2.split("<video ").collect::<Vec<&str>>()[2] let video_element = text2.split("<video ").collect::<Vec<&str>>()[2]
.split("</video>") .split("</video>")
.collect::<Vec<&str>>()[0]; .collect::<Vec<&str>>()[0];
let sources = video_element.split("<source ").collect::<Vec<&str>>()[1..].to_vec(); let sources = video_element.split("<source ").collect::<Vec<&str>>()[1..].to_vec();
for source in sources { for source in sources {
let title = source.split("title=\\\"").collect::<Vec<&str>>()[1] let title = source.split("title=\\\"").collect::<Vec<&str>>()[1]
.split("\\\"").collect::<Vec<&str>>()[0].to_string(); .split("\\\"")
.collect::<Vec<&str>>()[0]
.to_string();
let quality = title.split(" ").collect::<Vec<&str>>()[0].to_string(); let quality = title.split(" ").collect::<Vec<&str>>()[0].to_string();
let format = "mp4".to_string(); let format = "mp4".to_string();
let media_url = format!("https:{}", source.split("src=\\\"").collect::<Vec<&str>>()[1] let media_url = format!(
.split("\\\"").collect::<Vec<&str>>()[0]); "https:{}",
source.split("src=\\\"").collect::<Vec<&str>>()[1]
.split("\\\"")
.collect::<Vec<&str>>()[0]
);
formats.push(VideoFormat::new(media_url, quality, format).format_id(title.clone()).format_note(title.clone())); formats.push(
VideoFormat::new(media_url, quality, format)
.format_id(title.clone())
.format_note(title.clone()),
);
}
Ok((tags, formats))
} }
formats
} }
#[async_trait] #[async_trait]

22
src/providers/missav.rs
View File

@@ -3,6 +3,7 @@ use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait}; use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all; use futures::future::join_all;
use wreq::Version;
use crate::db; use crate::db;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
@@ -30,14 +31,16 @@ impl MissavProvider {
url: "https://missav.ws".to_string() url: "https://missav.ws".to_string()
} }
} }
async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> { async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
// Extract needed fields from options at the start // Extract needed fields from options at the start
let language = options.language.clone().unwrap(); let language = options.language.clone().unwrap();
let filter = options.filter.clone().unwrap(); let filter = options.filter.clone().unwrap();
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
if !sort.is_empty(){
let url_str = format!("{}/{}/{}?page={}&sort={}", self.url, language, filter, page, sort); sort = format!("&sort={}", sort);
}
let url_str = format!("{}/{}/{}?page={}{}", self.url, language, filter, page, sort);
let old_items = match cache.get(&url_str) { let old_items = match cache.get(&url_str) {
Some((time, items)) => { Some((time, items)) => {
@@ -53,7 +56,7 @@ impl MissavProvider {
} }
}; };
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap();
// Pass a reference to options if needed, or reconstruct as needed // Pass a reference to options if needed, or reconstruct as needed
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await;
if !video_items.is_empty() { if !video_items.is_empty() {
@@ -65,13 +68,16 @@ impl MissavProvider {
Ok(video_items) Ok(video_items)
} }
async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str, sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> { async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str, mut sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
// Extract needed fields from options at the start // Extract needed fields from options at the start
let language = options.language.clone().unwrap(); let language = options.language.clone().unwrap();
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let search_string = query.replace(" ", "%20"); let search_string = query.replace(" ", "%20");
if !sort.is_empty(){
sort = format!("&sort={}", sort);
}
let url_str = format!( let url_str = format!(
"{}/{}/search/{}?page={}&sort={}", "{}/{}/search/{}?page={}{}",
self.url, language, search_string, page, sort self.url, language, search_string, page, sort
); );
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -89,7 +95,7 @@ impl MissavProvider {
vec![] vec![]
} }
}; };
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await;
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url_str); cache.remove(&url_str);
@@ -148,7 +154,7 @@ impl MissavProvider {
} }
} }
drop(conn); drop(conn);
let vid = requester.get(&url_str).await.unwrap(); let vid = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap();
let mut title = vid.split("<meta property=\"og:title\" content=\"").collect::<Vec<&str>>()[1] let mut title = vid.split("<meta property=\"og:title\" content=\"").collect::<Vec<&str>>()[1]
.split("\"") .split("\"")
.collect::<Vec<&str>>()[0].trim() .collect::<Vec<&str>>()[0].trim()

5
src/providers/mod.rs
View File

@@ -37,6 +37,8 @@ pub mod pornxp;
pub mod rule34gen; pub mod rule34gen;
pub mod xxdbx; pub mod xxdbx;
pub mod hqporner; pub mod hqporner;
pub mod noodlemagazine;
pub mod pimpbunny;
// convenient alias // convenient alias
pub type DynProvider = Arc<dyn Provider>; pub type DynProvider = Arc<dyn Provider>;
@@ -50,6 +52,9 @@ pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|
m.insert("rule34gen", Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider); m.insert("rule34gen", Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider);
m.insert("xxdbx", Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider); m.insert("xxdbx", Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider);
m.insert("hqporner", Arc::new(hqporner::HqpornerProvider::new()) as DynProvider); m.insert("hqporner", Arc::new(hqporner::HqpornerProvider::new()) as DynProvider);
m.insert("pmvhaven", Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider);
m.insert("noodlemagazine", Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider);
m.insert("pimpbunny", Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider);
// add more here as you migrate them // add more here as you migrate them
m m
}); });

291
src/providers/noodlemagazine.rs Normal file
View File

@@ -0,0 +1,291 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use wreq::Version;
use std::{vec};
use titlecase::Titlecase;
error_chain! {
foreign_links {
Io(std::io::Error);
HttpRequest(wreq::Error);
}
}
#[derive(Debug, Clone)]
pub struct NoodlemagazineProvider {
url: String,
}
impl NoodlemagazineProvider {
pub fn new() -> Self {
let provider = NoodlemagazineProvider {
url: "https://noodlemagazine.com".to_string(),
};
provider
}
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion;
Channel {
id: "noodlemagazine".to_string(),
name: "Noodlemagazine".to_string(),
description: "The Best Search Engine of HD Videos".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=noodlemagazine.com"
.to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let _ = sort;
let video_url = format!(
"{}/popular/recent?sort_by=views&sort_order=desc&p={}",
self.url,
page - 1
);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone());
} else {
items.clone()
}
}
None => {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester.clone())
.await;
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.trim().replace(" ", "%20").to_string();
let video_url = format!("{}/video/{}?p={}", self.url, search_string, page - 1);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone());
} else {
let _ = cache.check().await;
return Ok(items.clone());
}
}
None => {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester.clone())
.await;
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
async fn get_video_items_from_html(
&self,
html: String,
requester: Requester,
) -> Vec<VideoItem> {
if html.is_empty() || html.contains("404 Not Found") {
return vec![];
}
let raw_videos = html.split(">Show more</div>").collect::<Vec<&str>>()[0]
.split("<div class=\"list_videos\" id=\"list_videos\">")
.collect::<Vec<&str>>()[1]
.split("<div class=\"item\">")
.collect::<Vec<&str>>()[1..]
.to_vec();
let futures = raw_videos
.into_iter()
.map(|el| self.get_video_item(el.to_string(), requester.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return video_items;
}
async fn get_video_item(
&self,
video_segment: String,
requester: Requester,
) -> Result<VideoItem> {
let video_url: String = format!(
"{}{}",
self.url,
video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string()
);
let mut title = video_segment
.split("<div class=\"title\">")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
// html decode
title = decode(title.as_bytes())
.to_string()
.unwrap_or(title)
.titlecase();
let id = video_url.split("/").collect::<Vec<&str>>()[4]
.split(".")
.collect::<Vec<&str>>()[0]
.to_string();
let thumb = video_segment.split("<img ").collect::<Vec<&str>>()[1]
.split("data-src=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let raw_duration = video_segment
.split("#clock-o\"></use></svg>")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let views = parse_abbreviated_number( video_segment
.split("<use xlink:href=\"#eye\"></use></svg>")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.trim()).unwrap_or(0);
let formats = match self.extract_media(&video_url, requester.clone()).await{
Some(f) => f,
None => return Err(Error::from("Failed to extract media formats")),
};
let video_item = VideoItem::new(
id,
title,
video_url,
"noodlemagazine".to_string(),
thumb,
duration,
)
.views(views)
.formats(formats)
;
return Ok(video_item);
}
async fn extract_media(&self, video_url: &String, mut requester: Requester) -> Option<Vec<VideoFormat>> {
let text = requester.get(video_url, Some(Version::HTTP_2)).await.unwrap_or("".to_string());
if text.is_empty() {
return None;
}
let mut formats = vec![];
let json_str = text.split("window.playlist = ")
.collect::<Vec<&str>>()[1]
.split(";")
.collect::<Vec<&str>>()[0];
let json: serde_json::Value = serde_json::from_str(json_str).unwrap();
let sources = json["sources"].as_array().unwrap();
for source in sources {
let file = source["file"].as_str().unwrap().to_string();
let label = source["label"].as_str().unwrap_or("unknown").to_string();
let format = VideoFormat::new(
file,
label.clone(),
"video/mp4".to_string(),
)
.format_id(label.clone())
.format_note(label.clone())
.http_header("Referer".to_string(), video_url.clone())
;
formats.push(format);
}
return Some(formats.into_iter().rev().collect());
}
}
#[async_trait]
impl Provider for NoodlemagazineProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
sort: String,
query: Option<String>,
page: String,
per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};
match videos {
Ok(v) => v,
Err(e) => {
println!("Error fetching videos: {}", e);
vec![]
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel {
self.build_channel(clientversion)
}
}

1
src/providers/okporn.rs
View File

@@ -44,7 +44,6 @@ impl OkpornProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()

1
src/providers/okxxx.rs
View File

@@ -45,7 +45,6 @@ impl OkxxxProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()

11
src/providers/omgxxx.rs
View File

@@ -84,7 +84,7 @@ impl OmgxxxProvider {
let mut requester = util::requester::Requester::new(); let mut requester = util::requester::Requester::new();
for page in [1..10].into_iter().flatten() { for page in [1..10].into_iter().flatten() {
let text = requester let text = requester
.get(format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str()) .get(format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str(), None)
.await .await
.unwrap(); .unwrap();
if text.contains("404 Not Found") || text.is_empty() { if text.contains("404 Not Found") || text.is_empty() {
@@ -122,7 +122,7 @@ impl OmgxxxProvider {
loop { loop {
page += 1; page += 1;
let text = requester let text = requester
.get(format!("{}/sites/{}/", &base_url, page).as_str()) .get(format!("{}/sites/{}/", &base_url, page).as_str(), None)
.await .await
.unwrap(); .unwrap();
if text.contains("404 Not Found") || text.is_empty() { if text.contains("404 Not Found") || text.is_empty() {
@@ -158,7 +158,7 @@ impl OmgxxxProvider {
async fn load_networks(base_url: &str, networks: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> { async fn load_networks(base_url: &str, networks: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new(); let mut requester = util::requester::Requester::new();
let text = requester.get(&base_url).await.unwrap(); let text = requester.get(&base_url, None).await.unwrap();
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>()[1] let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>()[1]
.split("</div>") .split("</div>")
.collect::<Vec<&str>>()[0]; .collect::<Vec<&str>>()[0];
@@ -333,7 +333,6 @@ impl OmgxxxProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -345,7 +344,7 @@ impl OmgxxxProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -393,7 +392,7 @@ impl OmgxxxProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

6
src/providers/paradisehill.rs
View File

@@ -51,7 +51,7 @@ impl ParadisehillProvider {
} }
}; };
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, None).await.unwrap();
// Pass a reference to options if needed, or reconstruct as needed // Pass a reference to options if needed, or reconstruct as needed
let video_items: Vec<VideoItem> = self let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester) .get_video_items_from_html(text.clone(), requester)
@@ -93,7 +93,7 @@ impl ParadisehillProvider {
vec![] vec![]
} }
}; };
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, None).await.unwrap();
let video_items: Vec<VideoItem> = self let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester) .get_video_items_from_html(text.clone(), requester)
.await; .await;
@@ -143,7 +143,7 @@ impl ParadisehillProvider {
} }
async fn get_video_item(&self, url_str: String, mut requester: Requester) -> Result<VideoItem> { async fn get_video_item(&self, url_str: String, mut requester: Requester) -> Result<VideoItem> {
let vid = requester.get(&url_str).await.unwrap(); let vid = requester.get(&url_str, None).await.unwrap();
let mut title = vid let mut title = vid
.split("<meta property=\"og:title\" content=\"") .split("<meta property=\"og:title\" content=\"")
.collect::<Vec<&str>>()[1] .collect::<Vec<&str>>()[1]

1
src/providers/perfectgirls.rs
View File

@@ -45,7 +45,6 @@ impl PerfectgirlsProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()

5
src/providers/perverzija.rs
View File

@@ -11,6 +11,7 @@ use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use serde::Deserialize; use serde::Deserialize;
use serde::Serialize; use serde::Serialize;
use wreq::Version;
use std::vec; use std::vec;
use wreq::Client; use wreq::Client;
use wreq_util::Emulation; use wreq_util::Emulation;
@@ -71,7 +72,7 @@ impl PerverzijaProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url_str); cache.remove(&url_str);
@@ -122,7 +123,7 @@ impl PerverzijaProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = match query_parse { let video_items: Vec<VideoItem> = match query_parse {
true => { true => {
self.get_video_items_from_html_query(text.clone(), pool) self.get_video_items_from_html_query(text.clone(), pool)

554
src/providers/pimpbunny.rs Normal file
View File

@@ -0,0 +1,554 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::sync::{Arc, RwLock};
use std::{thread, vec};
use titlecase::Titlecase;
use wreq::Version;
error_chain! {
foreign_links {
Io(std::io::Error);
HttpRequest(wreq::Error);
}
}
#[derive(Debug, Clone)]
pub struct PimpbunnyProvider {
url: String,
stars: Arc<RwLock<Vec<FilterOption>>>,
categories: Arc<RwLock<Vec<FilterOption>>>,
}
impl PimpbunnyProvider {
pub fn new() -> Self {
let provider = PimpbunnyProvider {
url: "https://pimpbunny.com".to_string(),
stars: Arc::new(RwLock::new(vec![])),
categories: Arc::new(RwLock::new(vec![])),
};
provider.spawn_initial_load();
provider
}
fn spawn_initial_load(&self) {
let url = self.url.clone();
let stars = Arc::clone(&self.stars);
let categories = Arc::clone(&self.categories);
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
rt.block_on(async move {
if let Err(e) = Self::load_stars(&url, stars).await {
eprintln!("load_stars failed: {e}");
}
if let Err(e) = Self::load_categories(&url, categories).await {
eprintln!("load_categories failed: {e}");
}
});
});
}
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(
format!("{}/onlyfans-models/?models_per_page=20", &base_url).as_str(),
Some(Version::HTTP_2),
)
.await
.unwrap();
let stars_div = text
.split("pb-list-models-block")
.collect::<Vec<&str>>()
.last()
.unwrap()
.split("pb-page-description")
.collect::<Vec<&str>>()[0];
for stars_element in stars_div
.split("<div class=\"col\">")
.collect::<Vec<&str>>()[1..]
.to_vec()
{
if stars_element.contains("pb-promoted-link") {
continue;
}
let star_id = stars_element
.split("href=\"https://pimpbunny.com/onlyfans-models/")
.collect::<Vec<&str>>()[1]
.split("/\"")
.collect::<Vec<&str>>()[0]
.to_string();
let star_name = stars_element
.split("<div class=\"pb-item-title-text\">")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
Self::push_unique(
&stars,
FilterOption {
id: star_id,
title: star_name,
},
);
}
return Ok(());
}
async fn load_categories(
base_url: &str,
categories: Arc<RwLock<Vec<FilterOption>>>,
) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(
format!("{}/categories/?items_per_page=120", &base_url).as_str(),
Some(Version::HTTP_2),
)
.await
.unwrap();
let categories_div = text
.split("list_categories_categories_list_items")
.collect::<Vec<&str>>()
.last()
.unwrap()
.split("pb-pagination-wrapper")
.collect::<Vec<&str>>()[0];
for categories_element in categories_div
.split("<div class=\"col\">")
.collect::<Vec<&str>>()[1..]
.to_vec()
{
let category_id = categories_element
.split("href=\"https://pimpbunny.com/categories/")
.collect::<Vec<&str>>()[1]
.split("/\"")
.collect::<Vec<&str>>()[0]
.to_string();
let category_name = categories_element
.split("<div class=\"pb-item-title pb-heading-h3 title\">")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.titlecase();
Self::push_unique(
&categories,
FilterOption {
id: category_id,
title: category_name,
},
);
}
return Ok(());
}
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion;
Channel {
id: "pimpbunny".to_string(),
name: "Pimpbunny".to_string(),
description: "Watch Porn!".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=pimpbunny.com".to_string(),
status: "active".to_string(),
categories: self
.categories
.read()
.unwrap()
.iter()
.map(|c| c.title.clone())
.collect(),
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "featured".into(),
title: "Featured".into(),
},
FilterOption {
id: "most recent".into(),
title: "Most Recent".into(),
},
FilterOption {
id: "most viewed".into(),
title: "Most Viewed".into(),
},
FilterOption {
id: "best rated".into(),
title: "Best Rated".into(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: None,
}
}
// Push one item with minimal lock time and dedup by id
fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
if let Ok(mut vec) = target.write() {
if !vec.iter().any(|x| x.id == item.id) {
vec.push(item);
// Optional: keep it sorted for nicer UX
// vec.sort_by(|a,b| a.title.cmp(&b.title));
}
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"best rated" => "&sort_by=rating",
"most viewed" => "&sort_by=video_viewed",
_ => "&sort_by=post_date",
};
let video_url = format!(
"{}/videos/{}/?videos_per_page=20{}",
self.url, page, sort_string
);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone());
} else {
items.clone()
}
}
None => {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.trim().to_string();
let mut video_url = format!(
"{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&videos_per_page=20&from_videos={}",
self.url, search_string.replace(" ","-"), page
);
let sort_string = match options.sort.as_deref().unwrap_or("") {
"best rated" => "&sort_by=rating",
"most viewed" => "&sort_by=video_viewed",
_ => "&sort_by=post_date",
};
if let Some(star) = self
.stars
.read()
.unwrap()
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/onlyfans-models/{}/{}/?videos_per_page=20{}",
self.url, star.id, page, sort_string
);
}
if let Some(cat) = self
.categories
.read()
.unwrap()
.iter()
.find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/categories/{}/{}/?videos_per_page=20{}",
self.url, cat.id, page, sort_string
);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone());
} else {
let _ = cache.check().await;
return Ok(items.clone());
}
}
None => {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
async fn get_video_items_from_html(
&self,
html: String,
requester: &mut Requester,
) -> Vec<VideoItem> {
if html.is_empty() || html.contains("404 Not Found") {
return vec![];
}
let raw_videos = html.split("pb-pagination-wrapper").collect::<Vec<&str>>()[0]
.split("pb-list-items")
.collect::<Vec<&str>>()[1]
.split("<div class=\"col\">")
.collect::<Vec<&str>>()[1..]
.to_vec();
let futures = raw_videos
.into_iter()
.map(|el| self.get_video_item(el.to_string(), requester.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return video_items;
}
async fn get_video_item(
&self,
video_segment: String,
mut requester: Requester,
) -> Result<VideoItem> {
let video_url: String = video_segment.split(" href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let mut title = video_segment.split("pb-item-title").collect::<Vec<&str>>()[1]
.split(">")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
// html decode
title = decode(title.as_bytes())
.to_string()
.unwrap_or(title)
.titlecase();
let id = video_url.split("/").collect::<Vec<&str>>()[4]
.split(".")
.collect::<Vec<&str>>()[0]
.to_string();
let mut thumb = video_segment.split("pb-thumbnail").collect::<Vec<&str>>()[1]
.split("src=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
if thumb.starts_with("data:image/jpg;base64") {
thumb = video_segment.split("pb-thumbnail").collect::<Vec<&str>>()[1]
.split("data-webp=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
}
let preview = video_segment.split("pb-thumbnail").collect::<Vec<&str>>()[1]
.split("data-preview=\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let (tags, formats, views, duration) = match self.extract_media(&video_url, &mut requester).await {
Ok((t, f, v, d)) => (t, f, v, d),
Err(_) => return Err(Error::from("Video media extraction failed")),
};
if formats.is_empty() {
return Err(Error::from("No formats found for video"));
}
let video_item = VideoItem::new(
id,
title,
video_url,
"pimpbunny".to_string(),
thumb,
duration,
)
.formats(formats)
.tags(tags)
.preview(preview)
.views(views)
;
return Ok(video_item);
}
async fn extract_media(
&self,
video_page_url: &str,
requester: &mut Requester,
) -> Result<(Vec<String>, Vec<VideoFormat>, u32, u32)> {
let mut formats = vec![];
let mut tags = vec![];
let text = requester
.get(&video_page_url, Some(Version::HTTP_2))
.await
.unwrap();
if text.contains("pb-video-models"){
let stars_elements = text.split("pb-video-models").collect::<Vec<&str>>()[1]
.split("pb-video-statistic")
.collect::<Vec<&str>>()[0]
.split("pb-models-item pb-models-item")
.collect::<Vec<&str>>()[1..]
.to_vec();
for star_el in stars_elements {
let star_id = star_el
.split("href=\"https://pimpbunny.com/onlyfans-models/")
.collect::<Vec<&str>>()[1]
.split("/\"")
.collect::<Vec<&str>>()[0]
.to_string();
let star_name = star_el
.split("<span class=\"pb-item-title pb-heading-h3\">")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
tags.push(star_name.clone());
Self::push_unique(
&self.stars,
FilterOption {
id: star_id,
title: star_name.clone(),
},
);
}
}
if text.contains("pb-video-tags") {
let categories_elements = text.split("pb-tags-list").collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0]
.split("href=\"https://pimpbunny.com/tags/")
.collect::<Vec<&str>>()[1..]
.to_vec();
for categories_el in categories_elements {
let category_id = categories_el.split("\"").collect::<Vec<&str>>()[0].to_string();
let category_name = categories_el.split("\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.titlecase();
tags.push(category_name.clone());
Self::push_unique(
&self.categories,
FilterOption {
id: category_id,
title: category_name.clone(),
},
);
}
}
let json_str = text
.split(";</script><script type=\"application/ld+json\">")
.collect::<Vec<&str>>()[1]
.split("</script>")
.collect::<Vec<&str>>()[0];
let json = serde_json::from_str::<serde_json::Value>(json_str).unwrap_or_default();
let video_url = json["contentUrl"].as_str().unwrap_or("").to_string();
let quality = video_url
.split("_")
.collect::<Vec<&str>>()
.last()
.map_or("", |v| v)
.split(".")
.collect::<Vec<&str>>()[0]
.to_string();
let views = json["interactionStatistic"].as_array().unwrap()[0]["userInteractionCount"]
.as_str().unwrap().parse::<u32>().unwrap_or(0);
let raw_duration = json["duration"].as_str().unwrap_or("00:00").replace("PT", "").replace("H", ":").replace("M", ":").replace("S", "");
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
formats.push(VideoFormat::new(
video_url,
quality.clone(),
"video/mp4".to_string(),
));
Ok((tags, formats, views, duration))
}
}
#[async_trait]
impl Provider for PimpbunnyProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
sort: String,
query: Option<String>,
page: String,
per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};
match videos {
Ok(v) => v,
Err(e) => {
println!("Error fetching videos: {}", e);
vec![]
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel {
self.build_channel(clientversion)
}
}

509
src/providers/pmvhaven.rs
View File

@@ -1,12 +1,15 @@
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use cute::c;
use error_chain::error_chain; use error_chain::error_chain;
// use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::sync::{Arc, RwLock};
use std::{vec};
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -15,294 +18,111 @@ error_chain! {
} }
} }
#[derive(serde::Serialize)]
struct PmvhavenRequest {
all: bool, //true,
pmv: bool, //false,
hmv: bool, //false,
hypno: bool, //false,
tiktok: bool, //false,
koreanbj: bool, //false,
other: bool, // false,
explicitContent: Option<bool>, //null,
sameSexContent: Option<bool>, //null,
transContent: Option<String>, //null
seizureWarning: Option<bool>, //null,
tags: Vec<String>, //[],
music: Vec<String>, //[],
stars: Vec<String>, //[],
creators: Vec<String>, //[],
range: Vec<u32>, //[0,40],
activeTime: String, //"All time",
activeQuality: String, //"Quality",
aspectRatio: String, //"Aspect Ratio",
activeView: String, //"Newest",
index: u32, //2,
showSubscriptionsOnly: bool, //false,
query: String, //"no",
profile: Option<String>, //null
}
impl PmvhavenRequest {
pub fn new(page: u32) -> Self {
PmvhavenRequest {
all: true,
pmv: false,
hmv: false,
hypno: false,
tiktok: false,
koreanbj: false,
other: false,
explicitContent: None,
sameSexContent: None,
transContent: None,
seizureWarning: None,
tags: vec![],
music: vec![],
stars: vec![],
creators: vec![],
range: vec![0, 40],
activeTime: "All time".to_string(),
activeQuality: "Quality".to_string(),
aspectRatio: "Aspect Ratio".to_string(),
activeView: "Newest".to_string(),
index: page,
showSubscriptionsOnly: false,
query: "no".to_string(),
profile: None,
}
}
fn hypno(&mut self) -> &mut Self {
self.all = false;
self.pmv = false;
self.hmv = false;
self.tiktok = false;
self.koreanbj = false;
self.other = false;
self.hypno = true;
self
}
fn pmv(&mut self) -> &mut Self {
self.all = false;
self.pmv = true;
self.hmv = false;
self.tiktok = false;
self.koreanbj = false;
self.other = false;
self.hypno = false;
self
}
fn hmv(&mut self) -> &mut Self {
self.all = false;
self.pmv = false;
self.hmv = true;
self.tiktok = false;
self.koreanbj = false;
self.other = false;
self.hypno = false;
self
}
fn tiktok(&mut self) -> &mut Self {
self.all = false;
self.pmv = false;
self.hmv = false;
self.tiktok = true;
self.koreanbj = false;
self.other = false;
self.hypno = false;
self
}
fn koreanbj(&mut self) -> &mut Self {
self.all = false;
self.pmv = false;
self.hmv = false;
self.tiktok = false;
self.koreanbj = true;
self.other = false;
self.hypno = false;
self
}
fn other(&mut self) -> &mut Self {
self.all = false;
self.pmv = false;
self.hmv = false;
self.tiktok = false;
self.koreanbj = false;
self.other = true;
self.hypno = false;
self
}
}
#[derive(serde::Serialize)]
struct PmvhavenSearch {
mode: String, //"DefaultMoreSearch",
data: String, //"pmv",
index: u32,
}
impl PmvhavenSearch {
fn new(search: String, page: u32) -> PmvhavenSearch {
PmvhavenSearch {
mode: "DefaultMoreSearch".to_string(),
data: search,
index: page,
}
}
}
#[derive(serde::Deserialize)]
struct PmvhavenVideo {
title: String, //JAV Addiction Therapy",
_uploader: Option<String>, //itonlygetsworse",
duration: f32, //259.093333,
_width: Option<String>, //3840",
_height: Option<String>, //2160",
_ratio: Option<u32>, //50,
thumbnails: Vec<Option<String>>, //[
// "placeholder",
// "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.png",
// "https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/thumbnail/webp320_686f24e96f7124f3dfbe90ab.webp"
// ],
views: u32, //1971,
_url: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/JAV Addiction Therapy_686f24e96f7124f3dfbe90ab.mp4",
previewUrlCompressed: Option<String>, //https://storage.pmvhaven.com/686f24e96f7124f3dfbe90ab/videoPreview/comus_686f24e96f7124f3dfbe90ab.mp4",
_seizureWarning: Option<bool>, //false,
_isoDate: Option<String>, //2025-07-10T02:52:26.000Z",
_gayContent: Option<bool>, //false,
_transContent: Option<bool>, //false,
creator: Option<String>, //itonlygetsworse",
_id: String, //686f2aeade2062f93d72931f",
_totalRaters: Option<u32>, //42,
_rating: Option<u32>, //164
}
impl PmvhavenVideo {
fn to_videoitem(self) -> VideoItem {
// let encoded_title = percent_encode_emojis(&self.title);
let thumbnail = self.thumbnails[self.thumbnails.len() - 1]
.clone()
.unwrap_or("".to_string());
// let video_id = thumbnail.split("_").collect::<Vec<&str>>().last().unwrap_or(&"").to_string().split('.').next().unwrap_or("").to_string();
let mut item = VideoItem::new(
self._id.clone(),
self.title.clone(),
format!(
"https://pmvhaven.com/video/{}_{}",
self.title.replace(" ", "-"),
self._id
),
"pmvhaven".to_string(),
thumbnail,
self.duration as u32,
)
.views(self.views);
item = match self.creator {
Some(c) => item.uploader(c),
_ => item,
};
item = match self.previewUrlCompressed {
Some(u) => item.preview(u),
_ => item,
};
return item;
}
}
#[derive(serde::Deserialize)]
struct PmvhavenResponse {
data: Vec<PmvhavenVideo>,
_count: Option<u32>,
}
impl PmvhavenResponse {
fn to_videoitems(self) -> Vec<VideoItem> {
return c![video.to_videoitem(), for video in self.data];
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct PmvhavenProvider { pub struct PmvhavenProvider {
url: String, url: String,
stars: Arc<RwLock<Vec<String>>>,
categories: Arc<RwLock<Vec<String>>>,
} }
impl PmvhavenProvider { impl PmvhavenProvider {
pub fn new() -> Self { pub fn new() -> Self {
PmvhavenProvider { let provider = PmvhavenProvider {
url: "https://pmvhaven.com".to_string(), url: "https://pmvhaven.com".to_string(),
} stars: Arc::new(RwLock::new(vec![])),
} categories: Arc::new(RwLock::new(vec![])),
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: String,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let category = options.category.unwrap_or("".to_string());
let index = format!("pmvhaven:{}:{}", page, category);
let url = format!("{}/api/getmorevideos", self.url);
let mut request = PmvhavenRequest::new(page as u32);
request.activeView = sort;
request = match category.as_str() {
"hypno" => {
request.hypno();
request
}
"pmv" => {
request.pmv();
request
}
"hmv" => {
request.hmv();
request
}
"tiktok" => {
request.tiktok();
request
}
"koreanbj" => {
request.koreanbj();
request
}
"other" => {
request.other();
request
}
_ => request,
}; };
provider
}
let old_items = match cache.get(&index) { fn build_channel(&self, clientversion: ClientVersion) -> Channel {
Some((time, items)) => { // if clientversion >= ClientVersion::new(22, 101, "22e".to_string()) {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { let _ = clientversion;
println!("Cache hit for URL: {}", url); Channel {
return Ok(items.clone()); id: "pmvhaven".to_string(),
} else { name: "PMVHaven".to_string(),
items.clone() description: "Best PMV Videos".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=pmvhaven.com".to_string(),
status: "active".to_string(),
categories: self.categories.read().unwrap().iter().map(|c| c.clone()).collect(),
options: vec![
ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "relevance".into(),
title: "Relevance".into(),
},
FilterOption {
id: "newest".into(),
title: "Newest".into(),
},
FilterOption {
id: "oldest".into(),
title: "Oldest".into(),
},
FilterOption {
id: "most viewed".into(),
title: "Most Viewed".into(),
},
FilterOption {
id: "most liked".into(),
title: "Most Liked".into(),
},
FilterOption {
id: "most disliked".into(),
title: "Most Disliked".into(),
},
],
multiSelect: false,
},
ChannelOption {
id: "duration".to_string(),
title: "Duration".to_string(),
description: "Length of the Videos".to_string(),
systemImage: "timer".to_string(),
colorName: "green".to_string(),
options: vec![
FilterOption {
id: "any".into(),
title: "Any".into(),
},
FilterOption {
id: "<4 min".into(),
title: "<4 min".into(),
},
FilterOption {
id: "4-20 min".into(),
title: "4-20 min".into(),
},
FilterOption {
id: "20-60 min".into(),
title: "20-60 min".into(),
},
FilterOption {
id: ">1 hour".into(),
title: ">1 hour".into(),
},
],
multiSelect: false,
},
],
nsfw: true,
cacheDuration: None,
} }
} }
None => {
vec![]
}
};
let mut requester = options.requester.clone().unwrap(); // Push one item with minimal lock time and dedup by id
let response = requester.post(&url, &request, vec![("Content-Type".to_string(),"text/plain;charset=UTF-8".to_string())]).await.unwrap(); fn push_unique(target: &Arc<RwLock<Vec<String>>>, item: String) {
let videos = match response.json::<PmvhavenResponse>().await { if let Ok(mut vec) = target.write() {
Ok(resp) => resp, if !vec.iter().any(|x| x == &item) {
Err(e) => { vec.push(item);
println!("Failed to parse PmvhavenResponse: {}", e);
return Ok(old_items);
} }
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
} }
return Ok(video_items);
} }
async fn query( async fn query(
@@ -312,11 +132,36 @@ impl PmvhavenProvider {
query: &str, query: &str,
options: ServerOptions, options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let index = format!("pmvhaven:{}:{}", query, page); let search_string = query.trim().to_string();
let url = format!("{}/api/v2/search", self.url); let sort_string = match options.sort.unwrap_or("".to_string()).as_str() {
let request = PmvhavenSearch::new(query.to_string(), page as u32); "newest" => "sort=-uploadDate",
// Check our Video Cache. If the result is younger than 1 hour, we return it. "oldest" => "sort=uploadDate",
let old_items = match cache.get(&index) { "most viewed" => "sort=-views",
"most liked" => "sort=-likes",
"most disliked" => "sort=-dislikes",
_ => "",
};
let duration_string = match options.duration.unwrap_or("".to_string()).as_str(){
"<4 min" => "durationMax=240",
"4-20 min" => "durationMin=240&durationMax=1200",
"20-60 min" => "durationMin=1200&durationMax=3600",
">1 hour" => "durationMin=3600",
_ => "",
};
let endpoint = if search_string.is_empty() {
"api/videos"
} else {
"api/videos/search"
};
let mut video_url = format!("{}/{}?limit=100&page={}&{}&{}", self.url, endpoint, page, duration_string, sort_string);
if let Some(star) = self.stars.read().unwrap().iter().find(|s| s.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
video_url = format!("{}&stars={}", video_url, star);
} else if let Some(category) = self.categories.read().unwrap().iter().find(|s| s.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
video_url = format!("{}&tagMode=AND&tags={}", video_url, category);
} else {
video_url = format!("{}&q={}", video_url, search_string);
}
let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone()); return Ok(items.clone());
@@ -331,22 +176,74 @@ impl PmvhavenProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let response = requester.post(&url, &request, vec![("Content-Type".to_string(),"text/plain;charset=UTF-8".to_string())]).await.unwrap();
let videos = match response.json::<PmvhavenResponse>().await { let text = requester.get(&video_url, None).await.unwrap();
Ok(resp) => resp, let json = serde_json::from_str::<serde_json::Value>(&text).unwrap_or(serde_json::Value::Null);
Err(e) => { let video_items: Vec<VideoItem> = self
println!("Failed to parse PmvhavenResponse: {}", e); .get_video_items_from_json(json)
return Ok(old_items); .await;
}
};
let video_items: Vec<VideoItem> = videos.to_videoitems();
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url); cache.remove(&video_url);
cache.insert(url.clone(), video_items.clone()); cache.insert(video_url.clone(), video_items.clone());
} else { } else {
return Ok(old_items); return Ok(old_items);
} }
return Ok(video_items); Ok(video_items)
}
async fn get_video_items_from_json(
&self,
json: serde_json::Value,
) -> Vec<VideoItem> {
if json.is_null() {
return vec![];
}
let mut items = vec![];
let success = json["success"].as_bool().unwrap_or(false);
if !success {
return items;
}
let videos = json["data"].as_array().cloned().unwrap_or_default();
if videos.is_empty() {
return items;
}
for video in videos.clone() {
let title = decode(video["title"].as_str().unwrap_or("").as_bytes()).to_string().unwrap_or("".to_string());
let id = video["_id"].as_str().unwrap_or(title.clone().as_str()).to_string();
let video_url = video["videoUrl"].as_str().unwrap_or("").to_string();
let views = video["views"].as_u64().unwrap_or(0);
let thumb = video["thumbnailUrl"].as_str().unwrap_or("").to_string();
let duration_str = video["duration"].as_str().unwrap_or("0");
let duration = parse_time_to_seconds(duration_str).unwrap_or(0);
let preview = video["previewUrl"].as_str().unwrap_or("").to_string();
let tags_array = video["tags"].as_array().cloned().unwrap_or_default();
for tag in tags_array.clone() {
let tag_str = decode(tag.as_str().unwrap_or("").as_bytes()).to_string().unwrap_or("".to_string());
Self::push_unique(&self.categories, tag_str.clone());
}
let stars_array = video["starsTags"].as_array().cloned().unwrap_or_default();
for tag in stars_array.clone() {
let tag_str = decode(tag.as_str().unwrap_or("").as_bytes()).to_string().unwrap_or("".to_string());
Self::push_unique(&self.stars, tag_str.clone());
}
let tags = stars_array.iter().chain(tags_array.iter()).cloned().collect::<Vec<_>>();
let video_item = VideoItem::new(
id,
title,
video_url.replace(" ", "%20").to_string(),
"pmvhaven".to_string(),
thumb,
duration as u32,
)
.views(views as u32)
.preview(preview)
.tags(tags.iter().map(|t| decode(t.as_str().unwrap_or("").as_bytes()).to_string().unwrap_or("".to_string())).collect());
items.push(video_item);
}
return items;
} }
} }
@@ -356,29 +253,16 @@ impl Provider for PmvhavenProvider {
&self, &self,
cache: VideoCache, cache: VideoCache,
pool: DbPool, pool: DbPool,
sort: String, _sort: String,
query: Option<String>, query: Option<String>,
page: String, page: String,
per_page: String, per_page: String,
options: ServerOptions, options: ServerOptions,
) -> Vec<VideoItem> { ) -> Vec<VideoItem> {
let _ = per_page; let _ = per_page;
let _ = pool; // Ignored in this implementation let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = self.query(cache, page.parse::<u8>().unwrap_or(1), query.unwrap_or("".to_string()).as_str(), options)
Some(q) => { .await;
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(
cache,
page.parse::<u8>().unwrap_or(1),
sort,
options,
)
.await
}
};
match videos { match videos {
Ok(v) => v, Ok(v) => v,
Err(e) => { Err(e) => {
@@ -387,4 +271,7 @@ impl Provider for PmvhavenProvider {
} }
} }
} }
fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel {
self.build_channel(clientversion)
}
} }

5
src/providers/porn00.rs
View File

@@ -49,7 +49,6 @@ impl Porn00Provider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -62,7 +61,7 @@ impl Porn00Provider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -99,7 +98,7 @@ impl Porn00Provider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

5
src/providers/pornhat.rs
View File

@@ -42,7 +42,6 @@ impl PornhatProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -53,7 +52,7 @@ impl PornhatProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -92,7 +91,7 @@ impl PornhatProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

5
src/providers/pornhub.rs
View File

@@ -37,7 +37,6 @@ impl PornhubProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -49,7 +48,7 @@ impl PornhubProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),"<ul id=\"video"); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),"<ul id=\"video");
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -116,7 +115,7 @@ impl PornhubProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),split_string); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),split_string);
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

5
src/providers/pornxp.rs
View File

@@ -84,7 +84,6 @@ impl PornxpProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -96,7 +95,7 @@ impl PornxpProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -142,7 +141,7 @@ impl PornxpProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

2
src/providers/pornzog.rs
View File

@@ -64,7 +64,7 @@ impl PornzogProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
println!("Fetching URL: {}", video_url); println!("Fetching URL: {}", video_url);
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

5
src/providers/redtube.rs
View File

@@ -39,7 +39,6 @@ impl RedtubeProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -50,7 +49,7 @@ impl RedtubeProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -87,7 +86,7 @@ impl RedtubeProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

4
src/providers/rule34gen.rs
View File

@@ -115,7 +115,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
}; };
} }
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap(); let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url); cache.remove(&url);
@@ -163,7 +163,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap(); let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url); cache.remove(&url);

4
src/providers/rule34video.rs
View File

@@ -67,7 +67,7 @@ impl Rule34videoProvider {
}; };
} }
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap(); let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url); cache.remove(&url);
@@ -115,7 +115,7 @@ impl Rule34videoProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap(); let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url); cache.remove(&url);

4
src/providers/sxyprn.rs
View File

@@ -81,7 +81,7 @@ impl SxyprnProvider {
} }
}; };
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, None).await.unwrap();
// Pass a reference to options if needed, or reconstruct as needed // Pass a reference to options if needed, or reconstruct as needed
let video_items: Vec<VideoItem> = self let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), pool, requester) .get_video_items_from_html(text.clone(), pool, requester)
@@ -131,7 +131,7 @@ impl SxyprnProvider {
vec![] vec![]
} }
}; };
let text = requester.get(&url_str).await.unwrap(); let text = requester.get(&url_str, None).await.unwrap();
let video_items: Vec<VideoItem> = self let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), pool, requester) .get_video_items_from_html(text.clone(), pool, requester)
.await; .await;

5
src/providers/tnaflix.rs
View File

@@ -368,7 +368,6 @@ impl TnaflixProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -380,7 +379,7 @@ impl TnaflixProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -427,7 +426,7 @@ impl TnaflixProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

5
src/providers/xxdbx.rs
View File

@@ -94,7 +94,6 @@ impl XxdbxProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -106,7 +105,7 @@ impl XxdbxProvider {
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -156,7 +155,7 @@ impl XxdbxProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

5
src/providers/xxthots.rs
View File

@@ -51,7 +51,6 @@ impl XxthotsProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -62,7 +61,7 @@ impl XxthotsProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -99,7 +98,7 @@ impl XxthotsProvider {
} }
}; };
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

5
src/providers/youjizz.rs
View File

@@ -46,7 +46,6 @@ impl YoujizzProvider {
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", video_url);
return Ok(items.clone()); return Ok(items.clone());
} else { } else {
items.clone() items.clone()
@@ -59,7 +58,7 @@ impl YoujizzProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
@@ -95,7 +94,7 @@ impl YoujizzProvider {
let mut requester = options.requester.clone().unwrap(); let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap(); let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);

2
src/proxies/sxyprn.rs
View File

@@ -40,7 +40,7 @@ impl SxyprnProxy {
) -> String { ) -> String {
let mut requester = requester.get_ref().clone(); let mut requester = requester.get_ref().clone();
let url = "https://sxyprn.com/".to_string() + &url; let url = "https://sxyprn.com/".to_string() + &url;
let text = requester.get(&url).await.unwrap_or("".to_string()); let text = requester.get(&url, None).await.unwrap_or("".to_string());
if text.is_empty() { if text.is_empty() {
return "".to_string(); return "".to_string();
} }

5
src/proxy.rs
View File

@@ -15,11 +15,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.route(web::post().to(crate::proxies::hanimecdn::get_image)) .route(web::post().to(crate::proxies::hanimecdn::get_image))
.route(web::get().to(crate::proxies::hanimecdn::get_image)), .route(web::get().to(crate::proxies::hanimecdn::get_image)),
) )
// .service(
// web::resource("/videos")
// // .route(web::get().to(videos_get))
// .route(web::post().to(videos_post)),
// )
; ;
} }

1
src/util/cache.rs
View File

@@ -49,7 +49,6 @@ impl VideoCache {
for (key, (time, _items)) in iter { for (key, (time, _items)) in iter {
if let Ok(elapsed) = time.elapsed() { if let Ok(elapsed) = time.elapsed() {
if elapsed > Duration::from_secs(60*60){ if elapsed > Duration::from_secs(60*60){
println!("Key: {}, elapsed: {:?}", key, elapsed);
self.remove(&key); self.remove(&key);
} }
} }

38
src/util/requester.rs
View File

@@ -83,7 +83,11 @@ impl Requester {
request.send().await request.send().await
} }
pub async fn get_raw_with_headers(&mut self, url: &str, headers: Vec<(String, String)>) -> Result<Response, wreq::Error> { pub async fn get_raw_with_headers(
&mut self,
url: &str,
headers: Vec<(String, String)>,
) -> Result<Response, wreq::Error> {
let client = Client::builder() let client = Client::builder()
.cert_verification(false) .cert_verification(false)
.emulation(Emulation::Firefox136) .emulation(Emulation::Firefox136)
@@ -106,7 +110,12 @@ impl Requester {
request.send().await request.send().await
} }
pub async fn post<S>(&mut self, url: &str, data: &S, headers: Vec<(String, String)>) -> Result<Response, wreq::Error> pub async fn post<S>(
&mut self,
url: &str,
data: &S,
headers: Vec<(String, String)>,
) -> Result<Response, wreq::Error>
where where
S: Serialize + ?Sized, S: Serialize + ?Sized,
{ {
@@ -134,21 +143,36 @@ impl Requester {
request.send().await request.send().await
} }
pub async fn get(&mut self, url: &str) -> Result<String, AnyErr> { pub async fn get(&mut self, url: &str, _http_version: Option<Version>) -> Result<String, AnyErr> {
let mut request = self.client.get(url).version(Version::HTTP_11); let http_version = match _http_version {
Some(v) => v,
None => Version::HTTP_11,
};
loop {
let mut request = self.client.get(url).version(http_version);
if self.proxy { if self.proxy {
if let Ok(proxy_url) = env::var("BURP_URL") { if let Ok(proxy_url) = env::var("BURP_URL") {
let proxy = Proxy::all(&proxy_url).unwrap(); let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy); request = request.proxy(proxy);
} }
} }
let response = request.send().await?; let response = request.send().await?;
if response.status().is_success() || response.status().as_u16() == 404 { if response.status().is_success() || response.status().as_u16() == 404 {
return Ok(response.text().await?); return Ok(response.text().await?);
} }
if response.status().as_u16() == 429 {
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
continue;
} else {
println!(
"Direct request to {} failed with status: {}",
url,
response.status()
);
break;
}
}
// If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately, // If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately,
// so no non-Send error value lives across later `.await`s. // so no non-Send error value lives across later `.await`s.

16
src/videos.rs
View File

@@ -48,6 +48,7 @@ pub struct ServerOptions {
pub stars: Option<String>, // pub stars: Option<String>, //
pub categories: Option<String>, // pub categories: Option<String>, //
pub duration: Option<String>, // pub duration: Option<String>, //
pub sort: Option<String>, //
} }
#[derive(serde::Serialize, Debug)] #[derive(serde::Serialize, Debug)]
@@ -128,6 +129,9 @@ impl VideoItem {
} }
} }
pub fn tags(mut self, tags: Vec<String>) -> Self { pub fn tags(mut self, tags: Vec<String>) -> Self {
if tags.is_empty(){
return self;
}
self.tags = Some(tags); self.tags = Some(tags);
self self
} }
@@ -156,6 +160,9 @@ impl VideoItem {
self self
} }
pub fn formats(mut self, formats: Vec<VideoFormat>) -> Self { pub fn formats(mut self, formats: Vec<VideoFormat>) -> Self {
if formats.is_empty(){
return self;
}
self.formats = Some(formats); self.formats = Some(formats);
self self
} }
@@ -270,6 +277,15 @@ impl VideoFormat {
headers.insert(key, value); headers.insert(key, value);
} }
} }
pub fn http_header(&mut self, key: String, value: String) -> Self {
if self.http_headers.is_none() {
self.http_headers = Some(HashMap::new());
}
if let Some(headers) = &mut self.http_headers {
headers.insert(key, value);
}
self.to_owned()
}
pub fn format_id(mut self, format_id: String) -> Self { pub fn format_id(mut self, format_id: String) -> Self {
self.format_id = Some(format_id); self.format_id = Some(format_id);
self self