simon/hottub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

database support

Browse Source
This commit is contained in:
Simon
2025-06-05 18:50:28 +00:00
parent 6d08362937
commit 175c9b748f
15 changed files with 333 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/providers/hanime.rs
View File

@@ -6,11 +6,13 @@ use htmlentity::entity::{decode, ICodedDataTrait};
use reqwest::{Proxy};
use futures::future::join_all;
use crate::db;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{self, Video_Embed, Video_Item}; // Make sure Provider trait is imported
use crate::videos::{self, Video_Embed, Video_Item};
use crate::DbPool; // Make sure Provider trait is imported
error_chain! {
foreign_links {
@@ -125,12 +127,30 @@ impl HanimeProvider {
}
}
async fn get_video_item(&self, hit: HanimeSearchResult) -> Result<(u64,Video_Item)> {
async fn get_video_item(&self, hit: HanimeSearchResult, pool: DbPool) -> Result<Video_Item> {
let mut conn = pool.get().expect("couldn't get db connection from pool");
let db_result = db::get_video(&mut conn,format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug.clone()));
drop(conn);
let id = hit.id.to_string();
let title = hit.name;
let thumb = hit.poster_url;
let duration = (hit.duration_in_ms / 1000) as u32; // Convert ms to seconds
let channel = "hanime".to_string(); // Placeholder, adjust as needed
match db_result {
Ok(Some(video_url)) => {
return Ok(Video_Item::new(id, title, video_url.clone(), channel, thumb, duration)
.tags(hit.tags)
.uploader(hit.brand)
.views(hit.views as u32)
.rating((hit.likes as f32 / (hit.likes + hit.dislikes)as f32) * 100 as f32)
.formats(vec![videos::Video_Format::new(video_url.clone(), "1080".to_string(), "m3u8".to_string())]));
}
Ok(None) => (),
Err(e) => {
println!("Error fetching video from database: {}", e);
// return Err(format!("Error fetching video from database: {}", e).into());
}
}
let client = match env::var("BURP_URL").as_deref() {
Ok(burp_url) =>
@@ -149,11 +169,14 @@ impl HanimeProvider {
let text = match response.status().is_success() {
true => {
response.text().await?},
response.text().await?
},
false => {
print!("Failed to fetch video item: {}\n\n", response.status());
return Err(format!("Failed to fetch video item: {}", response.status()).into());
} };
}
};
let urls = text.split("\"servers\"").collect::<Vec<&str>>()[1];
let mut url_vec = vec![];
@@ -163,16 +186,19 @@ impl HanimeProvider {
url_vec.push(url.to_string());
}
}
Ok((hit.created_at, Video_Item::new(id, title, url_vec[0].clone(), channel, thumb, duration)
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(&mut conn, &format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug.clone()), &url_vec[0].clone());
drop(conn);
Ok(Video_Item::new(id, title, url_vec[0].clone(), channel, thumb, duration)
.tags(hit.tags)
.uploader(hit.brand)
.views(hit.views as u32)
.rating((hit.likes as f32 / (hit.likes + hit.dislikes)as f32) * 100 as f32)
.formats(vec![videos::Video_Format::new(url_vec[0].clone(), "1080".to_string(), "m3u8".to_string())])))
.formats(vec![videos::Video_Format::new(url_vec[0].clone(), "1080".to_string(), "m3u8".to_string())]))
}
async fn get(&self, cache: VideoCache, page: u8, query: String) -> Result<Vec<Video_Item>> {
async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, query: String) -> Result<Vec<Video_Item>> {
let index = format!("{}:{}", query, page);
let old_items = match cache.get(&index) {
@@ -219,13 +245,12 @@ impl HanimeProvider {
let hits_json: Vec<HanimeSearchResult> = serde_json::from_str(hits.as_str())
.map_err(|e| format!("Failed to parse hits JSON: {}", e))?;
// let timeout_duration = Duration::from_secs(120);
let futures = hits_json.into_iter().map(|el| self.get_video_item(el.clone()));
let results: Vec<Result<(u64,Video_Item)>> = join_all(futures).await;
let mut items: Vec<(u64, Video_Item)> = results
let futures = hits_json.into_iter().map(|el| self.get_video_item(el.clone(), pool.clone()));
let results: Vec<Result<Video_Item>> = join_all(futures).await;
let video_items: Vec<Video_Item> = results
.into_iter()
.filter_map(Result::ok)
.collect();
let video_items: Vec<Video_Item> = items.into_iter().map(|(_, item)| item).collect();
.collect();
if !video_items.is_empty() {
cache.remove(&index);
cache.insert(index.clone(), video_items.clone());
@@ -241,6 +266,7 @@ impl Provider for HanimeProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
_channel: String,
sort: String,
query: Option<String>,
@@ -252,8 +278,8 @@ impl Provider for HanimeProvider {
let _ = per_page;
let _ = sort;
let videos: std::result::Result<Vec<Video_Item>, Error> = match query {
Some(q) => self.get(cache, page.parse::<u8>().unwrap_or(1), q).await,
None => self.get(cache, page.parse::<u8>().unwrap_or(1), "".to_string()).await,
Some(q) => self.get(cache, pool, page.parse::<u8>().unwrap_or(1), q).await,
None => self.get(cache, pool, page.parse::<u8>().unwrap_or(1), "".to_string()).await,
};
match videos {
Ok(v) => v,

10
src/providers/mod.rs
View File

@@ -1,10 +1,10 @@
use crate::{providers::{hanime::HanimeProvider, perverzija::PerverzijaProvider}, util::cache::VideoCache, videos::Video_Item};
use crate::{providers::{hanime::HanimeProvider, perverzija::PerverzijaProvider}, util::cache::VideoCache, videos::Video_Item, DbPool};
pub mod perverzija;
pub mod hanime;
pub trait Provider{
async fn get_videos(&self, cache: VideoCache ,channel: String, sort: String, query: Option<String>, page: String, per_page: String, featured: String) -> Vec<Video_Item>;
async fn get_videos(&self, cache: VideoCache, pool: DbPool, channel: String, sort: String, query: Option<String>, page: String, per_page: String, featured: String) -> Vec<Video_Item>;
}
pub enum AnyProvider {
@@ -12,10 +12,10 @@ pub enum AnyProvider {
Hanime(HanimeProvider),
}
impl Provider for AnyProvider {
async fn get_videos(&self, cache: VideoCache ,channel: String, sort: String, query: Option<String>, page: String, per_page: String, featured: String) -> Vec<Video_Item> {
async fn get_videos(&self, cache: VideoCache, pool:DbPool, channel: String, sort: String, query: Option<String>, page: String, per_page: String, featured: String) -> Vec<Video_Item> {
match self {
AnyProvider::Perverzija(p) => p.get_videos(cache ,channel, sort, query, page, per_page, featured).await,
AnyProvider::Hanime(p) => p.get_videos(cache ,channel, sort, query, page, per_page, featured).await,
AnyProvider::Perverzija(p) => p.get_videos(cache, pool, channel, sort, query, page, per_page, featured).await,
AnyProvider::Hanime(p) => p.get_videos(cache, pool, channel, sort, query, page, per_page, featured).await,
}
}
}

293
src/providers/perverzija.rs
View File

@@ -3,12 +3,16 @@ use std::env;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use reqwest::{Proxy};
use futures::future::join_all;
use ntex::web;
use crate::db;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{self, Video_Embed, Video_Item}; // Make sure Provider trait is imported
use crate::videos::{self, Video_Embed, Video_Item};
use crate::DbPool; // Make sure Provider trait is imported
error_chain! {
foreign_links {
@@ -26,9 +30,7 @@ impl PerverzijaProvider {
url: "https://tube.perverzija.com/".to_string(),
}
}
async fn get(&self, cache:VideoCache ,page: u8, featured: String) -> Result<Vec<Video_Item>> {
println!("get");
async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result<Vec<Video_Item>> {
//TODO
// let mut url = Url::parse("https://example.net")?;
// url.query_pairs_mut().append_pair("foo", "bar");
@@ -78,7 +80,7 @@ impl PerverzijaProvider {
// print!("Response: {:?}\n", response);
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<Video_Item> = self.get_video_items_from_html(text.clone());
let video_items: Vec<Video_Item> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
@@ -96,11 +98,10 @@ impl PerverzijaProvider {
maxTimeout: 60000,
})
.await;
println!("FlareSolverr result: {:?}", result);
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
self.get_video_items_from_html(res.solution.response, pool)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
@@ -116,22 +117,20 @@ impl PerverzijaProvider {
Ok(video_items)
}
}
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<Video_Item>> {
println!("query: {}", query);
async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result<Vec<Video_Item>> {
let search_string = query.replace(" ", "+");
let mut url = format!(
"{}advanced-search/?_sf_s={}&sf_paged={}",
self.url, search_string, page
"{}page/{}/?s={}",
self.url, page, search_string
);
if page == 1 {
url = format!("{}advanced-search/?_sf_s={}", self.url, search_string);
url = format!("{}?s={}", self.url, search_string);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
println!("Cache hit for URL: {}", url);
return Ok(items.clone());
}
else{
@@ -156,10 +155,9 @@ impl PerverzijaProvider {
};
let response = client.get(url.clone()).send().await?;
// print!("Response: {:?}\n", response);
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<Video_Item> = self.get_video_items_from_html_query(text.clone());
let video_items: Vec<Video_Item> = self.get_video_items_from_html_query(text.clone(), pool).await;
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
@@ -177,11 +175,9 @@ impl PerverzijaProvider {
maxTimeout: 60000,
})
.await;
println!("FlareSolverr result: {:?}", result);
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html_query(res.solution.response)
self.get_video_items_from_html_query(res.solution.response, pool).await
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
@@ -198,8 +194,7 @@ impl PerverzijaProvider {
}
}
fn get_video_items_from_html(&self, html: String) -> Vec<Video_Item> {
// println!("HTML: {}", html);
fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<Video_Item> {
if html.is_empty() {
println!("HTML is empty");
return vec![];
@@ -213,7 +208,6 @@ impl PerverzijaProvider {
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>();
if vid.len() > 20 {
println!("Skipping video segment with unexpected length: {}", vid.len());
continue;
}
let mut title = vid[1].split(">").collect::<Vec<&str>>()[1]
@@ -258,10 +252,14 @@ impl PerverzijaProvider {
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
// let referer_url = vid[1].split("data-url='").collect::<Vec<&str>>()[1]
// .split("'")
// .collect::<Vec<&str>>()[0]
// .to_string();
let id_url = vid[1].split("data-url='").collect::<Vec<&str>>()[1]
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(&mut conn, &id_url, &url);
drop(conn);
let referer_url = "https://xtremestream.xyz/".to_string();
let embed = Video_Embed::new(embed_html, url.clone());
@@ -298,97 +296,167 @@ impl PerverzijaProvider {
return items;
}
fn get_video_items_from_html_query(&self, html: String) -> Vec<Video_Item> {
let mut items: Vec<Video_Item> = Vec::new();
let video_listing_content = html.split("search-filter-results-").collect::<Vec<&str>>()[1];
let raw_videos = video_listing_content
async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec<Video_Item> {
let raw_videos = html
.split("video-item post")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>();
if vid.len() > 20 {
continue;
}
let mut title = vid[3].split("title='").collect::<Vec<&str>>()[1]
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let url = vid[4].split("iframe src=&quot;").collect::<Vec<&str>>()[1]
.split("&quot;")
.collect::<Vec<&str>>()[0]
.to_string().replace("index.php","xs1.php");
let id = url.split("data=").collect::<Vec<&str>>()[1]
.split("&")
.collect::<Vec<&str>>()[0]
.to_string();
let raw_duration = match vid.len() {
18 => vid[16].split("time_dur\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string(),
_ => "00:00".to_string(),
};
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb_index = match vid.len() {
18 => 14,
13 => 8,
_ => {
continue;
}
};
let thumb = match vid[thumb_index].contains("srcset=\"") {
true => vid[thumb_index].split(" ").collect::<Vec<&str>>()[0]
.to_string(),
false => vid[thumb_index].split("src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string(),
};
let embed_html = vid[4].split("data-embed='").collect::<Vec<&str>>()[1]
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
// let referer_url = vid[4].split("data-url='").collect::<Vec<&str>>()[1]
// .split("'")
// .collect::<Vec<&str>>()[0]
// .to_string();
let referer_url = "https://xtremestream.xyz/".to_string();
let embed = Video_Embed::new(embed_html, url.clone());
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
for tag in vid[0].split(" ").collect::<Vec<&str>>(){
if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1]
.to_string();
if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string());
}
}
}
let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone()));
let results: Vec<Result<Video_Item>> = join_all(futures).await;
let items: Vec<Video_Item> = results
.into_iter()
.filter_map(Result::ok)
.collect();
return items;
}
let mut video_item = Video_Item::new(
id,
title,
url.clone(),
"perverzija".to_string(),
thumb,
duration,
)
.tags(tags);
// .embed(embed.clone());
let mut format =
videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string());
format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() {
formats.push(format);
} else {
video_item.formats = Some(vec![format]);
}
items.push(video_item);
async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<Video_Item> {
let vid = snippet.split("\n").collect::<Vec<&str>>();
if vid.len() > 30 {
return Err("Unexpected video snippet length".into());
}
return items;
// for (index,line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
let mut title = vid[5].split(" title=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = match vid[6].split(" src=\"").collect::<Vec<&str>>().len(){
1=>{
for (index,line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line.to_string().trim());
}
return Err("Failed to parse thumbnail URL".into());
}
_ => vid[6].split(" src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string(),
};
let duration = 0;
let lookup_url = vid[5].split(" href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let referer_url = "https://xtremestream.xyz/".to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let db_result = db::get_video(&mut conn,lookup_url.clone());
match db_result {
Ok(Some(url)) => {
let mut id = url.split("data=").collect::<Vec<&str>>()[1]
.to_string();
if id.contains("&"){
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
}
let mut video_item = Video_Item::new(
id,
title,
url.clone(),
"perverzija".to_string(),
thumb,
duration,
);
let mut format =
videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string());
format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() {
formats.push(format);
} else {
video_item.formats = Some(vec![format]);
}
return Ok(video_item);
}
Ok(None) => (),
Err(e) => {
println!("Error fetching video from database: {}", e);
// return Err(format!("Error fetching video from database: {}", e).into());
}
}
drop(conn);
let client = match env::var("BURP_URL").as_deref() {
Ok(burp_url) =>
reqwest::Client::builder()
.user_agent("Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/33.0 Mobile/15E148 Safari/605.1.15")
.proxy(Proxy::https(burp_url).unwrap())
.danger_accept_invalid_certs(true)
.build()?,
Err(_) => reqwest::Client::builder()
.user_agent("Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/33.0 Mobile/15E148 Safari/605.1.15")
.danger_accept_invalid_certs(true)
.build()?,
};
let response = client.get(lookup_url.clone()).send().await?;
let text = match response.status().is_success(){
true => response.text().await?,
false => {
return Err("Failed to fetch video details".into());
}
};
let url = text.split("<iframe src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string().replace("index.php","xs1.php");
if url.starts_with("https://discord.com"){
return Err("Discord link found, skipping video".into());
}
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(&mut conn, &lookup_url, &url);
drop(conn);
if !url.contains("xtremestream.xyz"){
return Err("Video URL does not contain xtremestream.xyz".into());
}
let mut id = url.split("data=").collect::<Vec<&str>>()[1]
.to_string();
if id.contains("&"){
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
}
// if !vid[6].contains(" src=\""){
// for (index,line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
// }
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
for tag in vid[0].split(" ").collect::<Vec<&str>>(){
if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1]
.to_string();
if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string());
}
}
}
let mut video_item = Video_Item::new(
id,
title,
url.clone(),
"perverzija".to_string(),
thumb,
duration,
)
.tags(tags);
// .embed(embed.clone());
let mut format =
videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string());
format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() {
formats.push(format);
} else {
video_item.formats = Some(vec![format]);
}
return Ok(video_item);
}
}
@@ -396,6 +464,7 @@ impl Provider for PerverzijaProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
_channel: String,
sort: String,
query: Option<String>,
@@ -406,8 +475,8 @@ impl Provider for PerverzijaProvider {
let _ = per_page;
let _ = sort;
let videos: std::result::Result<Vec<Video_Item>, Error> = match query {
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
None => self.get(cache, page.parse::<u8>().unwrap_or(1), featured).await,
Some(q) => self.query(cache, pool, page.parse::<u8>().unwrap_or(1), &q).await,
None => self.get(cache, pool, page.parse::<u8>().unwrap_or(1), featured).await,
};
match videos {
Ok(v) => v,