simon/hottub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

database support

Browse Source
This commit is contained in:
Simon
2025-06-05 18:50:28 +00:00
parent 6d08362937
commit 175c9b748f
15 changed files with 333 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

293
src/providers/perverzija.rs
View File

@@ -3,12 +3,16 @@ use std::env;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use reqwest::{Proxy};
use futures::future::join_all;
use ntex::web;
use crate::db;
use crate::providers::Provider;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{self, Video_Embed, Video_Item}; // Make sure Provider trait is imported
use crate::videos::{self, Video_Embed, Video_Item};
use crate::DbPool; // Make sure Provider trait is imported
error_chain! {
foreign_links {
@@ -26,9 +30,7 @@ impl PerverzijaProvider {
url: "https://tube.perverzija.com/".to_string(),
}
}
async fn get(&self, cache:VideoCache ,page: u8, featured: String) -> Result<Vec<Video_Item>> {
println!("get");
async fn get(&self, cache:VideoCache, pool:DbPool, page: u8, featured: String) -> Result<Vec<Video_Item>> {
//TODO
// let mut url = Url::parse("https://example.net")?;
// url.query_pairs_mut().append_pair("foo", "bar");
@@ -78,7 +80,7 @@ impl PerverzijaProvider {
// print!("Response: {:?}\n", response);
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<Video_Item> = self.get_video_items_from_html(text.clone());
let video_items: Vec<Video_Item> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
@@ -96,11 +98,10 @@ impl PerverzijaProvider {
maxTimeout: 60000,
})
.await;
println!("FlareSolverr result: {:?}", result);
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html(res.solution.response)
self.get_video_items_from_html(res.solution.response, pool)
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
@@ -116,22 +117,20 @@ impl PerverzijaProvider {
Ok(video_items)
}
}
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<Video_Item>> {
println!("query: {}", query);
async fn query(&self, cache: VideoCache, pool:DbPool, page: u8, query: &str) -> Result<Vec<Video_Item>> {
let search_string = query.replace(" ", "+");
let mut url = format!(
"{}advanced-search/?_sf_s={}&sf_paged={}",
self.url, search_string, page
"{}page/{}/?s={}",
self.url, page, search_string
);
if page == 1 {
url = format!("{}advanced-search/?_sf_s={}", self.url, search_string);
url = format!("{}?s={}", self.url, search_string);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
println!("Cache hit for URL: {}", url);
return Ok(items.clone());
}
else{
@@ -156,10 +155,9 @@ impl PerverzijaProvider {
};
let response = client.get(url.clone()).send().await?;
// print!("Response: {:?}\n", response);
if response.status().is_success() {
let text = response.text().await?;
let video_items: Vec<Video_Item> = self.get_video_items_from_html_query(text.clone());
let video_items: Vec<Video_Item> = self.get_video_items_from_html_query(text.clone(), pool).await;
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
@@ -177,11 +175,9 @@ impl PerverzijaProvider {
maxTimeout: 60000,
})
.await;
println!("FlareSolverr result: {:?}", result);
let video_items = match result {
Ok(res) => {
// println!("FlareSolverr response: {}", res);
self.get_video_items_from_html_query(res.solution.response)
self.get_video_items_from_html_query(res.solution.response, pool).await
}
Err(e) => {
println!("Error solving FlareSolverr: {}", e);
@@ -198,8 +194,7 @@ impl PerverzijaProvider {
}
}
fn get_video_items_from_html(&self, html: String) -> Vec<Video_Item> {
// println!("HTML: {}", html);
fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<Video_Item> {
if html.is_empty() {
println!("HTML is empty");
return vec![];
@@ -213,7 +208,6 @@ impl PerverzijaProvider {
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>();
if vid.len() > 20 {
println!("Skipping video segment with unexpected length: {}", vid.len());
continue;
}
let mut title = vid[1].split(">").collect::<Vec<&str>>()[1]
@@ -258,10 +252,14 @@ impl PerverzijaProvider {
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
// let referer_url = vid[1].split("data-url='").collect::<Vec<&str>>()[1]
// .split("'")
// .collect::<Vec<&str>>()[0]
// .to_string();
let id_url = vid[1].split("data-url='").collect::<Vec<&str>>()[1]
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(&mut conn, &id_url, &url);
drop(conn);
let referer_url = "https://xtremestream.xyz/".to_string();
let embed = Video_Embed::new(embed_html, url.clone());
@@ -298,97 +296,167 @@ impl PerverzijaProvider {
return items;
}
fn get_video_items_from_html_query(&self, html: String) -> Vec<Video_Item> {
let mut items: Vec<Video_Item> = Vec::new();
let video_listing_content = html.split("search-filter-results-").collect::<Vec<&str>>()[1];
let raw_videos = video_listing_content
async fn get_video_items_from_html_query(&self, html: String, pool:DbPool) -> Vec<Video_Item> {
let raw_videos = html
.split("video-item post")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>();
if vid.len() > 20 {
continue;
}
let mut title = vid[3].split("title='").collect::<Vec<&str>>()[1]
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let url = vid[4].split("iframe src=&quot;").collect::<Vec<&str>>()[1]
.split("&quot;")
.collect::<Vec<&str>>()[0]
.to_string().replace("index.php","xs1.php");
let id = url.split("data=").collect::<Vec<&str>>()[1]
.split("&")
.collect::<Vec<&str>>()[0]
.to_string();
let raw_duration = match vid.len() {
18 => vid[16].split("time_dur\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string(),
_ => "00:00".to_string(),
};
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb_index = match vid.len() {
18 => 14,
13 => 8,
_ => {
continue;
}
};
let thumb = match vid[thumb_index].contains("srcset=\"") {
true => vid[thumb_index].split(" ").collect::<Vec<&str>>()[0]
.to_string(),
false => vid[thumb_index].split("src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string(),
};
let embed_html = vid[4].split("data-embed='").collect::<Vec<&str>>()[1]
.split("'")
.collect::<Vec<&str>>()[0]
.to_string();
// let referer_url = vid[4].split("data-url='").collect::<Vec<&str>>()[1]
// .split("'")
// .collect::<Vec<&str>>()[0]
// .to_string();
let referer_url = "https://xtremestream.xyz/".to_string();
let embed = Video_Embed::new(embed_html, url.clone());
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
for tag in vid[0].split(" ").collect::<Vec<&str>>(){
if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1]
.to_string();
if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string());
}
}
}
let futures = raw_videos.into_iter().map(|el| self.get_video_item(el, pool.clone()));
let results: Vec<Result<Video_Item>> = join_all(futures).await;
let items: Vec<Video_Item> = results
.into_iter()
.filter_map(Result::ok)
.collect();
return items;
}
let mut video_item = Video_Item::new(
id,
title,
url.clone(),
"perverzija".to_string(),
thumb,
duration,
)
.tags(tags);
// .embed(embed.clone());
let mut format =
videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string());
format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() {
formats.push(format);
} else {
video_item.formats = Some(vec![format]);
}
items.push(video_item);
async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<Video_Item> {
let vid = snippet.split("\n").collect::<Vec<&str>>();
if vid.len() > 30 {
return Err("Unexpected video snippet length".into());
}
return items;
// for (index,line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
let mut title = vid[5].split(" title=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = match vid[6].split(" src=\"").collect::<Vec<&str>>().len(){
1=>{
for (index,line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line.to_string().trim());
}
return Err("Failed to parse thumbnail URL".into());
}
_ => vid[6].split(" src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string(),
};
let duration = 0;
let lookup_url = vid[5].split(" href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let referer_url = "https://xtremestream.xyz/".to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let db_result = db::get_video(&mut conn,lookup_url.clone());
match db_result {
Ok(Some(url)) => {
let mut id = url.split("data=").collect::<Vec<&str>>()[1]
.to_string();
if id.contains("&"){
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
}
let mut video_item = Video_Item::new(
id,
title,
url.clone(),
"perverzija".to_string(),
thumb,
duration,
);
let mut format =
videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string());
format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() {
formats.push(format);
} else {
video_item.formats = Some(vec![format]);
}
return Ok(video_item);
}
Ok(None) => (),
Err(e) => {
println!("Error fetching video from database: {}", e);
// return Err(format!("Error fetching video from database: {}", e).into());
}
}
drop(conn);
let client = match env::var("BURP_URL").as_deref() {
Ok(burp_url) =>
reqwest::Client::builder()
.user_agent("Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/33.0 Mobile/15E148 Safari/605.1.15")
.proxy(Proxy::https(burp_url).unwrap())
.danger_accept_invalid_certs(true)
.build()?,
Err(_) => reqwest::Client::builder()
.user_agent("Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/33.0 Mobile/15E148 Safari/605.1.15")
.danger_accept_invalid_certs(true)
.build()?,
};
let response = client.get(lookup_url.clone()).send().await?;
let text = match response.status().is_success(){
true => response.text().await?,
false => {
return Err("Failed to fetch video details".into());
}
};
let url = text.split("<iframe src=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string().replace("index.php","xs1.php");
if url.starts_with("https://discord.com"){
return Err("Discord link found, skipping video".into());
}
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(&mut conn, &lookup_url, &url);
drop(conn);
if !url.contains("xtremestream.xyz"){
return Err("Video URL does not contain xtremestream.xyz".into());
}
let mut id = url.split("data=").collect::<Vec<&str>>()[1]
.to_string();
if id.contains("&"){
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
}
// if !vid[6].contains(" src=\""){
// for (index,line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
// }
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
for tag in vid[0].split(" ").collect::<Vec<&str>>(){
if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1]
.to_string();
if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string());
}
}
}
let mut video_item = Video_Item::new(
id,
title,
url.clone(),
"perverzija".to_string(),
thumb,
duration,
)
.tags(tags);
// .embed(embed.clone());
let mut format =
videos::Video_Format::new(url.clone(), "1080".to_string(), "m3u8".to_string());
format.add_http_header("Referer".to_string(), referer_url.clone());
if let Some(formats) = video_item.formats.as_mut() {
formats.push(format);
} else {
video_item.formats = Some(vec![format]);
}
return Ok(video_item);
}
}
@@ -396,6 +464,7 @@ impl Provider for PerverzijaProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
_channel: String,
sort: String,
query: Option<String>,
@@ -406,8 +475,8 @@ impl Provider for PerverzijaProvider {
let _ = per_page;
let _ = sort;
let videos: std::result::Result<Vec<Video_Item>, Error> = match query {
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
None => self.get(cache, page.parse::<u8>().unwrap_or(1), featured).await,
Some(q) => self.query(cache, pool, page.parse::<u8>().unwrap_or(1), &q).await,
None => self.get(cache, pool, page.parse::<u8>().unwrap_or(1), featured).await,
};
match videos {
Ok(v) => v,