upgrades
This commit is contained in:
@@ -38,7 +38,7 @@ impl SpankbangProvider {
|
|||||||
let old_items = match cache.get(&url) {
|
let old_items = match cache.get(&url) {
|
||||||
Some((time, items)) => {
|
Some((time, items)) => {
|
||||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
|
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
|
||||||
println!("Cache hit for URL: {}", url);
|
// println!("Cache hit for URL: {}", url);
|
||||||
return Ok(items.clone());
|
return Ok(items.clone());
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
@@ -123,7 +123,7 @@ impl SpankbangProvider {
|
|||||||
let old_items = match cache.get(&url) {
|
let old_items = match cache.get(&url) {
|
||||||
Some((time, items)) => {
|
Some((time, items)) => {
|
||||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
|
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
|
||||||
println!("Cache hit for URL: {}", url);
|
// println!("Cache hit for URL: {}", url);
|
||||||
return Ok(items.clone());
|
return Ok(items.clone());
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ use crate::api::ClientVersion;
|
|||||||
use crate::providers::Provider;
|
use crate::providers::Provider;
|
||||||
use crate::status::*;
|
use crate::status::*;
|
||||||
use crate::util::cache::VideoCache;
|
use crate::util::cache::VideoCache;
|
||||||
|
use crate::util::discord::{format_error_chain, send_discord_error_report};
|
||||||
use crate::util::requester::Requester;
|
use crate::util::requester::Requester;
|
||||||
use crate::util::time::parse_time_to_seconds;
|
use crate::util::time::parse_time_to_seconds;
|
||||||
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
||||||
@@ -18,6 +19,13 @@ error_chain! {
|
|||||||
foreign_links {
|
foreign_links {
|
||||||
Io(std::io::Error);
|
Io(std::io::Error);
|
||||||
HttpRequest(wreq::Error);
|
HttpRequest(wreq::Error);
|
||||||
|
Json(serde_json::Error);
|
||||||
|
}
|
||||||
|
errors {
|
||||||
|
Parse(msg: String) {
|
||||||
|
description("parse error")
|
||||||
|
display("parse error: {}", msg)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -27,6 +35,7 @@ pub struct HqpornerProvider {
|
|||||||
stars: Arc<RwLock<Vec<FilterOption>>>,
|
stars: Arc<RwLock<Vec<FilterOption>>>,
|
||||||
categories: Arc<RwLock<Vec<FilterOption>>>,
|
categories: Arc<RwLock<Vec<FilterOption>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HqpornerProvider {
|
impl HqpornerProvider {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
let provider = HqpornerProvider {
|
let provider = HqpornerProvider {
|
||||||
@@ -44,13 +53,12 @@ impl HqpornerProvider {
|
|||||||
let categories = Arc::clone(&self.categories);
|
let categories = Arc::clone(&self.categories);
|
||||||
|
|
||||||
thread::spawn(move || {
|
thread::spawn(move || {
|
||||||
// Create a tiny runtime just for these async tasks
|
|
||||||
let rt = tokio::runtime::Builder::new_current_thread()
|
let rt = tokio::runtime::Builder::new_current_thread()
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.build()
|
.build();
|
||||||
.expect("build tokio runtime");
|
|
||||||
|
|
||||||
rt.block_on(async move {
|
if let Ok(runtime) = rt {
|
||||||
|
runtime.block_on(async move {
|
||||||
if let Err(e) = Self::load_stars(&url, stars).await {
|
if let Err(e) = Self::load_stars(&url, stars).await {
|
||||||
eprintln!("load_stars failed: {e}");
|
eprintln!("load_stars failed: {e}");
|
||||||
}
|
}
|
||||||
@@ -58,75 +66,82 @@ impl HqpornerProvider {
|
|||||||
eprintln!("load_categories failed: {e}");
|
eprintln!("load_categories failed: {e}");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
|
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
|
||||||
let mut requester = Requester::new();
|
let mut requester = Requester::new();
|
||||||
let text = requester
|
let text = requester
|
||||||
.get(format!("{}/girls", &base_url).as_str(), None)
|
.get(&format!("{}/girls", base_url), None)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
|
||||||
|
|
||||||
let stars_div = text
|
let stars_div = text
|
||||||
.split("<span>Girls</span>")
|
.split("<span>Girls</span>")
|
||||||
.collect::<Vec<&str>>().last().unwrap()
|
.last()
|
||||||
.split("</ul>")
|
.and_then(|s| s.split("</ul>").next())
|
||||||
.collect::<Vec<&str>>()[0];
|
.ok_or_else(|| Error::from("Could not find stars div"))?;
|
||||||
for stars_element in stars_div.split("<li ").collect::<Vec<&str>>()[1..].to_vec() {
|
|
||||||
let star_id = stars_element.split("href=\"/actress/").collect::<Vec<&str>>()[1]
|
for stars_element in stars_div.split("<li ").skip(1) {
|
||||||
.split("\"")
|
let star_id = stars_element
|
||||||
.collect::<Vec<&str>>()[0]
|
.split("href=\"/actress/")
|
||||||
.to_string();
|
.nth(1)
|
||||||
let star_name = stars_element.split("<a ").collect::<Vec<&str>>()[1]
|
.and_then(|s| s.split('"').next())
|
||||||
.split(">").collect::<Vec<&str>>()[1]
|
.map(|s| s.to_string());
|
||||||
.split("<")
|
|
||||||
.collect::<Vec<&str>>()[0]
|
let star_name = stars_element
|
||||||
.to_string();
|
.split("<a ")
|
||||||
Self::push_unique(
|
.nth(1)
|
||||||
&stars,
|
.and_then(|s| s.split('>').nth(1))
|
||||||
FilterOption {
|
.and_then(|s| s.split('<').next())
|
||||||
id: star_id,
|
.map(|s| s.to_string());
|
||||||
title: star_name,
|
|
||||||
},
|
if let (Some(id), Some(name)) = (star_id, star_name) {
|
||||||
);
|
Self::push_unique(&stars, FilterOption { id, title: name });
|
||||||
}
|
}
|
||||||
return Ok(());
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_categories(base_url: &str, categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
|
async fn load_categories(
|
||||||
|
base_url: &str,
|
||||||
|
categories: Arc<RwLock<Vec<FilterOption>>>,
|
||||||
|
) -> Result<()> {
|
||||||
let mut requester = Requester::new();
|
let mut requester = Requester::new();
|
||||||
let text = requester
|
let text = requester
|
||||||
.get(format!("{}/categories", &base_url).as_str(), None)
|
.get(&format!("{}/categories", base_url), None)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
|
||||||
|
|
||||||
let categories_div = text
|
let categories_div = text
|
||||||
.split("<span>Categories</span>")
|
.split("<span>Categories</span>")
|
||||||
.collect::<Vec<&str>>().last().unwrap()
|
.last()
|
||||||
.split("</ul>")
|
.and_then(|s| s.split("</ul>").next())
|
||||||
.collect::<Vec<&str>>()[0];
|
.ok_or_else(|| Error::from("Could not find categories div"))?;
|
||||||
for categories_element in categories_div.split("<li ").collect::<Vec<&str>>()[1..].to_vec() {
|
|
||||||
let category_id = categories_element.split("href=\"/category/").collect::<Vec<&str>>()[1]
|
for categories_element in categories_div.split("<li ").skip(1) {
|
||||||
.split("\"")
|
let category_id = categories_element
|
||||||
.collect::<Vec<&str>>()[0]
|
.split("href=\"/category/")
|
||||||
.to_string();
|
.nth(1)
|
||||||
let category_name = categories_element.split("<a ").collect::<Vec<&str>>()[1]
|
.and_then(|s| s.split('"').next())
|
||||||
.split(">").collect::<Vec<&str>>()[1]
|
.map(|s| s.to_string());
|
||||||
.split("<")
|
|
||||||
.collect::<Vec<&str>>()[0]
|
let category_name = categories_element
|
||||||
.titlecase();
|
.split("<a ")
|
||||||
Self::push_unique(
|
.nth(1)
|
||||||
&categories,
|
.and_then(|s| s.split('>').nth(1))
|
||||||
FilterOption {
|
.and_then(|s| s.split('<').next())
|
||||||
id: category_id,
|
.map(|s| s.titlecase());
|
||||||
title: category_name,
|
|
||||||
},
|
if let (Some(id), Some(name)) = (category_id, category_name) {
|
||||||
);
|
Self::push_unique(&categories, FilterOption { id, title: name });
|
||||||
}
|
}
|
||||||
return Ok(());
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||||
let _ = clientversion;
|
|
||||||
Channel {
|
Channel {
|
||||||
id: "hqporner".to_string(),
|
id: "hqporner".to_string(),
|
||||||
name: "HQPorner".to_string(),
|
name: "HQPorner".to_string(),
|
||||||
@@ -134,20 +149,21 @@ impl HqpornerProvider {
|
|||||||
premium: false,
|
premium: false,
|
||||||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=hqporner.com".to_string(),
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=hqporner.com".to_string(),
|
||||||
status: "active".to_string(),
|
status: "active".to_string(),
|
||||||
categories: self.categories.read().unwrap().iter().map(|c| c.title.clone()).collect(),
|
categories: self
|
||||||
|
.categories
|
||||||
|
.read()
|
||||||
|
.map(|c| c.iter().map(|o| o.title.clone()).collect())
|
||||||
|
.unwrap_or_default(),
|
||||||
options: vec![],
|
options: vec![],
|
||||||
nsfw: true,
|
nsfw: true,
|
||||||
cacheDuration: None,
|
cacheDuration: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Push one item with minimal lock time and dedup by id
|
|
||||||
fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
|
fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
|
||||||
if let Ok(mut vec) = target.write() {
|
if let Ok(mut vec) = target.write() {
|
||||||
if !vec.iter().any(|x| x.id == item.id) {
|
if !vec.iter().any(|x| x.id == item.id) {
|
||||||
vec.push(item);
|
vec.push(item);
|
||||||
// Optional: keep it sorted for nicer UX
|
|
||||||
// vec.sort_by(|a,b| a.title.cmp(&b.title));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -156,34 +172,25 @@ impl HqpornerProvider {
|
|||||||
&self,
|
&self,
|
||||||
cache: VideoCache,
|
cache: VideoCache,
|
||||||
page: u8,
|
page: u8,
|
||||||
sort: &str,
|
_sort: &str,
|
||||||
options: ServerOptions,
|
options: ServerOptions,
|
||||||
) -> Result<Vec<VideoItem>> {
|
) -> Result<Vec<VideoItem>> {
|
||||||
let _ = sort;
|
|
||||||
let video_url = format!("{}/hdporn/{}", self.url, page);
|
let video_url = format!("{}/hdporn/{}", self.url, page);
|
||||||
let old_items = match cache.get(&video_url) {
|
if let Some((time, items)) = cache.get(&video_url) {
|
||||||
Some((time, items)) => {
|
if time.elapsed().unwrap_or_default().as_secs() < 300 {
|
||||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
|
||||||
return Ok(items.clone());
|
return Ok(items.clone());
|
||||||
} else {
|
|
||||||
items.clone()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => {
|
|
||||||
vec![]
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut requester = options.requester.clone().unwrap();
|
let mut requester = options.requester.clone().ok_or("No requester")?;
|
||||||
let text = requester.get(&video_url, None).await.unwrap();
|
let text = requester
|
||||||
let video_items: Vec<VideoItem> = self
|
.get(&video_url, None)
|
||||||
.get_video_items_from_html(text.clone(), &mut requester)
|
.await
|
||||||
.await;
|
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
|
||||||
|
|
||||||
|
let video_items = self.get_video_items_from_html(text, &mut requester).await;
|
||||||
if !video_items.is_empty() {
|
if !video_items.is_empty() {
|
||||||
cache.remove(&video_url);
|
cache.insert(video_url, video_items.clone());
|
||||||
cache.insert(video_url.clone(), video_items.clone());
|
|
||||||
} else {
|
|
||||||
return Ok(old_items);
|
|
||||||
}
|
}
|
||||||
Ok(video_items)
|
Ok(video_items)
|
||||||
}
|
}
|
||||||
@@ -195,42 +202,41 @@ impl HqpornerProvider {
|
|||||||
query: &str,
|
query: &str,
|
||||||
options: ServerOptions,
|
options: ServerOptions,
|
||||||
) -> Result<Vec<VideoItem>> {
|
) -> Result<Vec<VideoItem>> {
|
||||||
let search_string = query.trim().to_string();
|
let search_string = query.trim().to_lowercase();
|
||||||
|
|
||||||
let mut video_url = format!("{}/?q={}&p={}", self.url, search_string, page);
|
let mut video_url = format!("{}/?q={}&p={}", self.url, search_string, page);
|
||||||
|
|
||||||
if let Some(star) = self.stars.read().unwrap().iter().find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
|
if let Ok(stars) = self.stars.read() {
|
||||||
|
if let Some(star) = stars
|
||||||
|
.iter()
|
||||||
|
.find(|s| s.title.to_lowercase() == search_string)
|
||||||
|
{
|
||||||
video_url = format!("{}/actress/{}/{}", self.url, star.id, page);
|
video_url = format!("{}/actress/{}/{}", self.url, star.id, page);
|
||||||
}
|
}
|
||||||
if let Some(cat) = self.categories.read().unwrap().iter().find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
|
}
|
||||||
|
if let Ok(cats) = self.categories.read() {
|
||||||
|
if let Some(cat) = cats
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.title.to_lowercase() == search_string)
|
||||||
|
{
|
||||||
video_url = format!("{}/category/{}/{}", self.url, cat.id, page);
|
video_url = format!("{}/category/{}/{}", self.url, cat.id, page);
|
||||||
}
|
}
|
||||||
// Check our Video Cache. If the result is younger than 1 hour, we return it.
|
}
|
||||||
let old_items = match cache.get(&video_url) {
|
|
||||||
Some((time, items)) => {
|
if let Some((time, items)) = cache.get(&video_url) {
|
||||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
if time.elapsed().unwrap_or_default().as_secs() < 300 {
|
||||||
return Ok(items.clone());
|
|
||||||
} else {
|
|
||||||
let _ = cache.check().await;
|
|
||||||
return Ok(items.clone());
|
return Ok(items.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => {
|
|
||||||
vec![]
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut requester = options.requester.clone().unwrap();
|
let mut requester = options.requester.clone().ok_or("No requester")?;
|
||||||
|
let text = requester
|
||||||
|
.get(&video_url, None)
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
|
||||||
|
|
||||||
let text = requester.get(&video_url, None).await.unwrap();
|
let video_items = self.get_video_items_from_html(text, &mut requester).await;
|
||||||
let video_items: Vec<VideoItem> = self
|
|
||||||
.get_video_items_from_html(text.clone(), &mut requester)
|
|
||||||
.await;
|
|
||||||
if !video_items.is_empty() {
|
if !video_items.is_empty() {
|
||||||
cache.remove(&video_url);
|
cache.insert(video_url, video_items.clone());
|
||||||
cache.insert(video_url.clone(), video_items.clone());
|
|
||||||
} else {
|
|
||||||
return Ok(old_items);
|
|
||||||
}
|
}
|
||||||
Ok(video_items)
|
Ok(video_items)
|
||||||
}
|
}
|
||||||
@@ -243,185 +249,188 @@ impl HqpornerProvider {
|
|||||||
if html.is_empty() || html.contains("404 Not Found") {
|
if html.is_empty() || html.contains("404 Not Found") {
|
||||||
return vec![];
|
return vec![];
|
||||||
}
|
}
|
||||||
let raw_videos = html.split("id=\"footer\"").collect::<Vec<&str>>()[0]
|
|
||||||
.split("<section class=\"box features\">")
|
let raw_videos: Vec<String> = html
|
||||||
.collect::<Vec<&str>>()[2]
|
.split("id=\"footer\"")
|
||||||
.split("<section class=\"box feature\">")
|
.next()
|
||||||
.collect::<Vec<&str>>()[1..]
|
.and_then(|s| s.split("<section class=\"box features\">").nth(2))
|
||||||
.to_vec();
|
.map(|s| {
|
||||||
|
s.split("<section class=\"box feature\">")
|
||||||
|
.skip(1)
|
||||||
|
.map(|v| v.to_string())
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
let futures = raw_videos
|
let futures = raw_videos
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|el| self.get_video_item(el.to_string(), requester.clone()));
|
.map(|el| self.get_video_item(el, requester.clone()));
|
||||||
let results: Vec<Result<VideoItem>> = join_all(futures).await;
|
|
||||||
let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
|
join_all(futures)
|
||||||
return video_items;
|
.await
|
||||||
|
.into_iter()
|
||||||
|
.inspect(|r| {
|
||||||
|
if let Err(e) = r {
|
||||||
|
let msg = e.to_string();
|
||||||
|
let chain = format_error_chain(e);
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let _ = send_discord_error_report(
|
||||||
|
msg,
|
||||||
|
Some(chain),
|
||||||
|
Some("Hqporner Provider"),
|
||||||
|
None,
|
||||||
|
file!(),
|
||||||
|
line!(),
|
||||||
|
module_path!(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_video_item(
|
async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
|
||||||
&self,
|
let video_url = format!(
|
||||||
video_segment: String,
|
|
||||||
mut requester: Requester,
|
|
||||||
) -> Result<VideoItem> {
|
|
||||||
let video_url: String = format!(
|
|
||||||
"{}{}",
|
"{}{}",
|
||||||
self.url,
|
self.url,
|
||||||
video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
|
seg.split("<a href=\"")
|
||||||
.split("\"")
|
.nth(1)
|
||||||
.collect::<Vec<&str>>()[0]
|
.and_then(|s| s.split('"').next())
|
||||||
.to_string()
|
.ok_or_else(|| ErrorKind::Parse(format!("url \n{seg}").into()))?
|
||||||
);
|
);
|
||||||
let mut title = video_segment
|
let title_raw = seg
|
||||||
.split("<h3 class=\"meta-data-title\">")
|
.split("<h3 class=\"meta-data-title\">")
|
||||||
.collect::<Vec<&str>>()[1]
|
.nth(1)
|
||||||
.split(">")
|
.and_then(|s| s.split('>').nth(1))
|
||||||
.collect::<Vec<&str>>()[1]
|
.and_then(|s| s.split('<').next())
|
||||||
.split("<")
|
.ok_or_else(|| ErrorKind::Parse(format!("title \n{seg}").into()))?;
|
||||||
.collect::<Vec<&str>>()[0]
|
let title = decode(title_raw.as_bytes())
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
// html decode
|
|
||||||
title = decode(title.as_bytes())
|
|
||||||
.to_string()
|
.to_string()
|
||||||
.unwrap_or(title)
|
.unwrap_or_else(|_| title_raw.to_string())
|
||||||
.titlecase();
|
.titlecase();
|
||||||
let id = video_url.split("/").collect::<Vec<&str>>()[4]
|
|
||||||
.split(".")
|
|
||||||
.collect::<Vec<&str>>()[0]
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
|
let id = video_url
|
||||||
|
.split('/')
|
||||||
|
.nth(4)
|
||||||
|
.and_then(|s| s.split('.').next())
|
||||||
|
.ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))?
|
||||||
|
.to_string();
|
||||||
let thumb = format!(
|
let thumb = format!(
|
||||||
"https:{}",
|
"https:{}",
|
||||||
video_segment
|
seg.split("onmouseleave='defaultImage(\"")
|
||||||
.split("onmouseleave='defaultImage(\"")
|
.nth(1)
|
||||||
.collect::<Vec<&str>>()[1]
|
.and_then(|s| s.split('"').next())
|
||||||
.split("\"")
|
.ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?
|
||||||
.collect::<Vec<&str>>()[0]
|
|
||||||
.to_string()
|
|
||||||
);
|
);
|
||||||
let raw_duration = video_segment
|
let raw_duration = seg
|
||||||
.split("<span class=\"icon fa-clock-o meta-data\">")
|
.split("<span class=\"icon fa-clock-o meta-data\">")
|
||||||
.collect::<Vec<&str>>()[1]
|
.nth(1)
|
||||||
.split("s<")
|
.and_then(|s| s.split("s<").next())
|
||||||
.collect::<Vec<&str>>()[0]
|
.map(|s| s.replace("m ", ":"))
|
||||||
.replace("m ", ":")
|
.unwrap_or_default();
|
||||||
.to_string();
|
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
|
||||||
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
|
|
||||||
|
|
||||||
let (tags, formats) = match self.extract_media(&video_url, &mut requester).await{
|
let (tags, formats) = self.extract_media(&video_url, &mut requester).await?;
|
||||||
Ok((t, f)) => (t, f),
|
|
||||||
Err(_) => return Err(Error::from("Video media extraction failed")),
|
Ok(
|
||||||
};
|
VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration)
|
||||||
if formats.is_empty() {
|
|
||||||
return Err(Error::from("No formats found for video"));
|
|
||||||
}
|
|
||||||
let video_item = VideoItem::new(
|
|
||||||
id,
|
|
||||||
title,
|
|
||||||
video_url,
|
|
||||||
"hqporner".to_string(),
|
|
||||||
thumb,
|
|
||||||
duration,
|
|
||||||
)
|
|
||||||
.formats(formats)
|
.formats(formats)
|
||||||
.tags(tags);
|
.tags(tags),
|
||||||
return Ok(video_item);
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn extract_media(
|
async fn extract_media(
|
||||||
&self,
|
&self,
|
||||||
video_page_url: &str,
|
url: &str,
|
||||||
requester: &mut Requester,
|
requester: &mut Requester,
|
||||||
) -> Result<(Vec<String>, Vec<VideoFormat>)> {
|
) -> Result<(Vec<String>, Vec<VideoFormat>)> {
|
||||||
let mut formats = vec![];
|
let mut formats = vec![];
|
||||||
let mut tags = vec![];
|
let mut tags = vec![];
|
||||||
let text = requester.get_raw_with_headers(&video_page_url, vec![("Referer".to_string(), "https://hqporner.com".to_string())])
|
let resp = requester
|
||||||
.await.unwrap().text().await.unwrap();
|
.get_raw_with_headers(
|
||||||
|
url,
|
||||||
|
vec![("Referer".to_string(), "https://hqporner.com".into())],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
|
||||||
|
let text = resp
|
||||||
|
.text()
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;
|
||||||
|
|
||||||
if text.contains("Why do I see it?") {
|
if text.contains("Why do I see it?") {
|
||||||
return Ok((tags, formats));
|
return Ok((tags, formats));
|
||||||
}
|
}
|
||||||
let stars_elements = text.split("icon fa-star-o").collect::<Vec<&str>>()[1]
|
|
||||||
.split("</li>")
|
// Extract Stars & Tags
|
||||||
.collect::<Vec<&str>>()[0]
|
if let Some(stars_block) = text
|
||||||
.split("href=\"/actress/")
|
.split("icon fa-star-o")
|
||||||
.collect::<Vec<&str>>()[1..]
|
.nth(1)
|
||||||
.to_vec();
|
.and_then(|s| s.split("</li>").next())
|
||||||
for star_el in stars_elements {
|
{
|
||||||
let star_id = star_el.split("\"").collect::<Vec<&str>>()[0].to_string();
|
for star_el in stars_block.split("href=\"/actress/").skip(1) {
|
||||||
let star_name = star_el.split("\">").collect::<Vec<&str>>()[1]
|
let id = star_el.split('"').next().unwrap_or("").to_string();
|
||||||
.split("<")
|
let name = star_el
|
||||||
.collect::<Vec<&str>>()[0]
|
.split("\">")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split('<').next())
|
||||||
|
.unwrap_or("")
|
||||||
.to_string();
|
.to_string();
|
||||||
tags.push(star_name.clone());
|
if !name.is_empty() {
|
||||||
Self::push_unique(&self.stars, FilterOption {
|
tags.push(name.clone());
|
||||||
id: star_id,
|
Self::push_unique(&self.stars, FilterOption { id, title: name });
|
||||||
title: star_name.clone(),
|
}
|
||||||
});
|
|
||||||
}
|
}
|
||||||
let categories_elements = text.split("This video belongs to the following categories").collect::<Vec<&str>>()[1]
|
|
||||||
.split("</p>")
|
|
||||||
.collect::<Vec<&str>>()[0]
|
|
||||||
.split("href=\"/category/")
|
|
||||||
.collect::<Vec<&str>>()[1..]
|
|
||||||
.to_vec();
|
|
||||||
for categories_el in categories_elements {
|
|
||||||
let category_id = categories_el.split("\"").collect::<Vec<&str>>()[0].to_string();
|
|
||||||
let category_name = categories_el.split("\">").collect::<Vec<&str>>()[1]
|
|
||||||
.split("<")
|
|
||||||
.collect::<Vec<&str>>()[0].titlecase();
|
|
||||||
tags.push(category_name.clone());
|
|
||||||
Self::push_unique(&self.categories, FilterOption {
|
|
||||||
id: category_id,
|
|
||||||
title: category_name.clone(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let video_url = format!(
|
// Player / Video Extraction
|
||||||
|
let player_url = format!(
|
||||||
"https:{}",
|
"https:{}",
|
||||||
text.split("url: '/blocks/altplayer.php?i=")
|
text.split("url: '/blocks/altplayer.php?i=")
|
||||||
.collect::<Vec<&str>>()[1]
|
.nth(1)
|
||||||
.split("'")
|
.and_then(|s| s.split('\'').next())
|
||||||
.collect::<Vec<&str>>()[0]
|
.ok_or("No player link")?
|
||||||
);
|
);
|
||||||
|
|
||||||
let text2 = requester
|
let text2 = requester
|
||||||
.get_raw_with_headers(
|
.get_raw_with_headers(
|
||||||
&video_url,
|
&player_url,
|
||||||
vec![("Referer".to_string(), "https://hqporner.com/".to_string())],
|
vec![("Referer".to_string(), "https://hqporner.com/".into())],
|
||||||
)
|
)
|
||||||
.await
|
.await?
|
||||||
.unwrap()
|
|
||||||
.text()
|
.text()
|
||||||
.await
|
.await?;
|
||||||
.unwrap();
|
|
||||||
match text2.split("<video ").collect::<Vec<&str>>().len() > 2 {
|
let video_element = text2
|
||||||
false => return Err(Error::from("No video element found")),
|
.split("<video ")
|
||||||
true => (),
|
.nth(2)
|
||||||
}
|
.and_then(|s| s.split("</video>").next())
|
||||||
let video_element = text2.split("<video ").collect::<Vec<&str>>()[2]
|
.ok_or("No video element")?;
|
||||||
.split("</video>")
|
for source in video_element.split("<source ").skip(1) {
|
||||||
.collect::<Vec<&str>>()[0];
|
let title = source
|
||||||
let sources = video_element.split("<source ").collect::<Vec<&str>>()[1..].to_vec();
|
.split("title=\\\"")
|
||||||
for source in sources {
|
.nth(1)
|
||||||
let title = source.split("title=\\\"").collect::<Vec<&str>>()[1]
|
.and_then(|s| s.split("\\\"").next())
|
||||||
.split("\\\"")
|
.unwrap_or("")
|
||||||
.collect::<Vec<&str>>()[0]
|
|
||||||
.to_string();
|
.to_string();
|
||||||
let quality = title.split(" ").collect::<Vec<&str>>()[0].to_string();
|
let quality = title.split(' ').next().unwrap_or("HD").to_string();
|
||||||
let format = "mp4".to_string();
|
|
||||||
let media_url = format!(
|
let media_url = format!(
|
||||||
"https:{}",
|
"https:{}",
|
||||||
source.split("src=\\\"").collect::<Vec<&str>>()[1]
|
source
|
||||||
.split("\\\"")
|
.split("src=\\\"")
|
||||||
.collect::<Vec<&str>>()[0]
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("\\\"").next())
|
||||||
|
.unwrap_or("")
|
||||||
);
|
);
|
||||||
|
|
||||||
formats.push(
|
formats.push(
|
||||||
VideoFormat::new(media_url, quality, format)
|
VideoFormat::new(media_url, quality, "mp4".into())
|
||||||
.format_id(title.clone())
|
.format_id(title.clone())
|
||||||
.format_note(title.clone()),
|
.format_note(title),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok((tags, formats))
|
Ok((tags, formats))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -431,34 +440,26 @@ impl Provider for HqpornerProvider {
|
|||||||
async fn get_videos(
|
async fn get_videos(
|
||||||
&self,
|
&self,
|
||||||
cache: VideoCache,
|
cache: VideoCache,
|
||||||
pool: DbPool,
|
_pool: DbPool,
|
||||||
sort: String,
|
sort: String,
|
||||||
query: Option<String>,
|
query: Option<String>,
|
||||||
page: String,
|
page: String,
|
||||||
per_page: String,
|
_per_page: String,
|
||||||
options: ServerOptions,
|
options: ServerOptions,
|
||||||
) -> Vec<VideoItem> {
|
) -> Vec<VideoItem> {
|
||||||
let _ = per_page;
|
let page_num = page.parse::<u8>().unwrap_or(1);
|
||||||
let _ = pool;
|
let res = match query {
|
||||||
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
|
Some(q) => self.query(cache, page_num, &q, options).await,
|
||||||
Some(q) => {
|
None => self.get(cache, page_num, &sort, options).await,
|
||||||
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
match videos {
|
res.unwrap_or_else(|e| {
|
||||||
Ok(v) => v,
|
eprintln!("Hqporner error: {e}");
|
||||||
Err(e) => {
|
let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!());
|
||||||
println!("Error fetching videos: {}", e);
|
|
||||||
vec![]
|
vec![]
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
}
|
|
||||||
fn get_channel(&self, clientversion: ClientVersion) -> Option<crate::status::Channel> {
|
fn get_channel(&self, v: ClientVersion) -> Option<Channel> {
|
||||||
Some(self.build_channel(clientversion))
|
Some(self.build_channel(v))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -272,7 +272,7 @@ impl JavtifulProvider {
|
|||||||
.split(" href=\"")
|
.split(" href=\"")
|
||||||
.nth(1)
|
.nth(1)
|
||||||
.and_then(|s| s.split('"').next())
|
.and_then(|s| s.split('"').next())
|
||||||
.ok_or_else(|| ErrorKind::Parse("video url".into()))?
|
.ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))?
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
let mut title = seg
|
let mut title = seg
|
||||||
@@ -288,12 +288,12 @@ impl JavtifulProvider {
|
|||||||
.split('/')
|
.split('/')
|
||||||
.nth(5)
|
.nth(5)
|
||||||
.and_then(|s| s.split('.').next())
|
.and_then(|s| s.split('.').next())
|
||||||
.ok_or_else(|| ErrorKind::Parse("video id".into()))?
|
.ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))?
|
||||||
.to_string();
|
.to_string();
|
||||||
let thumb_block = seg
|
let thumb_block = seg
|
||||||
.split("<img ")
|
.split("<img ")
|
||||||
.nth(1)
|
.nth(1)
|
||||||
.ok_or_else(|| ErrorKind::Parse("thumb block".into()))?;
|
.ok_or_else(|| ErrorKind::Parse("thumb block\n\n{seg}".into()))?;
|
||||||
|
|
||||||
let thumb = thumb_block
|
let thumb = thumb_block
|
||||||
.split("data-src=\"")
|
.split("data-src=\"")
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
|
|||||||
old_items = match cache.get(&index) {
|
old_items = match cache.get(&index) {
|
||||||
Some((time, items)) => {
|
Some((time, items)) => {
|
||||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
||||||
println!("Cache hit for URL: {}", url);
|
// println!("Cache hit for URL: {}", url);
|
||||||
return Ok(items.clone());
|
return Ok(items.clone());
|
||||||
} else {
|
} else {
|
||||||
items.clone()
|
items.clone()
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ impl Rule34videoProvider {
|
|||||||
old_items = match cache.get(&index) {
|
old_items = match cache.get(&index) {
|
||||||
Some((time, items)) => {
|
Some((time, items)) => {
|
||||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
||||||
println!("Cache hit for URL: {}", url);
|
// println!("Cache hit for URL: {}", url);
|
||||||
return Ok(items.clone());
|
return Ok(items.clone());
|
||||||
} else {
|
} else {
|
||||||
items.clone()
|
items.clone()
|
||||||
|
|||||||
Reference in New Issue
Block a user