simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

provider refactors and fixes

Browse Source
This commit is contained in:
Simon
2026-03-05 13:28:38 +00:00
parent 060d8e7937
commit 8157e223fe
33 changed files with 3051 additions and 1694 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
src/providers/rule34gen.rs
View File

@@ -2,14 +2,13 @@ use crate::api::*;
use crate::status::*;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use std::time::{SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
error_chain! {
@@ -82,11 +81,6 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards");
let timestamp_millis = now.as_millis(); // u128
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"];
let sort = if expected_sorts.contains(&sort) {
sort
@@ -95,8 +89,11 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
};
let index = format!("rule34gen:{}:{}", page, sort);
let url = format!("{}/?mode=async&function=get_block&block_id=custom_list_videos_most_recent_videos&tag_ids=&sort_by={}&from={}&_={}", self.url, sort, page, timestamp_millis);
let url = if page <= 1 {
format!("{}/?sort_by={}", self.url, sort)
} else {
format!("{}/{}/?sort_by={}", self.url, page, sort)
};
let mut old_items: Vec<VideoItem> = vec![];
if !(sort == "pseudo_random") {
@@ -114,8 +111,20 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
}
};
}
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"rule34gen",
"get.request",
&format!("url={url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
@@ -133,10 +142,6 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards");
let timestamp_millis = now.as_millis(); // u128
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"];
let sort = if expected_sorts.contains(&sort) {
sort
@@ -145,8 +150,12 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
};
let index = format!("rule34gen:{}:{}:{}", page, sort, query);
let url = format!("{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}", self.url, query.replace(" ","-"), sort, page, page, timestamp_millis);
let search_slug = query.replace(" ", "-");
let url = if page <= 1 {
format!("{}/search/{}/?sort_by={}", self.url, search_slug, sort)
} else {
format!("{}/search/{}/{}/?sort_by={}", self.url, search_slug, page, sort)
};
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&index) {
@@ -162,16 +171,28 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"rule34gen",
"query.request",
&format!("url={url}; error={e}"),
)
.await;
return Ok(old_items);
}
Ok(video_items)
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -180,13 +201,99 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos").collect::<Vec<&str>>()[1].split("<div class=\"pagination\"").collect::<Vec<&str>>()[0].to_string();
if html.contains("cards__item") {
for video_segment in html.split("<div class=\"cards__item\"").skip(1) {
let video_url = video_segment
.split("href=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
if !video_url.contains("/video/") {
continue;
}
let id = video_url
.split("/video/")
.nth(1)
.and_then(|s| s.split('/').next())
.unwrap_or_default()
.to_string();
if id.is_empty() {
continue;
}
let mut title = video_segment
.split("title=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
if title.is_empty() {
title = video_segment
.split("<span class=\"card__title\">")
.nth(1)
.and_then(|s| s.split('<').next())
.unwrap_or_default()
.trim()
.to_string();
}
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = video_segment
.split("data-original=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
let preview = video_segment
.split("data-preview=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("card__label card__label--primary\">")
.nth(1)
.and_then(|s| s.split('<').next())
.unwrap_or_default()
.trim()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views_text = video_segment
.split("<small class=\"card__text\">")
.nth(2)
.and_then(|s| s.split("</small>").next())
.unwrap_or_default()
.replace("views", "")
.replace(' ', "")
.trim()
.to_string();
let views = parse_abbreviated_number(views_text.as_str()).unwrap_or(0) as u32;
items.push(
VideoItem::new(
id,
title,
video_url,
"rule34gen".to_string(),
thumb,
duration,
)
.views(views)
.preview(preview),
);
}
return items;
}
let video_listing_content = html.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let raw_videos = video_listing_content
.split("<div class=\"item thumb video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
// let vid = video_segment.split("\n").collect::<Vec<&str>>()[1]
// let vid = video_segment.split("\n").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
@@ -195,35 +302,35 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
continue;
}
let mut title = video_segment.split("<div class=\"thumb_title\">").collect::<Vec<&str>>()[1]
let mut title = video_segment.split("<div class=\"thumb_title\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_segment.split("https://rule34gen.com/video/").collect::<Vec<&str>>()[1].split("/").collect::<Vec<&str>>()[0].to_string();
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>()[1]
let id = video_segment.split("https://rule34gen.com/video/").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("/").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views = parse_abbreviated_number(&video_segment
.split("<div class=\"views\">").collect::<Vec<&str>>()[1].split("</svg>").collect::<Vec<&str>>()[1]
.split("<div class=\"views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("</svg>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]).unwrap_or(0);
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()).unwrap_or(0);
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>()[1].split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let url = video_segment.split("<a class=\"th js-open-popup\" href=\"").collect::<Vec<&str>>()[1]
let url = video_segment.split("<a class=\"th js-open-popup\" href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>()[1]
// let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
// .split("\"")
// .collect::<Vec<&str>>()[0]
// .collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
// .to_string();