simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pimpbunny fix

Browse Source
This commit is contained in:
Simon
2026-03-22 12:27:46 +00:00
parent a2d31d90a1
commit 50ea0e73b7
13 changed files with 646 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
build.rs
View File

@@ -231,6 +231,14 @@ fn main() {
println!("cargo:rerun-if-env-changed=HOT_TUB_PROVIDER");
println!("cargo:rerun-if-env-changed=HOTTUB_PROVIDER");
println!("cargo:rustc-check-cfg=cfg(hottub_single_provider)");
let provider_cfg_values = PROVIDERS
.iter()
.map(|provider| format!("\"{}\"", provider.id))
.collect::<Vec<_>>()
.join(", ");
println!(
"cargo:rustc-check-cfg=cfg(hottub_provider, values({provider_cfg_values}))"
);
let selected = env::var("HOT_TUB_PROVIDER")
.or_else(|_| env::var("HOTTUB_PROVIDER"))
@@ -247,6 +255,7 @@ fn main() {
panic!("Unknown provider `{selected_id}` from HOT_TUB_PROVIDER/HOTTUB_PROVIDER")
});
println!("cargo:rustc-cfg=hottub_single_provider");
println!("cargo:rustc-cfg=hottub_provider=\"{selected_id}\"");
vec![provider]
}
None => PROVIDERS.iter().collect(),

7
src/api.rs
View File

@@ -146,6 +146,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
}
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
#[cfg(feature = "debug")]
let trace_id = crate::util::flow_debug::next_trace_id("status");
let clientversion: ClientVersion = match req.headers().get("User-Agent") {
Some(v) => match v.to_str() {
@@ -175,6 +176,7 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
.to_string();
let public_url_base = format!("{}://{}", req.connection_info().scheme(), host);
let mut status = Status::new();
#[cfg(feature = "debug")]
let mut channel_count = 0usize;
for (provider_name, provider) in ALL_PROVIDERS.iter() {
@@ -191,7 +193,10 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
if channel.favicon.starts_with('/') {
channel.favicon = format!("{}{}", public_url_base, channel.favicon);
}
#[cfg(feature = "debug")]
{
channel_count += 1;
}
crate::flow_debug!(
"trace={} status added channel id={} provider={}",
trace_id,
@@ -407,6 +412,7 @@ async fn videos_post(
}
if let Some(literal_query) = literal_query.as_deref() {
#[cfg(feature = "debug")]
let before = video_items.len();
video_items.retain(|video| video_matches_literal_query(video, literal_query));
crate::flow_debug!(
@@ -434,6 +440,7 @@ async fn videos_post(
let per_page_clone = perPage.to_string();
let options_clone = options.clone();
let channel_clone = channel.clone();
#[cfg(feature = "debug")]
let prefetch_trace_id = trace_id.clone();
task::spawn_local(async move {
crate::flow_debug!(

24
src/db.rs
View File

@@ -1,10 +1,17 @@
use crate::models::DBVideo;
use diesel::prelude::*;
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "hanime",
hottub_provider = "hentaihaven",
hottub_provider = "missav",
hottub_provider = "perverzija",
))]
pub fn get_video(
conn: &mut SqliteConnection,
video_id: String,
) -> Result<Option<String>, diesel::result::Error> {
use crate::models::DBVideo;
use crate::schema::videos::dsl::*;
let result = videos
.filter(id.eq(video_id))
@@ -16,11 +23,19 @@ pub fn get_video(
}
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "hanime",
hottub_provider = "hentaihaven",
hottub_provider = "missav",
hottub_provider = "perverzija",
))]
pub fn insert_video(
conn: &mut SqliteConnection,
new_id: &str,
new_url: &str,
) -> Result<usize, diesel::result::Error> {
use crate::models::DBVideo;
use crate::schema::videos::dsl::*;
diesel::insert_into(videos)
.values(DBVideo {
@@ -30,6 +45,13 @@ pub fn insert_video(
.execute(conn)
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "hanime",
hottub_provider = "hentaihaven",
hottub_provider = "missav",
hottub_provider = "perverzija",
))]
pub fn delete_video(
conn: &mut SqliteConnection,
video_id: String,

5
src/main.rs
View File

@@ -57,10 +57,11 @@ async fn main() -> std::io::Result<()> {
);
let mut requester = util::requester::Requester::new();
requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string());
let proxy_enabled = env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string();
requester.set_proxy(proxy_enabled);
crate::flow_debug!(
"requester initialized proxy_enabled={}",
requester.proxy_enabled()
proxy_enabled
);
let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new()

1
src/providers/all.rs
View File

@@ -28,7 +28,6 @@ error_chain! {
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct AllProvider {}
impl AllProvider {

26
src/providers/hanime.rs
View File

@@ -37,7 +37,6 @@ struct HanimeSearchRequest {
page: u8,
}
#[allow(dead_code)]
impl HanimeSearchRequest {
pub fn new() -> Self {
HanimeSearchRequest {
@@ -51,26 +50,10 @@ impl HanimeSearchRequest {
page: 0,
}
}
pub fn tags(mut self, tags: Vec<String>) -> Self {
self.tags = tags;
self
}
pub fn search_text(mut self, search_text: String) -> Self {
self.search_text = search_text;
self
}
pub fn tags_mode(mut self, tags_mode: String) -> Self {
self.tags_mode = tags_mode;
self
}
pub fn brands(mut self, brands: Vec<String>) -> Self {
self.brands = brands;
self
}
pub fn blacklist(mut self, blacklist: Vec<String>) -> Self {
self.blacklist = blacklist;
self
}
pub fn order_by(mut self, order_by: String) -> Self {
self.order_by = order_by;
self
@@ -120,16 +103,11 @@ struct HanimeSearchResult {
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct HanimeProvider {
url: String,
}
pub struct HanimeProvider;
impl HanimeProvider {
pub fn new() -> Self {
HanimeProvider {
url: "https://hanime.tv/".to_string(),
}
HanimeProvider
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {

25
src/providers/mod.rs
View File

@@ -230,6 +230,10 @@ fn channel_group_order(group_id: &str) -> usize {
pub fn decorate_channel(channel: Channel) -> ChannelView {
let metadata = channel_metadata_for(&channel.id);
let ytdlp_command = match channel.id.as_str() {
"pimpbunny" => Some("yt-dlp --compat-options allow-unsafe-ext".to_string()),
_ => None,
};
ChannelView {
id: channel.id,
name: channel.name,
@@ -250,6 +254,7 @@ pub fn decorate_channel(channel: Channel) -> ChannelView {
.map(|tag| (*tag).to_string())
.collect()
}),
ytdlpCommand: ytdlp_command,
cacheDuration: channel.cacheDuration,
}
}
@@ -413,6 +418,7 @@ mod tests {
let channel = decorate_channel(base_channel("hsex"));
assert_eq!(channel.groupKey.as_deref(), Some("chinese"));
assert_eq!(channel.sortOrder, None);
assert_eq!(channel.ytdlpCommand, None);
assert_eq!(
channel.tags.as_deref(),
Some(
@@ -438,6 +444,15 @@ mod tests {
assert_eq!(groups[2].id, "jav");
}
#[test]
fn decorates_pimpbunny_with_ytdlp_command() {
let channel = decorate_channel(base_channel("pimpbunny"));
assert_eq!(
channel.ytdlpCommand.as_deref(),
Some("yt-dlp --compat-options allow-unsafe-ext")
);
}
#[test]
fn reflects_updated_group_moves() {
assert_eq!(
@@ -461,6 +476,7 @@ mod tests {
base_channel("missav"),
base_channel("hsex"),
base_channel("all"),
base_channel("pimpbunny"),
];
let json = serde_json::to_value(build_status_response(status)).expect("valid status json");
@@ -487,5 +503,14 @@ mod tests {
.find(|group| group["id"] == "chinese")
.expect("chinese group present");
assert_eq!(chinese_group["systemImage"], "globe");
let pimpbunny_channel = channels
.iter()
.find(|channel| channel["id"] == "pimpbunny")
.expect("pimpbunny channel present");
assert_eq!(
pimpbunny_channel["ytdlpCommand"],
"yt-dlp --compat-options allow-unsafe-ext"
);
}
}

374
src/providers/pimpbunny.rs
View File

@@ -49,8 +49,7 @@ impl PimpbunnyProvider {
const FIREFOX_USER_AGENT: &'static str =
"Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0";
const HTML_ACCEPT: &'static str =
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
pub fn new() -> Self {
let provider = Self {
url: "https://pimpbunny.com".to_string(),
@@ -235,39 +234,218 @@ impl PimpbunnyProvider {
format!("{}/", self.url.trim_end_matches('/'))
}
fn root_headers(&self) -> Vec<(String, String)> {
Self::html_headers_with_referer(&self.root_referer())
fn sort_by(sort: &str) -> &'static str {
match sort {
"best rated" => "rating",
"most viewed" => "video_viewed",
_ => "post_date",
}
}
fn html_headers_with_referer(referer: &str) -> Vec<(String, String)> {
vec![
("Referer".to_string(), referer.to_string()),
fn build_search_path_query(query: &str, separator: &str) -> String {
query.split_whitespace().collect::<Vec<_>>().join(separator)
}
fn append_archive_query(url: String, sort: &str) -> String {
let separator = if url.contains('?') { '&' } else { '?' };
format!("{url}{separator}sort_by={}", Self::sort_by(sort))
}
fn page_family_referer(&self, request_url: &str) -> String {
let Some(url) = Url::parse(request_url).ok() else {
return self.root_referer();
};
let path = url.path();
let referer_path = if path.starts_with("/videos/") {
"/videos/".to_string()
} else if path.starts_with("/search/") {
let parts: Vec<_> = path.trim_matches('/').split('/').collect();
if parts.len() >= 2 {
format!("/search/{}/", parts[1])
} else {
"/search/".to_string()
}
} else if path.starts_with("/categories/") {
let parts: Vec<_> = path.trim_matches('/').split('/').collect();
if parts.len() >= 2 {
format!("/categories/{}/", parts[1])
} else {
"/categories/".to_string()
}
} else if path.starts_with("/onlyfans-models/") {
let parts: Vec<_> = path.trim_matches('/').split('/').collect();
if parts.len() >= 2 {
format!("/onlyfans-models/{}/", parts[1])
} else {
"/onlyfans-models/".to_string()
}
} else {
"/".to_string()
};
format!("{}{}", self.url.trim_end_matches('/'), referer_path)
}
fn build_browse_url(&self, page: u8, sort: &str) -> String {
let base = if page <= 1 {
format!("{}/videos/", self.url)
} else {
format!("{}/videos/{page}/", self.url)
};
Self::append_archive_query(base, sort)
}
fn build_search_url(&self, query: &str, page: u8, sort: &str) -> String {
let path_query = Self::build_search_path_query(query, "-");
let base = if page <= 1 {
format!("{}/search/{path_query}/", self.url)
} else {
format!("{}/search/{path_query}/{page}/", self.url)
};
Self::append_archive_query(base, sort)
}
fn build_common_archive_url(&self, archive_path: &str, page: u8, sort: &str) -> String {
let canonical = format!(
"{}/{}",
self.url.trim_end_matches('/'),
archive_path.trim_start_matches('/')
);
let base = if page <= 1 {
canonical
} else {
format!("{}/{}", canonical.trim_end_matches('/'), page)
};
let base = if base.ends_with('/') {
base
} else {
format!("{base}/")
};
Self::append_archive_query(base, sort)
}
fn navigation_headers(
referer: Option<&str>,
sec_fetch_site: &'static str,
) -> Vec<(String, String)> {
let mut headers = vec![
(
"User-Agent".to_string(),
Self::FIREFOX_USER_AGENT.to_string(),
),
("Accept".to_string(), Self::HTML_ACCEPT.to_string()),
("Accept-Language".to_string(), "en-US,en;q=0.9".to_string()),
]
("Cache-Control".to_string(), "no-cache".to_string()),
("Pragma".to_string(), "no-cache".to_string()),
("Priority".to_string(), "u=0, i".to_string()),
("Connection".to_string(), "keep-alive".to_string()),
("TE".to_string(), "trailers".to_string()),
("Sec-Fetch-Dest".to_string(), "document".to_string()),
("Sec-Fetch-Mode".to_string(), "navigate".to_string()),
("Sec-Fetch-Site".to_string(), sec_fetch_site.to_string()),
("Sec-Fetch-User".to_string(), "?1".to_string()),
("Upgrade-Insecure-Requests".to_string(), "1".to_string()),
];
if let Some(referer) = referer {
headers.push(("Referer".to_string(), referer.to_string()));
}
headers
}
fn headers_with_cookies(
&self,
requester: &Requester,
request_url: &str,
referer: &str,
referer: Option<&str>,
sec_fetch_site: &'static str,
) -> Vec<(String, String)> {
let mut headers = Self::html_headers_with_referer(referer);
let mut headers = Self::navigation_headers(referer, sec_fetch_site);
if let Some(cookie) = requester.cookie_header_for_url(request_url) {
headers.push(("Cookie".to_string(), cookie));
}
headers
}
fn is_cloudflare_challenge(html: &str) -> bool {
html.contains("cf-turnstile-response")
|| html.contains("Performing security verification")
|| html.contains("__cf_chl_rt_tk")
|| html.contains("cUPMDTk:\"")
|| html.contains("Just a moment...")
}
fn extract_challenge_path(html: &str) -> Option<String> {
html.split("cUPMDTk:\"")
.nth(1)
.and_then(|s| s.split('"').next())
.map(str::to_string)
.or_else(|| {
html.split("__cf_chl_rt_tk=")
.nth(1)
.and_then(|s| s.split('"').next())
.map(|token| format!("/?__cf_chl_rt_tk={token}"))
})
}
fn absolute_site_url(&self, path_or_url: &str) -> String {
if path_or_url.starts_with("http://") || path_or_url.starts_with("https://") {
path_or_url.to_string()
} else {
format!(
"{}/{}",
self.url.trim_end_matches('/'),
path_or_url.trim_start_matches('/')
)
}
}
async fn fetch_html(
&self,
requester: &mut Requester,
request_url: &str,
referer: Option<&str>,
sec_fetch_site: &'static str,
) -> Result<String> {
let headers = self.headers_with_cookies(requester, request_url, referer, sec_fetch_site);
let response = requester
.get_raw_with_headers(request_url, headers.clone())
.await
.map_err(Error::from)?;
let status = response.status();
let body = response.text().await.map_err(Error::from)?;
if status.is_success() || status.as_u16() == 404 {
return Ok(body);
}
if status.as_u16() == 403 && Self::is_cloudflare_challenge(&body) {
if let Some(challenge_path) = Self::extract_challenge_path(&body) {
let challenge_url = self.absolute_site_url(&challenge_path);
let challenge_headers = self.headers_with_cookies(
requester,
&challenge_url,
Some(request_url),
"same-origin",
);
let _ = requester
.get_raw_with_headers(&challenge_url, challenge_headers)
.await;
}
}
let retry_headers =
self.headers_with_cookies(requester, request_url, referer, sec_fetch_site);
requester
.get_with_headers(request_url, retry_headers, Some(Version::HTTP_11))
.await
.map_err(|e| Error::from(format!("{e}")))
}
async fn warm_root_session(&self, requester: &mut Requester) {
let root_url = self.root_referer();
let _ = requester
.get_with_headers(&root_url, self.root_headers(), Some(Version::HTTP_11))
let _ = self
.fetch_html(requester, &root_url, None, "none")
.await;
}
@@ -276,7 +454,7 @@ impl PimpbunnyProvider {
let _ = requester
.get_with_headers(
&root_url,
Self::html_headers_with_referer(&root_url),
Self::navigation_headers(None, "none"),
Some(Version::HTTP_11),
)
.await;
@@ -288,7 +466,7 @@ impl PimpbunnyProvider {
let request_url = format!("{base}/onlyfans-models/?models_per_page=20");
let headers = {
let root_url = format!("{}/", base.trim_end_matches('/'));
let mut headers = Self::html_headers_with_referer(&root_url);
let mut headers = Self::navigation_headers(Some(&root_url), "same-origin");
if let Some(cookie) = requester.cookie_header_for_url(&request_url) {
headers.push(("Cookie".to_string(), cookie));
}
@@ -343,7 +521,7 @@ impl PimpbunnyProvider {
let request_url = format!("{base}/categories/?items_per_page=120");
let headers = {
let root_url = format!("{}/", base.trim_end_matches('/'));
let mut headers = Self::html_headers_with_referer(&root_url);
let mut headers = Self::navigation_headers(Some(&root_url), "same-origin");
if let Some(cookie) = requester.cookie_header_for_url(&request_url) {
headers.push(("Cookie".to_string(), cookie));
}
@@ -393,15 +571,7 @@ impl PimpbunnyProvider {
sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"best rated" => "&sort_by=rating",
"most viewed" => "&sort_by=video_viewed",
_ => "&sort_by=post_date",
};
let video_url = format!(
"{}/videos/{}/?videos_per_page=32{}",
self.url, page, sort_string
);
let video_url = self.build_browse_url(page, sort);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -417,9 +587,14 @@ impl PimpbunnyProvider {
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
self.warm_root_session(&mut requester).await;
let headers = self.headers_with_cookies(&requester, &video_url, &self.root_referer());
let text = match requester
.get_with_headers(&video_url, headers, Some(Version::HTTP_11))
let referer = self.page_family_referer(&video_url);
let text = match self
.fetch_html(
&mut requester,
&video_url,
Some(&referer),
"same-origin",
)
.await
{
Ok(text) => text,
@@ -451,27 +626,17 @@ impl PimpbunnyProvider {
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.trim().to_string();
let mut video_url = format!(
"{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&videos_per_page=32&from_videos={}",
self.url,
search_string.replace(" ", "-"),
page
);
let sort_string = match options.sort.as_deref().unwrap_or("") {
"best rated" => "&sort_by=rating",
"most viewed" => "&sort_by=video_viewed",
_ => "&sort_by=post_date",
};
let sort = options.sort.as_deref().unwrap_or("");
let mut video_url = self.build_search_url(&search_string, page, sort);
if let Ok(stars) = self.stars.read() {
if let Some(star) = stars
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/onlyfans-models/{}/{}/?videos_per_page=20{}",
self.url, star.id, page, sort_string
video_url = self.build_common_archive_url(
&format!("/onlyfans-models/{}/", star.id),
page,
sort,
);
}
} else {
@@ -486,10 +651,8 @@ impl PimpbunnyProvider {
.iter()
.find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/categories/{}/{}/?videos_per_page=20{}",
self.url, cat.id, page, sort_string
);
video_url =
self.build_common_archive_url(&format!("/categories/{}/", cat.id), page, sort);
}
} else {
crate::providers::report_provider_error_background(
@@ -516,10 +679,14 @@ impl PimpbunnyProvider {
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
self.warm_root_session(&mut requester).await;
println!("Fetching URL: {}", video_url);
let headers = self.headers_with_cookies(&requester, &video_url, &self.root_referer());
let text = match requester
.get_with_headers(&video_url, headers, Some(Version::HTTP_2))
let referer = self.page_family_referer(&video_url);
let text = match self
.fetch_html(
&mut requester,
&video_url,
Some(&referer),
"same-origin",
)
.await
{
Ok(text) => text,
@@ -710,9 +877,17 @@ mod tests {
use crate::videos::ServerOptions;
use std::sync::{Arc, RwLock};
fn test_provider() -> PimpbunnyProvider {
PimpbunnyProvider {
url: "https://pimpbunny.com".to_string(),
stars: Arc::new(RwLock::new(vec![])),
categories: Arc::new(RwLock::new(vec![])),
}
}
#[test]
fn rewrites_allowed_thumbs_to_proxy_urls() {
let provider = PimpbunnyProvider::new();
let provider = test_provider();
let options = ServerOptions {
featured: None,
category: None,
@@ -742,7 +917,7 @@ mod tests {
#[test]
fn rewrites_video_pages_to_redirect_proxy() {
let provider = PimpbunnyProvider::new();
let provider = test_provider();
let options = ServerOptions {
featured: None,
category: None,
@@ -772,11 +947,7 @@ mod tests {
#[test]
fn parses_listing_without_detail_requests() {
let provider = PimpbunnyProvider {
url: "https://pimpbunny.com".to_string(),
stars: Arc::new(RwLock::new(vec![])),
categories: Arc::new(RwLock::new(vec![])),
};
let provider = test_provider();
let options = ServerOptions {
featured: None,
category: None,
@@ -820,4 +991,91 @@ mod tests {
assert_eq!(items[0].views, Some(1200));
assert_eq!(items[0].formats.as_ref().map(|f| f.len()), Some(1));
}
#[test]
fn extracts_cloudflare_challenge_path() {
let html = r#"
<script type="text/javascript">
(function(){
window._cf_chl_opt = {
cUPMDTk:"/?mode=async&function=get_block&block_id=videos_videos_list&videos_per_page=8&sort_by=post_date&from=1&__cf_chl_tk=test-token"
};
}());
</script>
"#;
assert!(PimpbunnyProvider::is_cloudflare_challenge(html));
assert_eq!(
PimpbunnyProvider::extract_challenge_path(html).as_deref(),
Some(
"/?mode=async&function=get_block&block_id=videos_videos_list&videos_per_page=8&sort_by=post_date&from=1&__cf_chl_tk=test-token"
)
);
}
#[test]
fn builds_async_browse_url_instead_of_numbered_videos_path() {
let provider = test_provider();
assert_eq!(
provider.build_browse_url(1, "most recent"),
"https://pimpbunny.com/videos/?sort_by=post_date"
);
assert_eq!(
provider.build_browse_url(2, "most recent"),
"https://pimpbunny.com/videos/2/?sort_by=post_date"
);
}
#[test]
fn builds_search_url_with_query_and_pagination() {
let provider = test_provider();
assert_eq!(
provider.build_search_url("adriana chechik", 1, "most viewed"),
"https://pimpbunny.com/search/adriana-chechik/?sort_by=video_viewed"
);
assert_eq!(
provider.build_search_url("adriana chechik", 3, "most viewed"),
"https://pimpbunny.com/search/adriana-chechik/3/?sort_by=video_viewed"
);
}
#[test]
fn builds_common_archive_url_with_async_block() {
let provider = test_provider();
assert_eq!(
provider.build_common_archive_url("/categories/amateur/", 1, "best rated"),
"https://pimpbunny.com/categories/amateur/?sort_by=rating"
);
assert_eq!(
provider.build_common_archive_url("/categories/amateur/", 4, "best rated"),
"https://pimpbunny.com/categories/amateur/4/?sort_by=rating"
);
}
#[test]
fn derives_page_family_referer() {
let provider = test_provider();
assert_eq!(
provider.page_family_referer("https://pimpbunny.com/videos/2/?sort_by=post_date"),
"https://pimpbunny.com/videos/"
);
assert_eq!(
provider.page_family_referer(
"https://pimpbunny.com/categories/blowjob/2/?sort_by=post_date"
),
"https://pimpbunny.com/categories/blowjob/"
);
assert_eq!(
provider.page_family_referer(
"https://pimpbunny.com/search/adriana-chechik/3/?sort_by=video_viewed"
),
"https://pimpbunny.com/search/adriana-chechik/"
);
assert_eq!(
provider.page_family_referer(
"https://pimpbunny.com/onlyfans-models/momoitenshi/3/?sort_by=post_date"
),
"https://pimpbunny.com/onlyfans-models/momoitenshi/"
);
}
}

15
src/status.rs
View File

@@ -118,22 +118,9 @@ impl Status {
.to_string(),
}
}
#[allow(dead_code)]
pub fn add_notice(&mut self, notice: Notice) {
self.notices.push(notice);
}
#[allow(dead_code)]
pub fn add_channel(&mut self, channel: Channel) {
self.channels.push(channel);
}
#[allow(dead_code)]
pub fn add_option(&mut self, option: Options) {
self.options.push(option);
}
#[allow(dead_code)]
pub fn add_category(&mut self, category: String) {
self.categories.push(category);
}
}
#[derive(serde::Serialize)]
@@ -154,6 +141,8 @@ pub struct ChannelView {
#[serde(skip_serializing_if = "Option::is_none")]
pub tags: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ytdlpCommand: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cacheDuration: Option<u32>,
}

4
src/util/flow_debug.rs
View File

@@ -1,4 +1,5 @@
use std::sync::atomic::{AtomicU64, Ordering};
#[cfg(feature = "debug")]
use std::time::{SystemTime, UNIX_EPOCH};
static NEXT_TRACE_ID: AtomicU64 = AtomicU64::new(1);
@@ -17,9 +18,6 @@ pub fn emit(module: &str, line: u32, message: String) {
eprintln!("[debug][{millis}][{module}:{line}] {message}");
}
#[cfg(not(feature = "debug"))]
pub fn emit(_module: &str, _line: u32, _message: String) {}
pub fn preview(value: &str, limit: usize) -> String {
if value.len() <= limit {
return value.to_string();

1
src/util/mod.rs
View File

@@ -27,6 +27,7 @@ pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
.map(|n| (n * multiplier) as u32)
}
#[cfg(not(hottub_single_provider))]
pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {
let mut result = Vec::new();

141
src/util/requester.rs
View File

@@ -1,14 +1,14 @@
use serde::Serialize;
use std::env;
use std::fmt;
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
use wreq::Client;
use wreq::Proxy;
use wreq::Response;
use wreq::Uri;
use wreq::Version;
use wreq::cookie::{CookieStore, Cookies, Jar};
use wreq::header::{HeaderMap, HeaderValue, USER_AGENT};
use wreq::header::{HeaderMap, HeaderValue, SET_COOKIE, USER_AGENT};
use wreq::multipart::Form;
use wreq::redirect::Policy;
use wreq_util::Emulation;
@@ -45,6 +45,79 @@ impl fmt::Debug for Requester {
}
impl Requester {
fn shared_cookie_jar() -> Arc<Jar> {
static SHARED_COOKIE_JAR: OnceLock<Arc<Jar>> = OnceLock::new();
SHARED_COOKIE_JAR
.get_or_init(|| Arc::new(Jar::default()))
.clone()
}
fn origin_url_for_cookie_scope(url: &str) -> Option<url::Url> {
let parsed = url::Url::parse(url).ok()?;
let host = parsed.host_str()?;
let scheme = parsed.scheme();
url::Url::parse(&format!("{scheme}://{host}/")).ok()
}
fn store_response_cookies(&self, url: &str, response: &Response) {
let Some(origin) = Self::origin_url_for_cookie_scope(url) else {
return;
};
for value in response.headers().get_all(SET_COOKIE).iter() {
if let Ok(cookie) = value.to_str() {
self.cookie_jar.add_cookie_str(cookie, &origin.to_string());
}
}
}
fn store_flaresolverr_cookies(
&mut self,
request_url: &str,
cookies: &[crate::util::flaresolverr::FlaresolverrCookie],
) {
let fallback_origin = Self::origin_url_for_cookie_scope(request_url);
for cookie in cookies {
let origin = if !cookie.domain.is_empty() {
let scheme = fallback_origin
.as_ref()
.map(|url| url.scheme())
.unwrap_or("https");
let host = cookie.domain.trim_start_matches('.');
url::Url::parse(&format!("{scheme}://{host}/"))
.ok()
.or_else(|| fallback_origin.clone())
} else {
fallback_origin.clone()
};
let Some(origin) = origin else {
continue;
};
let mut cookie_string =
format!("{}={}; Path={}", cookie.name, cookie.value, cookie.path);
if !cookie.domain.is_empty() {
cookie_string.push_str(&format!("; Domain={}", cookie.domain));
}
if cookie.secure {
cookie_string.push_str("; Secure");
}
if cookie.httpOnly {
cookie_string.push_str("; HttpOnly");
}
if let Some(same_site) = cookie.sameSite.as_deref() {
if !same_site.is_empty() {
cookie_string.push_str(&format!("; SameSite={same_site}"));
}
}
self.cookie_jar
.add_cookie_str(&cookie_string, &origin.to_string());
}
}
fn debug_cookie_preview_from_owned_headers(
&self,
url: &str,
@@ -62,6 +135,7 @@ impl Requester {
.unwrap_or_else(|| "none".to_string())
}
#[cfg(any(not(hottub_single_provider), hottub_provider = "hypnotube"))]
fn debug_cookie_preview_from_borrowed_headers(
&self,
url: &str,
@@ -98,7 +172,7 @@ impl Requester {
}
pub fn new() -> Self {
let cookie_jar = Arc::new(Jar::default());
let cookie_jar = Self::shared_cookie_jar();
let client = Self::build_client(cookie_jar.clone(), None);
let requester = Requester {
@@ -122,14 +196,11 @@ impl Requester {
self.proxy = proxy;
}
pub fn proxy_enabled(&self) -> bool {
self.proxy
}
pub fn set_debug_trace_id(&mut self, debug_trace_id: Option<String>) {
self.debug_trace_id = debug_trace_id;
}
#[cfg(feature = "debug")]
pub fn debug_trace_id(&self) -> Option<&str> {
self.debug_trace_id.as_deref()
}
@@ -176,7 +247,9 @@ impl Requester {
}
}
request.send().await
let response = request.send().await?;
self.store_response_cookies(url, &response);
Ok(response)
}
pub async fn get_raw_with_headers(
@@ -209,7 +282,9 @@ impl Requester {
for (key, value) in headers.iter() {
request = request.header(key, value);
}
request.send().await
let response = request.send().await?;
self.store_response_cookies(url, &response);
Ok(response)
}
pub async fn post_json<S>(
@@ -246,9 +321,12 @@ impl Requester {
}
}
request.send().await
let response = request.send().await?;
self.store_response_cookies(url, &response);
Ok(response)
}
#[cfg(any(not(hottub_single_provider), hottub_provider = "hypnotube"))]
pub async fn post(
&mut self,
url: &str,
@@ -285,7 +363,9 @@ impl Requester {
}
}
request.send().await
let response = request.send().await?;
self.store_response_cookies(url, &response);
Ok(response)
}
pub async fn post_multipart(
@@ -325,7 +405,9 @@ impl Requester {
}
}
request.send().await
let response = request.send().await?;
self.store_response_cookies(url, &response);
Ok(response)
}
pub async fn get(
@@ -370,6 +452,7 @@ impl Requester {
}
}
let response = request.send().await?;
self.store_response_cookies(url, &response);
crate::flow_debug!(
"trace={} requester direct response url={} status={}",
self.debug_trace_id().unwrap_or("none"),
@@ -424,17 +507,9 @@ impl Requester {
.map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?;
// Rebuild client and apply UA/cookies from FlareSolverr
let cookie_origin = url.split('/').take(3).collect::<Vec<&str>>().join("/");
let useragent = res.solution.userAgent;
self.user_agent = Some(useragent);
if url::Url::parse(&cookie_origin).is_ok() {
for cookie in res.solution.cookies {
self.cookie_jar
.add_cookie_str(&format!("{}={}", cookie.name, cookie.value), &cookie_origin);
}
}
self.store_flaresolverr_cookies(url, &res.solution.cookies);
self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
crate::flow_debug!(
@@ -457,6 +532,7 @@ impl Requester {
}
let response = request.send().await?;
self.store_response_cookies(url, &response);
crate::flow_debug!(
"trace={} requester retry response url={} status={}",
self.debug_trace_id().unwrap_or("none"),
@@ -476,3 +552,26 @@ impl Requester {
Ok(res.solution.response)
}
}
#[cfg(test)]
mod tests {
use super::Requester;
#[test]
fn new_requesters_share_cookie_jar() {
let a = Requester::new();
let b = Requester::new();
let origin = "https://shared-cookie-requester-test.invalid/";
a.cookie_jar.add_cookie_str(
"shared_cookie=1; Path=/; SameSite=Lax",
origin,
);
let cookie_header = b
.cookie_header_for_url("https://shared-cookie-requester-test.invalid/path")
.unwrap_or_default();
assert!(cookie_header.contains("shared_cookie=1"));
}
}

152
src/videos.rs
View File

@@ -114,7 +114,6 @@ pub struct VideoItem {
#[serde(skip_serializing_if = "Option::is_none")]
pub aspectRatio: Option<f32>,
}
#[allow(dead_code)]
impl VideoItem {
pub fn new(
id: String,
@@ -145,9 +144,11 @@ impl VideoItem {
aspectRatio: None,
}
}
#[cfg(any(not(hottub_single_provider), hottub_provider = "hentaihaven"))]
pub fn from(s: String) -> Result<Self, serde_json::Error> {
serde_json::from_str::<VideoItem>(&s)
}
#[cfg(any(not(hottub_single_provider), hottub_provider = "hanime"))]
pub fn tags(mut self, tags: Vec<String>) -> Self {
if tags.is_empty() {
return self;
@@ -155,30 +156,113 @@ impl VideoItem {
self.tags = Some(tags);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "hanime",
hottub_provider = "heavyfetish",
hottub_provider = "porndish",
hottub_provider = "shooshtime",
hottub_provider = "spankbang",
hottub_provider = "chaturbate",
hottub_provider = "porn4fans",
hottub_provider = "xfree",
hottub_provider = "pornhub",
))]
pub fn uploader(mut self, uploader: String) -> Self {
self.uploader = Some(uploader);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "heavyfetish",
hottub_provider = "porndish",
hottub_provider = "shooshtime",
hottub_provider = "spankbang",
hottub_provider = "chaturbate",
))]
pub fn uploader_url(mut self, uploader_url: String) -> Self {
self.uploaderUrl = Some(uploader_url);
self
}
pub fn verified(mut self, verified: bool) -> Self {
self.verified = Some(verified);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "beeg",
hottub_provider = "chaturbate",
hottub_provider = "freepornvideosxxx",
hottub_provider = "hanime",
hottub_provider = "heavyfetish",
hottub_provider = "hentaihaven",
hottub_provider = "hypnotube",
hottub_provider = "javtiful",
hottub_provider = "noodlemagazine",
hottub_provider = "okxxx",
hottub_provider = "omgxxx",
hottub_provider = "perfectgirls",
hottub_provider = "pimpbunny",
hottub_provider = "pmvhaven",
hottub_provider = "porn00",
hottub_provider = "porn4fans",
hottub_provider = "porndish",
hottub_provider = "pornhat",
hottub_provider = "pornhub",
hottub_provider = "redtube",
hottub_provider = "rule34gen",
hottub_provider = "rule34video",
hottub_provider = "shooshtime",
hottub_provider = "spankbang",
hottub_provider = "sxyprn",
hottub_provider = "tnaflix",
hottub_provider = "tokyomotion",
hottub_provider = "viralxxxporn",
hottub_provider = "xfree",
hottub_provider = "xxthots",
hottub_provider = "yesporn",
hottub_provider = "youjizz",
))]
pub fn views(mut self, views: u32) -> Self {
self.views = Some(views);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "beeg",
hottub_provider = "hanime",
hottub_provider = "heavyfetish",
hottub_provider = "hsex",
hottub_provider = "porn4fans",
hottub_provider = "shooshtime",
hottub_provider = "spankbang",
hottub_provider = "tokyomotion",
hottub_provider = "vrporn",
hottub_provider = "yesporn",
))]
pub fn rating(mut self, rating: f32) -> Self {
self.rating = Some(rating);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "porndish",
hottub_provider = "shooshtime",
hottub_provider = "heavyfetish",
hottub_provider = "xfree",
))]
pub fn uploaded_at(mut self, uploaded_at: u64) -> Self {
self.uploadedAt = Some(uploaded_at);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "hanime",
hottub_provider = "heavyfetish",
hottub_provider = "hentaihaven",
hottub_provider = "hqporner",
hottub_provider = "javtiful",
hottub_provider = "noodlemagazine",
hottub_provider = "pimpbunny",
hottub_provider = "pmvhaven",
hottub_provider = "shooshtime",
))]
pub fn formats(mut self, formats: Vec<VideoFormat>) -> Self {
if formats.is_empty() {
return self;
@@ -186,27 +270,48 @@ impl VideoItem {
self.formats = Some(formats);
self
}
pub fn add_format(mut self, format: VideoFormat) {
if let Some(formats) = self.formats.as_mut() {
formats.push(format);
} else {
self.formats = Some(vec![format]);
}
}
pub fn embed(mut self, embed: VideoEmbed) -> Self {
self.embed = Some(embed);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "freepornvideosxxx",
hottub_provider = "heavyfetish",
hottub_provider = "homoxxx",
hottub_provider = "javtiful",
hottub_provider = "missav",
hottub_provider = "okxxx",
hottub_provider = "omgxxx",
hottub_provider = "perfectgirls",
hottub_provider = "pimpbunny",
hottub_provider = "pmvhaven",
hottub_provider = "pornhat",
hottub_provider = "redtube",
hottub_provider = "rule34gen",
hottub_provider = "shooshtime",
hottub_provider = "spankbang",
hottub_provider = "sxyprn",
hottub_provider = "tnaflix",
hottub_provider = "xfree",
hottub_provider = "xxdbx",
hottub_provider = "yesporn",
))]
pub fn preview(mut self, preview: String) -> Self {
self.preview = Some(preview);
self
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "hentaihaven",
hottub_provider = "hanime",
hottub_provider = "heavyfetish",
hottub_provider = "paradisehill",
hottub_provider = "xfree",
))]
pub fn aspect_ratio(mut self, aspect_ratio: f32) -> Self {
self.aspectRatio = Some(aspect_ratio);
self
}
#[cfg(any(not(hottub_single_provider), hottub_provider = "chaturbate"))]
pub fn is_live(mut self, is_live: bool) -> Self {
self.isLive = is_live;
self
@@ -294,6 +399,13 @@ impl VideoFormat {
http_headers: None,
}
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "vrporn",
hottub_provider = "perverzija",
hottub_provider = "porndish",
hottub_provider = "spankbang",
))]
pub fn add_http_header(&mut self, key: String, value: String) {
if self.http_headers.is_none() {
self.http_headers = Some(HashMap::new());
@@ -302,6 +414,14 @@ impl VideoFormat {
headers.insert(key, value);
}
}
#[cfg(any(
not(hottub_single_provider),
hottub_provider = "hentaihaven",
hottub_provider = "noodlemagazine",
hottub_provider = "shooshtime",
hottub_provider = "heavyfetish",
hottub_provider = "hsex",
))]
pub fn http_header(&mut self, key: String, value: String) -> Self {
if self.http_headers.is_none() {
self.http_headers = Some(HashMap::new());