From e37563e5bc781623465fb60ddaf07b29775af052 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 7 Sep 2026 08:57:43 +0000 Subject: [PATCH] hentaihaven fixes --- check.py | 4 + src/providers/hentaihaven.rs | 963 ++++++++++++++++------------------- 2 files changed, 433 insertions(+), 534 deletions(-) diff --git a/check.py b/check.py index f90ef70..1e21259 100644 --- a/check.py +++ b/check.py @@ -70,6 +70,10 @@ _CF_PROTECTED_HOSTS = { "www.camsoda.com", "camsoda.com", "assets.hotbunny.ai", + # Turnstile-gated: only the provider's wreq JA3 emulation gets a real page, + # plain requests/curl always see the "Just a moment" challenge. + "hentaihaven.xxx", + "www.hentaihaven.xxx", } diff --git a/src/providers/hentaihaven.rs b/src/providers/hentaihaven.rs index 4aae6b8..370fc9b 100644 --- a/src/providers/hentaihaven.rs +++ b/src/providers/hentaihaven.rs @@ -7,50 +7,28 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use crate::{DbPool, db}; use async_trait::async_trait; -use base64::{Engine as _, engine::general_purpose::STANDARD}; +use chrono::NaiveDateTime; use error_chain::error_chain; use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; +use regex::Regex; use serde::Deserialize; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, OnceLock, RwLock}; use std::vec; -use titlecase::Titlecase; use wreq::Version; -use wreq_util::Emulation; // How long a cached listing/search entry is considered usable at all. const HARD_TTL_SECS: u64 = 60 * 60 * 24; // Past this age we still answer instantly from cache/DB but trigger a -// background refresh so the next request gets fresh data / renewed signed URLs. +// background refresh so the next request gets fresh data. const SOFT_TTL_SECS: u64 = 60 * 60; -#[derive(Debug, Deserialize)] -struct PlayerSecureConfig { - en: String, - iv: String, - uri: String, -} - -#[derive(Debug, Deserialize)] -struct PlayerApiSource { - src: String, - #[serde(default)] - label: String, -} - -#[derive(Debug, Deserialize, Default)] -struct PlayerApiData { - #[serde(default)] - sources: Vec, -} - -#[derive(Debug, Deserialize)] -struct PlayerApiResponse { - status: bool, - #[serde(default)] - data: Option, -} +const SITE: &str = "https://hentaihaven.xxx"; +// Poster art lives on a separate image host; the catalogue only stores the path. +const IMG_BASE: &str = "https://img.hentaihaven.xxx/"; +// The catalogue API rejects the request outright above this page size. +const MAX_PER_PAGE: usize = 40; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { @@ -72,19 +50,70 @@ error_chain! { } } +#[derive(Debug, Deserialize, Default)] +struct ApiTitle { + #[serde(default)] + rendered: String, +} + +#[derive(Debug, Deserialize, Default)] +struct ApiMeta { + #[serde(default)] + vraven_remote_thumbnail: String, +} + +#[derive(Debug, Deserialize)] +struct ApiEntry { + id: u64, + #[serde(default)] + date: String, + slug: String, + #[serde(default)] + title: ApiTitle, + #[serde(default)] + meta: ApiMeta, +} + +#[derive(Debug, Deserialize, Default)] +struct ApiEngagement { + #[serde(default)] + rating: Option, + #[serde(default)] + views: Option, +} + +#[derive(Debug, Deserialize)] +struct ApiResponse { + #[serde(default)] + data: Vec, + #[serde(default)] + engagement: HashMap, +} + +/// Everything the catalogue API already tells us about a series, so resolving +/// an item only has to go looking for its episodes. +#[derive(Debug, Clone)] +struct ListingEntry { + id: u64, + slug: String, + url: String, + title: String, + thumb: String, + views: u32, + rating: f32, + uploaded_at: u64, +} + #[derive(Debug, Clone)] pub struct HentaihavenProvider { - url: String, categories: Arc>>, } impl HentaihavenProvider { pub fn new() -> Self { - let provider = Self { - url: "https://hentaihaven.xxx".to_string(), + Self { categories: Arc::new(RwLock::new(vec![])), - }; - provider + } } fn build_channel(&self, clientversion: ClientVersion) -> Channel { @@ -111,7 +140,7 @@ impl HentaihavenProvider { options: vec![], nsfw: true, cacheDuration: None, - ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()) + ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()), } } @@ -123,21 +152,46 @@ impl HentaihavenProvider { } } - async fn get( + /// Build the catalogue API URL backing a listing or search request. The URL + /// doubles as the cache key, so every parameter that changes the result set + /// has to be part of it. + fn api_url(page: u8, per_page: usize, sort: &str, query: Option<&str>) -> String { + let per_page = per_page.clamp(1, MAX_PER_PAGE); + let mut serializer = url::form_urlencoded::Serializer::new(String::new()); + serializer + .append_pair("per_page", &per_page.to_string()) + .append_pair("page", &page.max(1).to_string()); + if let Some(query) = query { + serializer.append_pair("search", query); + } else if matches!( + sort, + "views" | "popular" | "most-viewed" | "trending" | "hot" | "rating" + ) { + // The catalogue's ranked chart, which is what the site itself serves + // on /browse/trending/. + serializer + .append_pair("trending_period", "monthly") + .append_pair("live", "1"); + } + format!("{SITE}/api/manga/?{}", serializer.finish()) + } + + async fn list( &self, cache: VideoCache, page: u8, + per_page: usize, sort: &str, + query: Option<&str>, options: ServerOptions, pool: DbPool, ) -> Result> { - let _ = sort; - let video_url = format!("{}/hentai/page/{}/", self.url, page); + let api_url = Self::api_url(page, per_page, sort, query); // Fast path: a usable in-memory entry exists. Answer immediately; once it // is older than the soft TTL, kick a background refresh so the next caller // sees fresher data without anyone waiting on it now. - if let Some((time, items)) = cache.get(&video_url) { + if let Some((time, items)) = cache.get(&api_url) { let age = time.elapsed().unwrap_or_default().as_secs(); if age < HARD_TTL_SECS && !items.is_empty() { if age >= SOFT_TTL_SECS { @@ -146,227 +200,149 @@ impl HentaihavenProvider { module_path!(), "missing_requester", ); - self.spawn_refresh(requester, pool, cache, video_url, None, false); + self.spawn_refresh(requester, pool, cache, api_url, None); } return Ok(items); } } - // Fetch the listing page (a single cheap request) to learn which episode - // URLs belong on this page and in what order. + // One cheap JSON request gives us the whole page of series with titles, + // posters and engagement counts already attached. let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); - let text = match Self::get_with_retry(&mut requester, &video_url, 3).await { - Ok(text) => text, + let entries = match Self::fetch_listing(&mut requester, &api_url).await { + Ok(entries) => entries, Err(e) => { crate::providers::report_provider_error( "hentaihaven", - "get.request", - &format!("url={video_url}; error={e}"), + "list.request", + &format!("url={api_url}; error={e}"), ) .await; - return Ok(cache - .get(&video_url) - .map(|(_, items)| items) - .unwrap_or_default()); + return Ok(Self::cached(&cache, &api_url)); } }; - let urls = Self::parse_listing_urls(&text); - if urls.is_empty() { - return Ok(cache - .get(&video_url) - .map(|(_, items)| items) - .unwrap_or_default()); + if entries.is_empty() { + return Ok(Self::cached(&cache, &api_url)); } - // Serve whatever we have already resolved (from the DB) right away, then - // refresh the entire listing in the background. - let db_items = Self::items_from_db(&urls, &pool); - if !db_items.is_empty() { - cache.insert(video_url.clone(), db_items.clone()); - self.spawn_refresh(requester, pool, cache, video_url, Some(urls), false); + // Serve the page straight from the DB when it covers *every* series on it, + // then refresh the whole listing in the background. Partial coverage is + // not good enough: returning only the handful of already-resolved series + // would silently hand the client a short page. + let db_items = Self::items_from_db(&entries, &pool); + if db_items.len() == entries.len() { + cache.insert(api_url.clone(), db_items.clone()); + self.spawn_refresh(requester, pool, cache, api_url, Some(entries)); return Ok(db_items); } - // Cold start: nothing cached for any item yet, resolve synchronously this - // one time so the first ever request is not empty. - let items = self.resolve_urls(urls, &requester, pool).await; + // Cold start (or a page with newly published series): resolve now so the + // caller gets the full page. + let items = self.resolve_entries(entries, &requester, pool).await; if !items.is_empty() { - cache.insert(video_url.clone(), items.clone()); + cache.insert(api_url.clone(), items.clone()); return Ok(items); } - Ok(cache - .get(&video_url) - .map(|(_, items)| items) - .unwrap_or_default()) + Ok(Self::cached(&cache, &api_url)) } - async fn query( - &self, - cache: VideoCache, - page: u8, - query: &str, - options: ServerOptions, - pool: DbPool, - ) -> Result> { - if page > 1 { - return Ok(vec![]); - } - let video_url = format!("{}/?s={}", self.url, query.replace(" ", "+")); + fn cached(cache: &VideoCache, key: &str) -> Vec { + cache.get(key).map(|(_, items)| items).unwrap_or_default() + } - if let Some((time, items)) = cache.get(&video_url) { - let age = time.elapsed().unwrap_or_default().as_secs(); - if age < HARD_TTL_SECS && !items.is_empty() { - if age >= SOFT_TTL_SECS { - let requester = crate::providers::requester_or_default( - &options, - module_path!(), - "missing_requester", - ); - self.spawn_refresh(requester, pool, cache, video_url, None, true); + /// Fetch and flatten one page of the catalogue API. + async fn fetch_listing( + requester: &mut Requester, + api_url: &str, + ) -> std::result::Result, String> { + let body = Self::get_with_retry(requester, api_url, 3).await?; + let response: ApiResponse = serde_json::from_str(&body).map_err(|e| { + format!( + "failed to parse catalogue json: {e}; body={}", + body.chars().take(200).collect::() + ) + })?; + + Ok(response + .data + .into_iter() + .map(|entry| { + let engagement = response.engagement.get(&entry.id.to_string()); + let title = decode(entry.title.rendered.as_bytes()) + .to_string() + .unwrap_or_else(|_| entry.title.rendered.clone()); + ListingEntry { + id: entry.id, + url: format!("{SITE}/watch/{}/", entry.slug), + slug: entry.slug, + title, + thumb: Self::thumb_url(&entry.meta.vraven_remote_thumbnail), + views: engagement.and_then(|e| e.views).unwrap_or(0), + rating: engagement.and_then(|e| e.rating).unwrap_or(0.0), + uploaded_at: Self::parse_timestamp(&entry.date), } - return Ok(items); - } - } - - let mut requester = - crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); - let text = match Self::get_with_retry(&mut requester, &video_url, 3).await { - Ok(text) => text, - Err(e) => { - crate::providers::report_provider_error( - "hentaihaven", - "query.request", - &format!("url={video_url}; error={e}"), - ) - .await; - return Ok(cache - .get(&video_url) - .map(|(_, items)| items) - .unwrap_or_default()); - } - }; - let urls = Self::parse_search_urls(&text); - if urls.is_empty() { - return Ok(cache - .get(&video_url) - .map(|(_, items)| items) - .unwrap_or_default()); - } - - let db_items = Self::items_from_db(&urls, &pool); - if !db_items.is_empty() { - cache.insert(video_url.clone(), db_items.clone()); - self.spawn_refresh(requester, pool, cache, video_url, Some(urls), true); - return Ok(db_items); - } - - let items = self.resolve_urls(urls, &requester, pool).await; - if !items.is_empty() { - cache.insert(video_url.clone(), items.clone()); - return Ok(items); - } - Ok(cache - .get(&video_url) - .map(|(_, items)| items) - .unwrap_or_default()) - } - - fn extract_segment_url(seg: &str) -> Option { - seg.split("a href=\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .map(|s| s.to_string()) - } - - /// Extract the ordered list of episode page URLs from a listing page. - fn parse_listing_urls(html: &str) -> Vec { - if html.is_empty() || html.contains("404 Not Found") { - return vec![]; - } - let block = match html.split("previouspostslink").next().and_then(|s| { - s.split("vraven_manga_list").nth(1).or_else(|| { - s.find(r#"
"#) - .map(|idx| &s[idx..]) }) - }) { - Some(b) => b, - None => { - crate::providers::report_provider_error_background( - "hentaihaven", - "parse_listing.block", - "Failed to get block from listing html", - ); - return vec![]; - } - }; - block - .split("id=\"manga-item-") - .skip(1) - .filter_map(Self::extract_segment_url) - .collect() + .collect()) } - /// Extract the ordered list of result URLs from a search page. - fn parse_search_urls(html: &str) -> Vec { - if html.is_empty() || html.contains("404 Not Found") { - return vec![]; + /// Poster paths are stored raw and routinely contain spaces, so they have to + /// go through the URL parser rather than plain concatenation. + fn thumb_url(path: &str) -> String { + let path = path.trim(); + if path.is_empty() { + return String::new(); } - let block = match html - .split(" b, - None => { - crate::providers::report_provider_error_background( - "hentaihaven", - "parse_search.block", - "Failed to get block from search html", - ); - return vec![]; - } - }; - block - .split("c-tabs-item__content col-6 col-md-12") - .skip(1) - .filter_map(Self::extract_segment_url) - .collect() + if path.starts_with("http") { + return path.to_string(); + } + match url::Url::parse(&format!("{IMG_BASE}{}", path.trim_start_matches('/'))) { + Ok(url) => url.to_string(), + Err(_) => String::new(), + } + } + + /// Catalogue dates are naive local timestamps like `2026-09-01T01:27:10`. + fn parse_timestamp(date: &str) -> u64 { + NaiveDateTime::parse_from_str(date.trim(), "%Y-%m-%dT%H:%M:%S") + .map(|dt| dt.and_utc().timestamp().max(0) as u64) + .unwrap_or(0) } /// Build a response from already-resolved items stored in the DB, preserving - /// the order of `urls`. Items not yet in the DB are simply skipped. - fn items_from_db(urls: &[String], pool: &DbPool) -> Vec { + /// the order of `entries`. Items not yet in the DB are simply skipped. + fn items_from_db(entries: &[ListingEntry], pool: &DbPool) -> Vec { let mut conn = match pool.get() { Ok(conn) => conn, Err(_) => return vec![], }; - urls.iter() - .filter_map(|url| match db::get_video(&mut conn, url.clone()) { + entries + .iter() + .filter_map(|entry| match db::get_video(&mut conn, entry.url.clone()) { Ok(Some(json)) => VideoItem::from(json).ok(), _ => None, }) .collect() } - /// Resolve each episode page URL into a full `VideoItem`, persisting every - /// success to the DB. On failure we fall back to any stored copy so a - /// transient error does not drop the item from the page. - async fn resolve_urls( + /// Resolve each series into a full `VideoItem`, persisting every success to + /// the DB. On failure we fall back to any stored copy so a transient error + /// does not drop the item from the page. + async fn resolve_entries( &self, - urls: Vec, + entries: Vec, requester: &Requester, pool: DbPool, ) -> Vec { - stream::iter(urls.into_iter().map(|url| { + stream::iter(entries.into_iter().map(|entry| { let provider = self.clone(); let mut req = requester.clone(); let pool = pool.clone(); async move { - match provider.fetch_video_item(&url, &mut req).await { - Ok(item) => { + let url = entry.url.clone(); + match provider.fetch_video_item(&entry, &mut req).await { + Ok((item, complete)) => { if let Ok(mut conn) = pool.get() { - let new_len = item.formats.as_ref().map_or(0, |f| f.len()); let old_item = db::get_video(&mut conn, url.clone()) .ok() .flatten() @@ -375,7 +351,11 @@ impl HentaihavenProvider { .as_ref() .and_then(|o| o.formats.as_ref()) .map_or(0, |f| f.len()); - if new_len >= old_len { + let new_len = item.formats.as_ref().map_or(0, |f| f.len()); + if complete || new_len >= old_len { + // Every episode the series page advertises resolved, + // so this is authoritative even when it is shorter + // than what we stored earlier. let _ = db::upsert_video( &mut conn, &url, @@ -383,8 +363,7 @@ impl HentaihavenProvider { ); Some(item) } else { - // A partial refresh resolved fewer episodes than we - // already have (likely a transient outage) — keep the + // Some episode pages failed this round — keep the // richer stored copy rather than degrading it. old_item.or(Some(item)) } @@ -432,8 +411,8 @@ impl HentaihavenProvider { } } - /// Spawn a non-blocking refresh of a listing/search page. `urls` may be - /// supplied when the caller already fetched the listing; otherwise the + /// Spawn a non-blocking refresh of a listing/search page. `entries` may be + /// supplied when the caller already fetched the catalogue page; otherwise the /// refresh re-fetches it itself. fn spawn_refresh( &self, @@ -441,8 +420,7 @@ impl HentaihavenProvider { pool: DbPool, cache: VideoCache, key: String, - urls: Option>, - search: bool, + entries: Option>, ) { if !Self::try_begin_refresh(&key) { return; @@ -450,7 +428,7 @@ impl HentaihavenProvider { let provider = self.clone(); tokio::spawn(async move { provider - .refresh(requester, pool, cache, key.clone(), urls, search) + .refresh(requester, pool, cache, key.clone(), entries) .await; Self::end_refresh(&key); }); @@ -462,19 +440,12 @@ impl HentaihavenProvider { pool: DbPool, cache: VideoCache, key: String, - urls: Option>, - search: bool, + entries: Option>, ) { - let urls = match urls { - Some(urls) => urls, - None => match Self::get_with_retry(&mut requester, &key, 3).await { - Ok(text) => { - if search { - Self::parse_search_urls(&text) - } else { - Self::parse_listing_urls(&text) - } - } + let entries = match entries { + Some(entries) => entries, + None => match Self::fetch_listing(&mut requester, &key).await { + Ok(entries) => entries, Err(e) => { crate::providers::report_provider_error_background( "hentaihaven", @@ -485,235 +456,239 @@ impl HentaihavenProvider { } }, }; - if urls.is_empty() { + if entries.is_empty() { return; } - let items = self.resolve_urls(urls, &requester, pool).await; + let items = self.resolve_entries(entries, &requester, pool).await; if !items.is_empty() { cache.insert(key, items); } } - async fn fetch_video_item( - &self, - video_url: &str, - requester: &mut Requester, - ) -> Result { - let html = Self::get_with_retry(requester, video_url, 3) - .await - .map_err(|e| Error::from(format!("Failed to fetch video page: {}", e)))?; - - let mut title = html - .split("

") - .nth(1) - .and_then(|s| s.split("

").next()) - .ok_or_else(|| ErrorKind::Parse(format!("video title: {video_url}")))? - .trim() - .to_string(); - title = decode(title.as_bytes()) - .to_string() - .unwrap_or(title) - .titlecase(); - let id = video_url - .split('/') - .nth(4) - .and_then(|s| s.split('.').next()) - .ok_or_else(|| ErrorKind::Parse(format!("video id: {video_url}")))? - .to_string(); - let thumb = html - .split("og:image\" content=\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .unwrap_or("") - .to_string(); - let raw_tags: Vec = html - .split("Genre(s)") - .nth(1) - .unwrap_or_default() - .split("Release") - .nth(0) - .unwrap_or_default() - .split("a href=\"") - .skip(1) - .map(|tag_block| { - let id = tag_block - .split("\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .unwrap_or("") - .to_string(); - let title = tag_block - .split('>') - .nth(1) - .and_then(|s| s.split('<').next()) - .map(|s| { - decode(s.as_bytes()) - .to_string() - .unwrap_or(s.to_string()) - .titlecase() - }) - .unwrap_or("".to_string()); - FilterOption { - id: id.to_ascii_lowercase().replace(" ", "+"), - title: title.clone(), - } - }) - .collect::>(); - for tag in &raw_tags { - Self::push_unique(&self.categories, tag.clone()); - } - let tags = raw_tags.into_iter().map(|t| t.title).collect(); - let views = html - .split("Viewed") - .last() - .and_then(|s| s.split("summary-content\">").nth(1)) - .and_then(|s| s.split(" Total").nth(0)) - .map(|s| s.trim().parse::().unwrap_or(0)) - .unwrap_or(0); - let episode_block = html - .split("manga-chapters-holder") - .nth(1) - .unwrap_or_default() - .split("vraven_read") - .nth(0) - .unwrap_or_default(); - let episodes: Vec<(String, String)> = episode_block - .split("wp-manga-chapter") - .skip(1) - .filter_map(|episode| { - let href = episode - .split("a href=\"") - .nth(1) - .and_then(|s| s.split('"').next())? - .to_string(); - let title = episode - .split("
") - .nth(1) - .and_then(|s| s.split('<').next()) - .unwrap_or_default() - .trim() - .to_string(); - Some((title, href)) - }) - .collect(); - - let formats: Vec = stream::iter(episodes.into_iter().map(|(title, href)| { - let requester = requester.clone(); - let provider = self.clone(); - async move { provider.resolve_episode_format(title, href, requester).await } - })) - .buffered(1) - .filter_map(|result| async move { - match result { - Ok(format) => Some(format), - Err(e) => { - eprintln!("Hentai Haven Provider: Failed to resolve episode format: {e}"); - None - } - } - }) - .collect::>() - .await; - if formats.is_empty() { - return Err(Error::from(format!("No formats found for video URL: {}", video_url))); - } - if formats.len() > 1 { - title = format!("{} ({} Episodes)", title, formats.len()); - } - - Ok( - VideoItem::new(id, title, video_url.to_string(), "hentaihaven".into(), thumb, 0) - .formats(formats) - .tags(tags) - .views(views) - .aspect_ratio(0.715), - ) + fn episode_regex(slug: &str) -> Result { + Regex::new(&format!( + r#"href="(/watch/{}/([^"/]+)/)""#, + regex::escape(slug) + )) + .map_err(|e| Error::from(format!("failed to build episode regex: {e}"))) } - async fn resolve_episode_format( - &self, - title: String, - href: String, - mut requester: Requester, - ) -> Result { - let episode_html = Self::get_with_retry(&mut requester, &href, 4) - .await - .map_err(|e| Error::from(format!("Failed to fetch episode page {href}: {e}")))?; + fn genre_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new(r#"href="/series/([a-z0-9-]+)/"[^>]*>([^<]+)<"#) + .expect("valid hentaihaven genre regex") + }) + } - let player_url = episode_html - .split("iframe src=\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .map(|s| s.replace("&", "&")) - .ok_or_else(|| ErrorKind::Parse(format!("player iframe url: {href}")))?; + fn content_url_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new(r#""contentUrl"\s*:\s*"([^"]+)""#) + .expect("valid hentaihaven contentUrl regex") + }) + } - let player_html = Self::get_with_retry(&mut requester, &player_url, 4) - .await - .map_err(|e| Error::from(format!("Failed to fetch player page {player_url}: {e}")))?; + fn duration_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new(r#""duration"\s*:\s*"(PT[^"]+)""#) + .expect("valid hentaihaven duration regex") + }) + } - let token = player_html - .split("x-secure-token\" content=\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .ok_or_else(|| ErrorKind::Parse(format!("secure token: {href}")))?; - - let config = Self::decode_secure_token(token)?; - let api_base = if config.uri.starts_with("//") { - format!("https:{}", config.uri) - } else { - config.uri.clone() - }; - let api_url = format!("{api_base}api.php"); - - let body = Self::build_player_api_body(&config.en, &config.iv); - - let text = Self::post_ajax_with_retry( - &api_url, - &body, - vec![ - ("Content-Type", "application/x-www-form-urlencoded"), - ("Accept", "*/*"), - ("Accept-Language", "en-US,en;q=0.5"), - ("Referer", player_url.as_str()), - ("Origin", self.url.as_str()), - ("Sec-Fetch-Dest", "empty"), - ("Sec-Fetch-Mode", "cors"), - ("Sec-Fetch-Site", "same-origin"), - ("X-Requested-With", "XMLHttpRequest"), - ], - 4, - ) - .await - .map_err(|e| Error::from(format!("Failed to call player api {api_url}: {e}")))?; - - let api_response: PlayerApiResponse = serde_json::from_str(&text) - .map_err(|e| Error::from(format!("Failed to parse player api body {api_url}: {e}")))?; - if !api_response.status { - return Err(Error::from(format!("player api returned status=false for {href}"))); + /// The series page also renders sidebars full of *other* series' episodes, so + /// every candidate link is scoped to this series' own slug. + fn parse_episode_paths(html: &str, slug: &str) -> Result> { + let regex = Self::episode_regex(slug)?; + let mut seen: HashSet = HashSet::new(); + let mut episodes: Vec<(u32, String)> = vec![]; + for capture in regex.captures_iter(html) { + let path = capture[1].to_string(); + let segment = capture[2].to_string(); + if !segment.starts_with("episode") { + continue; + } + if !seen.insert(path.clone()) { + continue; + } + let number = segment + .rsplit('-') + .next() + .and_then(|s| s.parse::().ok()) + .unwrap_or(u32::MAX); + episodes.push((number, format!("{SITE}{path}"))); } - let source = api_response - .data - .and_then(|d| d.sources.into_iter().next()) - .ok_or_else(|| ErrorKind::Parse(format!("no sources in player api response: {href}")))?; + episodes.sort_by_key(|(number, _)| *number); + Ok(episodes) + } - let quality = if source.label.trim().is_empty() { - "auto".to_string() + fn parse_genres(html: &str) -> Vec { + let mut seen: HashSet = HashSet::new(); + Self::genre_regex() + .captures_iter(html) + .filter_map(|capture| { + let id = capture[1].to_string(); + let title = decode(capture[2].as_bytes()) + .to_string() + .unwrap_or_else(|_| capture[2].to_string()) + .trim() + .to_string(); + if title.is_empty() || !seen.insert(id.clone()) { + return None; + } + Some(FilterOption { id, title }) + }) + .collect() + } + + /// Parse an ISO-8601 duration such as `PT6M32S` into seconds. + fn parse_iso8601_duration(value: &str) -> u32 { + let mut seconds = 0u32; + let mut number = 0u32; + for c in value.trim_start_matches("PT").chars() { + match c { + '0'..='9' => number = number.saturating_mul(10).saturating_add(c as u32 - '0' as u32), + 'H' => { + seconds = seconds.saturating_add(number.saturating_mul(3600)); + number = 0; + } + 'M' => { + seconds = seconds.saturating_add(number.saturating_mul(60)); + number = 0; + } + 'S' => { + seconds = seconds.saturating_add(number); + number = 0; + } + _ => number = 0, + } + } + seconds + } + + /// Resolve one series into a `VideoItem`. The boolean is `true` when every + /// episode the series page advertises produced a playable format, which lets + /// the caller tell a genuinely short series from a partly failed fetch. + async fn fetch_video_item( + &self, + entry: &ListingEntry, + requester: &mut Requester, + ) -> Result<(VideoItem, bool)> { + let html = Self::get_with_retry(requester, &entry.url, 3) + .await + .map_err(|e| Error::from(format!("Failed to fetch series page: {e}")))?; + + let genres = Self::parse_genres(&html); + for genre in &genres { + Self::push_unique(&self.categories, genre.clone()); + } + let tags: Vec = genres.into_iter().map(|g| g.title).collect(); + + let episodes = Self::parse_episode_paths(&html, &entry.slug)?; + let expected = episodes.len(); + let formats: Vec<(VideoFormat, u32)> = if episodes.is_empty() { + // A handful of one-shot entries render the player straight onto the + // series page instead of linking out to an episode. + vec![Self::format_from_player_html(&html, "Episode 1").ok_or_else(|| { + ErrorKind::Parse(format!("no episodes and no player on {}", entry.url)) + })?] } else { - source.label.to_ascii_lowercase() + stream::iter(episodes.into_iter().map(|(number, url)| { + let mut req = requester.clone(); + async move { + let label = if number == u32::MAX { + "Episode".to_string() + } else { + format!("Episode {number}") + }; + match Self::get_with_retry(&mut req, &url, 3).await { + Ok(html) => Self::format_from_player_html(&html, &label), + Err(e) => { + eprintln!("Hentai Haven Provider: Failed to fetch episode {url}: {e}"); + None + } + } + } + })) + .buffered(2) + .filter_map(|format| async move { format }) + .collect::>() + .await }; - Ok( - VideoFormat::new(source.src, quality, "m3u8".to_string()) - .format_id(title.clone()) - .http_header( - "User-Agent".to_string(), - "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0" - .to_string(), - ) - .http_header("Referer".to_string(), self.url.clone()) - .http_header("Origin".to_string(), self.url.clone()) - .format_note(title), + if formats.is_empty() { + return Err(Error::from(format!( + "No formats found for series: {}", + entry.url + ))); + } + + // A multi-episode series plays its first episode by default, so that is + // the duration the client should show. + let format_count = formats.len(); + let duration = formats.first().map(|(_, secs)| *secs).unwrap_or(0); + let mut title = entry.title.clone(); + if format_count > 1 { + title = format!("{} ({} Episodes)", title, format_count); + } + + let mut item = VideoItem::new( + entry.id.to_string(), + title, + entry.url.clone(), + "hentaihaven".into(), + entry.thumb.clone(), + duration, ) + .formats(formats.into_iter().map(|(format, _)| format).collect()) + .tags(tags) + .views(entry.views) + .aspect_ratio(0.667); + + if entry.rating > 0.0 { + item = item.rating(entry.rating); + } + if entry.uploaded_at > 0 { + item = item.uploaded_at(entry.uploaded_at); + } + + // The single-player fallback has nothing to compare against, so it only + // gets here when it did produce a format — treat that as complete too. + let complete = expected == 0 || format_count == expected; + Ok((item, complete)) + } + + /// Episode pages publish the stream in their `VideoObject` JSON-LD block, so + /// no player handshake is needed any more. The page also carries an + /// `ImageObject` whose `contentUrl` is the poster and which is emitted + /// *before* the video, so the playlist has to be picked out by extension + /// rather than by taking the first match. + fn format_from_player_html(html: &str, label: &str) -> Option<(VideoFormat, u32)> { + let src = Self::content_url_regex() + .captures_iter(html) + .map(|c| c[1].to_string()) + .find(|url| Self::is_stream_url(url))?; + let duration = Self::duration_regex() + .captures(html) + .map(|c| Self::parse_iso8601_duration(&c[1])) + .unwrap_or(0); + let container = if src.to_lowercase().contains(".mp4") { + "mp4" + } else { + "m3u8" + }; + let format = VideoFormat::new(src, "auto".to_string(), container.to_string()) + .format_id(label.to_string()) + .format_note(label.to_string()); + Some((format, duration)) + } + + /// Distinguishes the `VideoObject` stream from the poster art that shares + /// the `contentUrl` key. + fn is_stream_url(url: &str) -> bool { + let path = url.split(['?', '#']).next().unwrap_or(url).to_lowercase(); + path.ends_with(".m3u8") || path.ends_with(".mp4") } async fn get_with_retry( @@ -734,94 +709,6 @@ impl HentaihavenProvider { } Err(last_err) } - - fn ajax_client() -> &'static wreq::Client { - static CLIENT: OnceLock = OnceLock::new(); - CLIENT.get_or_init(|| { - wreq::Client::builder() - .cert_verification(false) - .emulation(Emulation::Chrome137) - .build() - .expect("Failed to build hentaihaven AJAX client") - }) - } - - async fn post_ajax_with_retry( - url: &str, - body: &str, - headers: Vec<(&str, &str)>, - attempts: u32, - ) -> std::result::Result { - let mut last_err = String::new(); - for attempt in 0..attempts { - if attempt > 0 { - let backoff_ms = 500u64 * (1u64 << (attempt - 1).min(3)); - tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; - } - let mut request = Self::ajax_client() - .post(url) - .version(Version::HTTP_2) - .body(body.to_string()); - for (key, value) in headers.iter() { - request = request.header(*key, *value); - } - match request.send().await { - Ok(response) => { - let status = response.status(); - match response.text().await { - Ok(text) if status.is_success() => return Ok(text), - Ok(_) => last_err = format!("status {status}"), - Err(e) => last_err = e.to_string(), - } - } - Err(e) => last_err = e.to_string(), - } - } - Err(last_err) - } - - fn build_player_api_body(en: &str, iv: &str) -> String { - let mut serializer = url::form_urlencoded::Serializer::new(String::new()); - serializer - .append_pair("action", "zarat_get_data_player_ajax") - .append_pair("a", en) - .append_pair("b", iv); - serializer.finish() - } - - fn decode_secure_token(token: &str) -> Result { - let stripped = token.strip_prefix("sha512-").unwrap_or(token); - let mut data = Self::rot13(stripped); - data = Self::decode_base64_layer(&data)?; - data = Self::rot13(&data); - data = Self::decode_base64_layer(&data)?; - data = Self::rot13(&data); - data = Self::decode_base64_layer(&data)?; - serde_json::from_str(&data) - .map_err(|e| Error::from(format!("Failed to parse secure token json: {e}"))) - } - - fn decode_base64_layer(value: &str) -> Result { - let mut normalized = value.trim().to_string(); - while normalized.len() % 4 != 0 { - normalized.push('='); - } - let bytes = STANDARD - .decode(normalized) - .map_err(|e| Error::from(format!("base64 decode failed: {e}")))?; - String::from_utf8(bytes).map_err(|e| Error::from(format!("utf8 decode failed: {e}"))) - } - - fn rot13(input: &str) -> String { - input - .chars() - .map(|c| match c { - 'A'..='Z' => (((c as u8 - b'A' + 13) % 26) + b'A') as char, - 'a'..='z' => (((c as u8 - b'a' + 13) % 26) + b'a') as char, - other => other, - }) - .collect() - } } #[async_trait] @@ -833,15 +720,23 @@ impl Provider for HentaihavenProvider { sort: String, query: Option, page: String, - _per_page: String, + per_page: String, options: ServerOptions, ) -> Vec { let page = page.parse::().unwrap_or(1); + let per_page = per_page.parse::().unwrap_or(20); - let res = match query { - Some(q) => self.to_owned().query(cache, page, &q, options, pool).await, - None => self.get(cache, page, &sort, options, pool).await, - }; + let res = self + .list( + cache, + page, + per_page, + &sort, + query.as_deref(), + options, + pool, + ) + .await; res.unwrap_or_else(|e| { eprintln!("hentai haven error: {e}");