use crate::DbPool; use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error, report_provider_error_background, requester_or_default}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::requester::Requester; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use base64::{Engine, engine::general_purpose::STANDARD}; use chrono::DateTime; use error_chain::error_chain; use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use regex::Regex; use scraper::{ElementRef, Html, Selector}; use std::collections::HashSet; use std::sync::{Arc, RwLock}; use std::thread; use wreq::Version; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "hentai-animation", tags: &["hentai", "anime", "episodes"], }; error_chain! { foreign_links { Io(std::io::Error); } errors { Parse(msg: String) { description("parse error") display("parse error: {}", msg) } } } const CHANNEL_ID: &str = "hentaimama"; const BASE_URL: &str = "https://hentaimama.io"; // Static-file mirror the site's own "rtmp" player option points at. Serves // direct 200s with Accept-Ranges and no Referer/token needed, so the format // URL can be built locally from the AJAX response with no extra fetch. const MEDIA_HOST: &str = "https://gdvid.info/"; const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36"; #[derive(Debug, Clone)] enum Target { Latest, Genre(String), Studio(String), Search(String), } #[derive(Debug, Clone)] struct EpisodeCard { id: String, title: String, url: String, thumb: String, rating: Option, } #[derive(Debug, Clone)] pub struct HentaimamaProvider { genres: Arc>>, } impl HentaimamaProvider { pub fn new() -> Self { let provider = Self { genres: Arc::new(RwLock::new(vec![])), }; provider.spawn_initial_load(); provider } fn spawn_initial_load(&self) { let genres = Arc::clone(&self.genres); thread::spawn(move || { let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() { Ok(runtime) => runtime, Err(e) => { report_provider_error_background( CHANNEL_ID, "spawn_initial_load.runtime_build", &e.to_string(), ); return; } }; runtime.block_on(async move { if let Err(e) = Self::load_genres(genres).await { report_provider_error_background(CHANNEL_ID, "load_genres", &e.to_string()); } }); }); } async fn load_genres(genres: Arc>>) -> Result<()> { let mut requester = Requester::new(); let html = requester .get_with_headers( &format!("{BASE_URL}/genres-filter/"), Self::html_headers(BASE_URL), Some(Version::HTTP_2), ) .await .map_err(|e| Error::from(format!("genres fetch failed: {e}")))?; let document = Html::parse_document(&html); let selector = Self::selector("a.genreitem")?; let mut options = Vec::new(); let mut seen = HashSet::new(); for element in document.select(&selector) { let Some(href) = element.value().attr("href") else { continue; }; let slug = href.trim_end_matches('/').rsplit('/').next().unwrap_or("").to_string(); if slug.is_empty() || !seen.insert(slug.clone()) { continue; } let title = Self::decode_entities(&element.text().collect::()); if title.is_empty() { continue; } options.push(FilterOption { id: slug, title }); } if !options.is_empty() { if let Ok(mut guard) = genres.write() { *guard = options; } } Ok(()) } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { let genres = self.genres.read().map(|g| g.clone()).unwrap_or_default(); Channel { id: CHANNEL_ID.to_string(), name: "Hentaimama".to_string(), description: "Watch hentai episodes online free in HD.".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=hentaimama.io".to_string(), status: "active".to_string(), categories: genres.iter().map(|g| g.title.clone()).collect(), options: vec![ChannelOption { id: "categories".to_string(), title: "Genres".to_string(), description: "Filter by genre".to_string(), systemImage: "tag.fill".to_string(), colorName: "green".to_string(), options: genres, multiSelect: false, }], nsfw: true, cacheDuration: Some(1800), ytdlpCommand: Some("yt-dlp".to_string()), } } fn selector(value: &str) -> Result { Selector::parse(value).map_err(|e| Error::from(format!("selector `{value}` parse failed: {e}"))) } fn decode_entities(text: &str) -> String { decode(text.as_bytes()) .to_string() .unwrap_or_else(|_| text.to_string()) .split_whitespace() .collect::>() .join(" ") .trim() .to_string() } fn html_headers(referer: &str) -> Vec<(String, String)> { vec![ ("Referer".to_string(), referer.to_string()), ("User-Agent".to_string(), USER_AGENT.to_string()), ] } async fn fetch_html(requester: &mut Requester, url: &str, referer: &str) -> Result { requester .get_with_headers(url, Self::html_headers(referer), Some(Version::HTTP_2)) .await .map_err(|e| Error::from(format!("request failed for {url}: {e}"))) } fn slugify(value: &str) -> String { value .trim() .to_lowercase() .chars() .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) .collect::() .split('-') .filter(|s| !s.is_empty()) .collect::>() .join("-") } fn find_genre_slug(&self, value: &str) -> Option { let normalized = value.trim().to_lowercase(); let genres = self.genres.read().ok()?; genres .iter() .find(|g| g.id.eq_ignore_ascii_case(value.trim()) || g.title.to_lowercase() == normalized) .map(|g| g.id.clone()) } fn resolve_target(&self, query: Option<&str>, categories: Option<&str>) -> Target { if let Some(value) = categories { if let Some(slug) = self.find_genre_slug(value) { return Target::Genre(slug); } let slug = Self::slugify(value); if !slug.is_empty() { return Target::Genre(slug); } } if let Some(q) = query { let q = q.trim(); if let Some(rest) = q .strip_prefix("genre:") .or_else(|| q.strip_prefix("cat:")) .or_else(|| q.strip_prefix("category:")) { let slug = self.find_genre_slug(rest).unwrap_or_else(|| Self::slugify(rest)); return Target::Genre(slug); } if let Some(rest) = q.strip_prefix("studio:").or_else(|| q.strip_prefix("uploader:")) { return Target::Studio(Self::slugify(rest)); } if let Some(slug) = self.find_genre_slug(q) { return Target::Genre(slug); } if !q.is_empty() { return Target::Search(q.to_string()); } } Target::Latest } fn build_list_url(target: &Target, page: u32) -> String { match target { Target::Latest => { if page <= 1 { format!("{BASE_URL}/episodes/") } else { format!("{BASE_URL}/episodes/page/{page}/") } } Target::Genre(slug) => { if page <= 1 { format!("{BASE_URL}/genre/{slug}/") } else { format!("{BASE_URL}/genre/{slug}/page/{page}/") } } Target::Studio(slug) => { if page <= 1 { format!("{BASE_URL}/studio/{slug}/") } else { format!("{BASE_URL}/studio/{slug}/page/{page}/") } } Target::Search(query) => { let encoded = utf8_percent_encode(query, NON_ALPHANUMERIC).to_string(); if page <= 1 { format!("{BASE_URL}/?s={encoded}") } else { format!("{BASE_URL}/page/{page}/?s={encoded}") } } } } fn parse_episode_cards(html: &str) -> Result> { let document = Html::parse_document(html); let card_selector = Self::selector("article.se.episodes")?; let link_selector = Self::selector("div.season_m a")?; let img_selector = Self::selector("img")?; let rating_selector = Self::selector("div.rating")?; let rating_re = Regex::new(r"([0-9]+(?:\.[0-9]+)?)").unwrap(); let mut cards = Vec::new(); for card in document.select(&card_selector) { let Some(id) = card.value().attr("rel").filter(|s| !s.is_empty()) else { continue; }; let Some(link) = card.select(&link_selector).next() else { continue; }; let Some(url) = link.value().attr("href") else { continue; }; let title = Self::decode_entities(&link.text().collect::()); if title.is_empty() { continue; } let thumb = card .select(&img_selector) .next() .and_then(|img| img.value().attr("src")) .unwrap_or("") .to_string(); let rating = card .select(&rating_selector) .next() .map(|r| r.text().collect::()) .and_then(|text| rating_re.captures(&text).and_then(|c| c[1].parse::().ok())); cards.push(EpisodeCard { id: id.to_string(), title, url: url.to_string(), thumb, rating, }); } Ok(cards) } fn parse_series_card_urls(html: &str) -> Result> { let document = Html::parse_document(html); let selector = Self::selector("a.sc-poster")?; Ok(document .select(&selector) .filter_map(|e| e.value().attr("href").map(|s| s.to_string())) .collect()) } fn last_episode_url(html: &str) -> Result> { let document = Html::parse_document(html); let selector = Self::selector("a.dt-se-item")?; Ok(document .select(&selector) .filter_map(|e: ElementRef| e.value().attr("href").map(|s| s.to_string())) .last()) } fn post_id_regex() -> &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| Regex::new(r#"data-post-id="(\d+)""#).expect("valid regex")) } fn title_regex() -> &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| Regex::new(r#""#).expect("valid regex")) } fn thumb_regex() -> &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| Regex::new(r#" &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| Regex::new(r#"data-rating="([0-9.]+)""#).expect("valid regex")) } fn published_regex() -> &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| { Regex::new(r#" &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| { Regex::new(r#"href="https://hentaimama\.io/(?:genre|studio)/[a-z0-9-]+/"[^>]*rel="tag">([^<]+)"#) .expect("valid regex") }) } fn rtmp_p_regex() -> &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| Regex::new(r#"dt_embed=rtmp[^"]*?[?&]p=([A-Za-z0-9+/=]+)"#).expect("valid regex")) } fn iframe_src_regex() -> &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| Regex::new(r#"src="([^"]+)""#).expect("valid regex")) } fn jwplayer_file_regex() -> &'static Regex { static RE: std::sync::OnceLock = std::sync::OnceLock::new(); RE.get_or_init(|| Regex::new(r#"file:\s*"([^"]+)""#).expect("valid regex")) } fn pad_base64(value: &str) -> String { let padding = (4 - value.len() % 4) % 4; format!("{value}{}", "=".repeat(padding)) } /// The site's "rtmp" player option embeds the raw storage path as a base64 /// query param; the direct mirror URL is just that path appended to /// `MEDIA_HOST`, so no extra request is needed to resolve it. fn format_from_p_param(iframe_html: &str) -> Option { let captures = Self::rtmp_p_regex().captures(iframe_html)?; let decoded = STANDARD.decode(Self::pad_base64(&captures[1])).ok()?; let path = String::from_utf8(decoded).ok()?; let path = path.trim().trim_start_matches('/'); if path.is_empty() { return None; } Some(VideoFormat::new( format!("{MEDIA_HOST}{path}"), "auto".to_string(), "mp4".to_string(), )) } fn format_from_jwplayer(embed_html: &str) -> Option { let src = &Self::jwplayer_file_regex().captures(embed_html)?[1]; let container = if src.to_lowercase().contains(".m3u8") { "m3u8" } else { "mp4" }; Some(VideoFormat::new(src.to_string(), "auto".to_string(), container.to_string())) } async fn fetch_player_iframe( requester: &mut Requester, referer: &str, post_id: &str, slot: u8, ) -> Option { let body = format!("action=get_player_contents&a={post_id}&i={slot}"); let headers = vec![ ("Content-Type", "application/x-www-form-urlencoded"), ("X-Requested-With", "XMLHttpRequest"), ("Referer", referer), ("User-Agent", USER_AGENT), ]; let response = requester .post(&format!("{BASE_URL}/wp-admin/admin-ajax.php"), &body, headers) .await .ok()?; let text = response.text().await.ok()?; let slots: Vec = serde_json::from_str(&text).ok()?; slots.into_iter().find(|s| !s.trim().is_empty()) } async fn resolve_format(requester: &mut Requester, referer: &str, post_id: &str) -> Option { let iframe_html = Self::fetch_player_iframe(requester, referer, post_id, 1).await?; if let Some(format) = Self::format_from_p_param(&iframe_html) { return Some(format); } // Fallback for embed shapes we haven't seen: follow the iframe itself and // pull the jwplayer source out of its markup. let src = &Self::iframe_src_regex().captures(&iframe_html)?[1]; let embed_html = Self::fetch_html(requester, src, referer).await.ok()?; Self::format_from_jwplayer(&embed_html) } async fn build_item_from_episode_card(requester: &mut Requester, entry: &EpisodeCard) -> Option { let format = Self::resolve_format(requester, &entry.url, &entry.id).await?; let mut item = VideoItem::new( entry.id.clone(), entry.title.clone(), entry.url.clone(), CHANNEL_ID.to_string(), entry.thumb.clone(), 0, ) .formats(vec![format]) .aspect_ratio(16.0 / 9.0); if let Some(rating) = entry.rating { item = item.rating(rating * 10.0); } Some(item) } /// Search/genre/studio archives only expose series (show) cards, so a match /// is represented by its most recent episode - the same shape the latest /// feed already returns, just reached through one extra hop. async fn resolve_series_to_item(requester: &mut Requester, series_url: &str) -> Option { let html = Self::fetch_html(requester, series_url, BASE_URL).await.ok()?; let episode_url = Self::last_episode_url(&html).ok().flatten()?; Self::resolve_episode_url(requester, &episode_url).await } async fn resolve_episode_url(requester: &mut Requester, episode_url: &str) -> Option { let html = Self::fetch_html(requester, episode_url, BASE_URL).await.ok()?; let post_id = Self::post_id_regex().captures(&html)?[1].to_string(); let title = Self::decode_entities(&Self::title_regex().captures(&html)?[1]); if title.is_empty() { return None; } let thumb = Self::thumb_regex() .captures(&html) .map(|c| c[1].to_string()) .unwrap_or_default(); let rating = Self::rating_regex().captures(&html).and_then(|c| c[1].parse::().ok()); let uploaded_at = Self::published_regex() .captures(&html) .and_then(|c| DateTime::parse_from_rfc3339(&c[1]).ok()) .map(|dt| dt.timestamp().max(0) as u64); let mut seen = HashSet::new(); let tags: Vec = Self::tag_regex() .captures_iter(&html) .map(|c| Self::decode_entities(&c[1])) .filter(|t| !t.is_empty() && seen.insert(t.clone())) .collect(); let format = Self::resolve_format(requester, episode_url, &post_id).await?; let mut item = VideoItem::new( post_id, title, episode_url.to_string(), CHANNEL_ID.to_string(), thumb, 0, ) .formats(vec![format]) .tags(tags) .aspect_ratio(16.0 / 9.0); if let Some(rating) = rating { item = item.rating(rating * 10.0); } if let Some(uploaded_at) = uploaded_at { item = item.uploaded_at(uploaded_at); } Some(item) } async fn get( &self, cache: VideoCache, page: u32, per_page: usize, query: Option<&str>, options: ServerOptions, ) -> Result> { let target = self.resolve_target(query, options.categories.as_deref()); let list_url = Self::build_list_url(&target, page); if let Some((time, items)) = cache.get(&list_url) { if time.elapsed().unwrap_or_default().as_secs() < 300 { return Ok(items); } } let mut requester = requester_or_default(&options, CHANNEL_ID, "get_videos"); let html = Self::fetch_html(&mut requester, &list_url, BASE_URL).await?; let items = match target { Target::Latest => { let cards = Self::parse_episode_cards(&html)?; stream::iter(cards.into_iter().take(per_page.max(1)).map(|entry| { let mut req = requester.clone(); async move { Self::build_item_from_episode_card(&mut req, &entry).await } })) .buffer_unordered(6) .filter_map(|item| async move { item }) .collect::>() .await } _ => { let urls = Self::parse_series_card_urls(&html)?; stream::iter(urls.into_iter().take(per_page.max(1)).map(|url| { let mut req = requester.clone(); async move { Self::resolve_series_to_item(&mut req, &url).await } })) .buffer_unordered(4) .filter_map(|item| async move { item }) .collect::>() .await } }; if !items.is_empty() { cache.insert(list_url, items.clone()); } Ok(items) } } #[async_trait] impl Provider for HentaimamaProvider { async fn get_videos( &self, cache: VideoCache, _pool: DbPool, _sort: String, query: Option, page: String, per_page: String, options: ServerOptions, ) -> Vec { let page = page.parse::().unwrap_or(1).max(1); let per_page = per_page.parse::().unwrap_or(24); let query_ref = query.as_deref().filter(|q| !q.trim().is_empty()); match self.get(cache, page, per_page, query_ref, options).await { Ok(items) => items, Err(e) => { report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await; vec![] } } } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } } #[cfg(test)] mod tests { use super::*; fn provider() -> HentaimamaProvider { HentaimamaProvider { genres: Arc::new(RwLock::new(vec![FilterOption { id: "maid".to_string(), title: "Maid".to_string(), }])), } } #[test] fn builds_latest_urls() { assert_eq!( HentaimamaProvider::build_list_url(&Target::Latest, 1), "https://hentaimama.io/episodes/" ); assert_eq!( HentaimamaProvider::build_list_url(&Target::Latest, 2), "https://hentaimama.io/episodes/page/2/" ); } #[test] fn builds_genre_and_search_urls() { assert_eq!( HentaimamaProvider::build_list_url(&Target::Genre("maid".to_string()), 1), "https://hentaimama.io/genre/maid/" ); assert_eq!( HentaimamaProvider::build_list_url(&Target::Search("school girl".to_string()), 2), "https://hentaimama.io/page/2/?s=school%20girl" ); } #[test] fn routes_query_shortcuts() { let provider = provider(); assert!(matches!(provider.resolve_target(Some("maid"), None), Target::Genre(s) if s == "maid")); assert!(matches!(provider.resolve_target(Some("genre:blowjob"), None), Target::Genre(s) if s == "blowjob")); assert!(matches!(provider.resolve_target(Some("studio:majin"), None), Target::Studio(s) if s == "majin")); assert!(matches!(provider.resolve_target(Some("random keyword"), None), Target::Search(s) if s == "random keyword")); assert!(matches!(provider.resolve_target(None, None), Target::Latest)); } #[test] fn decodes_gdvid_path_from_p_param() { let iframe = r#""#; let format = HentaimamaProvider::format_from_p_param(iframe).unwrap(); assert_eq!( format.url, "https://gdvid.info/P/pure-x-holic-junketsu-otome-to-konin-kankei-the-animation-2.mp4" ); } }