diff --git a/docs/provider-fix-todo.md b/docs/provider-fix-todo.md new file mode 100644 index 0000000..380408b --- /dev/null +++ b/docs/provider-fix-todo.md @@ -0,0 +1,70 @@ +# Provider fix todo + +Source: `check.py` + search probe audit, 2026-10-10. Serial re-run of every flagged channel, +so timeouts below are not load artifacts. Update the status column as each item is fixed and +verified (`HOT_TUB_PROVIDER=` build, `/api/videos` feed + search + page 2, `yt-dlp` on +`video.url` or `formats[0].url`, `python3 check.py `). + +Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not reproducible / not a provider bug (reason noted). + +## 1. Channels returning no items + +| | Channel | Symptom | Notes | +|---|---|---|---| +| [x] | `javtiful` | feed times out (150s) then 0 items; search 0 | site redesign (`article.video-card`); rewrote parser, page URL as `video.url` (yt-dlp-native), no per-item detail fetches. 23 items, p2/search OK | +| [ ] | `sextb` | 0 items on feed, search and page 2 | | +| [ ] | `camsoda` | 0 items on every request | live cams, CF + Jina-only fetch (see memory) | +| [ ] | `hentaimama` | 0 items or 85-150s timeouts | | +| [ ] | `hotbunny` | 0 items; first request 78s | | + +## 2. Playback: page URL not resolvable by yt-dlp / formats not playable + +| | Channel | Symptom | Notes | +|---|---|---|---| +| [ ] | `thaiporntv` | page URL "Unsupported URL", no `formats` | populate formats from detail page | +| [ ] | `pornmz` | page URL "Unsupported URL", no `formats` | populate formats from detail page | +| [ ] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | check whether `formats` are populated for videos | +| [ ] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | | +| [ ] | `hypnotube` | page URLs 404 on all sampled items | URL construction | +| [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | | +| [ ] | `ripnsfw` | `doodstream` format 502 | | +| [ ] | `freepornvideosxxx` | yt-dlp 502 on page URL | may be transient, verify | +| [ ] | `hdporn92` | m3u8 formats time out (curl + yt-dlp) | morencius.com streams | + +## 3. Thumbnails + +| | Channel | Symptom | Notes | +|---|---|---|---| +| [ ] | `xxdbx` | malformed thumbs `https:/7004….jpg` | built as `"https:" + src` | +| [ ] | `ph-shorties` | thumbs return 403 | | +| [ ] | `noodlemagazine` | thumbs unreachable (-1 / 400) in check.py | thumb proxy | + +## 4. Search + +| | Channel | Symptom | Notes | +|---|---|---|---| +| [ ] | `fapello` | "girl" and "milf" both 0 | | +| [ ] | `chaturbate` | "girl" and "milf" both 0 | | +| [ ] | `cumlouder` | unknown tag takes 146s before returning 0 | own provider, retry path on 404 | +| [ ] | `vjav` | "girl" 0, "milf" 10 | may be term-specific | +| [ ] | `xxxtik` | "girl" 0 (not re-run serially) | | +| [ ] | `hypnotube` | "milf" 0, "girl" 20 | may be term-specific | + +## 5. Slow channels + +| | Channel | Symptom | Notes | +|---|---|---|---| +| [ ] | `aps` | 30-98s per request; check.py 30s timeout | | +| [ ] | `hdporn92` | feed about 66s per request | shares item with section 2 | +| [ ] | `porndish` | 14-23s feed/search | | + +## 6. Tooling + +| | Item | Notes | +|---|---|---| +| [ ] | `check.py` "url points to hottub server" | contradicts the hottub skill (proxy-needing videos use the proxy as `item.url`). Affects spankbang, hqporner, pornhd3x, pimpbunny, sxyprn, noodlemagazine. Decide: exempt proxy URLs that resolve, rather than hard-fail. | + +## Environmental / needs the production IP (not worked here) + +`porndish`, `supjav`, `jable`, `fapello`, `missav`, `wowxxx`, `hentaihaven` return Cloudflare 403 to +yt-dlp/curl from this machine while the server itself returns items. `hsex` returned 429 (rate limit). diff --git a/src/providers/javtiful.rs b/src/providers/javtiful.rs index 41c1769..7ffdca2 100644 --- a/src/providers/javtiful.rs +++ b/src/providers/javtiful.rs @@ -1,20 +1,16 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{Provider, build_proxy_url, strip_url_scheme}; +use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::discord::{format_error_chain, send_discord_error_report}; -use crate::util::requester::Requester; +use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; -use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; -use std::sync::{Arc, RwLock}; use std::vec; -use titlecase::Titlecase; use wreq::Version; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = @@ -40,16 +36,13 @@ error_chain! { #[derive(Debug, Clone)] pub struct JavtifulProvider { url: String, - categories: Arc>>, } impl JavtifulProvider { pub fn new() -> Self { - let provider = Self { + Self { url: "https://javtiful.com".to_string(), - categories: Arc::new(RwLock::new(vec![])), - }; - provider + } } fn build_channel(&self, clientversion: ClientVersion) -> Channel { @@ -61,18 +54,7 @@ impl JavtifulProvider { premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=javtiful.com".to_string(), status: "active".to_string(), - categories: self - .categories - .read() - .map(|categories| categories.iter().map(|c| c.title.clone()).collect()) - .unwrap_or_else(|e| { - crate::providers::report_provider_error_background( - "javtiful", - "build_channel.categories_read", - &e.to_string(), - ); - vec![] - }), + categories: vec![], options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), @@ -80,35 +62,85 @@ impl JavtifulProvider { systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ - FilterOption { - id: "relevance".into(), - title: "Relevance".into(), - }, FilterOption { id: "latest".into(), title: "Latest".into(), }, FilterOption { id: "popular".into(), - title: "Popular".into(), + title: "Popular This Week".into(), + }, + FilterOption { + id: "most_viewed".into(), + title: "Most Viewed".into(), + }, + FilterOption { + id: "most_liked".into(), + title: "Most Liked".into(), }, ], multiSelect: false, }], nsfw: true, cacheDuration: Some(1800), - ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()) + ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()), } } - fn push_unique(target: &Arc>>, item: FilterOption) { - if let Ok(mut vec) = target.write() { - if !vec.iter().any(|x| x.id == item.id) { - vec.push(item); - } + /// Maps the app's sort ids to the site's `?sort=` values. The default + /// listing (no parameter) is newest first. + fn sort_param(sort: &str) -> &'static str { + match sort { + "popular" => "sort=popular_week&", + "most_viewed" => "sort=most_viewed&", + "most_liked" => "sort=most_liked&", + _ => "", } } + async fn fetch_listing( + &self, + cache: VideoCache, + video_url: String, + page: u8, + options: &ServerOptions, + ) -> Result> { + let old_items = match cache.get(&video_url) { + Some((time, items)) if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 => { + return Ok(items.clone()); + } + Some((_, items)) => items.clone(), + None => vec![], + }; + + let mut requester = + crate::providers::requester_or_default(options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "javtiful", + "fetch_listing.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; + // Out-of-range pages are served as page 1 again; only trust the grid + // when the pager marks the requested page as current. + if page > 1 && !text.contains(&format!("aria-current=\"page\">{}<", page)) { + return Ok(vec![]); + } + let video_items = self.get_video_items_from_html(&text); + if video_items.is_empty() { + return Ok(old_items); + } + cache.remove(&video_url); + cache.insert(video_url, video_items.clone()); + Ok(video_items) + } + async fn get( &self, cache: VideoCache, @@ -116,58 +148,13 @@ impl JavtifulProvider { sort: &str, options: ServerOptions, ) -> Result> { - let sort_string = match sort { - "latest" => "sort=latest&", - "popular" => "sort=popular&", - _ => "", - }; - let video_url = format!("{}/videos?{}page={}", self.url, sort_string, page); - println!("Fetching URL: {}", video_url); - let old_items = match cache.get(&video_url) { - Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { - return Ok(items.clone()); - } else { - items.clone() - } - } - None => { - vec![] - } - }; - - let mut requester = - crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); - let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { - Ok(text) => text, - Err(e) => { - crate::providers::report_provider_error( - "javtiful", - "get.request", - &format!("url={video_url}; error={e}"), - ) - .await; - return Ok(old_items); - } - }; - if page > 1 - && !text.contains(&format!( - "aria-current=\"page\">{}", - page - )) - { - return Ok(vec![]); - } - let video_items: Vec = self - .get_video_items_from_html(text.clone(), &mut requester, &options) - .await; - if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + let video_url = format!( + "{}/videos?{}page={}", + self.url, + Self::sort_param(sort), + page + ); + self.fetch_listing(cache, video_url, page, &options).await } async fn query( @@ -177,271 +164,83 @@ impl JavtifulProvider { query: &str, options: ServerOptions, ) -> Result> { - let sort_string = match options.sort.as_deref().unwrap_or("") { - "latest" => "sort=latest&", - "popular" => "sort=popular&", - _ => "", - }; + let sort = options.sort.clone().unwrap_or_default(); let video_url = format!( "{}/search?{}q={}&page={}", self.url, - sort_string, - query.replace(" ", "+"), + Self::sort_param(&sort), + query.trim().replace(' ', "+"), page ); - // Check our Video Cache. If the result is younger than 1 hour, we return it. - let old_items = match cache.get(&video_url) { - Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { - return Ok(items.clone()); - } else { - let _ = cache.check().await; - return Ok(items.clone()); - } - } - None => { - vec![] - } - }; - - let mut requester = - crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); - let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { - Ok(text) => text, - Err(e) => { - crate::providers::report_provider_error( - "javtiful", - "query.request", - &format!("url={video_url}; error={e}"), - ) - .await; - return Ok(old_items); - } - }; - if page > 1 - && !text.contains(&format!( - "aria-current=\"page\">{}", - page - )) - { - return Ok(vec![]); - } - let video_items: Vec = self - .get_video_items_from_html(text.clone(), &mut requester, &options) - .await; - if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + self.fetch_listing(cache, video_url, page, &options).await } - async fn get_video_items_from_html( - &self, - html: String, - requester: &mut Requester, - options: &ServerOptions, - ) -> Vec { - if html.is_empty() || html.contains("404 Not Found") { - return vec![]; - } + fn between<'a>(text: &'a str, start: &str, end: &str) -> Option<&'a str> { + let from = text.find(start)? + start.len(); + let rest = &text[from..]; + Some(&rest[..rest.find(end)?]) + } - let block = match html - .split("front-pagination") - .next() - .and_then(|s| s.split("front-video-grid").nth(1)) - { - Some(b) => b, - None => { - eprint!("Javtiful Provider: Failed to get block from html"); - let e = Error::from(ErrorKind::Parse("html".into())); - send_discord_error_report( - e.to_string(), - Some(format_error_chain(&e)), - Some("Javtiful Provider"), - Some(&format!("Failed to get block from html:\n```{html}\n```")), - file!(), - line!(), - module_path!(), - ) - .await; - return vec![]; - } - }; - - let futures = block - .split("\"front-video-card\"") + fn get_video_items_from_html(&self, html: &str) -> Vec { + // Cards are `
`; sponsored slots add a + // `video-card--partner` modifier and link off-site, so skip them. + html.split("
Result { - let video_url = format!( - "{}{}", - self.url, - seg.split(" href=\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .ok_or_else(|| ErrorKind::Parse(format!("video url\n\n{seg}")))? - .to_string() - ); + fn get_video_item(&self, seg: &str) -> Option { + let href = Self::between(seg, "href=\"", "\"")?; + if !href.starts_with("/video/") { + return None; + } + let video_url = format!("{}{}", self.url, href); + let id = href.trim_start_matches("/video/").split('/').next()?.to_string(); + if id.is_empty() { + return None; + } - let mut title = match seg.contains("front-video-title") { - true => seg - .split("front-video-title") - .nth(1) - .and_then(|s| s.split('>').nth(1)) - .and_then(|s| s.split('<').next()) - .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}")))? - .trim() - .to_string(), - false => seg - .split("alt=\"") - .nth(1) - .and_then(|s| s.split('\"').next()) - .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}")))? - .trim() - .to_string(), - }; - - title = decode(title.as_bytes()) + let title_raw = Self::between(seg, "class=\"video-card__title\" title=\"", "\"") + .or_else(|| Self::between(seg, "alt=\"Thumbnail for ", "\""))? + .trim(); + let title = decode(title_raw.as_bytes()) .to_string() - .unwrap_or(title) - .titlecase(); - let id = video_url - .split("/video/") - .filter(|s| !s.is_empty()) - .nth(1) - .ok_or_else(|| ErrorKind::Parse(format!("video id\n\n{seg}")))? - .to_string(); - let thumb_block = seg.split(" format!("{}{}", self.url,block - .split("data-front-lazy-src=\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .unwrap_or("") - .to_string()), - None => "".to_string(), - }; - let mut preview = seg - .split("data-front-video-preview-src=\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .unwrap_or("") - .to_string(); - let raw_duration = seg - .split("class=\"front-duration-tag\">") - .nth(1) - .and_then(|s| s.split('<').next()) - .unwrap_or("") - .to_string(); - let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let (tags, views) = self.extract_media(&video_url, &mut requester).await?; - - if preview.len() == 0 { - preview = format!("https://trailers.jav.si/preview/{id}.mp4"); - } - let proxy_url = build_proxy_url( - options, - "javtiful", - &strip_url_scheme(video_url.clone().as_str()), - ); - let video_item = VideoItem::new(id, title, proxy_url, "javtiful".into(), thumb, duration) - .tags(tags) - .preview(preview) - .views(views); - Ok(video_item) - } - - async fn extract_media( - &self, - url: &str, - requester: &mut Requester, - ) -> Result<(Vec, u32)> { - let text = requester - .get(url, Some(Version::HTTP_2)) - .await - .map_err(|e| Error::from(format!("{}", e)))?; - let tags = text - .split("related-actress") - .next() - .and_then(|s| s.split("video-comments").next()) - .and_then(|s| s.split(">Tags").nth(1)) - .and_then(|s| s.split("").next()) - .map(|tag_block| { - tag_block - .split("') - .nth(1) - .and_then(|s| s.split('<').next()) - .map(|s| { - decode(s.as_bytes()) - .to_string() - .unwrap_or(s.to_string()) - .titlecase() - }) - }) - .collect() + let thumb = Self::between(seg, "data-front-lazy-src=\"", "\"") + .map(|t| { + if t.starts_with("http") { + t.to_string() + } else { + format!("{}{}", self.url, t) + } }) - .unwrap_or_else(|| vec![]); - for tag in &tags { - Self::push_unique( - &self.categories, - FilterOption { - id: tag.to_ascii_lowercase().replace(" ", "+"), - title: tag.to_string(), - }, - ); - } - let views = text - .split(" views") - .next() - .and_then(|s| s.split('>').last()) - .and_then(|s| s.trim().replace(".", "").replace(",", "").parse::().ok()) + .unwrap_or_default(); + let preview = Self::between(seg, "data-front-video-preview-src=\"", "\"") + .unwrap_or("") + .to_string(); + + // `FHD02:49:46` + let duration = Self::between(seg, "class=\"video-card__details\">", "") + .and_then(|d| d.rsplit("").next()) + .and_then(|d| parse_time_to_seconds(d.trim())) + .unwrap_or(0) as u32; + + // `
12.9K views · 6 hours ago
` + let views = Self::between(seg, "class=\"video-card__meta\">", "views") + .and_then(|v| parse_abbreviated_number(&v.trim().replace(',', ""))) .unwrap_or(0); - Ok((tags, views)) + // The page URL resolves natively in yt-dlp (html5 ``), so no + // proxy and no formats are needed. + let mut item = VideoItem::new(id, title, video_url, "javtiful".into(), thumb, duration) + .views(views); + if !preview.is_empty() { + item = item.preview(preview); + } + Some(item) } } @@ -474,3 +273,32 @@ impl Provider for JavtifulProvider { Some(self.build_channel(v)) } } + +#[cfg(test)] +mod tests { + use super::JavtifulProvider; + + #[test] + fn parses_cards_and_skips_partner_slots() { + let html = r#"
+ +
Partner thumbnail for Ad
+
"#; + let items = JavtifulProvider::new().get_video_items_from_html(html); + assert_eq!(items.len(), 1); + assert_eq!(items[0].id, "114451"); + assert_eq!(items[0].url, "https://javtiful.com/video/114451/royd-358"); + assert_eq!(items[0].thumb, "https://javtiful.com/uploads/t.jpg"); + assert_eq!(items[0].duration, 2 * 3600 + 49 * 60 + 46); + assert_eq!(items[0].views, Some(12900)); + assert!(items[0].formats.is_none()); + } + + #[test] + fn maps_sort_values() { + assert_eq!(JavtifulProvider::sort_param("latest"), ""); + assert_eq!(JavtifulProvider::sort_param("popular"), "sort=popular_week&"); + assert_eq!(JavtifulProvider::sort_param("most_viewed"), "sort=most_viewed&"); + } +}