diff --git a/docs/provider-fix-todo.md b/docs/provider-fix-todo.md index 4a4bb55..db3fbb8 100644 --- a/docs/provider-fix-todo.md +++ b/docs/provider-fix-todo.md @@ -26,8 +26,8 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro | [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end | | [x] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | sponsored `channel` cards carry a stale `data-video-id` (404) while the card link is the live page; id/URL now come from the link. Signed `pix-cdn77` thumbs 403 without a Referer: new host-restricted `/proxy/aylo-thumb` (also fixes ph-shorties). check.py passes | | [x] | `hypnotube` | page URLs 404 on all sampled items | the 404 was a check.py artifact (server answers HEAD 404, GET 200); check.py now confirms with a ranged GET. Real bug found on the way: the site returns ONE results page per searchgate POST, so any 2nd search on the shared session (and every page 2) came back empty. Each (query, page) now uses its own isolated session. 3 consecutive searches + page 2 verified | -| [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | | -| [ ] | `ripnsfw` | `doodstream` format 502 | | +| [x] | `fullpornerorg` | `lulustream` proxy 403 / 502 | three separate causes: (1) hoster files deleted ("File is no longer available") but still listed -> now probed and dropped; (2) `myvidplay.com` embeds lost their host (bare ID sent to lulustream, wrong namespace) -> routed to the doodstream proxy with host kept, which now allows myvidplay/playmogo; (3) tnmr.org m3u8 is TLS-fingerprint gated (works with chrome TLS = the channel's `--impersonate chrome-120`; plain yt-dlp 403s are the checker's expected CF warnings). myvidplay playback could not be verified: the embed now returns a Turnstile captcha shell to this client | +| [-] | `ripnsfw` | `doodstream` format 502 | lulustream format on the same items resolves (302). The doodstream format cannot be verified here: `*.dood.video` resolves to 127.0.0.1 on this host's resolver, so the CDN is unreachable. Needs the production network | | [ ] | `freepornvideosxxx` | yt-dlp 502 on page URL | may be transient, verify | | [ ] | `hdporn92` | m3u8 formats time out (curl + yt-dlp) | morencius.com streams | diff --git a/src/providers/fullpornerorg.rs b/src/providers/fullpornerorg.rs index a0f8744..65964d1 100644 --- a/src/providers/fullpornerorg.rs +++ b/src/providers/fullpornerorg.rs @@ -11,6 +11,8 @@ use futures::stream::{self, StreamExt}; use regex::Regex; use scraper::{Html, Selector}; +const DELETED_EMBED: &str = "hoster file deleted"; + pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "mainstream-tube", @@ -311,8 +313,28 @@ impl FullpornerorgProvider { None } - /// Builds the lulustream proxy URL for a given embed ID. - fn build_proxy_url(embed_id: &str, options: &ServerOptions) -> String { + /// The hoster serves a tiny stub page for removed files. Listings on + /// fullporner.org keep pointing at them, so detect it and skip the item. + fn embed_is_deleted(html: &str) -> bool { + let lower = html.to_ascii_lowercase(); + lower.contains("no longer available") + || lower.contains("has been deleted") + // DoodStream-protocol hosts (myvidplay.com) title the stub this way. + || lower.contains("video not found") + } + + /// Builds the proxy URL for an embed. The ID alone is only unique per + /// hoster: `myvidplay.com` is a DoodStream-protocol host, so those embeds + /// must keep their host and go through the doodstream proxy; resolving the + /// same ID on lulustream finds nothing (or a different, deleted file). + fn build_proxy_url(embed_url: &str, embed_id: &str, options: &ServerOptions) -> String { + if embed_url.contains("myvidplay.com/") { + return crate::providers::build_proxy_url( + options, + "doodstream", + &format!("myvidplay.com/e/{embed_id}"), + ); + } crate::providers::build_proxy_url(options, "lulustream", embed_id) } @@ -324,9 +346,17 @@ impl FullpornerorgProvider { let embed_id = Self::extract_embed_id(&embed_url) .ok_or_else(|| Error::from(format!("could not parse embed id from {embed_url}")))?; + // Skip files the hoster has already removed (fail open if the probe + // itself is blocked or times out). + if let Ok(embed_html) = Self::fetch_html(options, &embed_url, BASE_URL).await { + if Self::embed_is_deleted(&embed_html) { + return Err(Error::from(DELETED_EMBED)); + } + } + // Step 2: build the proxy URL (will hit /proxy/lulustream/{embed_id} which // resolves the actual HLS URL via the lulustream proxy). - let proxy_url = Self::build_proxy_url(&embed_id, options); + let proxy_url = Self::build_proxy_url(&embed_url, &embed_id, options); // Build the format. The proxy will return a redirect to the actual HLS URL. let mut format = VideoFormat::m3u8(proxy_url, "auto".to_string(), "m3u8".to_string()); @@ -371,11 +401,13 @@ impl FullpornerorgProvider { match Self::enrich_card(stub, &options).await { Ok(item) => Some(item), Err(error) => { - report_provider_error_background( - CHANNEL_ID, - "fetch_page.enrich_card", - &error.to_string(), - ); + if error.to_string() != DELETED_EMBED { + report_provider_error_background( + CHANNEL_ID, + "fetch_page.enrich_card", + &error.to_string(), + ); + } None } } @@ -482,6 +514,36 @@ mod tests { ); } + #[test] + fn myvidplay_embeds_route_to_doodstream_with_host() { + let options = ServerOptions { + client_version: None, featured: None, category: None, sites: None, filter: None, + language: None, public_url_base: None, requester: None, network: None, stars: None, + categories: None, duration: None, sort: None, sexuality: None, + }; + assert_eq!( + FullpornerorgProvider::build_proxy_url("https://myvidplay.com/e/abc123", "abc123", &options), + "/proxy/doodstream/myvidplay.com/e/abc123" + ); + assert_eq!( + FullpornerorgProvider::build_proxy_url("https://luluvdo.com/e/abc123", "abc123", &options), + "/proxy/lulustream/abc123" + ); + } + + #[test] + fn detects_deleted_embed_stub() { + assert!(FullpornerorgProvider::embed_is_deleted( + "<body>File is no longer available as it expired or has been deleted.</body>" + )); + assert!(FullpornerorgProvider::embed_is_deleted( + "<html><head><title>Video not found | DoodStream" + )); + assert!(!FullpornerorgProvider::embed_is_deleted( + "Payton Preslee - Lulustream.mp4" + )); + } + #[test] fn picks_search_for_query() { match FullpornerorgProvider::pick_target(Some("test")) { diff --git a/src/proxies/doodstream.rs b/src/proxies/doodstream.rs index beea01b..c0f35ec 100644 --- a/src/proxies/doodstream.rs +++ b/src/proxies/doodstream.rs @@ -63,6 +63,10 @@ impl DoodstreamProxy { | "www.streamhg.com" | "doodstream.com" | "www.doodstream.com" + | "myvidplay.com" + | "www.myvidplay.com" + | "playmogo.com" + | "www.playmogo.com" ) }