Fix tube8 ids and Referer-gated Aylo thumbnails

Sponsored channel cards carry a stale data-video-id that 404s; use the
card link target. The signed pix-cdn77 thumbnail hosts 403 without a
Referer, so route them (tube8, ph-shorties) through a new host-restricted
aylo-thumb proxy. Also correct the pornhub-thumb route that sat under a
doubled /proxy prefix.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi

docs/provider-fix-todo.md

@@ -24,7 +24,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
 | [x] | `thaiporntv` | page URL "Unsupported URL", no `formats` | player attribute renamed `data-enc` -> `data-video-sources` (reversed base64 JSON `{quality,url}`); decoder handles both. 3 formats/item, yt-dlp OK, check.py passes |
 | [x] | `pornmz` | page URL "Unsupported URL", no `formats` | site changed `itemprop="contentURL"` -> `contentUrl`; selector now case-insensitive. Enrichment concurrency 10 + 25s per-item timeout. Origin is slow (15s+/detail) so first feed 25-50s. check.py passes |
 | [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end |
-| [ ] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | |
+| [x] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | sponsored `channel` cards carry a stale `data-video-id` (404) while the card link is the live page; id/URL now come from the link. Signed `pix-cdn77` thumbs 403 without a Referer: new host-restricted `/proxy/aylo-thumb` (also fixes ph-shorties). check.py passes |
 | [ ] | `hypnotube` | page URLs 404 on all sampled items | URL construction |
 | [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | |
 | [ ] | `ripnsfw` | `doodstream` format 502 | |
@@ -36,7 +36,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
 | | Channel | Symptom | Notes |
 |---|---|---|---|
 | [ ] | `xxdbx` | malformed thumbs `https:/7004….jpg` | built as `"https:" + src` |
-| [ ] | `ph-shorties` | thumbs return 403 | |
+| [x] | `ph-shorties` | thumbs return 403 | same Referer-gated `pix-cdn77.phncdn.com` thumbs; routed through `aylo-thumb` proxy. check.py passes |
 | [ ] | `noodlemagazine` | thumbs unreachable (-1 / 400) in check.py | thumb proxy |
 
 ## 4. Search
@@ -64,6 +64,11 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
 |---|---|---|
 | [ ] | `check.py` "url points to hottub server" | contradicts the hottub skill (proxy-needing videos use the proxy as `item.url`). Affects spankbang, hqporner, pornhd3x, pimpbunny, sxyprn, noodlemagazine. Decide: exempt proxy URLs that resolve, rather than hard-fail. |
 
+## Found while fixing (pre-existing, unrelated)
+
+- `src/proxy.rs`: the `pornhub-thumb` resource was registered as `/proxy/pornhub-thumb` inside the `/proxy` scope (unreachable); fixed alongside `aylo-thumb`. No provider uses it today.
+- Failing unit tests, untouched files: `proxies::hanimecdn::tests::prefixes_relative_paths_with_hanime_cdn_host`, `proxies::vidara::tests::normalizes_detail_request_with_path_only`.
+
 ## Environmental / needs the production IP (not worked here)
 
 `porndish`, `supjav`, `jable`, `fapello`, `missav`, `wowxxx`, `hentaihaven` return Cloudflare 403 to

src/providers/coomer.rs

@@ -489,15 +489,6 @@ impl CoomerProvider {
     }
 }
 
-fn target_label(target: &Target) -> &'static str {
-    match target {
-        Target::Latest => "latest",
-        Target::Popular => "popular",
-        Target::Search { .. } => "search",
-        Target::Uploader { .. } => "uploader",
-    }
-}
-
 fn percent_encode(value: &str) -> String {
     let mut out = String::with_capacity(value.len());
     for byte in value.as_bytes() {

src/providers/mod.rs

@@ -644,6 +644,25 @@ pub fn requester_or_default(
 }
 
 #[allow(dead_code)]
+/// Routes signed Aylo CDN thumbnails (`pix-cdn77.t8cdn.com`,
+/// `pix-cdn77.phncdn.com`) through the local `aylo-thumb` proxy, because the
+/// CDN answers 403 unless a matching Referer is sent. Other URLs are returned
+/// unchanged.
+pub fn proxy_aylo_thumb(options: &ServerOptions, thumb: &str) -> String {
+    let needs_proxy = thumb
+        .strip_prefix("https://")
+        .or_else(|| thumb.strip_prefix("http://"))
+        .and_then(|rest| rest.split('/').next())
+        .is_some_and(|host| {
+            host.starts_with("pix-cdn77.")
+                && (host.ends_with(".t8cdn.com") || host.ends_with(".phncdn.com"))
+        });
+    if !needs_proxy {
+        return thumb.to_string();
+    }
+    build_proxy_url(options, "aylo-thumb", &strip_url_scheme(thumb))
+}
+
 pub fn strip_url_scheme(url: &str) -> String {
     url.strip_prefix("https://")
         .or_else(|| url.strip_prefix("http://"))

src/providers/pornhub_shorties.rs

@@ -1,6 +1,6 @@
 use crate::DbPool;
 use crate::api::ClientVersion;
-use crate::providers::{Provider, report_provider_error, requester_or_default};
+use crate::providers::{Provider, proxy_aylo_thumb, report_provider_error, requester_or_default};
 use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::videos::{ServerOptions, VideoItem};
@@ -408,6 +408,11 @@ impl PornhubShortiesProvider {
         let items: Vec<VideoItem> = shorties
             .into_iter()
             .filter_map(Self::item_to_video)
+            .map(|mut video| {
+                // The signed pix-cdn77 thumbnails 403 without a Referer.
+                video.thumb = proxy_aylo_thumb(&options, &video.thumb);
+                video
+            })
             .collect();
 
         if items.is_empty() {

src/providers/tube8.rs

@@ -1,6 +1,8 @@
 use crate::DbPool;
 use crate::api::ClientVersion;
-use crate::providers::{Provider, build_proxy_url, report_provider_error, requester_or_default};
+use crate::providers::{
+    Provider, build_proxy_url, proxy_aylo_thumb, report_provider_error, requester_or_default,
+};
 use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::parse_abbreviated_number;
@@ -257,11 +259,24 @@ impl Tube8Provider {
         let mut items = Vec::new();
 
         for card in document.select(&card_sel) {
-            let id = card
+            // The card's own link is authoritative: sponsored `channel` cards
-                .value()
+            // carry a `data-video-id` that 404s while the link target
-                .attr("data-video-id")
+            // (`/porn-video/<other id>/`) is the live page.
-                .map(|v| v.to_string())
+            let link_id = link_sel
-                .filter(|v| !v.is_empty())
+                .as_ref()
+                .and_then(|s| card.select(s).next())
+                .and_then(|v| v.value().attr("href"))
+                .and_then(|href| href.split("/porn-video/").nth(1))
+                .and_then(|rest| rest.split('/').next())
+                .filter(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_digit()))
+                .map(|v| v.to_string());
+            let id = link_id
+                .or_else(|| {
+                    card.value()
+                        .attr("data-video-id")
+                        .map(|v| v.to_string())
+                        .filter(|v| !v.is_empty())
+                })
                 .unwrap_or_default();
             if id.is_empty() {
                 continue;
@@ -296,6 +311,7 @@ impl Tube8Provider {
                         .or_else(|| v.value().attr("src"))
                 })
                 .map(|v| self.normalize_url(v))
+                .map(|v| proxy_aylo_thumb(options, &v))
                 .unwrap_or_default();
 
             // Preview flipbook clip from the thumbnail link's data-mediabook
@@ -536,6 +552,27 @@ mod tests {
         );
     }
 
+    #[test]
+    fn uses_link_target_id_when_card_video_id_is_stale() {
+        let p = provider();
+        let html = r#"
+        <article class="video-box pc js_video-box js-pop" data-video-id="170549881"
+             data-uploader-type="channel" aria-label="Sponsored clip">
+            <a href="/porn-video/67452091/" data-testid="plw_video_thumbnail_link">
+                <img class="thumb-image js_lazy" data-src="https://ei-ph.t8cdn.com/t.jpg"/>
+            </a>
+        </article>"#;
+        let opts = ServerOptions {
+            client_version: None, featured: None, category: None, sites: None, filter: None,
+            language: None, public_url_base: None, requester: None, network: None, stars: None,
+            categories: None, duration: None, sort: None, sexuality: None,
+        };
+        let items = p.parse_items(html, &opts);
+        assert_eq!(items.len(), 1);
+        assert_eq!(items[0].id, "67452091");
+        assert_eq!(items[0].url, "https://www.tube8.com/porn-video/67452091/");
+    }
+
     #[test]
     fn parses_video_cards() {
         let p = provider();

src/proxies/aylothumb.rs

@@ -0,0 +1,126 @@
+use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
+use ntex::{
+    http::Response,
+    web::{self, HttpRequest, error},
+};
+use url::Url;
+
+use crate::util::requester::Requester;
+
+/// Referer the image CDN expects, or `None` when the host is not allowed.
+///
+/// The signed `pix-cdn77.*` thumbnail hosts used by tube8 and the pornhub
+/// shorties feed answer 403 without a matching Referer, so apps that load
+/// the URL directly get broken images.
+fn referer_for_host(host: &str) -> Option<&'static str> {
+    let host = host.to_ascii_lowercase();
+    if host == "t8cdn.com" || host.ends_with(".t8cdn.com") {
+        Some("https://www.tube8.com/")
+    } else if host == "phncdn.com" || host.ends_with(".phncdn.com") {
+        Some("https://www.pornhub.com/")
+    } else {
+        None
+    }
+}
+
+fn is_image_path(path: &str) -> bool {
+    let path = path.to_ascii_lowercase();
+    // The signed thumbnail paths end in processing options such as
+    // `/plain/rs:fit:320:180/vts:3`, not a file extension, so accept those too.
+    [".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"]
+        .iter()
+        .any(|ext| path.ends_with(ext))
+        || path.contains("/plain/")
+}
+
+fn upstream_url(req: &HttpRequest) -> Option<(String, &'static str)> {
+    let endpoint = req.match_info().query("endpoint").trim_start_matches('/');
+    let mut image_url = if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
+        endpoint.to_string()
+    } else {
+        format!("https://{endpoint}")
+    };
+
+    // The signed URL's own query (`hash`, `validto`) arrives as the request query.
+    let query = req.query_string();
+    if !query.is_empty() && !image_url.contains('?') {
+        image_url.push('?');
+        image_url.push_str(query);
+    }
+
+    let parsed = Url::parse(&image_url).ok()?;
+    if parsed.scheme() != "https" {
+        return None;
+    }
+    let referer = referer_for_host(parsed.host_str()?)?;
+    if !is_image_path(parsed.path()) {
+        return None;
+    }
+    Some((image_url, referer))
+}
+
+pub async fn get_image(
+    req: HttpRequest,
+    requester: web::types::State<Requester>,
+) -> Result<impl web::Responder, web::Error> {
+    let Some((image_url, referer)) = upstream_url(&req) else {
+        return Ok(web::HttpResponse::BadRequest().finish());
+    };
+
+    let upstream = match requester
+        .get_ref()
+        .clone()
+        .get_raw_with_headers(
+            image_url.as_str(),
+            vec![("Referer".to_string(), referer.to_string())],
+        )
+        .await
+    {
+        Ok(response) => response,
+        Err(_) => return Ok(web::HttpResponse::NotFound().finish()),
+    };
+
+    let status = upstream.status();
+    let headers = upstream.headers().clone();
+    let bytes = upstream.bytes().await.map_err(error::ErrorBadGateway)?;
+
+    let mut resp = Response::build(status);
+    if let Some(ct) = headers.get(CONTENT_TYPE) {
+        if let Ok(ct_str) = ct.to_str() {
+            resp.set_header(CONTENT_TYPE, ct_str);
+        }
+    }
+    if let Some(cl) = headers.get(CONTENT_LENGTH) {
+        if let Ok(cl_str) = cl.to_str() {
+            resp.set_header(CONTENT_LENGTH, cl_str);
+        }
+    }
+
+    Ok(resp.body(bytes.to_vec()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn maps_hosts_to_referers_and_rejects_others() {
+        assert_eq!(
+            referer_for_host("pix-cdn77.t8cdn.com"),
+            Some("https://www.tube8.com/")
+        );
+        assert_eq!(
+            referer_for_host("pix-cdn77.phncdn.com"),
+            Some("https://www.pornhub.com/")
+        );
+        assert_eq!(referer_for_host("evil.example.com"), None);
+        assert_eq!(referer_for_host("t8cdn.com.evil.example"), None);
+    }
+
+    #[test]
+    fn accepts_signed_processing_paths_and_extensions() {
+        assert!(is_image_path("/c6531/videos/1/original.mov/plain/rs:fit:320:180/vts:3"));
+        assert!(is_image_path("/videos/thumb.JPG"));
+        assert!(!is_image_path("/videos/clip.mp4"));
+    }
+}

src/proxies/mod.rs

@@ -18,6 +18,7 @@ use crate::proxies::lulustream::LulustreamProxy;
 use crate::proxies::thaiporntv::ThaipornTvProxy;
 
 pub mod allpornstream;
+pub mod aylothumb;
 pub mod melonstube;
 pub mod animeidhentai;
 pub mod archivebate;

src/proxy.rs

@@ -149,7 +149,12 @@ pub fn config(cfg: &mut web::ServiceConfig) {
             .route(web::get().to(crate::proxies::hotbunnythumb::get_image)),
     );
     cfg.service(
-        web::resource("/proxy/pornhub-thumb/{endpoint}*")
+        web::resource("/aylo-thumb/{endpoint}*")
+            .route(web::post().to(crate::proxies::aylothumb::get_image))
+            .route(web::get().to(crate::proxies::aylothumb::get_image)),
+    );
+    cfg.service(
+        web::resource("/pornhub-thumb/{endpoint}*")
             .route(web::post().to(crate::proxies::pornhubthumb::get_image))
             .route(web::get().to(crate::proxies::pornhubthumb::get_image)),
     );