sxyprn fix?

src/api.rs

@@ -19,7 +19,7 @@ use std::process::Command;
 use tokio::task;
 use url::Url;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct ClientVersion {
     version: u32,
     subversion: u32,
@@ -568,6 +568,7 @@ async fn videos_post(
         duration: Some(duration),
         sort: Some(sort.clone()),
         sexuality: Some(sexuality),
+        client_version: Some(clientversion.clone()),
     };
 
     if let Some(query_url) = query.as_deref().and_then(normalize_query_url) {
@@ -580,7 +581,9 @@ async fn videos_post(
         if let Some(mut video_items) =
             videos_from_ytdlp_query_url(&channel, &query_url, perPage as usize)
         {
-            if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
+            if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string())
+                && clientversion.version == 38
+            {
                 video_items = video_items
                     .into_iter()
                     .filter_map(|video| {
@@ -674,8 +677,10 @@ async fn videos_post(
         }
     }
 
-    // There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine
+    // There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine.
-    if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
+    // ClientVersion equality only compares the client name, so this also checks `.version` directly
+    // to make sure the workaround stays scoped to actual v38 clients.
+    if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) && clientversion.version == 38 {
         // filter out videos without preview for old clients
         video_items = video_items
             .into_iter()
@@ -809,6 +814,7 @@ async fn uploaders_post(
         duration: None,
         sort: None,
         sexuality: None,
+        client_version: None,
     };
 
     crate::flow_debug!(

src/providers/allpornstream.rs

@@ -471,6 +471,7 @@ mod tests {
 
     fn make_options() -> ServerOptions {
         ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/archivebate.rs

@@ -148,6 +148,7 @@ impl ArchivebateProvider {
 
             runtime.block_on(async move {
                 let options = ServerOptions {
+                    client_version: None,
                     featured: None,
                     category: None,
                     sites: None,

src/providers/hsex.rs

@@ -1343,6 +1343,7 @@ mod tests {
     async fn fetches_page_two_items() {
         let provider = provider();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/jable.rs

@@ -605,6 +605,7 @@ mod tests {
     #[test]
     fn resolve_lang_defaults_to_en() {
         let opts = ServerOptions {
+            client_version: None,
             language: None,
             sort: None, featured: None, category: None, sites: None,
             filter: None, public_url_base: None, requester: None,
@@ -651,6 +652,7 @@ mod tests {
     #[test]
     fn picks_target_from_sort() {
         let opts = ServerOptions {
+            client_version: None,
             sort: Some("hot".to_string()),
             featured: None,
             category: None,
@@ -674,6 +676,7 @@ mod tests {
     #[test]
     fn picks_tag_target_from_query_prefix() {
         let opts = ServerOptions {
+            client_version: None,
             sort: None,
             featured: None,
             category: None,

src/providers/mod.rs

@@ -258,6 +258,7 @@ async fn run_provider_validation(provider_id: &str) -> Result<(), String> {
         .and_then(|value| value.to_u8())
         .unwrap_or(VALIDATION_RESULTS_REQUIRED as u8);
     let options = ServerOptions {
+        client_version: Some(validation_client_version()),
         featured: request.featured.clone(),
         category: request.category.clone(),
         sites: request.sites.clone(),

src/providers/noodlemagazine.rs

@@ -604,6 +604,7 @@ mod tests {
 
     fn options() -> ServerOptions {
         ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/pimpbunny.rs

@@ -871,6 +871,7 @@ mod tests {
     fn rewrites_allowed_thumbs_to_proxy_urls() {
         let provider = test_provider();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,
@@ -901,6 +902,7 @@ mod tests {
     fn rewrites_video_pages_to_redirect_proxy() {
         let provider = test_provider();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,
@@ -931,6 +933,7 @@ mod tests {
     fn parses_listing_without_detail_requests() {
         let provider = test_provider();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/porndish.rs

@@ -1430,6 +1430,7 @@ mod tests {
         let provider = PorndishProvider::new();
         let mut requester = Requester::new();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/pornhd3x.rs

@@ -485,6 +485,7 @@ impl Pornhd3xProvider {
     async fn load_home_catalogs(&self) -> Result<()> {
         let mut requester = requester_or_default(
             &ServerOptions {
+                client_version: None,
                 featured: None,
                 category: None,
                 sites: None,
@@ -551,6 +552,7 @@ impl Pornhd3xProvider {
     async fn load_sitemap_catalogs(&self) -> Result<()> {
         let mut requester = requester_or_default(
             &ServerOptions {
+                client_version: None,
                 featured: None,
                 category: None,
                 sites: None,
@@ -609,6 +611,7 @@ impl Pornhd3xProvider {
     async fn load_recent_tags(&self) -> Result<()> {
         let mut requester = requester_or_default(
             &ServerOptions {
+                client_version: None,
                 featured: None,
                 category: None,
                 sites: None,
@@ -1333,6 +1336,7 @@ mod tests {
     fn builds_proxied_video_urls() {
         let provider = Pornhd3xProvider::new_for_tests();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/redgifs.rs

@@ -607,6 +607,7 @@ mod tests {
 
     fn empty_options() -> ServerOptions {
         ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/shooshtime.rs

@@ -1335,6 +1335,7 @@ mod tests {
                 html,
                 "https://shooshtime.com/videos/example/123/",
                 &crate::videos::ServerOptions {
+                    client_version: None,
                     featured: None,
                     category: None,
                     sites: None,
@@ -1363,6 +1364,7 @@ mod tests {
     fn builds_proxied_video_urls() {
         let provider = ShooshtimeProvider::new();
         let options = crate::videos::ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/supjav.rs

@@ -1758,6 +1758,7 @@ mod tests {
     fn ignores_sort_filter_when_resolving_archive_target() {
         let provider = test_provider();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: Some("all".to_string()),
             sites: None,
@@ -1780,6 +1781,7 @@ mod tests {
     fn normalizes_relative_archive_targets() {
         let provider = test_provider();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: Some("/category/censored-jav".to_string()),
             sites: None,
@@ -1874,6 +1876,7 @@ mod tests {
         let cache = VideoCache::new().max_size(1_000).to_owned();
         let pool = test_db_pool();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/sxyprn.rs

@@ -5,7 +5,6 @@ use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::discord::format_error_chain;
 use crate::util::discord::send_discord_error_report;
-use crate::util::hoster_proxy::{proxy_name_for_url, rewrite_hoster_url};
 use crate::util::requester::Requester;
 use crate::util::time::parse_time_to_seconds;
 use crate::videos::ServerOptions;
@@ -142,9 +141,24 @@ impl SxyprnProvider {
             sort_string
         );
 
-        let old_items = match cache.get(&url_str) {
+        // App requests carry eagerly-resolved (and shorter-lived) CDN URLs in
+        // `formats`, so they're cached separately from the plain listing and
+        // expire sooner, since the resolved URLs are more likely to go stale.
+        let is_app_client = options
+            .client_version
+            .as_ref()
+            .map(|cv| *cv == ClientVersion::new(0, 0, "Hot%20Tub".to_string()))
+            .unwrap_or(false);
+        let cache_key = if is_app_client {
+            format!("{url_str}#app")
+        } else {
+            url_str.clone()
+        };
+        let cache_ttl_secs: u64 = if is_app_client { 30 * 60 } else { 60 * 60 };
+
+        let old_items = match cache.get(&cache_key) {
             Some((time, items)) => {
-                if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
+                if time.elapsed().unwrap_or_default().as_secs() < cache_ttl_secs {
                     return Ok(items.clone());
                 } else {
                     items.clone()
@@ -192,8 +206,8 @@ impl SxyprnProvider {
         //     .get_video_items_from_html(text.clone(), pool, requester)
         //     .await;
         if !video_items.is_empty() {
-            cache.remove(&url_str);
+            cache.remove(&cache_key);
-            cache.insert(url_str.clone(), video_items.clone());
+            cache.insert(cache_key.clone(), video_items.clone());
         } else {
             return Ok(old_items);
         }
@@ -225,10 +239,24 @@ impl SxyprnProvider {
             ((page as u32) - 1) * 20,
             sort_string
         );
-        // Check our Video Cache. If the result is younger than 1 hour, we return it.
+        // App requests carry eagerly-resolved (and shorter-lived) CDN URLs in
-        let old_items = match cache.get(&url_str) {
+        // `formats`, so they're cached separately from the plain listing and
+        // expire sooner, since the resolved URLs are more likely to go stale.
+        let is_app_client = options
+            .client_version
+            .as_ref()
+            .map(|cv| *cv == ClientVersion::new(0, 0, "Hot%20Tub".to_string()))
+            .unwrap_or(false);
+        let cache_key = if is_app_client {
+            format!("{url_str}#app")
+        } else {
+            url_str.clone()
+        };
+        let cache_ttl_secs: u64 = if is_app_client { 30 * 60 } else { 60 * 60 };
+        // Check our Video Cache. If the result is younger than the TTL, we return it.
+        let old_items = match cache.get(&cache_key) {
             Some((time, items)) => {
-                if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
+                if time.elapsed().unwrap_or_default().as_secs() < cache_ttl_secs {
                     return Ok(items.clone());
                 } else {
                     let _ = cache.check().await;
@@ -277,8 +305,8 @@ impl SxyprnProvider {
         //     .get_video_items_from_html(text.clone(), pool, requester)
         //     .await;
         if !video_items.is_empty() {
-            cache.remove(&url_str);
+            cache.remove(&cache_key);
-            cache.insert(url_str.clone(), video_items.clone());
+            cache.insert(cache_key.clone(), video_items.clone());
         } else {
             return Ok(old_items);
         }
@@ -289,13 +317,23 @@ impl SxyprnProvider {
         &self,
         html: String,
         _pool: DbPool,
-        _requester: Requester,
+        requester: Requester,
         options: &ServerOptions,
     ) -> Result<Vec<VideoItem>> {
         if html.is_empty() {
             return Ok(vec![]);
         }
 
+        // The Hottub app can resolve directly-playable format URLs itself, so for
+        // app requests we serve the real sxyprn.com page as `url` and eagerly
+        // resolve every mirror CDN URL into `formats`. Other clients keep the
+        // lazy `/proxy/sxyprn/post/{id}` redirect and get no formats.
+        let is_app_client = options
+            .client_version
+            .as_ref()
+            .map(|cv| *cv == ClientVersion::new(0, 0, "Hot%20Tub".to_string()))
+            .unwrap_or(false);
+
         // take content before "<script async"
         let main_content = html
             .split("main_content")
@@ -310,6 +348,7 @@ impl SxyprnProvider {
         }
 
         let mut items = Vec::new();
+        let mut slugs: Vec<String> = Vec::new();
 
         for video_segment in raw_videos {
             // url id
@@ -331,12 +370,6 @@ impl SxyprnProvider {
                 .and_then(|s| s.split("</div>").next())
                 .ok_or_else(|| ErrorKind::Parse("failed to extract title_parts".into()))?;
 
-            let title_links: Vec<String> = video_segment
-                .split("href='https://")
-                .skip(1)
-                .filter_map(|part| part.split("'").next().map(|u| u.to_string()))
-                .collect();
-
             let document = Html::parse_document(title_parts);
             let selector = Selector::parse("*")
                 .map_err(|e| ErrorKind::Parse(format!("selector parse failed: {e}")))?;
@@ -470,29 +503,13 @@ impl SxyprnProvider {
 
             let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
 
-            // Start with the sxyprn proxy URL; doodstream or vidara overwrite it if present.
+            // Default to the lazy proxy redirect; app clients get the real
-            let sxyprn_url = format!(
+            // sxyprn.com page URL plus eagerly-resolved formats below.
+            let proxy_video_url = format!(
                 "{}/proxy/sxyprn/post/{}",
                 options.public_url_base.as_deref().unwrap_or(""),
                 id
             );
-            let video_url = sxyprn_url;
-
-            // if let Some(dood_url) = title_links
-            //     .iter()
-            //     .find(|u| proxy_name_for_url(u).as_deref() == Some("doodstream"))
-            //     .map(|u| rewrite_hoster_url(options, u))
-            // {
-            //     video_url = dood_url;
-            // }
-
-            // if let Some(vidara_url) = title_links
-            //     .iter()
-            //     .find(|u| proxy_name_for_url(u).as_deref() == Some("vidara"))
-            //     .map(|u| rewrite_hoster_url(options, u))
-            // {
-            //     video_url = vidara_url;
-            // }
 
             let mut headers = std::collections::HashMap::new();
             headers.insert("Referer".to_string(), "https://sxyprn.com/".to_string());
@@ -500,7 +517,7 @@ impl SxyprnProvider {
             let mut video_item = VideoItem::new(
                 id.clone(),
                 title,
-                video_url,
+                proxy_video_url,
                 "sxyprn".to_string(),
                 thumb,
                 duration,
@@ -517,6 +534,53 @@ impl SxyprnProvider {
                 video_item.preview = preview;
             }
             items.push(video_item);
+            slugs.push(id);
+        }
+
+        if is_app_client && !items.is_empty() {
+            let tasks: Vec<_> = slugs
+                .iter()
+                .cloned()
+                .map(|slug| {
+                    let mut item_requester = requester.clone();
+                    let base_url = self.url.clone();
+                    tokio::spawn(async move {
+                        crate::proxies::sxyprn::resolve_all_media_urls(
+                            &mut item_requester,
+                            &base_url,
+                            &slug,
+                        )
+                        .await
+                    })
+                })
+                .collect();
+
+            for ((video_item, slug), task) in
+                items.iter_mut().zip(slugs.iter()).zip(tasks)
+            {
+                let resolved_urls = task.await.unwrap_or_default();
+                if resolved_urls.is_empty() {
+                    continue;
+                }
+
+                video_item.url = format!("{}/post/{}", self.url, slug);
+                video_item.formats = Some(
+                    resolved_urls
+                        .into_iter()
+                        .map(|media_url| {
+                            let mut format_headers = std::collections::HashMap::new();
+                            format_headers
+                                .insert("Referer".to_string(), "https://sxyprn.com/".to_string());
+                            crate::videos::VideoFormat::new(
+                                media_url,
+                                "auto".to_string(),
+                                "mp4".to_string(),
+                            )
+                            .http_headers(format_headers)
+                        })
+                        .collect(),
+                );
+            }
         }
 
         Ok(items)

src/providers/thaiporntv.rs

@@ -742,6 +742,7 @@ mod tests {
     async fn fetches_and_parses_archive() {
         let provider = provider();
         let options = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/tube8.rs

@@ -565,6 +565,7 @@ mod tests {
         "#;
 
         let opts = ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/providers/vrporn.rs

@@ -537,6 +537,7 @@ impl VrpornProvider {
         if query.is_empty() {
             return self.resolve_option_target(
                 &ServerOptions {
+                    client_version: None,
                     featured: None,
                     category: None,
                     sites: None,

src/proxies/sxyprn.rs

@@ -21,7 +21,7 @@ fn boo(sum1: u32, sum2: u32) -> String {
 }
 
 /// Extracts all CDN path values from the data-vnfo JSON attribute.
-fn extract_all_cdn_paths(html: &str) -> Vec<String> {
+pub(crate) fn extract_all_cdn_paths(html: &str) -> Vec<String> {
     let json_str = match html.split("data-vnfo='").nth(1) {
         Some(s) => s.split('\'').next().unwrap_or(""),
         None => return vec![],
@@ -53,7 +53,7 @@ fn extract_all_cdn_paths(html: &str) -> Vec<String> {
 }
 
 /// Applies the sxyprn segment transformation to produce the pre-redirect CDN URL.
-fn transform_cdn_path(path: &str) -> Option<String> {
+pub(crate) fn transform_cdn_path(path: &str) -> Option<String> {
     let mut tmp: Vec<String> = path.split('/').map(|s| s.to_string()).collect();
     if tmp.len() < 8 {
         return None;
@@ -101,6 +101,68 @@ async fn race_cdn_urls(candidate_urls: Vec<String>) -> String {
         .unwrap_or_default()
 }
 
+/// Resolves all candidate CDN mirror URLs concurrently, collecting every
+/// mirror that redirects successfully within the timeout window. Unlike
+/// `race_cdn_urls`, this does not stop at the first success -- the mirrors
+/// are redundant copies of the same stream, and callers that want to serve
+/// directly-playable format URLs need as many working ones as possible.
+pub(crate) async fn resolve_all_cdn_urls(candidate_urls: Vec<String>) -> Vec<String> {
+    if candidate_urls.is_empty() {
+        return vec![];
+    }
+
+    let handles: Vec<_> = candidate_urls
+        .into_iter()
+        .map(|cdn_url| {
+            tokio::spawn(async move {
+                tokio::task::spawn_blocking(move || {
+                    crate::util::get_redirect_location(&cdn_url)
+                        .ok()
+                        .flatten()
+                        .map(|loc| format!("https:{}", loc))
+                })
+                .await
+                .ok()
+                .flatten()
+            })
+        })
+        .collect();
+
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
+    let mut resolved = Vec::new();
+    for handle in handles {
+        if let Ok(Ok(Some(url))) = tokio::time::timeout_at(deadline, handle).await {
+            if !resolved.contains(&url) {
+                resolved.push(url);
+            }
+        }
+    }
+    resolved
+}
+
+/// Fetches the sxyprn detail page for `slug` and resolves every mirror CDN
+/// URL it advertises. Used by the provider to eagerly populate `formats`
+/// for app clients, instead of the lazy single-mirror `/proxy/sxyprn/...`
+/// redirect used by other clients.
+pub(crate) async fn resolve_all_media_urls(
+    requester: &mut Requester,
+    base_url: &str,
+    slug: &str,
+) -> Vec<String> {
+    let full_url = format!("{}/post/{}", base_url, slug);
+    let text = requester.get(&full_url, None).await.unwrap_or_default();
+    if text.is_empty() {
+        return vec![];
+    }
+
+    let candidate_urls: Vec<String> = extract_all_cdn_paths(&text)
+        .iter()
+        .filter_map(|p| transform_cdn_path(p))
+        .collect();
+
+    resolve_all_cdn_urls(candidate_urls).await
+}
+
 #[derive(Debug, Clone)]
 pub struct SxyprnProxy {}
 

src/util/hoster_proxy.rs

@@ -56,6 +56,7 @@ mod tests {
 
     fn options() -> ServerOptions {
         ServerOptions {
+            client_version: None,
             featured: None,
             category: None,
             sites: None,

src/videos.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 
+use crate::api::ClientVersion;
 use crate::util::requester::Requester;
 use serde;
 
@@ -68,6 +69,7 @@ pub struct ServerOptions {
     pub duration: Option<String>,   //
     pub sort: Option<String>,       //
     pub sexuality: Option<String>,  //
+    pub client_version: Option<ClientVersion>,
 }
 
 #[derive(serde::Serialize, Debug)]