Fix coomer: offset pagination, per-query cache key, video-only posts

The API ignores page= (every page 2+ repeated page 1), searches shared a
single cache entry, image-only posts were listed as unplayable videos, and
gzip/br bodies were not decoded because wreq had no decompression features.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi

Cargo.toml

@@ -20,7 +20,7 @@ ntex-files = "3"
 serde = "1.0.228"
 serde_json = "1.0.145"
 tokio = { version = "1.49", features = ["full"] }
-wreq =  { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json"] }
+wreq =  { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json", "gzip", "brotli", "deflate", "zstd"] }
 wreq-util = "3.0.0-rc.14"
 percent-encoding = "2.3.2"
 capitalize = "0.3.4"

docs/provider-fix-todo.md

@@ -23,7 +23,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
 |---|---|---|---|
 | [x] | `thaiporntv` | page URL "Unsupported URL", no `formats` | player attribute renamed `data-enc` -> `data-video-sources` (reversed base64 JSON `{quality,url}`); decoder handles both. 3 formats/item, yt-dlp OK, check.py passes |
 | [x] | `pornmz` | page URL "Unsupported URL", no `formats` | site changed `itemprop="contentURL"` -> `contentUrl`; selector now case-insensitive. Enrichment concurrency 10 + 25s per-item timeout. Origin is slow (15s+/detail) so first feed 25-50s. check.py passes |
-| [ ] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | check whether `formats` are populated for videos |
+| [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end |
 | [ ] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | |
 | [ ] | `hypnotube` | page URLs 404 on all sampled items | URL construction |
 | [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | |

src/providers/coomer.rs

@@ -37,6 +37,8 @@ const REFERER: &str = "https://coomer.st/";
 const JSON_ACCEPT: &str = "text/css,*/*;q=0.1";
 const CACHE_TTL_SECS: u64 = 60 * 15;
 const MAX_PAGE: u16 = 50;
+/// The API pages by offset (`o`), in blocks of this many posts. `page=` is ignored.
+const API_PAGE_SIZE: u16 = 50;
 const DEFAULT_PER_PAGE: usize = 30;
 const USER_POSTS_FETCH_CONCURRENCY: usize = 4;
 
@@ -239,19 +241,18 @@ impl CoomerProvider {
         }
     }
 
+    fn offset_for(page: u16) -> u32 {
+        (page.max(1).min(MAX_PAGE) as u32 - 1) * API_PAGE_SIZE as u32
+    }
+
     fn build_main_url(target: &Target, page: u16) -> String {
-        let page = page.max(1).min(MAX_PAGE);
+        let offset = Self::offset_for(page);
         match target {
-            Target::Latest => format!("{}/api/v1/posts?limit=50&page={page}", BASE_URL),
+            Target::Latest => format!("{}/api/v1/posts?o={offset}", BASE_URL),
-            Target::Popular => format!("{}/api/v1/posts/popular?limit=50&page={page}", BASE_URL),
+            Target::Popular => format!("{}/api/v1/posts/popular?o={offset}", BASE_URL),
-            // Search is non-paginated: ?page=N is ignored by the server for
-            // q=... requests, but we still emit a `page=1` for consistency.
             Target::Search { query } => {
                 let encoded = percent_encode(query);
-                format!(
+                format!("{}/api/v1/posts?q={encoded}&o={offset}", BASE_URL)
-                    "{}/api/v1/posts?q={encoded}&limit=50&page=1",
-                    BASE_URL
-                )
             }
             // Uploader URL is built separately.
             Target::Uploader { .. } => String::new(),
@@ -259,14 +260,12 @@ impl CoomerProvider {
     }
 
     fn build_uploader_url(service: &Service, username: &str, page: u16) -> String {
-        let page = page.max(1).min(MAX_PAGE);
-        // The user-posts endpoint is also non-paginated, but we send page=1 for
-        // clarity. Service comes from the enum so the path is stable.
         format!(
-            "{}/api/v1/{}/user/{}/posts?limit=50&page={page}",
+            "{}/api/v1/{}/user/{}/posts?o={}",
             BASE_URL,
             service.as_str(),
             percent_encode(username),
+            Self::offset_for(page),
         )
     }
 
@@ -308,25 +307,21 @@ impl CoomerProvider {
         per_page_limit: usize,
         options: &ServerOptions,
     ) -> Result<Vec<VideoItem>> {
-        let cache_key = match &target {
-            Target::Uploader { service, username } => {
-                format!("uploader::{}::{}::{}", service.as_str(), username, page)
-            }
-            _ => format!("main::{}::{}", target_label(&target), page),
-        };
-
-        if let Some((time, items)) = cache.get(&cache_key) {
-            if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS {
-                return Ok(items.clone());
-            }
-        }
-
         let url = match &target {
             Target::Uploader { service, username } => {
                 Self::build_uploader_url(service, username, page)
             }
             _ => Self::build_main_url(&target, page),
         };
+        // Key on the full request URL: the search query must be part of it,
+        // otherwise every search shares one cached page.
+        let cache_key = format!("coomer::{url}");
+
+        if let Some((time, items)) = cache.get(&cache_key) {
+            if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS {
+                return Ok(items.clone());
+            }
+        }
 
         let posts = self.fetch_json(options, "coomer.fetch_items", &url).await?;
         eprintln!("[coomer] fetched {} posts from {}", posts.len(), url);
@@ -389,47 +384,46 @@ impl CoomerProvider {
             });
         let page_url = format!("{}/{}/user/{}/post/{}", BASE_URL, service, user, id);
 
-        // Coomer returns the file list in `attachments`; the main image/video
+        // Coomer returns the main file in `file` and the rest in `attachments`.
-        // lives in `file` for the first attachment. The `file.path` is the
+        // Pick the first *video* across both: a video can sit in an attachment
-        // data-path (without the /data/ prefix); we have to prepend it.
+        // while `file` is a cover image. Image-only posts have nothing
-        let preview_file = post.file.as_ref();
+        // playable (and yt-dlp can't resolve the post page), so they are
-        let preview_url: Option<String> = preview_file
+        // dropped from this video channel.
+        let is_video_ext =
+            |ext: &str| matches!(ext.to_ascii_lowercase().as_str(), "mp4" | "webm" | "mov" | "mkv" | "m4v");
+        let mut candidates: Vec<&ApiFile> = Vec::new();
+        if let Some(file) = post.file.as_ref() {
+            candidates.push(file);
+        }
+        if let Some(attachments) = post.attachments.as_ref() {
+            candidates.extend(attachments.iter());
+        }
+        let video_file = candidates.into_iter().find(|file| {
+            file.path.as_deref().is_some_and(|path| !path.is_empty())
+                && file
+                    .name
+                    .as_deref()
+                    .and_then(extension_from_name)
+                    .or_else(|| file.path.as_deref().and_then(extension_from_name))
+                    .is_some_and(|ext| is_video_ext(ext))
+        })?;
+
+        let preview_url: Option<String> = post
+            .file
+            .as_ref()
             .and_then(|file| file.path.as_deref())
+            .filter(|path| !path.is_empty())
             .map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
-        let mut format_url: Option<String> = None;
+        let format_url: Option<String> = video_file
-        let mut ext_hint: Option<String> = None;
+            .path
-
+            .as_deref()
-        if let Some(file) = preview_file {
+            .map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
-            if let Some(name) = file.name.as_deref() {
+        let ext_hint: Option<String> = video_file
-                if let Some(ext) = extension_from_name(name) {
+            .name
-                    ext_hint = Some(ext.to_string());
+            .as_deref()
-                }
+            .and_then(extension_from_name)
-            }
+            .or_else(|| video_file.path.as_deref().and_then(extension_from_name))
-            // The main file is the playable media. For images it's still
+            .map(|ext| ext.to_ascii_lowercase());
-            // served as the media URL; for videos it's the actual .mp4.
-            if let Some(path) = file.path.as_deref() {
-                format_url = Some(format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
-            }
-        }
-
-        if format_url.is_none() {
-            if let Some(attach) = post.attachments.as_ref().and_then(|a| a.first()) {
-                if let Some(path) = attach.path.as_deref() {
-                    format_url = Some(format!(
-                        "{}/data/{}",
-                        BASE_URL,
-                        path.trim_start_matches('/')
-                    ));
-                }
-                if ext_hint.is_none() {
-                    if let Some(name) = attach.name.as_deref() {
-                        if let Some(ext) = extension_from_name(name) {
-                            ext_hint = Some(ext.to_string());
-                        }
-                    }
-                }
-            }
-        }
 
         // Final fallback for the thumbnail: the preview file's data URL, or
         // the format URL if the file itself is the only image.
@@ -461,7 +455,7 @@ impl CoomerProvider {
 
         if let Some(ref url) = format_url {
             let ext = ext_hint.unwrap_or_else(|| "mp4".to_string());
-            let is_video = matches!(ext.as_str(), "mp4" | "webm" | "mov" | "mkv");
+            let is_video = is_video_ext(&ext);
             // An image-only post (very common - many creators post photos,
             // not just videos) has nothing playable; emitting a "format"
             // pointing at a static jpg/png just gives players a broken
@@ -660,11 +654,11 @@ mod tests {
     fn builds_latest_url() {
         assert_eq!(
             CoomerProvider::build_main_url(&Target::Latest, 1),
-            "https://coomer.st/api/v1/posts?limit=50&page=1"
+            "https://coomer.st/api/v1/posts?o=0"
         );
         assert_eq!(
             CoomerProvider::build_main_url(&Target::Latest, 3),
-            "https://coomer.st/api/v1/posts?limit=50&page=3"
+            "https://coomer.st/api/v1/posts?o=100"
         );
     }
 
@@ -672,7 +666,7 @@ mod tests {
     fn builds_popular_url() {
         assert_eq!(
             CoomerProvider::build_main_url(&Target::Popular, 1),
-            "https://coomer.st/api/v1/posts/popular?limit=50&page=1"
+            "https://coomer.st/api/v1/posts/popular?o=0"
         );
     }
 
@@ -683,22 +677,24 @@ mod tests {
         };
         let url = CoomerProvider::build_main_url(&target, 1);
         assert!(url.contains("q=milf%20%2B%20solo"));
-        assert!(url.contains("limit=50"));
+        assert!(url.ends_with("&o=0"));
+        let page_three = CoomerProvider::build_main_url(&target, 3);
+        assert!(page_three.ends_with("&o=100"));
     }
 
     #[test]
     fn builds_uploader_url_per_service() {
         assert_eq!(
             CoomerProvider::build_uploader_url(&Service::Onlyfans, "tabycatxoxo", 1),
-            "https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?limit=50&page=1"
+            "https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?o=0"
         );
         assert_eq!(
             CoomerProvider::build_uploader_url(&Service::Fansly, "ruby", 2),
-            "https://coomer.st/api/v1/fansly/user/ruby/posts?limit=50&page=2"
+            "https://coomer.st/api/v1/fansly/user/ruby/posts?o=50"
         );
         assert_eq!(
             CoomerProvider::build_uploader_url(&Service::Candfans, "model", 1),
-            "https://coomer.st/api/v1/candfans/user/model/posts?limit=50&page=1"
+            "https://coomer.st/api/v1/candfans/user/model/posts?o=0"
         );
     }
 
@@ -835,11 +831,25 @@ mod tests {
             "file": {"name": "p.jpg", "path": "/x/p.jpg"}
         }"#;
         let post: ApiPost = serde_json::from_str(json).expect("parse");
+        // Image-only posts have nothing playable and are dropped.
+        assert!(CoomerProvider::build_video_item(post).is_none());
+    }
+
+    #[test]
+    fn build_video_item_uses_video_attachment_when_file_is_image() {
+        let json = r#"{
+            "id": "43",
+            "user": "ruby",
+            "service": "fansly",
+            "title": "clip",
+            "file": {"name": "cover.jpg", "path": "/c/cover.jpg"},
+            "attachments": [{"name": "a.jpg", "path": "/a/a.jpg"}, {"name": "v.MP4", "path": "/v/v.mp4"}]
+        }"#;
+        let post: ApiPost = serde_json::from_str(json).expect("parse");
         let item = CoomerProvider::build_video_item(post).expect("item");
-        assert_eq!(item.title, "Hello there friends");
+        let formats = item.formats.expect("video attachment must yield a format");
-        assert_eq!(item.thumb, "https://coomer.st/data/x/p.jpg");
+        assert_eq!(formats[0].url, "https://coomer.st/data/v/v.mp4");
-        // For an image attachment, no formats are emitted (preview is enough).
+        assert_eq!(item.thumb, "https://coomer.st/data/c/cover.jpg");
-        assert!(item.formats.is_none());
     }
 
     #[test]