Fix hypnotube search sessions; check.py confirms HEAD failures with GET

The site serves one results page per searchgate POST, so later searches
and page 2 returned nothing on the shared cookie session. Use an isolated
session per (query, page). check.py now retries a refused HEAD (403/404/
405/501) with a ranged GET before reporting a URL as unreachable.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi

check.py

@@ -149,13 +149,15 @@ def _is_cf_protected(url: str) -> bool:
 
 
 def http_ok(url: str, headers: dict | None = None) -> tuple[bool, int]:
-    """Return (ok, http_status). Tries HEAD then ranged GET on 405."""
+    """Return (ok, http_status). Tries HEAD, then a ranged GET when HEAD is refused."""
     h = {"User-Agent": _BROWSER_UA, **(headers or {})}
     try:
         r = requests.head(url, headers=h, timeout=HTTP_TIMEOUT, allow_redirects=True)
         if r.status_code in (200, 206):
             return True, r.status_code
-        if r.status_code == 405:
+        # Some servers mishandle HEAD (405/501, or even 404/403 while GET is 200);
+        # confirm with a ranged GET before reporting the URL as broken.
+        if r.status_code in (403, 404, 405, 501):
             r2 = requests.get(
                 url,
                 headers={**h, "Range": "bytes=0-1023"},

docs/provider-fix-todo.md

@@ -25,7 +25,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
 | [x] | `pornmz` | page URL "Unsupported URL", no `formats` | site changed `itemprop="contentURL"` -> `contentUrl`; selector now case-insensitive. Enrichment concurrency 10 + 25s per-item timeout. Origin is slow (15s+/detail) so first feed 25-50s. check.py passes |
 | [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end |
 | [x] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | sponsored `channel` cards carry a stale `data-video-id` (404) while the card link is the live page; id/URL now come from the link. Signed `pix-cdn77` thumbs 403 without a Referer: new host-restricted `/proxy/aylo-thumb` (also fixes ph-shorties). check.py passes |
-| [ ] | `hypnotube` | page URLs 404 on all sampled items | URL construction |
+| [x] | `hypnotube` | page URLs 404 on all sampled items | the 404 was a check.py artifact (server answers HEAD 404, GET 200); check.py now confirms with a ranged GET. Real bug found on the way: the site returns ONE results page per searchgate POST, so any 2nd search on the shared session (and every page 2) came back empty. Each (query, page) now uses its own isolated session. 3 consecutive searches + page 2 verified |
 | [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | |
 | [ ] | `ripnsfw` | `doodstream` format 502 | |
 | [ ] | `freepornvideosxxx` | yt-dlp 502 on page URL | may be transient, verify |
@@ -48,7 +48,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
 | [ ] | `cumlouder` | unknown tag takes 146s before returning 0 | own provider, retry path on 404 |
 | [ ] | `vjav` | "girl" 0, "milf" 10 | may be term-specific |
 | [ ] | `xxxtik` | "girl" 0 (not re-run serially) | |
-| [ ] | `hypnotube` | "milf" 0, "girl" 20 | may be term-specific |
+| [x] | `hypnotube` | "milf" 0, "girl" 20 | same root cause as above (second search on a session returned nothing), not term-specific |
 
 ## 5. Slow channels
 

src/providers/hypnotube.rs

@@ -14,7 +14,11 @@ use htmlentity::entity::{ICodedDataTrait, decode};
 use std::sync::{Arc, RwLock};
 use std::{thread, vec};
 use titlecase::Titlecase;
+use wreq::Client;
 use wreq::Version;
+use wreq::cookie::Jar;
+use wreq::redirect::Policy;
+use wreq_util::Emulation;
 
 pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
     crate::providers::ProviderChannelMetadata {
@@ -246,6 +250,43 @@ impl HypnotubeProvider {
         video_items
     }
 
+    /// Fetches one page of search results.
+    ///
+    /// The site hands out exactly one results page per `searchgate.php` POST
+    /// (a second GET of the same search, or any later search on the same
+    /// session, comes back empty), so every (query, page) uses its own
+    /// isolated cookie session instead of the shared requester jar. Page 1 is
+    /// the redirect target of the POST; page N>1 is fetched directly.
+    async fn fetch_search_html(&self, query: &str, page: u8) -> Result<String> {
+        let client = Client::builder()
+            .tls_cert_verification(false)
+            .emulation(Emulation::Firefox136)
+            .cookie_provider(Arc::new(Jar::default()))
+            .redirect(if page <= 1 { Policy::default() } else { Policy::none() })
+            .build()?;
+
+        let body = format!("q={}&type=videos", query.trim().replace(' ', "+"));
+        let gate = client
+            .post(format!("{}/searchgate.php", self.url))
+            .header("Content-Type", "application/x-www-form-urlencoded")
+            .header("Referer", format!("{}/", self.url))
+            .body(body)
+            .send()
+            .await?;
+        if page <= 1 {
+            return Ok(gate.text().await?);
+        }
+
+        let slug = query.trim().replace(' ', "%20");
+        let page_url = format!("{}/search/videos/{}/page{}.html", self.url, slug, page);
+        let response = client
+            .get(page_url)
+            .header("Referer", format!("{}/search/videos/{}/", self.url, slug))
+            .send()
+            .await?;
+        Ok(response.text().await?)
+    }
+
     async fn query(
         &self,
         cache: VideoCache,
@@ -281,41 +322,18 @@ impl HypnotubeProvider {
             }
         };
 
-        let mut requester =
-            crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
-        let post_response = match requester
-            .post(
-                format!("{}/searchgate.php", self.url).as_str(),
-                format!("q={}&type=videos", query.replace(" ", "+")).as_str(),
-                vec![("Content-Type", "application/x-www-form-urlencoded")],
-            )
-            .await
-        {
-            Ok(response) => response,
+        let text = match self.fetch_search_html(query, page).await {
+            Ok(text) => text,
             Err(e) => {
                 crate::providers::report_provider_error(
                     "hypnotube",
-                    "query.search_post",
+                    "query.search",
                     &format!("url={video_url}; error={e}"),
                 )
                 .await;
                 return old_items;
             }
         };
-        let text = match post_response.text().await {
-            Ok(t) => t,
-            Err(e) => {
-                eprint!("Hypnotube search POST request failed: {}", e);
-                crate::providers::report_provider_error_background(
-                    "hypnotube",
-                    "query.search_post.text",
-                    &e.to_string(),
-                );
-                return vec![];
-            }
-        };
-        // println!("Hypnotube search POST response status: {}", p.text().await.unwrap_or_default());
-        // let text = requester.get(&video_url, Some(Version::HTTP_11)).await.unwrap();
         if text.contains("Sorry, no results were found.") {
             return vec![];
         }