upgraded wreq

Cargo.toml

@@ -20,8 +20,8 @@ ntex-files = "3"
 serde = "1.0.228"
 serde_json = "1.0.145"
 tokio = { version = "1.49", features = ["full"] }
-wreq =  { version = "5.3.0", features = ["cookies", "multipart", "json"] }
+wreq =  { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json"] }
-wreq-util = "2"
+wreq-util = "3.0.0-rc.14"
 percent-encoding = "2.3.2"
 capitalize = "0.3.4"
 url = "2.5.7"

src/providers/freeuseporn.rs

@@ -203,7 +203,7 @@ impl FreeusepornProvider {
             .await
             .map_err(|error| format!("search submit failed url={search_url}; error={error}"))?;
 
-        Ok(response.url().to_string().trim_end_matches('/').to_string())
+        Ok(response.uri().to_string().trim_end_matches('/').to_string())
     }
 
     fn build_formats(&self, id: &str) -> Vec<VideoFormat> {

src/providers/homoxxx.rs

@@ -96,7 +96,7 @@ impl HomoxxxProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 
@@ -208,7 +208,7 @@ impl HomoxxxProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 

src/providers/melonstube.rs

@@ -406,7 +406,7 @@ impl MelonstubeProvider {
             .await
             .map_err(|e| Error::from(format!("search submit failed url={search_url}: {e}")))?;
 
-        Ok(response.url().to_string().trim_end_matches('/').to_string())
+        Ok(response.uri().to_string().trim_end_matches('/').to_string())
     }
 
     async fn fetch_and_parse(

src/providers/okporn.rs

@@ -96,7 +96,7 @@ impl OkpornProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 
@@ -194,7 +194,7 @@ impl OkpornProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 

src/providers/okxxx.rs

@@ -151,7 +151,7 @@ impl OkxxxProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 
@@ -252,7 +252,7 @@ impl OkxxxProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 

src/providers/perfectgirls.rs

@@ -151,7 +151,7 @@ impl PerfectgirlsProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 
@@ -252,7 +252,7 @@ impl PerfectgirlsProvider {
 
         // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
         let client = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Firefox136)
             .build()?;
 

src/providers/yesporn.rs

@@ -406,7 +406,7 @@ impl YespornProvider {
             )));
         }
 
-        let canonical_url = response.url().to_string();
+        let canonical_url = response.uri().to_string();
         let body = response
             .text()
             .await

src/proxies/allpornstream.rs

@@ -24,7 +24,7 @@ impl AllPornStreamProxy {
     fn build_chrome_client() -> Option<wreq::Client> {
         let jar = Arc::new(Jar::default());
         wreq::Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Chrome120)
             .cookie_provider(jar)
             .redirect(Policy::default())

src/proxies/doodstream.rs

@@ -366,7 +366,7 @@ impl DoodstreamProxy {
     fn build_chrome_client() -> Option<wreq::Client> {
         let jar = Arc::new(Jar::default());
         wreq::Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Chrome120)
             .cookie_provider(jar)
             .redirect(Policy::default())
@@ -387,7 +387,7 @@ impl DoodstreamProxy {
         if !response.status().is_success() {
             return None;
         }
-        let effective_url = response.url().to_string();
+        let effective_url = response.uri().to_string();
         let html = response.text().await.ok()?;
 
         if let Some(url) = Self::extract_stream_url(&html) {
@@ -434,7 +434,7 @@ impl crate::proxies::Proxy for DoodstreamProxy {
             .await
         {
             Ok(response) if response.status().is_success() => {
-                let effective_url = response.url().to_string();
+                let effective_url = response.uri().to_string();
                 match response.text().await {
                     Ok(text) => (text, effective_url),
                     Err(_) => return String::new(),

src/proxies/lulustream.rs

@@ -62,7 +62,7 @@ impl LulustreamProxy {
     fn build_chrome_client() -> Option<wreq::Client> {
         let jar = Arc::new(Jar::default());
         wreq::Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Chrome120)
             .cookie_provider(jar)
             .redirect(Policy::default())

src/proxies/melonstube.rs

@@ -113,7 +113,7 @@ impl MelonstubeProxy {
 
     fn redirect_client() -> Option<wreq::Client> {
         wreq::Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Chrome120)
             .redirect(Policy::default())
             .build()
@@ -122,7 +122,7 @@ impl MelonstubeProxy {
 
     fn no_redirect_client() -> Option<wreq::Client> {
         wreq::Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
             .emulation(Emulation::Chrome120)
             .redirect(Policy::none())
             .build()

src/util/flaresolverr.rs

@@ -183,48 +183,33 @@ impl Flaresolverr {
         &self,
         request: FlareSolverrRequest,
     ) -> Result<FlareSolverrResponse, Box<dyn std::error::Error + Send + Sync>> {
-        // Keep one ready session globally and rotate it per solve:
+        // Keep exactly one long-lived session globally and reuse it:
-        // - solve with current ready session
+        // - solve with the current ready session
-        // - create replacement session in parallel
+        // - on success, keep it for the next solve
-        // - destroy old session
+        // - on failure, drop it so the next solve starts from a fresh browser
-        // - keep replacement as new ready session
+        //
+        // An earlier version rotated the session on every solve (create a
+        // replacement, destroy the old one). That leaked a browser instance
+        // whenever the solve future was cancelled — a client timeout mid-solve
+        // left both the active and the replacement session alive on the
+        // FlareSolverr host, and the accumulated Chrome instances eventually made
+        // every solve time out.
         let session_state = global_session_state().clone();
         let mut state = session_state.lock().await;
         let active_session = self.ensure_ready_session_locked(&mut state).await?;
 
-        let replacement_creator = {
-            let solver = self.clone();
-            tokio::spawn(async move { solver.create_session().await })
-        };
-
         let solve_result = self.solve_with_session(request, &active_session).await;
 
-        let replacement_session = match replacement_creator.await {
+        if solve_result.is_err() {
-            Ok(Ok(session)) => session,
+            state.ready_session = None;
-            Ok(Err(error)) => {
-                eprintln!(
-                    "FlareSolverr replacement session creation failed, retrying inline: {}",
-                    error
-                );
-                self.create_session().await?
-            }
-            Err(join_error) => {
-                eprintln!(
-                    "FlareSolverr replacement task join failed, retrying inline: {}",
-                    join_error
-                );
-                self.create_session().await?
-            }
-        };
-
             if let Err(error) = self.destroy_session(&active_session).await {
                 eprintln!(
                     "FlareSolverr session cleanup failed for session '{}': {}",
                     active_session, error
                 );
             }
+        }
 
-        state.ready_session = Some(replacement_session);
         solve_result
     }
 }

src/util/proxy.rs

@@ -173,7 +173,7 @@ fn protocol_from_list_url(list_url: &str) -> Result<&'static str, ProxyParseErro
 async fn verify_proxy(proxy: Proxy) -> Result<Proxy, ProxyParseError> {
     let wreq_proxy = proxy.to_wreq_proxy()?;
     let client = Client::builder()
-        .cert_verification(false)
+        .tls_cert_verification(false)
         .build()
         .map_err(|e| ProxyParseError::new(format!("failed to build http client: {e}")))?;
 

src/util/requester.rs

@@ -1,14 +1,15 @@
 use serde::Serialize;
+use std::collections::HashMap;
 use std::env;
 use std::fmt;
-use std::sync::{Arc, OnceLock};
+use std::sync::{Arc, OnceLock, RwLock};
 use std::time::Duration;
 use wreq::Client;
 use wreq::Proxy;
 use wreq::Response;
-use wreq::Url;
 use wreq::Version;
-use wreq::cookie::{CookieStore, Jar};
+use wreq::Uri;
+use wreq::cookie::{Cookies, CookieStore, Jar};
 use wreq::header::{HeaderMap, HeaderValue, SET_COOKIE, USER_AGENT};
 use wreq::multipart::Form;
 use wreq::redirect::Policy;
@@ -53,6 +54,63 @@ impl Requester {
             .clone()
     }
 
+    /// User agents recovered from FlareSolverr solves, keyed by host.
+    ///
+    /// The cookie jar is already process-wide, but the solved user agent used to
+    /// live only on the `Requester` that ran the solve. Providers clone their
+    /// requester for every concurrent detail fetch, so those clones went out with
+    /// the default user agent and Cloudflare rejected them — forcing another
+    /// (~15s) solve per page. Keeping the identity here lets every later request
+    /// to that host replay the solved user agent alongside the shared cookies and
+    /// stay on the direct path.
+    fn flaresolverr_user_agents() -> &'static RwLock<HashMap<String, String>> {
+        static USER_AGENTS: OnceLock<RwLock<HashMap<String, String>>> = OnceLock::new();
+        USER_AGENTS.get_or_init(|| RwLock::new(HashMap::new()))
+    }
+
+    fn host_key(url: &str) -> Option<String> {
+        url::Url::parse(url)
+            .ok()?
+            .host_str()
+            .map(|host| host.trim_start_matches("www.").to_ascii_lowercase())
+    }
+
+    fn remember_flaresolverr_user_agent(url: &str, user_agent: &str) {
+        if user_agent.is_empty() {
+            return;
+        }
+        let Some(host) = Self::host_key(url) else {
+            return;
+        };
+        if let Ok(mut agents) = Self::flaresolverr_user_agents().write() {
+            agents.insert(host, user_agent.to_string());
+        }
+    }
+
+    fn remembered_flaresolverr_user_agent(url: &str) -> Option<String> {
+        let host = Self::host_key(url)?;
+        Self::flaresolverr_user_agents()
+            .read()
+            .ok()?
+            .get(&host)
+            .cloned()
+    }
+
+    /// Adopt a previously solved user agent for `url` when this requester does not
+    /// already carry one, so the direct attempt reuses the FlareSolverr identity.
+    /// Returns true when the client was rebuilt.
+    fn adopt_remembered_identity(&mut self, url: &str) -> bool {
+        if self.user_agent.is_some() {
+            return false;
+        }
+        let Some(user_agent) = Self::remembered_flaresolverr_user_agent(url) else {
+            return false;
+        };
+        self.user_agent = Some(user_agent);
+        self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
+        true
+    }
+
     fn origin_url_for_cookie_scope(url: &str) -> Option<url::Url> {
         let parsed = url::Url::parse(url).ok()?;
         let host = parsed.host_str()?;
@@ -73,7 +131,7 @@ impl Requester {
             // register the cookie against https://foo.com/ so the Jar returns it for
             // sub.foo.com requests as well.
             let origin = Self::cookie_domain_origin(cookie, &default_origin);
-            self.cookie_jar.add_cookie_str(cookie, &origin);
+            self.cookie_jar.add(cookie, origin.as_str());
         }
     }
 
@@ -139,8 +197,7 @@ impl Requester {
                 }
             }
 
-            self.cookie_jar
+            self.cookie_jar.add(cookie_string.as_str(), origin.as_str());
-                .add_cookie_str(&cookie_string, &origin);
         }
     }
 
@@ -225,8 +282,8 @@ impl Requester {
 
     fn build_client(cookie_jar: Arc<Jar>, user_agent: Option<&str>) -> Client {
         let mut builder = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
-            .emulation(Emulation::Firefox136)
+            .emulation(Emulation::Firefox151)
             .cookie_provider(cookie_jar)
             .redirect(Policy::default());
 
@@ -270,22 +327,28 @@ impl Requester {
         self.debug_trace_id = debug_trace_id;
     }
 
-    #[cfg(feature = "debug")]
     pub fn debug_trace_id(&self) -> Option<&str> {
         self.debug_trace_id.as_deref()
     }
 
     pub fn cookie_header_for_url(&self, url: &str) -> Option<String> {
-        let parsed = url.parse::<Url>().ok()?;
+        let uri = url.parse::<Uri>().ok()?;
-        let joined = self.cookie_jar.cookies(&parsed)
+        // HTTP/1.1 asks the store for the combined `a=1; b=2` form; HTTP/2 would
-            .into_iter()
+        // hand back one field value per cookie instead.
-            .filter_map(|c| c.to_str().ok().map(ToOwned::to_owned))
+        let joined = match self.cookie_jar.cookies(&uri, Version::HTTP_11) {
+            Cookies::Compressed(value) => value.to_str().ok()?.to_string(),
+            Cookies::Uncompressed(values) => values
+                .iter()
+                .filter_map(|value| value.to_str().ok())
                 .collect::<Vec<_>>()
-            .join("; ");
+                .join("; "),
+            _ => return None,
+        };
         (!joined.is_empty()).then_some(joined)
     }
 
     pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
+        self.adopt_remembered_identity(url);
         let cookie_preview = self
             .cookie_header_for_url(url)
             .map(|cookie| crate::util::flow_debug::preview(&cookie, 160))
@@ -322,6 +385,7 @@ impl Requester {
     /// a bare `wreq::Client`, which Cloudflare is far more likely to flag,
     /// especially from datacenter/production IPs.
     pub async fn get_no_redirect_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
+        self.adopt_remembered_identity(url);
         let cookie_preview = self
             .cookie_header_for_url(url)
             .map(|cookie| crate::util::flow_debug::preview(&cookie, 160))
@@ -336,8 +400,8 @@ impl Requester {
             self.proxy
         );
         let mut builder = Client::builder()
-            .cert_verification(false)
+            .tls_cert_verification(false)
-            .emulation(Emulation::Firefox136)
+            .emulation(Emulation::Firefox151)
             .cookie_provider(self.cookie_jar.clone())
             .redirect(Policy::none());
         if let Some(user_agent) = self.user_agent.as_deref() {
@@ -546,6 +610,14 @@ impl Requester {
         headers: Vec<(String, String)>,
         _http_version: Option<Version>,
     ) -> Result<String, AnyErr> {
+        if self.adopt_remembered_identity(url) {
+            crate::flow_debug!(
+                "trace={} requester reusing flaresolverr identity url={} user_agent={}",
+                self.debug_trace_id().unwrap_or("none"),
+                crate::util::flow_debug::preview(url, 120),
+                crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
+            );
+        }
         let cookie_preview = self.debug_cookie_preview_from_owned_headers(url, &headers);
         #[cfg(not(feature = "debug"))]
         let _ = &cookie_preview;
@@ -558,10 +630,12 @@ impl Requester {
             _http_version,
             self.proxy
         );
-        let http_version = match _http_version {
+        // Only pin the protocol when the caller asked for one. Defaulting to
-            Some(v) => v,
+        // HTTP/1.1 used to force every request onto h1 while the TLS handshake
-            None => Version::HTTP_11,
+        // still advertised the browser emulation's h2 ALPN — a mismatch
-        };
+        // Cloudflare scores as a bot. On jable.tv it is a deterministic 403:
+        // the identical client returns 200 over the negotiated h2 and 403 the
+        // moment the version is pinned to HTTP/1.1.
         let cookie_jar = self.cookie_jar.clone();
         let user_agent = self.user_agent.clone();
         let proxy_enabled = self.proxy;
@@ -571,7 +645,10 @@ impl Requester {
             Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await
         });
         loop {
-            let mut request = self.client.get(url).version(http_version);
+            let mut request = self.client.get(url);
+            if let Some(version) = _http_version {
+                request = request.version(version);
+            }
             for (key, value) in headers.iter() {
                 request = request.header(key, value);
             }
@@ -746,8 +823,11 @@ impl Requester {
             .await
             .map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?;
 
-        // Rebuild client and apply UA/cookies from FlareSolverr
+        // Rebuild client and apply UA/cookies from FlareSolverr. Both are also
+        // published process-wide (cookies via the shared jar, user agent by host)
+        // so later requests can go direct instead of solving again.
         let useragent = res.solution.userAgent;
+        Self::remember_flaresolverr_user_agent(url, &useragent);
         self.user_agent = Some(useragent);
         self.store_flaresolverr_cookies(url, &res.solution.cookies);
 
@@ -759,8 +839,10 @@ impl Requester {
             crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
         );
 
-        // Retry the original URL with the updated client & (optional) proxy
+        // Retry the original URL with the updated client & (optional) proxy.
-        let mut request = self.client.get(url).version(Version::HTTP_11);
+        // Left unpinned so the protocol matches the emulated ALPN, same as the
+        // direct attempt.
+        let mut request = self.client.get(url);
         for (key, value) in headers.iter() {
             request = request.header(key, value);
         }
@@ -814,7 +896,7 @@ mod tests {
         let origin = "https://shared-cookie-requester-test.invalid/";
 
         a.cookie_jar
-            .add_cookie_str("shared_cookie=1; Path=/; SameSite=Lax", &url::Url::parse(origin).unwrap());
+            .add("shared_cookie=1; Path=/; SameSite=Lax", origin);
 
         let cookie_header = b
             .cookie_header_for_url("https://shared-cookie-requester-test.invalid/path")