upgraded wreq

src/util/requester.rs

@@ -1,14 +1,15 @@
 use serde::Serialize;
+use std::collections::HashMap;
 use std::env;
 use std::fmt;
-use std::sync::{Arc, OnceLock};
+use std::sync::{Arc, OnceLock, RwLock};
 use std::time::Duration;
 use wreq::Client;
 use wreq::Proxy;
 use wreq::Response;
-use wreq::Url;
 use wreq::Version;
-use wreq::cookie::{CookieStore, Jar};
+use wreq::Uri;
+use wreq::cookie::{Cookies, CookieStore, Jar};
 use wreq::header::{HeaderMap, HeaderValue, SET_COOKIE, USER_AGENT};
 use wreq::multipart::Form;
 use wreq::redirect::Policy;
@@ -53,6 +54,63 @@ impl Requester {
             .clone()
     }
 
+    /// User agents recovered from FlareSolverr solves, keyed by host.
+    ///
+    /// The cookie jar is already process-wide, but the solved user agent used to
+    /// live only on the `Requester` that ran the solve. Providers clone their
+    /// requester for every concurrent detail fetch, so those clones went out with
+    /// the default user agent and Cloudflare rejected them — forcing another
+    /// (~15s) solve per page. Keeping the identity here lets every later request
+    /// to that host replay the solved user agent alongside the shared cookies and
+    /// stay on the direct path.
+    fn flaresolverr_user_agents() -> &'static RwLock<HashMap<String, String>> {
+        static USER_AGENTS: OnceLock<RwLock<HashMap<String, String>>> = OnceLock::new();
+        USER_AGENTS.get_or_init(|| RwLock::new(HashMap::new()))
+    }
+
+    fn host_key(url: &str) -> Option<String> {
+        url::Url::parse(url)
+            .ok()?
+            .host_str()
+            .map(|host| host.trim_start_matches("www.").to_ascii_lowercase())
+    }
+
+    fn remember_flaresolverr_user_agent(url: &str, user_agent: &str) {
+        if user_agent.is_empty() {
+            return;
+        }
+        let Some(host) = Self::host_key(url) else {
+            return;
+        };
+        if let Ok(mut agents) = Self::flaresolverr_user_agents().write() {
+            agents.insert(host, user_agent.to_string());
+        }
+    }
+
+    fn remembered_flaresolverr_user_agent(url: &str) -> Option<String> {
+        let host = Self::host_key(url)?;
+        Self::flaresolverr_user_agents()
+            .read()
+            .ok()?
+            .get(&host)
+            .cloned()
+    }
+
+    /// Adopt a previously solved user agent for `url` when this requester does not
+    /// already carry one, so the direct attempt reuses the FlareSolverr identity.
+    /// Returns true when the client was rebuilt.
+    fn adopt_remembered_identity(&mut self, url: &str) -> bool {
+        if self.user_agent.is_some() {
+            return false;
+        }
+        let Some(user_agent) = Self::remembered_flaresolverr_user_agent(url) else {
+            return false;
+        };
+        self.user_agent = Some(user_agent);
+        self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
+        true
+    }
+
     fn origin_url_for_cookie_scope(url: &str) -> Option<url::Url> {
         let parsed = url::Url::parse(url).ok()?;
         let host = parsed.host_str()?;
@@ -73,7 +131,7 @@ impl Requester {
             // register the cookie against https://foo.com/ so the Jar returns it for
             // sub.foo.com requests as well.
             let origin = Self::cookie_domain_origin(cookie, &default_origin);
-            self.cookie_jar.add_cookie_str(cookie, &origin);
+            self.cookie_jar.add(cookie, origin.as_str());
         }
     }
 
@@ -139,8 +197,7 @@ impl Requester {
                 }
             }
 
-            self.cookie_jar
-                .add_cookie_str(&cookie_string, &origin);
+            self.cookie_jar.add(cookie_string.as_str(), origin.as_str());
         }
     }
 
@@ -225,8 +282,8 @@ impl Requester {
 
     fn build_client(cookie_jar: Arc<Jar>, user_agent: Option<&str>) -> Client {
         let mut builder = Client::builder()
-            .cert_verification(false)
-            .emulation(Emulation::Firefox136)
+            .tls_cert_verification(false)
+            .emulation(Emulation::Firefox151)
             .cookie_provider(cookie_jar)
             .redirect(Policy::default());
 
@@ -270,22 +327,28 @@ impl Requester {
         self.debug_trace_id = debug_trace_id;
     }
 
-    #[cfg(feature = "debug")]
     pub fn debug_trace_id(&self) -> Option<&str> {
         self.debug_trace_id.as_deref()
     }
 
     pub fn cookie_header_for_url(&self, url: &str) -> Option<String> {
-        let parsed = url.parse::<Url>().ok()?;
-        let joined = self.cookie_jar.cookies(&parsed)
-            .into_iter()
-            .filter_map(|c| c.to_str().ok().map(ToOwned::to_owned))
-            .collect::<Vec<_>>()
-            .join("; ");
+        let uri = url.parse::<Uri>().ok()?;
+        // HTTP/1.1 asks the store for the combined `a=1; b=2` form; HTTP/2 would
+        // hand back one field value per cookie instead.
+        let joined = match self.cookie_jar.cookies(&uri, Version::HTTP_11) {
+            Cookies::Compressed(value) => value.to_str().ok()?.to_string(),
+            Cookies::Uncompressed(values) => values
+                .iter()
+                .filter_map(|value| value.to_str().ok())
+                .collect::<Vec<_>>()
+                .join("; "),
+            _ => return None,
+        };
         (!joined.is_empty()).then_some(joined)
     }
 
     pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
+        self.adopt_remembered_identity(url);
         let cookie_preview = self
             .cookie_header_for_url(url)
             .map(|cookie| crate::util::flow_debug::preview(&cookie, 160))
@@ -322,6 +385,7 @@ impl Requester {
     /// a bare `wreq::Client`, which Cloudflare is far more likely to flag,
     /// especially from datacenter/production IPs.
     pub async fn get_no_redirect_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
+        self.adopt_remembered_identity(url);
         let cookie_preview = self
             .cookie_header_for_url(url)
             .map(|cookie| crate::util::flow_debug::preview(&cookie, 160))
@@ -336,8 +400,8 @@ impl Requester {
             self.proxy
         );
         let mut builder = Client::builder()
-            .cert_verification(false)
-            .emulation(Emulation::Firefox136)
+            .tls_cert_verification(false)
+            .emulation(Emulation::Firefox151)
             .cookie_provider(self.cookie_jar.clone())
             .redirect(Policy::none());
         if let Some(user_agent) = self.user_agent.as_deref() {
@@ -546,6 +610,14 @@ impl Requester {
         headers: Vec<(String, String)>,
         _http_version: Option<Version>,
     ) -> Result<String, AnyErr> {
+        if self.adopt_remembered_identity(url) {
+            crate::flow_debug!(
+                "trace={} requester reusing flaresolverr identity url={} user_agent={}",
+                self.debug_trace_id().unwrap_or("none"),
+                crate::util::flow_debug::preview(url, 120),
+                crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
+            );
+        }
         let cookie_preview = self.debug_cookie_preview_from_owned_headers(url, &headers);
         #[cfg(not(feature = "debug"))]
         let _ = &cookie_preview;
@@ -558,10 +630,12 @@ impl Requester {
             _http_version,
             self.proxy
         );
-        let http_version = match _http_version {
-            Some(v) => v,
-            None => Version::HTTP_11,
-        };
+        // Only pin the protocol when the caller asked for one. Defaulting to
+        // HTTP/1.1 used to force every request onto h1 while the TLS handshake
+        // still advertised the browser emulation's h2 ALPN — a mismatch
+        // Cloudflare scores as a bot. On jable.tv it is a deterministic 403:
+        // the identical client returns 200 over the negotiated h2 and 403 the
+        // moment the version is pinned to HTTP/1.1.
         let cookie_jar = self.cookie_jar.clone();
         let user_agent = self.user_agent.clone();
         let proxy_enabled = self.proxy;
@@ -571,7 +645,10 @@ impl Requester {
             Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await
         });
         loop {
-            let mut request = self.client.get(url).version(http_version);
+            let mut request = self.client.get(url);
+            if let Some(version) = _http_version {
+                request = request.version(version);
+            }
             for (key, value) in headers.iter() {
                 request = request.header(key, value);
             }
@@ -746,8 +823,11 @@ impl Requester {
             .await
             .map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?;
 
-        // Rebuild client and apply UA/cookies from FlareSolverr
+        // Rebuild client and apply UA/cookies from FlareSolverr. Both are also
+        // published process-wide (cookies via the shared jar, user agent by host)
+        // so later requests can go direct instead of solving again.
         let useragent = res.solution.userAgent;
+        Self::remember_flaresolverr_user_agent(url, &useragent);
         self.user_agent = Some(useragent);
         self.store_flaresolverr_cookies(url, &res.solution.cookies);
 
@@ -759,8 +839,10 @@ impl Requester {
             crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
         );
 
-        // Retry the original URL with the updated client & (optional) proxy
-        let mut request = self.client.get(url).version(Version::HTTP_11);
+        // Retry the original URL with the updated client & (optional) proxy.
+        // Left unpinned so the protocol matches the emulated ALPN, same as the
+        // direct attempt.
+        let mut request = self.client.get(url);
         for (key, value) in headers.iter() {
             request = request.header(key, value);
         }
@@ -814,7 +896,7 @@ mod tests {
         let origin = "https://shared-cookie-requester-test.invalid/";
 
         a.cookie_jar
-            .add_cookie_str("shared_cookie=1; Path=/; SameSite=Lax", &url::Url::parse(origin).unwrap());
+            .add("shared_cookie=1; Path=/; SameSite=Lax", origin);
 
         let cookie_header = b
             .cookie_header_for_url("https://shared-cookie-requester-test.invalid/path")