From 5941a6e607016ffaf94493bcea12ba23cedf615f Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 7 Sep 2026 13:00:19 +0000 Subject: [PATCH] upgraded wreq --- Cargo.toml | 4 +- src/providers/freeuseporn.rs | 2 +- src/providers/homoxxx.rs | 4 +- src/providers/melonstube.rs | 2 +- src/providers/okporn.rs | 4 +- src/providers/okxxx.rs | 4 +- src/providers/perfectgirls.rs | 4 +- src/providers/yesporn.rs | 2 +- src/proxies/allpornstream.rs | 2 +- src/proxies/doodstream.rs | 6 +- src/proxies/lulustream.rs | 2 +- src/proxies/melonstube.rs | 4 +- src/util/flaresolverr.rs | 47 ++++-------- src/util/proxy.rs | 2 +- src/util/requester.rs | 134 +++++++++++++++++++++++++++------- 15 files changed, 145 insertions(+), 78 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0dc8a99..fdd9539 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,8 +20,8 @@ ntex-files = "3" serde = "1.0.228" serde_json = "1.0.145" tokio = { version = "1.49", features = ["full"] } -wreq = { version = "5.3.0", features = ["cookies", "multipart", "json"] } -wreq-util = "2" +wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json"] } +wreq-util = "3.0.0-rc.14" percent-encoding = "2.3.2" capitalize = "0.3.4" url = "2.5.7" diff --git a/src/providers/freeuseporn.rs b/src/providers/freeuseporn.rs index 77a3106..988415d 100644 --- a/src/providers/freeuseporn.rs +++ b/src/providers/freeuseporn.rs @@ -203,7 +203,7 @@ impl FreeusepornProvider { .await .map_err(|error| format!("search submit failed url={search_url}; error={error}"))?; - Ok(response.url().to_string().trim_end_matches('/').to_string()) + Ok(response.uri().to_string().trim_end_matches('/').to_string()) } fn build_formats(&self, id: &str) -> Vec { diff --git a/src/providers/homoxxx.rs b/src/providers/homoxxx.rs index 212a1da..2ae7f82 100644 --- a/src/providers/homoxxx.rs +++ b/src/providers/homoxxx.rs @@ -96,7 +96,7 @@ impl HomoxxxProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; @@ -208,7 +208,7 @@ impl HomoxxxProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; diff --git a/src/providers/melonstube.rs b/src/providers/melonstube.rs index 22cff4d..1e869bb 100644 --- a/src/providers/melonstube.rs +++ b/src/providers/melonstube.rs @@ -406,7 +406,7 @@ impl MelonstubeProvider { .await .map_err(|e| Error::from(format!("search submit failed url={search_url}: {e}")))?; - Ok(response.url().to_string().trim_end_matches('/').to_string()) + Ok(response.uri().to_string().trim_end_matches('/').to_string()) } async fn fetch_and_parse( diff --git a/src/providers/okporn.rs b/src/providers/okporn.rs index cd56756..78efbd6 100644 --- a/src/providers/okporn.rs +++ b/src/providers/okporn.rs @@ -96,7 +96,7 @@ impl OkpornProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; @@ -194,7 +194,7 @@ impl OkpornProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; diff --git a/src/providers/okxxx.rs b/src/providers/okxxx.rs index 9a6e1c5..7b3766e 100644 --- a/src/providers/okxxx.rs +++ b/src/providers/okxxx.rs @@ -151,7 +151,7 @@ impl OkxxxProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; @@ -252,7 +252,7 @@ impl OkxxxProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; diff --git a/src/providers/perfectgirls.rs b/src/providers/perfectgirls.rs index e608b98..2b09056 100644 --- a/src/providers/perfectgirls.rs +++ b/src/providers/perfectgirls.rs @@ -151,7 +151,7 @@ impl PerfectgirlsProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; @@ -252,7 +252,7 @@ impl PerfectgirlsProvider { // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Firefox136) .build()?; diff --git a/src/providers/yesporn.rs b/src/providers/yesporn.rs index 4f5cc62..495761b 100644 --- a/src/providers/yesporn.rs +++ b/src/providers/yesporn.rs @@ -406,7 +406,7 @@ impl YespornProvider { ))); } - let canonical_url = response.url().to_string(); + let canonical_url = response.uri().to_string(); let body = response .text() .await diff --git a/src/proxies/allpornstream.rs b/src/proxies/allpornstream.rs index 35087d4..ed0be61 100644 --- a/src/proxies/allpornstream.rs +++ b/src/proxies/allpornstream.rs @@ -24,7 +24,7 @@ impl AllPornStreamProxy { fn build_chrome_client() -> Option { let jar = Arc::new(Jar::default()); wreq::Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Chrome120) .cookie_provider(jar) .redirect(Policy::default()) diff --git a/src/proxies/doodstream.rs b/src/proxies/doodstream.rs index 89de450..beea01b 100644 --- a/src/proxies/doodstream.rs +++ b/src/proxies/doodstream.rs @@ -366,7 +366,7 @@ impl DoodstreamProxy { fn build_chrome_client() -> Option { let jar = Arc::new(Jar::default()); wreq::Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Chrome120) .cookie_provider(jar) .redirect(Policy::default()) @@ -387,7 +387,7 @@ impl DoodstreamProxy { if !response.status().is_success() { return None; } - let effective_url = response.url().to_string(); + let effective_url = response.uri().to_string(); let html = response.text().await.ok()?; if let Some(url) = Self::extract_stream_url(&html) { @@ -434,7 +434,7 @@ impl crate::proxies::Proxy for DoodstreamProxy { .await { Ok(response) if response.status().is_success() => { - let effective_url = response.url().to_string(); + let effective_url = response.uri().to_string(); match response.text().await { Ok(text) => (text, effective_url), Err(_) => return String::new(), diff --git a/src/proxies/lulustream.rs b/src/proxies/lulustream.rs index 5f4dd5c..10fab66 100644 --- a/src/proxies/lulustream.rs +++ b/src/proxies/lulustream.rs @@ -62,7 +62,7 @@ impl LulustreamProxy { fn build_chrome_client() -> Option { let jar = Arc::new(Jar::default()); wreq::Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Chrome120) .cookie_provider(jar) .redirect(Policy::default()) diff --git a/src/proxies/melonstube.rs b/src/proxies/melonstube.rs index f75af96..d9ffdd8 100644 --- a/src/proxies/melonstube.rs +++ b/src/proxies/melonstube.rs @@ -113,7 +113,7 @@ impl MelonstubeProxy { fn redirect_client() -> Option { wreq::Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Chrome120) .redirect(Policy::default()) .build() @@ -122,7 +122,7 @@ impl MelonstubeProxy { fn no_redirect_client() -> Option { wreq::Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .emulation(Emulation::Chrome120) .redirect(Policy::none()) .build() diff --git a/src/util/flaresolverr.rs b/src/util/flaresolverr.rs index dcc3e7a..4a0e401 100644 --- a/src/util/flaresolverr.rs +++ b/src/util/flaresolverr.rs @@ -183,48 +183,33 @@ impl Flaresolverr { &self, request: FlareSolverrRequest, ) -> Result> { - // Keep one ready session globally and rotate it per solve: - // - solve with current ready session - // - create replacement session in parallel - // - destroy old session - // - keep replacement as new ready session + // Keep exactly one long-lived session globally and reuse it: + // - solve with the current ready session + // - on success, keep it for the next solve + // - on failure, drop it so the next solve starts from a fresh browser + // + // An earlier version rotated the session on every solve (create a + // replacement, destroy the old one). That leaked a browser instance + // whenever the solve future was cancelled — a client timeout mid-solve + // left both the active and the replacement session alive on the + // FlareSolverr host, and the accumulated Chrome instances eventually made + // every solve time out. let session_state = global_session_state().clone(); let mut state = session_state.lock().await; let active_session = self.ensure_ready_session_locked(&mut state).await?; - let replacement_creator = { - let solver = self.clone(); - tokio::spawn(async move { solver.create_session().await }) - }; - let solve_result = self.solve_with_session(request, &active_session).await; - let replacement_session = match replacement_creator.await { - Ok(Ok(session)) => session, - Ok(Err(error)) => { + if solve_result.is_err() { + state.ready_session = None; + if let Err(error) = self.destroy_session(&active_session).await { eprintln!( - "FlareSolverr replacement session creation failed, retrying inline: {}", - error + "FlareSolverr session cleanup failed for session '{}': {}", + active_session, error ); - self.create_session().await? } - Err(join_error) => { - eprintln!( - "FlareSolverr replacement task join failed, retrying inline: {}", - join_error - ); - self.create_session().await? - } - }; - - if let Err(error) = self.destroy_session(&active_session).await { - eprintln!( - "FlareSolverr session cleanup failed for session '{}': {}", - active_session, error - ); } - state.ready_session = Some(replacement_session); solve_result } } diff --git a/src/util/proxy.rs b/src/util/proxy.rs index 40ba02d..74c9798 100644 --- a/src/util/proxy.rs +++ b/src/util/proxy.rs @@ -173,7 +173,7 @@ fn protocol_from_list_url(list_url: &str) -> Result<&'static str, ProxyParseErro async fn verify_proxy(proxy: Proxy) -> Result { let wreq_proxy = proxy.to_wreq_proxy()?; let client = Client::builder() - .cert_verification(false) + .tls_cert_verification(false) .build() .map_err(|e| ProxyParseError::new(format!("failed to build http client: {e}")))?; diff --git a/src/util/requester.rs b/src/util/requester.rs index e24649b..624553d 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -1,14 +1,15 @@ use serde::Serialize; +use std::collections::HashMap; use std::env; use std::fmt; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, OnceLock, RwLock}; use std::time::Duration; use wreq::Client; use wreq::Proxy; use wreq::Response; -use wreq::Url; use wreq::Version; -use wreq::cookie::{CookieStore, Jar}; +use wreq::Uri; +use wreq::cookie::{Cookies, CookieStore, Jar}; use wreq::header::{HeaderMap, HeaderValue, SET_COOKIE, USER_AGENT}; use wreq::multipart::Form; use wreq::redirect::Policy; @@ -53,6 +54,63 @@ impl Requester { .clone() } + /// User agents recovered from FlareSolverr solves, keyed by host. + /// + /// The cookie jar is already process-wide, but the solved user agent used to + /// live only on the `Requester` that ran the solve. Providers clone their + /// requester for every concurrent detail fetch, so those clones went out with + /// the default user agent and Cloudflare rejected them — forcing another + /// (~15s) solve per page. Keeping the identity here lets every later request + /// to that host replay the solved user agent alongside the shared cookies and + /// stay on the direct path. + fn flaresolverr_user_agents() -> &'static RwLock> { + static USER_AGENTS: OnceLock>> = OnceLock::new(); + USER_AGENTS.get_or_init(|| RwLock::new(HashMap::new())) + } + + fn host_key(url: &str) -> Option { + url::Url::parse(url) + .ok()? + .host_str() + .map(|host| host.trim_start_matches("www.").to_ascii_lowercase()) + } + + fn remember_flaresolverr_user_agent(url: &str, user_agent: &str) { + if user_agent.is_empty() { + return; + } + let Some(host) = Self::host_key(url) else { + return; + }; + if let Ok(mut agents) = Self::flaresolverr_user_agents().write() { + agents.insert(host, user_agent.to_string()); + } + } + + fn remembered_flaresolverr_user_agent(url: &str) -> Option { + let host = Self::host_key(url)?; + Self::flaresolverr_user_agents() + .read() + .ok()? + .get(&host) + .cloned() + } + + /// Adopt a previously solved user agent for `url` when this requester does not + /// already carry one, so the direct attempt reuses the FlareSolverr identity. + /// Returns true when the client was rebuilt. + fn adopt_remembered_identity(&mut self, url: &str) -> bool { + if self.user_agent.is_some() { + return false; + } + let Some(user_agent) = Self::remembered_flaresolverr_user_agent(url) else { + return false; + }; + self.user_agent = Some(user_agent); + self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); + true + } + fn origin_url_for_cookie_scope(url: &str) -> Option { let parsed = url::Url::parse(url).ok()?; let host = parsed.host_str()?; @@ -73,7 +131,7 @@ impl Requester { // register the cookie against https://foo.com/ so the Jar returns it for // sub.foo.com requests as well. let origin = Self::cookie_domain_origin(cookie, &default_origin); - self.cookie_jar.add_cookie_str(cookie, &origin); + self.cookie_jar.add(cookie, origin.as_str()); } } @@ -139,8 +197,7 @@ impl Requester { } } - self.cookie_jar - .add_cookie_str(&cookie_string, &origin); + self.cookie_jar.add(cookie_string.as_str(), origin.as_str()); } } @@ -225,8 +282,8 @@ impl Requester { fn build_client(cookie_jar: Arc, user_agent: Option<&str>) -> Client { let mut builder = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) + .tls_cert_verification(false) + .emulation(Emulation::Firefox151) .cookie_provider(cookie_jar) .redirect(Policy::default()); @@ -270,22 +327,28 @@ impl Requester { self.debug_trace_id = debug_trace_id; } - #[cfg(feature = "debug")] pub fn debug_trace_id(&self) -> Option<&str> { self.debug_trace_id.as_deref() } pub fn cookie_header_for_url(&self, url: &str) -> Option { - let parsed = url.parse::().ok()?; - let joined = self.cookie_jar.cookies(&parsed) - .into_iter() - .filter_map(|c| c.to_str().ok().map(ToOwned::to_owned)) - .collect::>() - .join("; "); + let uri = url.parse::().ok()?; + // HTTP/1.1 asks the store for the combined `a=1; b=2` form; HTTP/2 would + // hand back one field value per cookie instead. + let joined = match self.cookie_jar.cookies(&uri, Version::HTTP_11) { + Cookies::Compressed(value) => value.to_str().ok()?.to_string(), + Cookies::Uncompressed(values) => values + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>() + .join("; "), + _ => return None, + }; (!joined.is_empty()).then_some(joined) } pub async fn get_raw(&mut self, url: &str) -> Result { + self.adopt_remembered_identity(url); let cookie_preview = self .cookie_header_for_url(url) .map(|cookie| crate::util::flow_debug::preview(&cookie, 160)) @@ -322,6 +385,7 @@ impl Requester { /// a bare `wreq::Client`, which Cloudflare is far more likely to flag, /// especially from datacenter/production IPs. pub async fn get_no_redirect_raw(&mut self, url: &str) -> Result { + self.adopt_remembered_identity(url); let cookie_preview = self .cookie_header_for_url(url) .map(|cookie| crate::util::flow_debug::preview(&cookie, 160)) @@ -336,8 +400,8 @@ impl Requester { self.proxy ); let mut builder = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) + .tls_cert_verification(false) + .emulation(Emulation::Firefox151) .cookie_provider(self.cookie_jar.clone()) .redirect(Policy::none()); if let Some(user_agent) = self.user_agent.as_deref() { @@ -546,6 +610,14 @@ impl Requester { headers: Vec<(String, String)>, _http_version: Option, ) -> Result { + if self.adopt_remembered_identity(url) { + crate::flow_debug!( + "trace={} requester reusing flaresolverr identity url={} user_agent={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96) + ); + } let cookie_preview = self.debug_cookie_preview_from_owned_headers(url, &headers); #[cfg(not(feature = "debug"))] let _ = &cookie_preview; @@ -558,10 +630,12 @@ impl Requester { _http_version, self.proxy ); - let http_version = match _http_version { - Some(v) => v, - None => Version::HTTP_11, - }; + // Only pin the protocol when the caller asked for one. Defaulting to + // HTTP/1.1 used to force every request onto h1 while the TLS handshake + // still advertised the browser emulation's h2 ALPN — a mismatch + // Cloudflare scores as a bot. On jable.tv it is a deterministic 403: + // the identical client returns 200 over the negotiated h2 and 403 the + // moment the version is pinned to HTTP/1.1. let cookie_jar = self.cookie_jar.clone(); let user_agent = self.user_agent.clone(); let proxy_enabled = self.proxy; @@ -571,7 +645,10 @@ impl Requester { Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await }); loop { - let mut request = self.client.get(url).version(http_version); + let mut request = self.client.get(url); + if let Some(version) = _http_version { + request = request.version(version); + } for (key, value) in headers.iter() { request = request.header(key, value); } @@ -746,8 +823,11 @@ impl Requester { .await .map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?; - // Rebuild client and apply UA/cookies from FlareSolverr + // Rebuild client and apply UA/cookies from FlareSolverr. Both are also + // published process-wide (cookies via the shared jar, user agent by host) + // so later requests can go direct instead of solving again. let useragent = res.solution.userAgent; + Self::remember_flaresolverr_user_agent(url, &useragent); self.user_agent = Some(useragent); self.store_flaresolverr_cookies(url, &res.solution.cookies); @@ -759,8 +839,10 @@ impl Requester { crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96) ); - // Retry the original URL with the updated client & (optional) proxy - let mut request = self.client.get(url).version(Version::HTTP_11); + // Retry the original URL with the updated client & (optional) proxy. + // Left unpinned so the protocol matches the emulated ALPN, same as the + // direct attempt. + let mut request = self.client.get(url); for (key, value) in headers.iter() { request = request.header(key, value); } @@ -814,7 +896,7 @@ mod tests { let origin = "https://shared-cookie-requester-test.invalid/"; a.cookie_jar - .add_cookie_str("shared_cookie=1; Path=/; SameSite=Lax", &url::Url::parse(origin).unwrap()); + .add("shared_cookie=1; Path=/; SameSite=Lax", origin); let cookie_header = b .cookie_header_for_url("https://shared-cookie-requester-test.invalid/path")