simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

upgraded wreq

Browse Source
This commit is contained in:
Simon
2026-09-07 13:00:19 +00:00
parent e37563e5bc
commit 5941a6e607
15 changed files with 145 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.toml
View File

@@ -20,8 +20,8 @@ ntex-files = "3"
serde = "1.0.228"
serde_json = "1.0.145"
tokio = { version = "1.49", features = ["full"] }
wreq = { version = "5.3.0", features = ["cookies", "multipart", "json"] }
wreq-util = "2"
wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json"] }
wreq-util = "3.0.0-rc.14"
percent-encoding = "2.3.2"
capitalize = "0.3.4"
url = "2.5.7"

2
src/providers/freeuseporn.rs
View File

@@ -203,7 +203,7 @@ impl FreeusepornProvider {
.await
.map_err(|error| format!("search submit failed url={search_url}; error={error}"))?;
Ok(response.url().to_string().trim_end_matches('/').to_string())
Ok(response.uri().to_string().trim_end_matches('/').to_string())
}
fn build_formats(&self, id: &str) -> Vec<VideoFormat> {

4
src/providers/homoxxx.rs
View File

@@ -96,7 +96,7 @@ impl HomoxxxProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
@@ -208,7 +208,7 @@ impl HomoxxxProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;

2
src/providers/melonstube.rs
View File

@@ -406,7 +406,7 @@ impl MelonstubeProvider {
.await
.map_err(|e| Error::from(format!("search submit failed url={search_url}: {e}")))?;
Ok(response.url().to_string().trim_end_matches('/').to_string())
Ok(response.uri().to_string().trim_end_matches('/').to_string())
}
async fn fetch_and_parse(

4
src/providers/okporn.rs
View File

@@ -96,7 +96,7 @@ impl OkpornProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
@@ -194,7 +194,7 @@ impl OkpornProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;

4
src/providers/okxxx.rs
View File

@@ -151,7 +151,7 @@ impl OkxxxProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
@@ -252,7 +252,7 @@ impl OkxxxProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;

4
src/providers/perfectgirls.rs
View File

@@ -151,7 +151,7 @@ impl PerfectgirlsProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
@@ -252,7 +252,7 @@ impl PerfectgirlsProvider {
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;

2
src/providers/yesporn.rs
View File

@@ -406,7 +406,7 @@ impl YespornProvider {
)));
}
let canonical_url = response.url().to_string();
let canonical_url = response.uri().to_string();
let body = response
.text()
.await

2
src/proxies/allpornstream.rs
View File

@@ -24,7 +24,7 @@ impl AllPornStreamProxy {
fn build_chrome_client() -> Option<wreq::Client> {
let jar = Arc::new(Jar::default());
wreq::Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Chrome120)
.cookie_provider(jar)
.redirect(Policy::default())

6
src/proxies/doodstream.rs
View File

@@ -366,7 +366,7 @@ impl DoodstreamProxy {
fn build_chrome_client() -> Option<wreq::Client> {
let jar = Arc::new(Jar::default());
wreq::Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Chrome120)
.cookie_provider(jar)
.redirect(Policy::default())
@@ -387,7 +387,7 @@ impl DoodstreamProxy {
if !response.status().is_success() {
return None;
}
let effective_url = response.url().to_string();
let effective_url = response.uri().to_string();
let html = response.text().await.ok()?;
if let Some(url) = Self::extract_stream_url(&html) {
@@ -434,7 +434,7 @@ impl crate::proxies::Proxy for DoodstreamProxy {
.await
{
Ok(response) if response.status().is_success() => {
let effective_url = response.url().to_string();
let effective_url = response.uri().to_string();
match response.text().await {
Ok(text) => (text, effective_url),
Err(_) => return String::new(),

2
src/proxies/lulustream.rs
View File

@@ -62,7 +62,7 @@ impl LulustreamProxy {
fn build_chrome_client() -> Option<wreq::Client> {
let jar = Arc::new(Jar::default());
wreq::Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Chrome120)
.cookie_provider(jar)
.redirect(Policy::default())

4
src/proxies/melonstube.rs
View File

@@ -113,7 +113,7 @@ impl MelonstubeProxy {
fn redirect_client() -> Option<wreq::Client> {
wreq::Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Chrome120)
.redirect(Policy::default())
.build()
@@ -122,7 +122,7 @@ impl MelonstubeProxy {
fn no_redirect_client() -> Option<wreq::Client> {
wreq::Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.emulation(Emulation::Chrome120)
.redirect(Policy::none())
.build()

43
src/util/flaresolverr.rs
View File

@@ -183,48 +183,33 @@ impl Flaresolverr {
&self,
request: FlareSolverrRequest,
) -> Result<FlareSolverrResponse, Box<dyn std::error::Error + Send + Sync>> {
// Keep one ready session globally and rotate it per solve:
// - solve with current ready session
// - create replacement session in parallel
// - destroy old session
// - keep replacement as new ready session
// Keep exactly one long-lived session globally and reuse it:
// - solve with the current ready session
// - on success, keep it for the next solve
// - on failure, drop it so the next solve starts from a fresh browser
//
// An earlier version rotated the session on every solve (create a
// replacement, destroy the old one). That leaked a browser instance
// whenever the solve future was cancelled — a client timeout mid-solve
// left both the active and the replacement session alive on the
// FlareSolverr host, and the accumulated Chrome instances eventually made
// every solve time out.
let session_state = global_session_state().clone();
let mut state = session_state.lock().await;
let active_session = self.ensure_ready_session_locked(&mut state).await?;
let replacement_creator = {
let solver = self.clone();
tokio::spawn(async move { solver.create_session().await })
};
let solve_result = self.solve_with_session(request, &active_session).await;
let replacement_session = match replacement_creator.await {
Ok(Ok(session)) => session,
Ok(Err(error)) => {
eprintln!(
"FlareSolverr replacement session creation failed, retrying inline: {}",
error
);
self.create_session().await?
}
Err(join_error) => {
eprintln!(
"FlareSolverr replacement task join failed, retrying inline: {}",
join_error
);
self.create_session().await?
}
};
if solve_result.is_err() {
state.ready_session = None;
if let Err(error) = self.destroy_session(&active_session).await {
eprintln!(
"FlareSolverr session cleanup failed for session '{}': {}",
active_session, error
);
}
}
state.ready_session = Some(replacement_session);
solve_result
}
}

2
src/util/proxy.rs
View File

@@ -173,7 +173,7 @@ fn protocol_from_list_url(list_url: &str) -> Result<&'static str, ProxyParseErro
async fn verify_proxy(proxy: Proxy) -> Result<Proxy, ProxyParseError> {
let wreq_proxy = proxy.to_wreq_proxy()?;
let client = Client::builder()
.cert_verification(false)
.tls_cert_verification(false)
.build()
.map_err(|e| ProxyParseError::new(format!("failed to build http client: {e}")))?;

132
src/util/requester.rs
View File

@@ -1,14 +1,15 @@
use serde::Serialize;
use std::collections::HashMap;
use std::env;
use std::fmt;
use std::sync::{Arc, OnceLock};
use std::sync::{Arc, OnceLock, RwLock};
use std::time::Duration;
use wreq::Client;
use wreq::Proxy;
use wreq::Response;
use wreq::Url;
use wreq::Version;
use wreq::cookie::{CookieStore, Jar};
use wreq::Uri;
use wreq::cookie::{Cookies, CookieStore, Jar};
use wreq::header::{HeaderMap, HeaderValue, SET_COOKIE, USER_AGENT};
use wreq::multipart::Form;
use wreq::redirect::Policy;
@@ -53,6 +54,63 @@ impl Requester {
.clone()
}
/// User agents recovered from FlareSolverr solves, keyed by host.
///
/// The cookie jar is already process-wide, but the solved user agent used to
/// live only on the `Requester` that ran the solve. Providers clone their
/// requester for every concurrent detail fetch, so those clones went out with
/// the default user agent and Cloudflare rejected them — forcing another
/// (~15s) solve per page. Keeping the identity here lets every later request
/// to that host replay the solved user agent alongside the shared cookies and
/// stay on the direct path.
fn flaresolverr_user_agents() -> &'static RwLock<HashMap<String, String>> {
static USER_AGENTS: OnceLock<RwLock<HashMap<String, String>>> = OnceLock::new();
USER_AGENTS.get_or_init(|| RwLock::new(HashMap::new()))
}
fn host_key(url: &str) -> Option<String> {
url::Url::parse(url)
.ok()?
.host_str()
.map(|host| host.trim_start_matches("www.").to_ascii_lowercase())
}
fn remember_flaresolverr_user_agent(url: &str, user_agent: &str) {
if user_agent.is_empty() {
return;
}
let Some(host) = Self::host_key(url) else {
return;
};
if let Ok(mut agents) = Self::flaresolverr_user_agents().write() {
agents.insert(host, user_agent.to_string());
}
}
fn remembered_flaresolverr_user_agent(url: &str) -> Option<String> {
let host = Self::host_key(url)?;
Self::flaresolverr_user_agents()
.read()
.ok()?
.get(&host)
.cloned()
}
/// Adopt a previously solved user agent for `url` when this requester does not
/// already carry one, so the direct attempt reuses the FlareSolverr identity.
/// Returns true when the client was rebuilt.
fn adopt_remembered_identity(&mut self, url: &str) -> bool {
if self.user_agent.is_some() {
return false;
}
let Some(user_agent) = Self::remembered_flaresolverr_user_agent(url) else {
return false;
};
self.user_agent = Some(user_agent);
self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
true
}
fn origin_url_for_cookie_scope(url: &str) -> Option<url::Url> {
let parsed = url::Url::parse(url).ok()?;
let host = parsed.host_str()?;
@@ -73,7 +131,7 @@ impl Requester {
// register the cookie against https://foo.com/ so the Jar returns it for
// sub.foo.com requests as well.
let origin = Self::cookie_domain_origin(cookie, &default_origin);
self.cookie_jar.add_cookie_str(cookie, &origin);
self.cookie_jar.add(cookie, origin.as_str());
}
}
@@ -139,8 +197,7 @@ impl Requester {
}
}
self.cookie_jar
.add_cookie_str(&cookie_string, &origin);
self.cookie_jar.add(cookie_string.as_str(), origin.as_str());
}
}
@@ -225,8 +282,8 @@ impl Requester {
fn build_client(cookie_jar: Arc<Jar>, user_agent: Option<&str>) -> Client {
let mut builder = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.tls_cert_verification(false)
.emulation(Emulation::Firefox151)
.cookie_provider(cookie_jar)
.redirect(Policy::default());
@@ -270,22 +327,28 @@ impl Requester {
self.debug_trace_id = debug_trace_id;
}
#[cfg(feature = "debug")]
pub fn debug_trace_id(&self) -> Option<&str> {
self.debug_trace_id.as_deref()
}
pub fn cookie_header_for_url(&self, url: &str) -> Option<String> {
let parsed = url.parse::<Url>().ok()?;
let joined = self.cookie_jar.cookies(&parsed)
.into_iter()
.filter_map(|c| c.to_str().ok().map(ToOwned::to_owned))
let uri = url.parse::<Uri>().ok()?;
// HTTP/1.1 asks the store for the combined `a=1; b=2` form; HTTP/2 would
// hand back one field value per cookie instead.
let joined = match self.cookie_jar.cookies(&uri, Version::HTTP_11) {
Cookies::Compressed(value) => value.to_str().ok()?.to_string(),
Cookies::Uncompressed(values) => values
.iter()
.filter_map(|value| value.to_str().ok())
.collect::<Vec<_>>()
.join("; ");
.join("; "),
_ => return None,
};
(!joined.is_empty()).then_some(joined)
}
pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
self.adopt_remembered_identity(url);
let cookie_preview = self
.cookie_header_for_url(url)
.map(|cookie| crate::util::flow_debug::preview(&cookie, 160))
@@ -322,6 +385,7 @@ impl Requester {
/// a bare `wreq::Client`, which Cloudflare is far more likely to flag,
/// especially from datacenter/production IPs.
pub async fn get_no_redirect_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
self.adopt_remembered_identity(url);
let cookie_preview = self
.cookie_header_for_url(url)
.map(|cookie| crate::util::flow_debug::preview(&cookie, 160))
@@ -336,8 +400,8 @@ impl Requester {
self.proxy
);
let mut builder = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.tls_cert_verification(false)
.emulation(Emulation::Firefox151)
.cookie_provider(self.cookie_jar.clone())
.redirect(Policy::none());
if let Some(user_agent) = self.user_agent.as_deref() {
@@ -546,6 +610,14 @@ impl Requester {
headers: Vec<(String, String)>,
_http_version: Option<Version>,
) -> Result<String, AnyErr> {
if self.adopt_remembered_identity(url) {
crate::flow_debug!(
"trace={} requester reusing flaresolverr identity url={} user_agent={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
);
}
let cookie_preview = self.debug_cookie_preview_from_owned_headers(url, &headers);
#[cfg(not(feature = "debug"))]
let _ = &cookie_preview;
@@ -558,10 +630,12 @@ impl Requester {
_http_version,
self.proxy
);
let http_version = match _http_version {
Some(v) => v,
None => Version::HTTP_11,
};
// Only pin the protocol when the caller asked for one. Defaulting to
// HTTP/1.1 used to force every request onto h1 while the TLS handshake
// still advertised the browser emulation's h2 ALPN — a mismatch
// Cloudflare scores as a bot. On jable.tv it is a deterministic 403:
// the identical client returns 200 over the negotiated h2 and 403 the
// moment the version is pinned to HTTP/1.1.
let cookie_jar = self.cookie_jar.clone();
let user_agent = self.user_agent.clone();
let proxy_enabled = self.proxy;
@@ -571,7 +645,10 @@ impl Requester {
Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await
});
loop {
let mut request = self.client.get(url).version(http_version);
let mut request = self.client.get(url);
if let Some(version) = _http_version {
request = request.version(version);
}
for (key, value) in headers.iter() {
request = request.header(key, value);
}
@@ -746,8 +823,11 @@ impl Requester {
.await
.map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?;
// Rebuild client and apply UA/cookies from FlareSolverr
// Rebuild client and apply UA/cookies from FlareSolverr. Both are also
// published process-wide (cookies via the shared jar, user agent by host)
// so later requests can go direct instead of solving again.
let useragent = res.solution.userAgent;
Self::remember_flaresolverr_user_agent(url, &useragent);
self.user_agent = Some(useragent);
self.store_flaresolverr_cookies(url, &res.solution.cookies);
@@ -759,8 +839,10 @@ impl Requester {
crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
);
// Retry the original URL with the updated client & (optional) proxy
let mut request = self.client.get(url).version(Version::HTTP_11);
// Retry the original URL with the updated client & (optional) proxy.
// Left unpinned so the protocol matches the emulated ALPN, same as the
// direct attempt.
let mut request = self.client.get(url);
for (key, value) in headers.iter() {
request = request.header(key, value);
}
@@ -814,7 +896,7 @@ mod tests {
let origin = "https://shared-cookie-requester-test.invalid/";
a.cookie_jar
.add_cookie_str("shared_cookie=1; Path=/; SameSite=Lax", &url::Url::parse(origin).unwrap());
.add("shared_cookie=1; Path=/; SameSite=Lax", origin);
let cookie_header = b
.cookie_header_for_url("https://shared-cookie-requester-test.invalid/path")