diff --git a/src/util/flaresolverr.rs b/src/util/flaresolverr.rs index 70d352e..dcc3e7a 100644 --- a/src/util/flaresolverr.rs +++ b/src/util/flaresolverr.rs @@ -19,15 +19,24 @@ pub struct FlaresolverrCookie { pub value: String, pub domain: String, pub path: String, + #[serde(default)] pub expires: f64, + #[serde(default)] pub size: u64, + #[serde(default)] pub httpOnly: bool, + #[serde(default)] pub secure: bool, + #[serde(default)] pub session: bool, pub sameSite: Option, + #[serde(default)] pub priority: String, + #[serde(default)] pub sameParty: bool, + #[serde(default)] pub sourceScheme: String, + #[serde(default)] pub sourcePort: u32, pub partitionKey: Option, } diff --git a/src/util/requester.rs b/src/util/requester.rs index 36131a7..1bc0e36 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -535,15 +535,25 @@ impl Requester { } let response = match request.send().await { Ok(response) => response, - Err(error) => { + Err(_error) => { crate::flow_debug!( "trace={} requester direct transport failed url={} error={} using_jina_fallback=true", _trace_id, crate::util::flow_debug::preview(url, 120), - crate::util::flow_debug::preview(&error.to_string(), 160) + crate::util::flow_debug::preview(&_error.to_string(), 160) ); match jina_handle.await { - Ok(Ok(body)) => return Ok(body), + Ok(Ok(body)) => { + if Self::looks_like_cloudflare_challenge(&body) { + crate::flow_debug!( + "trace={} requester jina fallback returned cloudflare challenge after transport error url={} using_flaresolverr=true", + _trace_id, + crate::util::flow_debug::preview(url, 120) + ); + } else { + return Ok(body); + } + } Ok(Err(_jina_error)) => { crate::flow_debug!( "trace={} requester jina fallback failed after transport error url={} error={}", @@ -551,7 +561,6 @@ impl Requester { crate::util::flow_debug::preview(url, 120), crate::util::flow_debug::preview(&_jina_error.to_string(), 160) ); - return Err(error.into()); } Err(_join_error) => { crate::flow_debug!( @@ -560,9 +569,9 @@ impl Requester { crate::util::flow_debug::preview(url, 120), crate::util::flow_debug::preview(&_join_error.to_string(), 160) ); - return Err(error.into()); } } + break; } }; self.store_response_cookies(url, &response); @@ -714,7 +723,17 @@ impl Requester { } } - let response = request.send().await?; + let response = match request.send().await { + Ok(r) => r, + Err(_retry_err) => { + crate::flow_debug!( + "trace={} requester post-flaresolverr retry transport failed url={} using_flaresolverr_body=true", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120) + ); + return Ok(res.solution.response); + } + }; self.store_response_cookies(url, &response); crate::flow_debug!( "trace={} requester retry response url={} status={}",