simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed a bug in requester

Browse Source
This commit is contained in:
Simon
2026-06-23 19:17:16 +00:00
parent 228fdf7a21
commit b7ebb0d768
1 changed files with 58 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/util/requester.rs
View File

@@ -136,6 +136,13 @@ impl Requester {
.unwrap_or_else(|| "none".to_string())
}
fn looks_like_cloudflare_challenge(body: &str) -> bool {
body.contains("Just a moment...")
|| body.contains("cf-turnstile-response")
|| body.contains("challenges.cloudflare.com")
|| body.contains("Enable JavaScript and cookies to continue")
}
fn jina_mirror_url(url: &str) -> Option<String> {
if url.trim().is_empty() {
return None;
@@ -542,8 +549,47 @@ impl Requester {
response.status()
);
if response.status().is_success() || response.status().as_u16() == 404 {
let status = response.status();
let body = response.text().await?;
if status.is_success() && Self::looks_like_cloudflare_challenge(&body) {
crate::flow_debug!(
"trace={} requester direct response looks like cloudflare challenge url={} using_jina_fallback=true",
_trace_id,
crate::util::flow_debug::preview(url, 120)
);
match jina_handle.await {
Ok(Ok(jina_body)) => {
if Self::looks_like_cloudflare_challenge(&jina_body) {
crate::flow_debug!(
"trace={} requester jina fallback also returned cloudflare challenge url={}",
_trace_id,
crate::util::flow_debug::preview(url, 120)
);
} else {
return Ok(jina_body);
}
}
Ok(Err(_jina_error)) => {
crate::flow_debug!(
"trace={} requester jina fallback failed after cloudflare challenge url={} error={}",
_trace_id,
crate::util::flow_debug::preview(url, 120),
crate::util::flow_debug::preview(&_jina_error.to_string(), 160)
);
}
Err(_join_error) => {
crate::flow_debug!(
"trace={} requester jina task join failed after cloudflare challenge url={} error={}",
_trace_id,
crate::util::flow_debug::preview(url, 120),
crate::util::flow_debug::preview(&_join_error.to_string(), 160)
);
}
}
break;
}
jina_handle.abort();
return Ok(response.text().await?);
return Ok(body);
}
if response.status().as_u16() == 429 {
crate::flow_debug!(
@@ -561,7 +607,17 @@ impl Requester {
response.status()
);
match jina_handle.await {
Ok(Ok(body)) => return Ok(body),
Ok(Ok(body)) => {
if Self::looks_like_cloudflare_challenge(&body) {
crate::flow_debug!(
"trace={} requester jina fallback returned cloudflare challenge url={}",
_trace_id,
crate::util::flow_debug::preview(url, 120)
);
} else {
return Ok(body);
}
}
Ok(Err(_error)) => {
crate::flow_debug!(
"trace={} requester jina fallback failed url={} error={}",