From b7ebb0d768c9775441f84b0a265edb7eaa79de34 Mon Sep 17 00:00:00 2001
From: Simon <simon.huenecke@gmail.com>
Date: Tue, 23 Jun 2026 19:17:16 +0000
Subject: [PATCH] fixed a bug in requester

---
 src/util/requester.rs | 60 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 58 insertions(+), 2 deletions(-)

diff --git a/src/util/requester.rs b/src/util/requester.rs
index eaf9a41..7706770 100644
--- a/src/util/requester.rs
+++ b/src/util/requester.rs
@@ -136,6 +136,13 @@ impl Requester {
             .unwrap_or_else(|| "none".to_string())
     }
 
+    fn looks_like_cloudflare_challenge(body: &str) -> bool {
+        body.contains("Just a moment...")
+            || body.contains("cf-turnstile-response")
+            || body.contains("challenges.cloudflare.com")
+            || body.contains("Enable JavaScript and cookies to continue")
+    }
+
     fn jina_mirror_url(url: &str) -> Option<String> {
         if url.trim().is_empty() {
             return None;
@@ -542,8 +549,47 @@ impl Requester {
                 response.status()
             );
             if response.status().is_success() || response.status().as_u16() == 404 {
+                let status = response.status();
+                let body = response.text().await?;
+                if status.is_success() && Self::looks_like_cloudflare_challenge(&body) {
+                    crate::flow_debug!(
+                        "trace={} requester direct response looks like cloudflare challenge url={} using_jina_fallback=true",
+                        _trace_id,
+                        crate::util::flow_debug::preview(url, 120)
+                    );
+                    match jina_handle.await {
+                        Ok(Ok(jina_body)) => {
+                            if Self::looks_like_cloudflare_challenge(&jina_body) {
+                                crate::flow_debug!(
+                                    "trace={} requester jina fallback also returned cloudflare challenge url={}",
+                                    _trace_id,
+                                    crate::util::flow_debug::preview(url, 120)
+                                );
+                            } else {
+                                return Ok(jina_body);
+                            }
+                        }
+                        Ok(Err(_jina_error)) => {
+                            crate::flow_debug!(
+                                "trace={} requester jina fallback failed after cloudflare challenge url={} error={}",
+                                _trace_id,
+                                crate::util::flow_debug::preview(url, 120),
+                                crate::util::flow_debug::preview(&_jina_error.to_string(), 160)
+                            );
+                        }
+                        Err(_join_error) => {
+                            crate::flow_debug!(
+                                "trace={} requester jina task join failed after cloudflare challenge url={} error={}",
+                                _trace_id,
+                                crate::util::flow_debug::preview(url, 120),
+                                crate::util::flow_debug::preview(&_join_error.to_string(), 160)
+                            );
+                        }
+                    }
+                    break;
+                }
                 jina_handle.abort();
-                return Ok(response.text().await?);
+                return Ok(body);
             }
             if response.status().as_u16() == 429 {
                 crate::flow_debug!(
@@ -561,7 +607,17 @@ impl Requester {
                     response.status()
                 );
                 match jina_handle.await {
-                    Ok(Ok(body)) => return Ok(body),
+                    Ok(Ok(body)) => {
+                        if Self::looks_like_cloudflare_challenge(&body) {
+                            crate::flow_debug!(
+                                "trace={} requester jina fallback returned cloudflare challenge url={}",
+                                _trace_id,
+                                crate::util::flow_debug::preview(url, 120)
+                            );
+                        } else {
+                            return Ok(body);
+                        }
+                    }
                     Ok(Err(_error)) => {
                         crate::flow_debug!(
                             "trace={} requester jina fallback failed url={} error={}",