debugging and single provider compime

src/util/requester.rs

@@ -26,6 +26,8 @@ pub struct Requester {
     client: Client,
     #[serde(skip)]
     cookie_jar: Arc<Jar>,
+    #[serde(skip)]
+    debug_trace_id: Option<String>,
     proxy: bool,
     flaresolverr_session: Option<String>,
     user_agent: Option<String>,
@@ -35,6 +37,7 @@ impl fmt::Debug for Requester {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("Requester")
             .field("proxy", &self.proxy)
+            .field("debug_trace_id", &self.debug_trace_id)
             .field("flaresolverr_session", &self.flaresolverr_session)
             .field("user_agent", &self.user_agent)
             .finish()
@@ -67,6 +70,7 @@ impl Requester {
         let requester = Requester {
             client,
             cookie_jar,
+            debug_trace_id: None,
             proxy: false,
             flaresolverr_session: None,
             user_agent: None,
@@ -84,6 +88,18 @@ impl Requester {
         self.proxy = proxy;
     }
 
+    pub fn proxy_enabled(&self) -> bool {
+        self.proxy
+    }
+
+    pub fn set_debug_trace_id(&mut self, debug_trace_id: Option<String>) {
+        self.debug_trace_id = debug_trace_id;
+    }
+
+    pub fn debug_trace_id(&self) -> Option<&str> {
+        self.debug_trace_id.as_deref()
+    }
+
     pub fn cookie_header_for_url(&self, url: &str) -> Option<String> {
         let parsed = url.parse::<Uri>().ok()?;
         match self.cookie_jar.cookies(&parsed) {
@@ -102,6 +118,12 @@ impl Requester {
     }
 
     pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
+        crate::flow_debug!(
+            "trace={} requester get_raw url={} proxy={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            self.proxy
+        );
         let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
 
         let mut request = client.get(url).version(Version::HTTP_11);
@@ -121,6 +143,13 @@ impl Requester {
         url: &str,
         headers: Vec<(String, String)>,
     ) -> Result<Response, wreq::Error> {
+        crate::flow_debug!(
+            "trace={} requester get_raw_with_headers url={} headers={} proxy={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            headers.len(),
+            self.proxy
+        );
         let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
 
         let mut request = client.get(url).version(Version::HTTP_11);
@@ -147,6 +176,13 @@ impl Requester {
     where
         S: Serialize + ?Sized,
     {
+        crate::flow_debug!(
+            "trace={} requester post_json url={} headers={} proxy={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            headers.len(),
+            self.proxy
+        );
         let mut request = self.client.post(url).version(Version::HTTP_11).json(data);
 
         // Set custom headers
@@ -170,6 +206,14 @@ impl Requester {
         data: &str,
         headers: Vec<(&str, &str)>,
     ) -> Result<Response, wreq::Error> {
+        crate::flow_debug!(
+            "trace={} requester post url={} headers={} body_len={} proxy={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            headers.len(),
+            data.len(),
+            self.proxy
+        );
         let mut request = self
             .client
             .post(url)
@@ -198,6 +242,13 @@ impl Requester {
         headers: Vec<(String, String)>,
         _http_version: Option<Version>,
     ) -> Result<Response, wreq::Error> {
+        crate::flow_debug!(
+            "trace={} requester post_multipart url={} headers={} proxy={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            headers.len(),
+            self.proxy
+        );
         let http_version = match _http_version {
             Some(v) => v,
             None => Version::HTTP_11,
@@ -234,6 +285,14 @@ impl Requester {
         headers: Vec<(String, String)>,
         _http_version: Option<Version>,
     ) -> Result<String, AnyErr> {
+        crate::flow_debug!(
+            "trace={} requester get_with_headers start url={} headers={} http_version={:?} proxy={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            headers.len(),
+            _http_version,
+            self.proxy
+        );
         let http_version = match _http_version {
             Some(v) => v,
             None => Version::HTTP_11,
@@ -250,10 +309,21 @@ impl Requester {
                 }
             }
             let response = request.send().await?;
+            crate::flow_debug!(
+                "trace={} requester direct response url={} status={}",
+                self.debug_trace_id().unwrap_or("none"),
+                crate::util::flow_debug::preview(url, 120),
+                response.status()
+            );
             if response.status().is_success() || response.status().as_u16() == 404 {
                 return Ok(response.text().await?);
             }
             if response.status().as_u16() == 429 {
+                crate::flow_debug!(
+                    "trace={} requester direct retry url={} status=429",
+                    self.debug_trace_id().unwrap_or("none"),
+                    crate::util::flow_debug::preview(url, 120)
+                );
                 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
                 continue;
             } else {
@@ -276,6 +346,12 @@ impl Requester {
         if self.proxy && env::var("BURP_URL").is_ok() {
             flare.set_proxy(true);
         }
+        crate::flow_debug!(
+            "trace={} requester flaresolverr url={} proxy={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            self.proxy
+        );
 
         let res = flare
             .solve(FlareSolverrRequest {
@@ -300,6 +376,12 @@ impl Requester {
         }
 
         self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
+        crate::flow_debug!(
+            "trace={} requester flaresolverr solved url={} user_agent={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
+        );
 
         // Retry the original URL with the updated client & (optional) proxy
         let mut request = self.client.get(url).version(Version::HTTP_11);
@@ -314,11 +396,22 @@ impl Requester {
         }
 
         let response = request.send().await?;
+        crate::flow_debug!(
+            "trace={} requester retry response url={} status={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120),
+            response.status()
+        );
         if response.status().is_success() {
             return Ok(response.text().await?);
         }
 
         // Fall back to FlareSolverr-provided body
+        crate::flow_debug!(
+            "trace={} requester fallback body url={}",
+            self.debug_trace_id().unwrap_or("none"),
+            crate::util::flow_debug::preview(url, 120)
+        );
         Ok(res.solution.response)
     }
 }