some upgrades

src/util/mod.rs

@@ -3,6 +3,7 @@ pub mod flaresolverr;
 pub mod cache;
 pub mod requester;
 pub mod discord;
+pub mod proxy;
 
 pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
     let s = s.trim();
@@ -42,4 +43,4 @@ pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {
     }
 
     result
-}
+}

src/util/proxy.rs

@@ -0,0 +1,311 @@
+use std::fmt;
+use std::sync::Arc;
+use std::str::FromStr;
+
+use serde::Serialize;
+use tokio::sync::{OnceCell, RwLock};
+use tokio::time::{self, Duration};
+use url::Url;
+use wreq::Proxy as WreqProxy;
+use wreq::{Client, Version};
+
+use crate::util::requester::Requester;
+
+pub static ALL_PROXIES: OnceCell<Arc<RwLock<Vec<Proxy>>>> = OnceCell::const_new();
+static ALL_PROXY_KEYS: OnceCell<Arc<RwLock<std::collections::HashSet<String>>>> =
+    OnceCell::const_new();
+
+const PROXY_LIST: [&str; 3] = [
+    "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt",
+    "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt",
+    "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt",
+];
+const IFCONFIG_URL: &str = "https://ifconfig.co";
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct Proxy {
+    pub protocol: String,
+    pub host: String,
+    pub port: u16,
+    pub username: Option<String>,
+    pub password: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ProxyParseError {
+    message: String,
+}
+
+impl ProxyParseError {
+    fn new(message: impl Into<String>) -> Self {
+        Self {
+            message: message.into(),
+        }
+    }
+}
+
+pub fn all_proxies_handle() -> Option<Arc<RwLock<Vec<Proxy>>>> {
+    ALL_PROXIES.get().cloned()
+}
+
+pub async fn all_proxies_snapshot() -> Option<Vec<Proxy>> {
+    let handle = ALL_PROXIES.get()?.clone();
+    let proxies = handle.read().await;
+    Some(proxies.clone())
+}
+
+impl fmt::Display for ProxyParseError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl std::error::Error for ProxyParseError {}
+
+impl fmt::Display for Proxy {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match (&self.username, &self.password) {
+            (Some(username), Some(password)) => write!(
+                f,
+                "{}://{}:{}@{}:{}",
+                self.protocol, username, password, self.host, self.port
+            ),
+            (Some(username), None) => write!(
+                f,
+                "{}://{}@{}:{}",
+                self.protocol, username, self.host, self.port
+            ),
+            (None, Some(_)) => write!(
+                f,
+                "{}://{}:{}",
+                self.protocol, self.host, self.port
+            ),
+            (None, None) => write!(f, "{}://{}:{}", self.protocol, self.host, self.port),
+        }
+    }
+}
+
+impl Proxy {
+    pub fn parse(input: &str) -> Result<Self, ProxyParseError> {
+        input.parse()
+    }
+
+    pub fn to_wreq_proxy(&self) -> Result<WreqProxy, ProxyParseError> {
+        let base_url = format!("{}://{}:{}", self.protocol, self.host, self.port);
+        let proxy_url = Url::parse(&base_url)
+            .map_err(|e| ProxyParseError::new(format!("invalid proxy url: {e}")))?;
+
+        let mut proxy = WreqProxy::all(proxy_url.as_str())
+            .map_err(|e| ProxyParseError::new(format!("failed to build wreq proxy: {e}")))?;
+
+        if let Some(username) = &self.username {
+            let password = self.password.as_deref().unwrap_or("");
+            proxy = proxy.basic_auth(username, password);
+        }
+
+        Ok(proxy)
+    }
+}
+
+pub async fn fetch_proxies(
+    requester: &mut Requester,
+    proxy_list_url: &str,
+    proxy_sink: Arc<RwLock<Vec<Proxy>>>,
+) -> Result<usize, ProxyParseError> {
+    let default_protocol = protocol_from_list_url(proxy_list_url)?;
+    let body = requester
+        .get(proxy_list_url, None)
+        .await
+        .map_err(|e| ProxyParseError::new(format!("failed to fetch proxy list: {e}")))?;
+    let mut tasks = tokio::task::JoinSet::new();
+    for (line_index, line) in body.lines().enumerate() {
+        let trimmed = line.trim();
+        if trimmed.is_empty() {
+            continue;
+        }
+
+        let proxy_input = if trimmed.contains("://") {
+            trimmed.to_string()
+        } else {
+            format!("{default_protocol}://{trimmed}")
+        };
+
+        let proxy = Proxy::parse(&proxy_input).map_err(|e| {
+            ProxyParseError::new(format!("invalid proxy on line {}: {e}", line_index + 1))
+        })?;
+
+        tasks.spawn(async move { verify_proxy(proxy).await });
+    }
+
+    let mut added = 0usize;
+    while let Some(result) = tasks.join_next().await {
+        match result {
+            Ok(Ok(proxy)) => {
+                if let Some(proxy_keys) = ALL_PROXY_KEYS.get() {
+                    let key = proxy_key(&proxy);
+                    let mut keys = proxy_keys.write().await;
+                    if !keys.insert(key) {
+                        continue;
+                    }
+                    let mut proxies = proxy_sink.write().await;
+                    proxies.push(proxy);
+                    added += 1;
+                }
+            }
+            Ok(Err(err)) => {
+                // eprintln!("Proxy verification failed: {err}");
+            }
+            Err(err) => {
+                eprintln!("Proxy verification task failed: {err}");
+            }
+        }
+    }
+
+    Ok(added)
+}
+
+fn protocol_from_list_url(list_url: &str) -> Result<&'static str, ProxyParseError> {
+    if list_url.contains("http.txt") {
+        Ok("http")
+    } else if list_url.contains("socks4.txt") {
+        Ok("socks4")
+    } else if list_url.contains("socks5.txt") {
+        Ok("socks5")
+    } else {
+        Err(ProxyParseError::new(format!(
+            "unknown proxy list protocol for url: {list_url}"
+        )))
+    }
+}
+
+async fn verify_proxy(proxy: Proxy) -> Result<Proxy, ProxyParseError> {
+    let wreq_proxy = proxy.to_wreq_proxy()?;
+    let client = Client::builder()
+        .cert_verification(false)
+        .build()
+        .map_err(|e| ProxyParseError::new(format!("failed to build http client: {e}")))?;
+
+    let response = client
+        .get(IFCONFIG_URL)
+        .version(Version::HTTP_11)
+        .proxy(wreq_proxy)
+        .timeout(Duration::from_secs(5))
+        .send()
+        .await
+        .map_err(|e| ProxyParseError::new(format!("proxy request failed: {e}")))?;
+
+    if response.status().is_success() {
+        Ok(proxy)
+    } else {
+        Err(ProxyParseError::new(format!(
+            "proxy returned status {}",
+            response.status()
+        )))
+    }
+}
+
+pub fn init_all_proxies_background(requester: Requester) {
+    if ALL_PROXIES.get().is_some() {
+        return;
+    }
+
+    if tokio::runtime::Handle::try_current().is_err() {
+        eprintln!("Skipping proxy list init: no Tokio runtime available");
+        return;
+    }
+
+    let proxy_cache = Arc::new(RwLock::new(Vec::new()));
+    let proxy_keys = Arc::new(RwLock::new(std::collections::HashSet::new()));
+    if ALL_PROXIES.set(proxy_cache.clone()).is_err() {
+        return;
+    }
+    let _ = ALL_PROXY_KEYS.set(proxy_keys);
+
+    tokio::spawn(async move {
+        let mut interval = time::interval(Duration::from_secs(60 * 60));
+        loop {
+            let mut tasks = tokio::task::JoinSet::new();
+            for list in PROXY_LIST {
+                let proxy_cache = proxy_cache.clone();
+                let mut requester = requester.clone();
+                tasks
+                    .spawn(async move { fetch_proxies(&mut requester, list, proxy_cache).await });
+            }
+
+            while let Some(result) = tasks.join_next().await {
+                match result {
+                    Ok(Ok(_added)) => {}
+                    Ok(Err(err)) => {
+                        eprintln!("Failed to fetch proxy list: {err}");
+                    }
+                    Err(err) => {
+                        eprintln!("Proxy list task failed: {err}");
+                    }
+                }
+            }
+
+            interval.tick().await;
+        }
+    });
+}
+
+impl FromStr for Proxy {
+    type Err = ProxyParseError;
+
+    fn from_str(input: &str) -> Result<Self, Self::Err> {
+        let trimmed = input.trim();
+        if trimmed.is_empty() {
+            return Err(ProxyParseError::new("proxy string is empty"));
+        }
+
+        let with_scheme = if trimmed.contains("://") {
+            trimmed.to_string()
+        } else {
+            format!("http://{trimmed}")
+        };
+
+        let url = Url::parse(&with_scheme)
+            .map_err(|e| ProxyParseError::new(format!("invalid proxy url: {e}")))?;
+        if !(url.path().is_empty() || url.path() == "/")
+            || url.query().is_some()
+            || url.fragment().is_some()
+        {
+            println!("Parsed proxy URL: {:?}", url);
+            return Err(ProxyParseError::new(format!(
+                "proxy url must not include path, query, or fragment: {:?}",
+                input
+            )));
+        }
+
+        let host = url
+            .host_str()
+            .ok_or_else(|| ProxyParseError::new("proxy url is missing host"))?
+            .to_string();
+
+        let port = url
+            .port()
+            .unwrap_or(80);
+
+        Ok(Proxy {
+            protocol: url.scheme().to_string(),
+            host,
+            port,
+            username: match url.username() {
+                "" => None,
+                username => Some(username.to_string()),
+            },
+            password: url.password().map(|password| password.to_string()),
+        })
+    }
+}
+
+fn proxy_key(proxy: &Proxy) -> String {
+    format!(
+        "{}://{}:{}@{}:{}",
+        proxy.protocol,
+        proxy.username.as_deref().unwrap_or(""),
+        proxy.password.as_deref().unwrap_or(""),
+        proxy.host,
+        proxy.port
+    )
+}

src/util/requester.rs

@@ -8,7 +8,9 @@ use wreq::Version;
 use wreq::header::HeaderValue;
 use wreq::redirect::Policy;
 use wreq_util::Emulation;
+use rand::seq::SliceRandom;
 
+use crate::util::proxy;
 use crate::util::flaresolverr::FlareSolverrRequest;
 use crate::util::flaresolverr::Flaresolverr;
 
@@ -21,6 +23,7 @@ pub struct Requester {
     client: Client,
     proxy: bool,
     flaresolverr_session: Option<String>,
+    use_random_proxy: bool,
 }
 
 impl Requester {
@@ -33,11 +36,16 @@ impl Requester {
             .build()
             .expect("Failed to create HTTP client");
 
-        Requester {
+        let requester = Requester {
             client,
             proxy: false,
             flaresolverr_session: None,
-        }
+            use_random_proxy: false,
+        };
+
+        proxy::init_all_proxies_background(requester.clone());
+
+        requester
     }
 
     pub fn set_proxy(&mut self, proxy: bool) {
@@ -47,6 +55,10 @@ impl Requester {
         self.proxy = proxy;
     }
 
+    pub fn set_random_proxy(&mut self, random: bool) {
+        self.use_random_proxy = random;
+    }
+
     pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
         let client = Client::builder()
             .cert_verification(false)
@@ -67,6 +79,26 @@ impl Requester {
         request.send().await
     }
 
+    pub async fn get_raw_with_proxy(
+        &mut self,
+        url: &str,
+        proxy: Proxy,
+    ) -> Result<Response, wreq::Error> {
+        let client = Client::builder()
+            .cert_verification(false)
+            .emulation(Emulation::Firefox136)
+            .cookie_store(true)
+            .build()
+            .expect("Failed to create HTTP client");
+
+        client
+            .get(url)
+            .version(Version::HTTP_11)
+            .proxy(proxy)
+            .send()
+            .await
+    }
+
     pub async fn get_raw_with_headers(
         &mut self,
         url: &str,
@@ -185,7 +217,14 @@ pub async fn post_json<S>(
                     let proxy = Proxy::all(&proxy_url).unwrap();
                     request = request.proxy(proxy);
                 }
-            }
+            } 
+            // else if self.use_random_proxy {
+            //     let proxies = proxy::all_proxies_snapshot().await.unwrap_or_default();
+            //     if !proxies.is_empty() {
+            //         let mut random_proxy = proxies.choose_mut(&mut rand::thread_rng()).unwrap().clone();
+            //         request = request.proxy(random_proxy);
+            //     }
+            // }
             let response = request.send().await?;
             if response.status().is_success() || response.status().as_u16() == 404 {
                 return Ok(response.text().await?);