diff --git a/src/providers/spankbang.rs b/src/providers/spankbang.rs index c6196cc..790a854 100644 --- a/src/providers/spankbang.rs +++ b/src/providers/spankbang.rs @@ -3,6 +3,7 @@ use std::env; use error_chain::error_chain; use futures::future::join_all; use htmlentity::entity::{decode, ICodedDataTrait}; +use wreq::Proxy; use crate::db; use crate::providers::Provider; use crate::util::cache::VideoCache; @@ -23,11 +24,13 @@ error_chain! { #[derive(Debug, Clone)] pub struct SpankbangProvider { url: String, + proxy: Proxy, } impl SpankbangProvider { pub fn new() -> Self { SpankbangProvider { url: "https://spankbang.com/".to_string(), + proxy: wreq::Proxy::https(std::env::var("BURP_URL").unwrap()).unwrap() } } async fn get(&self, cache:VideoCache, pool: DbPool, page: u8, sort: String) -> Result> { @@ -52,9 +55,10 @@ impl SpankbangProvider { let client = Client::builder() .emulation(Emulation::Firefox136) + .cert_verification(false) .build()?; - let response = client.get(url.clone()).send().await?; + let response = client.get(url.clone()).proxy(self.proxy.clone()).send().await?; let mut cookies_string = String::new(); if let Some(_) = response.headers().get_all("set-cookie").iter().next() { for _ in response.headers().get_all("set-cookie").iter() { @@ -77,10 +81,7 @@ impl SpankbangProvider { } if response.status().is_success() { let text = response.text().await?; - println!("Response text: {}", text); let video_items: Vec = self.get_video_items_from_html(text.clone(), &client, cookies_string, pool.clone()).await; - - if !video_items.is_empty() { cache.remove(&url); cache.insert(url.clone(), video_items.clone()); @@ -139,9 +140,10 @@ impl SpankbangProvider { let client = Client::builder() .emulation(Emulation::Firefox136) + .cert_verification(false) .build()?; - let response = client.get(url.clone()).send().await?; + let response = client.get(url.clone()).proxy(self.proxy.clone()).send().await?; let mut cookies_string = String::new(); if let Some(_) = response.headers().get_all("set-cookie").iter().next() { for _ in response.headers().get_all("set-cookie").iter() { @@ -217,7 +219,7 @@ impl SpankbangProvider { // return Err(format!("Error fetching video from database: {}", e).into()); } } - let response = client.get(url.clone()).header("Cookie", cookies.clone()).send().await?; + let response = client.get(url.clone()).header("Cookie", cookies.clone()).proxy(self.proxy.clone()).send().await?; let mut response = response; while response.status().as_u16() == 429 { @@ -327,7 +329,7 @@ impl SpankbangProvider { println!("Could not find video-list in HTML"); return items; } - let video_listing_content = split_html[1]; + let video_listing_content = format!("{}{}", split_html[1], split_html.get(2).unwrap_or(&"")); let raw_videos_vec = video_listing_content .split("data-testid=\"video-item\"") .collect::>(); @@ -336,6 +338,7 @@ impl SpankbangProvider { return items; } let raw_videos = raw_videos_vec[1..].to_vec(); + println!("Found {} video items", raw_videos.len()); let futures = raw_videos.into_iter().map(|el| self.parse_video_item(el.to_string(), client, cookies.clone(), pool.clone())); let results: Vec> = join_all(futures).await; let video_items: Vec = results