simon/hottub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed bug with spankbang where only 7 video items where shown

Browse Source
This commit is contained in:
Simon
2025-07-16 13:49:20 +00:00
parent a63e260dac
commit bd565e044a
1 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/providers/spankbang.rs
View File

@@ -3,6 +3,7 @@ use std::env;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use htmlentity::entity::{decode, ICodedDataTrait}; use htmlentity::entity::{decode, ICodedDataTrait};
use wreq::Proxy;
use crate::db; use crate::db;
use crate::providers::Provider; use crate::providers::Provider;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
@@ -23,11 +24,13 @@ error_chain! {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct SpankbangProvider { pub struct SpankbangProvider {
url: String, url: String,
proxy: Proxy,
} }
impl SpankbangProvider { impl SpankbangProvider {
pub fn new() -> Self { pub fn new() -> Self {
SpankbangProvider { SpankbangProvider {
url: "https://spankbang.com/".to_string(), url: "https://spankbang.com/".to_string(),
proxy: wreq::Proxy::https(std::env::var("BURP_URL").unwrap()).unwrap()
} }
} }
async fn get(&self, cache:VideoCache, pool: DbPool, page: u8, sort: String) -> Result<Vec<VideoItem>> { async fn get(&self, cache:VideoCache, pool: DbPool, page: u8, sort: String) -> Result<Vec<VideoItem>> {
@@ -52,9 +55,10 @@ impl SpankbangProvider {
let client = Client::builder() let client = Client::builder()
.emulation(Emulation::Firefox136) .emulation(Emulation::Firefox136)
.cert_verification(false)
.build()?; .build()?;
let response = client.get(url.clone()).send().await?; let response = client.get(url.clone()).proxy(self.proxy.clone()).send().await?;
let mut cookies_string = String::new(); let mut cookies_string = String::new();
if let Some(_) = response.headers().get_all("set-cookie").iter().next() { if let Some(_) = response.headers().get_all("set-cookie").iter().next() {
for _ in response.headers().get_all("set-cookie").iter() { for _ in response.headers().get_all("set-cookie").iter() {
@@ -77,10 +81,7 @@ impl SpankbangProvider {
} }
if response.status().is_success() { if response.status().is_success() {
let text = response.text().await?; let text = response.text().await?;
println!("Response text: {}", text);
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), &client, cookies_string, pool.clone()).await; let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), &client, cookies_string, pool.clone()).await;
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&url); cache.remove(&url);
cache.insert(url.clone(), video_items.clone()); cache.insert(url.clone(), video_items.clone());
@@ -139,9 +140,10 @@ impl SpankbangProvider {
let client = Client::builder() let client = Client::builder()
.emulation(Emulation::Firefox136) .emulation(Emulation::Firefox136)
.cert_verification(false)
.build()?; .build()?;
let response = client.get(url.clone()).send().await?; let response = client.get(url.clone()).proxy(self.proxy.clone()).send().await?;
let mut cookies_string = String::new(); let mut cookies_string = String::new();
if let Some(_) = response.headers().get_all("set-cookie").iter().next() { if let Some(_) = response.headers().get_all("set-cookie").iter().next() {
for _ in response.headers().get_all("set-cookie").iter() { for _ in response.headers().get_all("set-cookie").iter() {
@@ -217,7 +219,7 @@ impl SpankbangProvider {
// return Err(format!("Error fetching video from database: {}", e).into()); // return Err(format!("Error fetching video from database: {}", e).into());
} }
} }
let response = client.get(url.clone()).header("Cookie", cookies.clone()).send().await?; let response = client.get(url.clone()).header("Cookie", cookies.clone()).proxy(self.proxy.clone()).send().await?;
let mut response = response; let mut response = response;
while response.status().as_u16() == 429 { while response.status().as_u16() == 429 {
@@ -327,7 +329,7 @@ impl SpankbangProvider {
println!("Could not find video-list in HTML"); println!("Could not find video-list in HTML");
return items; return items;
} }
let video_listing_content = split_html[1]; let video_listing_content = format!("{}{}", split_html[1], split_html.get(2).unwrap_or(&""));
let raw_videos_vec = video_listing_content let raw_videos_vec = video_listing_content
.split("data-testid=\"video-item\"") .split("data-testid=\"video-item\"")
.collect::<Vec<&str>>(); .collect::<Vec<&str>>();
@@ -336,6 +338,7 @@ impl SpankbangProvider {
return items; return items;
} }
let raw_videos = raw_videos_vec[1..].to_vec(); let raw_videos = raw_videos_vec[1..].to_vec();
println!("Found {} video items", raw_videos.len());
let futures = raw_videos.into_iter().map(|el| self.parse_video_item(el.to_string(), client, cookies.clone(), pool.clone())); let futures = raw_videos.into_iter().map(|el| self.parse_video_item(el.to_string(), client, cookies.clone(), pool.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await; let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results let video_items: Vec<VideoItem> = results