diff --git a/src/providers/spankbang.rs b/src/providers/spankbang.rs index b03542b..092e71c 100644 --- a/src/providers/spankbang.rs +++ b/src/providers/spankbang.rs @@ -14,6 +14,8 @@ use crate::videos::{self, VideoItem}; use crate::DbPool; use crate::USER_AGENT; // Make sure Provider trait is imported use std::collections::HashMap; +use std::time::Duration; +use tokio::time::sleep; error_chain! { foreign_links { @@ -34,6 +36,7 @@ impl SpankbangProvider { } } async fn get(&self, cache:VideoCache, pool: DbPool, page: u8, sort: String) -> Result> { + let mut url = format!("{}{}/{}/", self.url, sort, page); let old_items = match cache.get(&url) { @@ -237,7 +240,15 @@ impl SpankbangProvider { // return Err(format!("Error fetching video from database: {}", e).into()); } } - let response = client.get(url.clone()).header("Cookie", cookies).send().await?; + let response = client.get(url.clone()).header("Cookie", cookies.clone()).send().await?; + + let mut response = response; + while response.status().as_u16() == 429 { + println!("Received 429 Too Many Requests. Waiting 10 seconds before retrying..."); + sleep(Duration::from_secs(60)).await; + response = client.get(url.clone()).header("Cookie", cookies.clone()).send().await?; + } + if response.status().is_success() { let text = response.text().await?; let lines = text.split("\n").collect::>(); @@ -260,10 +271,10 @@ impl SpankbangProvider { cookies: String, pool: DbPool ) -> Result { - let vid = html.split("\n").collect::>(); - // for (index,line) in vid.iter().enumerate(){ - // println!("Line {}: {}\n\n", index, line); - // } + let vid = html.split("\n").collect::>(); + if vid.len() > 200 { + return Err("Video item has too many lines".into()); + } let title_line = vid.iter() .find(|s| s.trim_start().starts_with(" = Vec::new(); - let split_html = html.split("class=\"video-list ").collect::>(); + let mut split_html = html.split("\"video-list").collect::>(); if split_html.len() < 2 { println!("Could not find video-list in HTML"); return items; } let video_listing_content = split_html[1]; let raw_videos_vec = video_listing_content - .split("class=\"video-item\"") + .split("data-testid=\"video-item\"") .collect::>(); if raw_videos_vec.len() < 2 { println!("Could not find video-item in HTML");