spankbang fix

src/providers/spankbang.rs

@@ -5,11 +5,13 @@ use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::parse_abbreviated_number;
 use crate::util::time::parse_time_to_seconds;
-use crate::videos::{ServerOptions, VideoItem};
+use crate::videos::{ServerOptions, VideoFormat, VideoItem};
 use async_trait::async_trait;
 use error_chain::error_chain;
 use htmlentity::entity::{ICodedDataTrait, decode};
 use scraper::{ElementRef, Html, Selector};
+use std::process::Command;
+use std::time::Duration;
 use url::form_urlencoded::byte_serialize;
 
 pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
@@ -144,6 +146,177 @@ impl SpankbangProvider {
         vec![("Referer".to_string(), format!("{}/", self.url))]
     }
 
+    fn is_cloudflare_block(text: &str) -> bool {
+        let lowercase = text.to_ascii_lowercase();
+        lowercase.contains("attention required")
+            || lowercase.contains("you have been blocked")
+            || lowercase.contains("cloudflare ray id")
+    }
+
+    fn fallback_items_from_ytdlp(&self, page_url: &str, limit: usize) -> Vec<VideoItem> {
+        let output = match Command::new("yt-dlp")
+            .arg("-J")
+            .arg("--flat-playlist")
+            .arg("--extractor-args")
+            .arg("generic:impersonate=chrome")
+            .arg(page_url)
+            .output()
+        {
+            Ok(output) if output.status.success() => output,
+            _ => return vec![],
+        };
+
+        let payload: serde_json::Value = match serde_json::from_slice(&output.stdout) {
+            Ok(payload) => payload,
+            Err(_) => return vec![],
+        };
+
+        let entries = match payload.get("entries").and_then(|value| value.as_array()) {
+            Some(entries) => entries,
+            None => return vec![],
+        };
+
+        let mut items = Vec::new();
+        for (index, entry) in entries.iter().take(limit).enumerate() {
+            let Some(url) = entry.get("url").and_then(|value| value.as_str()) else {
+                continue;
+            };
+            if !(url.starts_with("https://") || url.starts_with("http://")) {
+                continue;
+            }
+
+            let id = entry
+                .get("id")
+                .and_then(|value| value.as_str())
+                .filter(|value| !value.is_empty())
+                .map(ToOwned::to_owned)
+                .unwrap_or_else(|| format!("spankbang-fallback-{}", index + 1));
+            let title = entry
+                .get("title")
+                .and_then(|value| value.as_str())
+                .filter(|value| !value.is_empty())
+                .map(Self::decode_html)
+                .unwrap_or_else(|| format!("SpankBang Video {}", index + 1));
+            let thumb = entry
+                .get("thumbnail")
+                .and_then(|value| value.as_str())
+                .unwrap_or("")
+                .to_string();
+            let duration = entry
+                .get("duration")
+                .and_then(|value| value.as_u64())
+                .and_then(|value| u32::try_from(value).ok())
+                .unwrap_or(0);
+
+            let format_kind = if url.contains(".m3u8") {
+                "m3u8"
+            } else {
+                "video/mp4"
+            };
+            let mut format = VideoFormat::new(url.to_string(), "auto".to_string(), format_kind.to_string());
+            if let Some(headers) = entry.get("http_headers").and_then(|value| value.as_object()) {
+                for (key, value) in headers {
+                    if let Some(value) = value.as_str() {
+                        format.add_http_header(key.to_string(), value.to_string());
+                    }
+                }
+            }
+            if entry
+                .get("http_headers")
+                .and_then(|value| value.as_object())
+                .is_none()
+            {
+                format.add_http_header("Referer".to_string(), format!("{}/", self.url));
+            }
+
+            let mut item = VideoItem::new(
+                id,
+                title,
+                url.to_string(),
+                "spankbang".to_string(),
+                thumb,
+                duration,
+            )
+            .formats(vec![format]);
+
+            if let Some(views) = entry
+                .get("view_count")
+                .and_then(|value| value.as_u64())
+                .and_then(|value| u32::try_from(value).ok())
+            {
+                item = item.views(views);
+            }
+            if let Some(uploader) = entry
+                .get("uploader")
+                .and_then(|value| value.as_str())
+                .filter(|value| !value.is_empty())
+            {
+                item = item.uploader(uploader.to_string());
+            }
+            items.push(item);
+        }
+
+        items
+    }
+
+    async fn fallback_items_with_working_media(
+        &self,
+        page_url: &str,
+        options: &ServerOptions,
+    ) -> Vec<VideoItem> {
+        let fallback_items = self.fallback_items_from_ytdlp(page_url, 72);
+        if fallback_items.is_empty() {
+            return vec![];
+        }
+
+        let mut requester = requester_or_default(
+            options,
+            "spankbang",
+            "spankbang.fallback_items_with_working_media.missing_requester",
+        );
+        let mut working_items = Vec::new();
+
+        for item in fallback_items {
+            let format_headers = item
+                .formats
+                .as_ref()
+                .and_then(|formats| formats.first())
+                .map(|format| format.http_headers_pairs())
+                .unwrap_or_default();
+            let media_url = item
+                .formats
+                .as_ref()
+                .and_then(|formats| formats.first())
+                .map(|format| format.url.clone())
+                .unwrap_or_else(|| item.url.clone());
+            if media_url.is_empty() {
+                continue;
+            }
+
+            let mut headers = format_headers;
+            if !headers
+                .iter()
+                .any(|(key, _)| key.eq_ignore_ascii_case("range"))
+            {
+                headers.push(("Range".to_string(), "bytes=0-2047".to_string()));
+            }
+
+            let is_working = match requester
+                .get_raw_with_headers_timeout(&media_url, headers, Some(Duration::from_secs(20)))
+                .await
+            {
+                Ok(response) => response.status().is_success(),
+                Err(_) => false,
+            };
+
+            if is_working {
+                working_items.push(item);
+            }
+        }
+
+        working_items
+    }
+
     fn build_query_url(&self, query: &str, page: u32, sort: &str) -> String {
         let encoded_query = Self::encode_search_query(query);
         let mut url = if page > 1 {
@@ -432,6 +605,14 @@ impl SpankbangProvider {
                     &format!("url={video_url}; error={e}"),
                 )
                 .await;
+                let fallback_items = self
+                    .fallback_items_with_working_media(&video_url, &options)
+                    .await;
+                if !fallback_items.is_empty() {
+                    cache.remove(&video_url);
+                    cache.insert(video_url.clone(), fallback_items.clone());
+                    return Ok(fallback_items);
+                }
                 return Ok(old_items);
             }
         };
@@ -443,6 +624,32 @@ impl SpankbangProvider {
                 &format!("url={video_url}"),
             )
             .await;
+            let fallback_items = self
+                .fallback_items_with_working_media(&video_url, &options)
+                .await;
+            if !fallback_items.is_empty() {
+                cache.remove(&video_url);
+                cache.insert(video_url.clone(), fallback_items.clone());
+                return Ok(fallback_items);
+            }
+            return Ok(old_items);
+        }
+
+        if Self::is_cloudflare_block(&text) {
+            report_provider_error(
+                "spankbang",
+                "get.cloudflare_block",
+                &format!("url={video_url}"),
+            )
+            .await;
+            let fallback_items = self
+                .fallback_items_with_working_media(&video_url, &options)
+                .await;
+            if !fallback_items.is_empty() {
+                cache.remove(&video_url);
+                cache.insert(video_url.clone(), fallback_items.clone());
+                return Ok(fallback_items);
+            }
             return Ok(old_items);
         }
 
@@ -490,6 +697,14 @@ impl SpankbangProvider {
                     &format!("url={video_url}; error={e}"),
                 )
                 .await;
+                let fallback_items = self
+                    .fallback_items_with_working_media(&video_url, &options)
+                    .await;
+                if !fallback_items.is_empty() {
+                    cache.remove(&video_url);
+                    cache.insert(video_url.clone(), fallback_items.clone());
+                    return Ok(fallback_items);
+                }
                 return Ok(old_items);
             }
         };
@@ -501,6 +716,32 @@ impl SpankbangProvider {
                 &format!("url={video_url}"),
             )
             .await;
+            let fallback_items = self
+                .fallback_items_with_working_media(&video_url, &options)
+                .await;
+            if !fallback_items.is_empty() {
+                cache.remove(&video_url);
+                cache.insert(video_url.clone(), fallback_items.clone());
+                return Ok(fallback_items);
+            }
+            return Ok(old_items);
+        }
+
+        if Self::is_cloudflare_block(&text) {
+            report_provider_error(
+                "spankbang",
+                "query.cloudflare_block",
+                &format!("url={video_url}"),
+            )
+            .await;
+            let fallback_items = self
+                .fallback_items_with_working_media(&video_url, &options)
+                .await;
+            if !fallback_items.is_empty() {
+                cache.remove(&video_url);
+                cache.insert(video_url.clone(), fallback_items.clone());
+                return Ok(fallback_items);
+            }
             return Ok(old_items);
         }