diff --git a/src/providers/xgroovy.rs b/src/providers/xgroovy.rs index a748376..d5a77b0 100644 --- a/src/providers/xgroovy.rs +++ b/src/providers/xgroovy.rs @@ -19,6 +19,11 @@ pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = const BASE_URL: &str = "https://xgroovy.com"; const CHANNEL_ID: &str = "xgroovy"; +// Hostnames to skip during listing parse. xgroovy listing cards normally link +// only to xgroovy.com, but the site occasionally mixes in syndicated/promoted +// cards that link out to other domains. Add such hosts here as they're found. +const BLACKLISTED_HOSTS: &[&str] = &[]; + const FIREFOX_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"; const HTML_ACCEPT: &str = @@ -190,6 +195,26 @@ impl XgroovyProvider { .join(" ") } + fn extract_host(url: &str) -> Option { + url.split("://") + .nth(1)? + .split('/') + .next() + .map(|h| h.to_ascii_lowercase()) + } + + fn is_blacklisted_host(host: &str, blacklist: &[&str]) -> bool { + blacklist + .iter() + .any(|b| host.eq_ignore_ascii_case(b) || host.ends_with(&format!(".{b}"))) + } + + fn is_blacklisted_url(url: &str) -> bool { + Self::extract_host(url) + .map(|host| Self::is_blacklisted_host(&host, BLACKLISTED_HOSTS)) + .unwrap_or(false) + } + fn normalize_key(s: &str) -> String { s.trim() .replace(['-', '_'], " ") @@ -273,6 +298,10 @@ impl XgroovyProvider { format!("{BASE_URL}{href}") }; + if Self::is_blacklisted_url(&page_url) { + continue; + } + let img = card.select(&img_sel).next(); let thumb = img .as_ref() @@ -556,6 +585,32 @@ mod tests { )); } + #[test] + fn matches_blacklisted_hosts_including_subdomains() { + let blacklist = &["spam-ads.example"]; + assert!(XgroovyProvider::is_blacklisted_host( + "spam-ads.example", + blacklist + )); + assert!(XgroovyProvider::is_blacklisted_host( + "cdn.spam-ads.example", + blacklist + )); + assert!(!XgroovyProvider::is_blacklisted_host( + "xgroovy.com", + blacklist + )); + } + + #[test] + fn extracts_host_from_url() { + assert_eq!( + XgroovyProvider::extract_host("https://xgroovy.com/videos/1/"), + Some("xgroovy.com".to_string()) + ); + assert_eq!(XgroovyProvider::extract_host("/relative/path/"), None); + } + #[test] fn parses_listing_card() { let html = r#"