clapdat image fix
This commit is contained in:
@@ -88,7 +88,8 @@ impl ClapdatProvider {
|
|||||||
Channel {
|
Channel {
|
||||||
id: CHANNEL_ID.to_string(),
|
id: CHANNEL_ID.to_string(),
|
||||||
name: "ClapDat".to_string(),
|
name: "ClapDat".to_string(),
|
||||||
description: "ClapDat trending/recent feeds with tag and uploader shortcuts.".to_string(),
|
description: "ClapDat trending/recent feeds with tag and uploader shortcuts."
|
||||||
|
.to_string(),
|
||||||
premium: false,
|
premium: false,
|
||||||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=clapdat.com".to_string(),
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=clapdat.com".to_string(),
|
||||||
status: "active".to_string(),
|
status: "active".to_string(),
|
||||||
@@ -221,7 +222,12 @@ impl ClapdatProvider {
|
|||||||
.unwrap_or(0)
|
.unwrap_or(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_card(&self, card: &ElementRef<'_>, link_sel: &Selector, img_sel: &Selector) -> Option<StubVideo> {
|
fn parse_card(
|
||||||
|
&self,
|
||||||
|
card: &ElementRef<'_>,
|
||||||
|
link_sel: &Selector,
|
||||||
|
img_sel: &Selector,
|
||||||
|
) -> Option<StubVideo> {
|
||||||
let link = card.select(link_sel).next()?;
|
let link = card.select(link_sel).next()?;
|
||||||
let href = link.value().attr("href")?;
|
let href = link.value().attr("href")?;
|
||||||
let url = self.normalize_url(href);
|
let url = self.normalize_url(href);
|
||||||
@@ -240,14 +246,24 @@ impl ClapdatProvider {
|
|||||||
.select(&Self::selector("span").ok()?)
|
.select(&Self::selector("span").ok()?)
|
||||||
.filter_map(|node| {
|
.filter_map(|node| {
|
||||||
let value = Self::normalize_text(&node.text().collect::<Vec<_>>().join(" "));
|
let value = Self::normalize_text(&node.text().collect::<Vec<_>>().join(" "));
|
||||||
if value.contains(':') { Some(value) } else { None }
|
if value.contains(':') {
|
||||||
|
Some(value)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.next()
|
.next()
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let thumb = card
|
let thumb = card
|
||||||
.select(img_sel)
|
.select(img_sel)
|
||||||
.find_map(|img| img.value().attr("src").or_else(|| img.value().attr("data-src")))
|
.filter_map(|img| {
|
||||||
|
// Collect the attributes we are looking for
|
||||||
|
img.value()
|
||||||
|
.attr("src")
|
||||||
|
.or_else(|| img.value().attr("data-src"))
|
||||||
|
})
|
||||||
|
.nth(1) // Skips the first match (index 0) and takes the second match (index 1)
|
||||||
.map(|value| self.normalize_url(value))
|
.map(|value| self.normalize_url(value))
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
@@ -327,10 +343,9 @@ impl ClapdatProvider {
|
|||||||
) -> Result<Vec<StubVideo>> {
|
) -> Result<Vec<StubVideo>> {
|
||||||
let mut requester = requester_or_default(options, CHANNEL_ID, "search_videos");
|
let mut requester = requester_or_default(options, CHANNEL_ID, "search_videos");
|
||||||
let offset = page.saturating_sub(1) as usize * per_page;
|
let offset = page.saturating_sub(1) as usize * per_page;
|
||||||
let query_encoded = url::form_urlencoded::byte_serialize(query.as_bytes()).collect::<String>();
|
let query_encoded =
|
||||||
let search_url = format!(
|
url::form_urlencoded::byte_serialize(query.as_bytes()).collect::<String>();
|
||||||
"{SEARCH_URL}?q={query_encoded}&limit={per_page}&offset={offset}"
|
let search_url = format!("{SEARCH_URL}?q={query_encoded}&limit={per_page}&offset={offset}");
|
||||||
);
|
|
||||||
let auth_header = format!("Bearer {SEARCH_KEY}");
|
let auth_header = format!("Bearer {SEARCH_KEY}");
|
||||||
let headers = vec![
|
let headers = vec![
|
||||||
("accept".to_string(), "application/json".to_string()),
|
("accept".to_string(), "application/json".to_string()),
|
||||||
@@ -376,15 +391,18 @@ impl ClapdatProvider {
|
|||||||
Option<String>,
|
Option<String>,
|
||||||
Option<u64>,
|
Option<u64>,
|
||||||
) {
|
) {
|
||||||
let uploader_name = Self::regex(r#"<a href="/user/([^"]+)"[^>]*>[^<]*<img[^>]*>\s*<p[^>]*>([^<]+)</p>"#)
|
let uploader_name =
|
||||||
.ok()
|
Self::regex(r#"<a href="/user/([^"]+)"[^>]*>[^<]*<img[^>]*>\s*<p[^>]*>([^<]+)</p>"#)
|
||||||
.and_then(|re| re.captures(html))
|
.ok()
|
||||||
.and_then(|caps| {
|
.and_then(|re| re.captures(html))
|
||||||
let slug = caps.get(1)?.as_str().to_string();
|
.and_then(|caps| {
|
||||||
let name = Self::normalize_text(caps.get(2)?.as_str());
|
let slug = caps.get(1)?.as_str().to_string();
|
||||||
if name.is_empty() { return None; }
|
let name = Self::normalize_text(caps.get(2)?.as_str());
|
||||||
Some((name, slug))
|
if name.is_empty() {
|
||||||
});
|
return None;
|
||||||
|
}
|
||||||
|
Some((name, slug))
|
||||||
|
});
|
||||||
|
|
||||||
let uploader = uploader_name.as_ref().map(|v| v.0.clone());
|
let uploader = uploader_name.as_ref().map(|v| v.0.clone());
|
||||||
let uploader_url = uploader_name
|
let uploader_url = uploader_name
|
||||||
@@ -394,13 +412,14 @@ impl ClapdatProvider {
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|v| format!("{CHANNEL_ID}:{}", v.1));
|
.map(|v| format!("{CHANNEL_ID}:{}", v.1));
|
||||||
|
|
||||||
let uploaded_at = Self::regex(r#"<p class="s-1he9h8j">([A-Za-z]{3}\s+\d{1,2},\s+\d{4})</p>"#)
|
let uploaded_at =
|
||||||
.ok()
|
Self::regex(r#"<p class="s-1he9h8j">([A-Za-z]{3}\s+\d{1,2},\s+\d{4})</p>"#)
|
||||||
.and_then(|re| re.captures(html))
|
.ok()
|
||||||
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()))
|
.and_then(|re| re.captures(html))
|
||||||
.and_then(|value| NaiveDate::parse_from_str(&value, "%b %e, %Y").ok())
|
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()))
|
||||||
.and_then(|date| date.and_hms_opt(0, 0, 0))
|
.and_then(|value| NaiveDate::parse_from_str(&value, "%b %e, %Y").ok())
|
||||||
.and_then(|dt| u64::try_from(dt.and_utc().timestamp()).ok());
|
.and_then(|date| date.and_hms_opt(0, 0, 0))
|
||||||
|
.and_then(|dt| u64::try_from(dt.and_utc().timestamp()).ok());
|
||||||
|
|
||||||
let tag_re = Self::regex(r#"<a href="/tag/[^"]+"[^>]*>([^<]+)</a>"#).ok();
|
let tag_re = Self::regex(r#"<a href="/tag/[^"]+"[^>]*>([^<]+)</a>"#).ok();
|
||||||
let tags = tag_re
|
let tags = tag_re
|
||||||
@@ -474,7 +493,10 @@ impl Provider for ClapdatProvider {
|
|||||||
let target = self.resolve_target(&query_value, &sort_value);
|
let target = self.resolve_target(&query_value, &sort_value);
|
||||||
|
|
||||||
let stubs = match &target {
|
let stubs = match &target {
|
||||||
Target::Search { query } => match self.search_videos(&options, query, page_num, per_page_num).await {
|
Target::Search { query } => match self
|
||||||
|
.search_videos(&options, query, page_num, per_page_num)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(items) => items,
|
Ok(items) => items,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
report_provider_error(CHANNEL_ID, "search_videos", &error.to_string()).await;
|
report_provider_error(CHANNEL_ID, "search_videos", &error.to_string()).await;
|
||||||
@@ -497,13 +519,18 @@ impl Provider for ClapdatProvider {
|
|||||||
_ => self.parse_listing_html(&html),
|
_ => self.parse_listing_html(&html),
|
||||||
};
|
};
|
||||||
match parsed {
|
match parsed {
|
||||||
Ok(items) => items,
|
Ok(items) => items,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
report_provider_error(CHANNEL_ID, "parse_listing_html", &error.to_string()).await;
|
report_provider_error(
|
||||||
vec![]
|
CHANNEL_ID,
|
||||||
|
"parse_listing_html",
|
||||||
|
&error.to_string(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
report_provider_error(CHANNEL_ID, "fetch_html", &error.to_string()).await;
|
report_provider_error(CHANNEL_ID, "fetch_html", &error.to_string()).await;
|
||||||
vec![]
|
vec![]
|
||||||
|
|||||||
Reference in New Issue
Block a user