simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix sextb: fetch HTML directly instead of the dead Jina markdown mirror

Browse Source 
r.jina.ai now answers 403/429, so every request returned nothing. The
site serves plain HTML to the emulating client; parse cards, detail rows
and option lists from HTML, and return empty for no-hit searches (the
site renders a recommendation tray under the notice).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
2026-10-10 07:21:50 +00:00
parent 5b62023d99
commit 39314c9a53
2 changed files with 210 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/provider-fix-todo.md
View File

@@ -12,7 +12,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
| | Channel | Symptom | Notes | | | Channel | Symptom | Notes |
|---|---|---|---| |---|---|---|---|
| [x] | `javtiful` | feed times out (150s) then 0 items; search 0 | site redesign (`article.video-card`); rewrote parser, page URL as `video.url` (yt-dlp-native), no per-item detail fetches. 23 items, p2/search OK | | [x] | `javtiful` | feed times out (150s) then 0 items; search 0 | site redesign (`article.video-card`); rewrote parser, page URL as `video.url` (yt-dlp-native), no per-item detail fetches. 23 items, p2/search OK |
| [ ] | `sextb` | 0 items on feed, search and page 2 | | | [x] | `sextb` | 0 items on feed, search and page 2 | Jina mirror now 403/429; site is plain HTML reachable with the emulating client. Rewrote listing/detail/option parsers for HTML, removed mirror. Remaining check.py errors are tooling items (proxy URL policy; imggle.net thumbs are TLS-gated for `requests`) |
| [ ] | `camsoda` | 0 items on every request | live cams, CF + Jina-only fetch (see memory) | | [ ] | `camsoda` | 0 items on every request | live cams, CF + Jina-only fetch (see memory) |
| [ ] | `hentaimama` | 0 items or 85-150s timeouts | | | [ ] | `hentaimama` | 0 items or 85-150s timeouts | |
| [ ] | `hotbunny` | 0 items; first request 78s | | | [ ] | `hotbunny` | 0 items; first request 78s | |

471
src/providers/sextb.rs
View File

@@ -13,6 +13,7 @@ use async_trait::async_trait;
use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc}; use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc};
use error_chain::error_chain; use error_chain::error_chain;
use futures::stream::{self, StreamExt}; use futures::stream::{self, StreamExt};
use htmlentity::entity::ICodedDataTrait;
use regex::Regex; use regex::Regex;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::time::Duration as StdDuration; use std::time::Duration as StdDuration;
@@ -40,7 +41,6 @@ error_chain! {
} }
const BASE_URL: &str = "https://sextb.net"; const BASE_URL: &str = "https://sextb.net";
const MIRROR_PREFIX: &str = "https://r.jina.ai/http://";
const CHANNEL_ID: &str = "sextb"; const CHANNEL_ID: &str = "sextb";
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -242,18 +242,6 @@ impl SextbProvider {
.to_string() .to_string()
} }
fn mirror_url(&self, url: &str) -> String {
// `url` is already an absolute `https://sextb.net/...` URL, and
// MIRROR_PREFIX already supplies a scheme — concatenating them
// directly produces a malformed `.../http://https://...` target
// that Jina rejects, so strip the scheme first.
let stripped = url
.strip_prefix("https://")
.or_else(|| url.strip_prefix("http://"))
.unwrap_or(url);
format!("{MIRROR_PREFIX}{stripped}")
}
fn slugify(value: &str) -> String { fn slugify(value: &str) -> String {
let mut output = String::new(); let mut output = String::new();
let mut needs_dash = false; let mut needs_dash = false;
@@ -336,35 +324,36 @@ impl SextbProvider {
} }
} }
async fn fetch_markdown(&self, requester: &mut Requester, url: &str) -> Result<String> { /// The site serves plain HTML; fetch it directly. (It used to be read
let mirror_url = self.mirror_url(url); /// through the r.jina.ai markdown mirror, which now answers 403/429.)
async fn fetch_html(&self, requester: &mut Requester, url: &str) -> Result<String> {
requester requester
.get(&mirror_url, Some(Version::HTTP_11)) .get(url, Some(Version::HTTP_2))
.await .await
.map_err(|error| Error::from(format!("request failed for {url}: {error}"))) .map_err(|error| Error::from(format!("request failed for {url}: {error}")))
} }
fn pagination_last_page(markdown: &str) -> Option<u16> { fn pagination_last_page(html: &str) -> Option<u16> {
let regex = Regex::new(r"\[(?P<page>\d+)\]\(https://sextb\.net/[^)]*pg-(?P=page)\)") let regex = Regex::new(r#"/pg-(?P<page>\d+)['"]"#).ok()?;
.ok()?;
regex regex
.captures_iter(markdown) .captures_iter(html)
.filter_map(|captures| captures.name("page")?.as_str().parse::<u16>().ok()) .filter_map(|captures| captures.name("page")?.as_str().parse::<u16>().ok())
.max() .max()
} }
fn parse_genre_options(markdown: &str) -> Result<Vec<FilterOption>> { fn collect_options(html: &str, pattern: &str) -> Result<Vec<FilterOption>> {
let regex = Self::regex( let regex = Self::regex(pattern)?;
r#"(?m)^\*\s+\[(?P<title>[^\]]+)\]\(https://sextb\.net/genre/(?P<slug>[^)\s]+)[^)]*\)\((?P<count>\d+)\)$"#,
)?;
let mut items = Vec::new(); let mut items = Vec::new();
for captures in regex.captures_iter(markdown) { for captures in regex.captures_iter(html) {
let title = Self::normalize_title(captures.name("title").map(|value| value.as_str()).unwrap_or_default());
let slug = captures let slug = captures
.name("slug") .name("slug")
.map(|value| value.as_str()) .map(|value| value.as_str().trim().to_string())
.unwrap_or_default() .unwrap_or_default();
.to_string(); let raw = captures.name("title").map(|value| value.as_str()).unwrap_or_default();
let decoded = htmlentity::entity::decode(raw.as_bytes())
.to_string()
.unwrap_or_else(|_| raw.to_string());
let title = Self::normalize_title(decoded.trim_start_matches('#'));
if title.is_empty() || slug.is_empty() { if title.is_empty() || slug.is_empty() {
continue; continue;
} }
@@ -373,46 +362,25 @@ impl SextbProvider {
Ok(items) Ok(items)
} }
fn parse_studio_options(markdown: &str) -> Result<Vec<FilterOption>> { fn parse_genre_options(html: &str) -> Result<Vec<FilterOption>> {
let regex = Self::regex( Self::collect_options(
r#"(?m)^\[(?P<title>[^\]]+)\]\(https://sextb\.net/studio/(?P<slug>[^)\s]+)[^)]*\)"#, html,
)?; r#"<li><i class="fas fa-folder"[^>]*></i>\s*<a href="/genre/(?P<slug>[^"?]+)"[^>]*>(?P<title>[^<]+)</a>"#,
let mut items = Vec::new(); )
for captures in regex.captures_iter(markdown) {
let title =
Self::normalize_title(captures.name("title").map(|value| value.as_str()).unwrap_or_default());
let slug = captures
.name("slug")
.map(|value| value.as_str())
.unwrap_or_default()
.to_string();
if title.is_empty() || slug.is_empty() {
continue;
}
items.push(FilterOption { id: slug, title });
}
Ok(items)
} }
fn parse_actress_options(markdown: &str) -> Result<Vec<FilterOption>> { fn parse_studio_options(html: &str) -> Result<Vec<FilterOption>> {
let regex = Self::regex( Self::collect_options(
r#"(?m)^\[\!\[Image \d+: (?P<title>[^\]]+)\]\(https://[^)]+\)\s+(?P=title)\s+[0-9.]+[KM]\s+\d{2}/\d{2}/\d{4}\]\(https://sextb\.net/actress/[^)\s]+\)$"#, html,
)?; r#"<div class="tray-item tray-item-list">\s*<a href="/studio/(?P<slug>[^"?]+)"[^>]*>(?:<i[^>]*></i>)?\s*(?P<title>[^<]+)</a>"#,
let mut items = Vec::new(); )
for captures in regex.captures_iter(markdown) { }
let title =
Self::normalize_title(captures.name("title").map(|value| value.as_str()).unwrap_or_default()); fn parse_actress_options(html: &str) -> Result<Vec<FilterOption>> {
if title.is_empty() { Self::collect_options(
continue; html,
} r#"<div class="tray-item-actress">\s*<a href="/actress/(?P<slug>[^"?]+)">.*?<div class="tray-item-actress-title">(?P<title>[^<]+)</div>"#,
let slug_source = title.split('(').next().unwrap_or(&title).trim(); )
let slug = Self::slugify(slug_source);
if slug.is_empty() {
continue;
}
items.push(FilterOption { id: slug, title: slug_source.to_string() });
}
Ok(items)
} }
async fn load_genres(base_url: &str, genres: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> { async fn load_genres(base_url: &str, genres: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
@@ -423,8 +391,8 @@ impl SextbProvider {
actresses: Arc::new(RwLock::new(vec![])), actresses: Arc::new(RwLock::new(vec![])),
}; };
let mut requester = Requester::new(); let mut requester = Requester::new();
let markdown = provider.fetch_markdown(&mut requester, &format!("{base_url}/genres")).await?; let html = provider.fetch_html(&mut requester, &format!("{base_url}/genres")).await?;
for item in Self::parse_genre_options(&markdown)? { for item in Self::parse_genre_options(&html)? {
Self::push_unique(&genres, item); Self::push_unique(&genres, item);
} }
Ok(()) Ok(())
@@ -438,9 +406,9 @@ impl SextbProvider {
actresses: Arc::new(RwLock::new(vec![])), actresses: Arc::new(RwLock::new(vec![])),
}; };
let mut requester = Requester::new(); let mut requester = Requester::new();
let markdown = let html =
provider.fetch_markdown(&mut requester, &format!("{base_url}/list-studios")).await?; provider.fetch_html(&mut requester, &format!("{base_url}/list-studios")).await?;
for item in Self::parse_studio_options(&markdown)? { for item in Self::parse_studio_options(&html)? {
Self::push_unique(&studios, item); Self::push_unique(&studios, item);
} }
Ok(()) Ok(())
@@ -458,17 +426,17 @@ impl SextbProvider {
}; };
let mut requester = Requester::new(); let mut requester = Requester::new();
let first_page = let first_page =
provider.fetch_markdown(&mut requester, &format!("{base_url}/list-actress")).await?; provider.fetch_html(&mut requester, &format!("{base_url}/list-actress")).await?;
for item in Self::parse_actress_options(&first_page)? { for item in Self::parse_actress_options(&first_page)? {
Self::push_unique(&actresses, item); Self::push_unique(&actresses, item);
} }
let last_page = Self::pagination_last_page(&first_page).unwrap_or(1).min(80); let last_page = Self::pagination_last_page(&first_page).unwrap_or(1).min(80);
for page in 2..=last_page { for page in 2..=last_page {
let markdown = provider let html = provider
.fetch_markdown(&mut requester, &format!("{base_url}/list-actress/pg-{page}")) .fetch_html(&mut requester, &format!("{base_url}/list-actress/pg-{page}"))
.await?; .await?;
for item in Self::parse_actress_options(&markdown)? { for item in Self::parse_actress_options(&html)? {
Self::push_unique(&actresses, item); Self::push_unique(&actresses, item);
} }
} }
@@ -600,120 +568,63 @@ impl SextbProvider {
} }
} }
fn parse_list_prefix_tags(prefix: &str) -> Vec<String> { fn parse_list_items(html: &str) -> Result<Vec<ListItemSeed>> {
let cleaned = Self::collapse_whitespace(prefix); // A no-hit search still renders a "recommended" tray under the notice.
if cleaned.is_empty() { if html.contains("No Video were found") {
return vec![]; return Ok(vec![]);
} }
let bracket_regex = Regex::new(r"\[(?P<tag>[^\]]+)\]").ok();
let mut tags = Vec::new();
if let Some(regex) = bracket_regex {
for captures in regex.captures_iter(&cleaned) {
if let Some(tag) = captures.name("tag").map(|value| value.as_str().trim()) {
if !tag.is_empty() {
tags.push(tag.to_string());
}
}
}
}
let without_brackets = Regex::new(r"\[[^\]]+\]")
.ok()
.map(|regex| regex.replace_all(&cleaned, " ").to_string())
.unwrap_or(cleaned.clone());
for token in without_brackets.split(" ") {
let value = Self::collapse_whitespace(token);
if !value.is_empty() {
tags.push(value);
}
}
tags.sort();
tags.dedup();
tags
}
fn parse_list_items(markdown: &str) -> Result<Vec<ListItemSeed>> {
let regex = Self::regex(
r#"(?s)\[\!\[Image \d+: (?P<alt>.*?)\]\((?P<thumb>https?://[^)]+)\)(?P<body>.*?)\]\((?P<url>https://sextb\.net/[^)]+)\)"#,
)?;
let meta_regex = Self::regex(
r#"(?s)^(?P<head>.*?)\s+(?P<quality>HD|SD)\s+(?P<duration>\d+)\s+min\s+(?P<code>[A-Za-z0-9\-]+)$"#,
)?;
let mut items = Vec::new(); let mut items = Vec::new();
for captures in regex.captures_iter(markdown) { // Cards are `<div class="tray-item">`, optionally with modifiers such
let url = captures // as `tray-item tray-item-uncensored`; `tray-item-title` etc. must not match.
.name("url") let card_start = Self::regex(r#"<div class="tray-item(?: [^"]*)?">"#)?;
.map(|value| value.as_str()) for card in card_start.split(html).skip(1) {
.unwrap_or_default()
.to_string(); let Some(href) = Self::attr_between(card, "<a href=\"", "\"") else {
if !url.starts_with("https://sextb.net/") { continue;
};
let slug = href.trim_matches('/').to_string();
if slug.is_empty() || slug.contains('/') {
continue; continue;
} }
let url = format!("{BASE_URL}/{slug}");
let body = Self::collapse_whitespace( let title_raw = Self::attr_between(card, "class=\"tray-item-title\">", "</div>")
captures.name("body").map(|value| value.as_str()).unwrap_or_default(), .or_else(|| Self::attr_between(card, "alt=\"", "\" "))
.unwrap_or_default();
let title = Self::normalize_title(
&htmlentity::entity::decode(title_raw.as_bytes())
.to_string()
.unwrap_or_else(|_| title_raw.to_string()),
); );
let meta = match meta_regex.captures(&body) {
Some(meta) => meta,
None => continue,
};
let alt_title = Self::normalize_title(
captures.name("alt").map(|value| value.as_str()).unwrap_or_default(),
);
let head = Self::normalize_title(meta.name("head").map(|value| value.as_str()).unwrap_or_default());
let title = if !alt_title.is_empty() && head.contains(&alt_title) {
alt_title.clone()
} else if !alt_title.is_empty() && !alt_title.eq_ignore_ascii_case("Julia") {
alt_title.clone()
} else {
head.clone()
};
if title.is_empty() { if title.is_empty() {
continue; continue;
} }
let prefix = if !alt_title.is_empty() && head.contains(&alt_title) { let thumb = Self::attr_between(card, "data-src=\"", "\"")
let prefix = head.split(&alt_title).next().unwrap_or_default(); .or_else(|| Self::attr_between(card, "data-poster=\"", "\""))
prefix.to_string()
} else {
String::new()
};
let slug = url
.trim_start_matches("https://sextb.net/")
.trim_end_matches('/')
.to_string();
if slug.is_empty() {
continue;
}
let thumb = captures
.name("thumb")
.map(|value| value.as_str())
.unwrap_or_default() .unwrap_or_default()
.to_string(); .to_string();
if thumb.is_empty() { if thumb.is_empty() {
continue; continue;
} }
let quality = meta let quality = if card.contains("class=\"hd\"") {
.name("quality") Some("HD".to_string())
.map(|value| value.as_str().to_string()) } else if card.contains("class=\"sd\"") {
.filter(|value| !value.is_empty()); Some("SD".to_string())
let code = meta } else {
.name("code") None
.map(|value| value.as_str().to_string()) };
.filter(|value| !value.is_empty()); let duration = Self::attr_between(card, "class=\"tray-item-runtime\">", "</div>")
let duration = meta .map(|value| value.replace("min", ""))
.name("duration") .map(|value| Self::parse_duration(&value))
.map(|value| Self::parse_duration(value.as_str()))
.unwrap_or(0); .unwrap_or(0);
let code = Self::attr_between(card, "class=\"tray-item-code\">", "</div>")
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty());
let mut tags = Self::parse_list_prefix_tags(&prefix); let mut tags = Vec::new();
if let Some(value) = quality.clone() { if let Some(value) = quality.clone() {
tags.push(value); tags.push(value);
} }
@@ -739,13 +650,30 @@ impl SextbProvider {
Ok(items) Ok(items)
} }
fn extract_link_texts(line: &str) -> Result<Vec<(String, String)>> { fn attr_between<'a>(text: &'a str, start: &str, end: &str) -> Option<&'a str> {
let regex = Self::regex(r#"\[(?P<title>[^\]]+)\]\((?P<url>https://sextb\.net/[^)\s]+)"#)?; let from = text.find(start)? + start.len();
let rest = &text[from..];
Some(&rest[..rest.find(end)?])
}
/// Returns `(title, absolute-url)` for every `<a ...><strong>Title</strong></a>`.
fn extract_link_texts(fragment: &str) -> Result<Vec<(String, String)>> {
let regex = Self::regex(
r#"<a href="(?P<url>[^"]+)"[^>]*>\s*<strong>(?P<title>[^<]+)</strong>\s*</a>"#,
)?;
Ok(regex Ok(regex
.captures_iter(line) .captures_iter(fragment)
.filter_map(|captures| { .filter_map(|captures| {
let title = captures.name("title")?.as_str().trim().trim_matches('*').to_string(); let raw = captures.name("title")?.as_str().trim();
let url = captures.name("url")?.as_str().to_string(); let title = htmlentity::entity::decode(raw.as_bytes())
.to_string()
.unwrap_or_else(|_| raw.to_string());
let url = captures.name("url")?.as_str();
let url = if url.starts_with('/') {
format!("{BASE_URL}{url}")
} else {
url.to_string()
};
Some((title, url)) Some((title, url))
}) })
.collect()) .collect())
@@ -757,27 +685,30 @@ impl SextbProvider {
Some(path.trim_start_matches('/').trim_end_matches('/').to_string()) Some(path.trim_start_matches('/').trim_end_matches('/').to_string())
} }
fn parse_detail(markdown: &str) -> Result<DetailMetadata> { fn strong_text(fragment: &str) -> String {
Self::attr_between(fragment, "<strong>", "</strong>")
.map(|value| value.trim().to_string())
.unwrap_or_default()
}
fn parse_detail(html: &str) -> Result<DetailMetadata> {
let mut metadata = DetailMetadata::default(); let mut metadata = DetailMetadata::default();
let aliases_regex = Self::regex(r#"(?m)^# \*\*(?P<body>.+?)\*\*$"#)?; let tag_regex = Self::regex(r"<[^>]+>")?;
if let Some(captures) = aliases_regex.captures(markdown) {
let body = captures.name("body").map(|value| value.as_str()).unwrap_or_default();
let link_regex = Self::regex(r#"\[(?P<alias>[^\]]+)\]\(https://sextb\.net/search/[^)]+\)"#)?;
for alias_match in link_regex.captures_iter(body) {
if let Some(alias) = alias_match.name("alias").map(|value| value.as_str().trim()) {
if !alias.is_empty() {
metadata.aliases.push(alias.to_string());
}
}
}
}
let line_regex = Self::regex(r#"(?m)^\s*(?P<key>Director|Label|Studio|Cast\(s\)|Genre\(s\)|Quality|Release Date|Runtimes|Added|Viewed|Description):\s*(?P<value>.+)$"#)?; // Info rows are `<div class="description"><i ..></i> Key: <strong>..</strong></div>`.
for captures in line_regex.captures_iter(markdown) { for block in html.split("<div class=\"description").skip(1) {
let key = captures.name("key").map(|value| value.as_str()).unwrap_or_default(); let block = block.split("</div>").next().unwrap_or(block);
let value = captures.name("value").map(|value| value.as_str()).unwrap_or_default().trim(); // Drop the remainder of the opening tag (extra classes, `>`).
match key { let Some((_, block)) = block.split_once('>') else {
continue;
};
let Some((key_part, value)) = block.split_once(':') else {
continue;
};
let key = tag_regex.replace_all(key_part, " ").trim().to_string();
let value = value.trim();
match key.as_str() {
"Director" => { "Director" => {
metadata.director = Self::extract_link_texts(value)? metadata.director = Self::extract_link_texts(value)?
.first() .first()
@@ -796,71 +727,48 @@ impl SextbProvider {
} }
} }
"Cast(s)" => { "Cast(s)" => {
let mut cast = Vec::new();
for (title, url) in Self::extract_link_texts(value)? { for (title, url) in Self::extract_link_texts(value)? {
let slug = Self::detail_link_slug(&url, "actress") let slug = Self::detail_link_slug(&url, "actress")
.unwrap_or_else(|| Self::slugify(&title)); .unwrap_or_else(|| Self::slugify(&title));
cast.push((title, slug)); metadata.cast.push((title, slug));
} }
metadata.cast = cast;
} }
"Genre(s)" => { "Genre(s)" => {
let mut genres = Vec::new();
for (title, url) in Self::extract_link_texts(value)? { for (title, url) in Self::extract_link_texts(value)? {
let slug = Self::detail_link_slug(&url, "genre") let slug = Self::detail_link_slug(&url, "genre")
.unwrap_or_else(|| Self::slugify(&title)); .unwrap_or_else(|| Self::slugify(&title));
genres.push((title, slug)); metadata.genres.push((title, slug));
} }
metadata.genres = genres;
} }
"Quality" => { "Quality" => {
let quality = value.trim_matches('*').trim().to_string(); let quality = Self::strong_text(value);
if !quality.is_empty() { if !quality.is_empty() {
metadata.quality = Some(quality); metadata.quality = Some(quality);
} }
} }
"Release Date" => { "Release Date" => {
let cleaned = value.trim_matches('*').trim(); metadata.release_date = Self::parse_absolute_date(&Self::strong_text(value));
metadata.release_date = Self::parse_absolute_date(cleaned);
} }
"Runtimes" => {}
"Added" => { "Added" => {
let cleaned = value.trim_matches('*').trim(); metadata.added_at = Self::parse_relative_or_date(&Self::strong_text(value));
metadata.added_at = Self::parse_relative_or_date(cleaned);
} }
"Viewed" => { "Viewed" => {
metadata.views = Self::parse_abbrev(value.trim()); let text = tag_regex.replace_all(value, " ").to_string();
metadata.views = Self::parse_abbrev(text.trim());
} }
"Description" => {}
_ => {} _ => {}
} }
} }
let stats_regex = if metadata.views.is_none() {
Self::regex(r#"(?m)(?P<likes>[0-9.]+[KM]?)\s+Liked(?:\s+(?P<dislikes>[0-9.]+[KM]?)\s+Disliked)?(?:\s+(?P<favorites>[0-9.]+[KM]?)\s+Favorited)?\s+(?P<views>[0-9.]+[KM]?)\s+Views"#)?; let views_regex = Self::regex(r"(?P<views>[0-9][0-9.,]*[KM]?)\s+Views")?;
if let Some(captures) = stats_regex.captures(markdown) { metadata.views = views_regex
if metadata.views.is_none() { .captures(html)
metadata.views = captures .and_then(|captures| captures.name("views"))
.name("views") .and_then(|value| Self::parse_abbrev(value.as_str()));
.and_then(|value| Self::parse_abbrev(value.as_str()));
}
if let (Some(likes), Some(dislikes)) = (
captures
.name("likes")
.and_then(|value| Self::parse_abbrev(value.as_str())),
captures
.name("dislikes")
.and_then(|value| Self::parse_abbrev(value.as_str())),
) {
let total = likes + dislikes;
if total > 0 {
metadata.rating = Some((likes as f32 / total as f32) * 100.0);
}
}
} }
metadata.aliases.sort(); // Like/dislike counts are filled in client-side, so no rating here.
metadata.aliases.dedup();
Ok(metadata) Ok(metadata)
} }
@@ -1078,12 +986,12 @@ impl SextbProvider {
let mut requester = requester_or_default(options, CHANNEL_ID, "enrich_seed"); let mut requester = requester_or_default(options, CHANNEL_ID, "enrich_seed");
let detail_fetch = timeout( let detail_fetch = timeout(
StdDuration::from_secs(10), StdDuration::from_secs(10),
self.fetch_markdown(&mut requester, &seed.url), self.fetch_html(&mut requester, &seed.url),
) )
.await; .await;
let mut item = match detail_fetch { let mut item = match detail_fetch {
Ok(Ok(markdown)) => match Self::parse_detail(&markdown) { Ok(Ok(html)) => match Self::parse_detail(&html) {
Ok(detail) => self.apply_detail_metadata(seed.clone(), detail), Ok(detail) => self.apply_detail_metadata(seed.clone(), detail),
Err(error) => { Err(error) => {
report_provider_error_background(CHANNEL_ID, "parse_detail", &error.to_string()); report_provider_error_background(CHANNEL_ID, "parse_detail", &error.to_string());
@@ -1130,14 +1038,14 @@ impl SextbProvider {
} }
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_items_for_url"); let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_items_for_url");
let markdown = timeout( let html = timeout(
StdDuration::from_secs(12), StdDuration::from_secs(12),
self.fetch_markdown(&mut requester, &url), self.fetch_html(&mut requester, &url),
) )
.await .await
.map_err(|_| Error::from(format!("list request timed out for {url}")))??; .map_err(|_| Error::from(format!("list request timed out for {url}")))??;
let seeds = Self::parse_list_items(&markdown)? let seeds = Self::parse_list_items(&html)?
.into_iter() .into_iter()
.take(per_page_limit.max(1)) .take(per_page_limit.max(1))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
@@ -1266,15 +1174,6 @@ mod tests {
); );
} }
#[test]
fn builds_jina_mirror_url_without_double_scheme() {
let url = provider().mirror_url("https://sextb.net/censored?genre=all");
assert_eq!(
url,
"https://r.jina.ai/http://sextb.net/censored?genre=all"
);
}
#[test] #[test]
fn builds_search_page_two_url() { fn builds_search_page_two_url() {
let url = provider().build_url_for_target( let url = provider().build_url_for_target(
@@ -1308,33 +1207,81 @@ mod tests {
#[test] #[test]
fn parses_detail_metadata_block() { fn parses_detail_metadata_block() {
let markdown = r#" let html = r#"
57 Liked 16 Disliked 88 Favorited 45.2K Views <div class="col-7">
<div class="description"> <i class="fa fa-user"></i> Director: <a href="/director/himurokku" title="Himurokku"><strong>Himurokku</strong></a> <i class="fas fa-heart favorite-director" data-id="1"></i> </div>
<div class="description"> <i class="fa fa-tag"></i> Label: <a href="/label/madonna-x1" title="Madonna"><strong>Madonna</strong></a> </div>
<div class="description"> <i class="fa fa-camera"></i> Studio: <a href="/studio/madonna-abc123" title="MADONNA"><strong>Madonna</strong></a> </div>
<div class="description"> <i class="fa fa-users"></i> Cast(s): <a href="/actress/miyuu-imai" title="Miyuu Imai"><strong>Miyuu Imai</strong></a> <i class="fas fa-heart"></i> </div>
<div class="description"> <i class="fa fa-list"></i> Genre(s): <a href="/genre/big-tits-aa11" title="Big Tits"><strong>Big Tits</strong></a>, <a href="/genre/creampie-bb22" title="Creampie"><strong>Creampie</strong></a> </div>
<div class="description"><i class="fa fa-film"></i> Quality: <strong>HD</strong></div>
<div class="description"><i class="fa fa-calendar"></i> Release Date: <strong>Mar. 24, 2026</strong></div>
<div class="description"><i class="fa fa-calendar"></i> Added: <strong>2 days ago</strong></div>
<div class="description visible-sm"><i class="fa fa-eye"></i> Viewed: 45.2K</div>
</div>"#;
Director: [**Himurokku**](https://sextb.net/director/himurokku "Himurokku") let detail = SextbProvider::parse_detail(html).expect("detail should parse");
Label: [**Madonna**](https://sextb.net/label/madonna "Madonna")
Studio: [**Madonna**](https://sextb.net/studio/madonna "MADONNA")
Cast(s): [**Miyuu Imai**](https://sextb.net/actress/miyuu-imai "Miyuu Imai")
Genre(s): [**Big Tits**](https://sextb.net/genre/big-tits "Big Tits"), [**Creampie**](https://sextb.net/genre/creampie "Creampie")
Quality: **HD**
Release Date: **Mar. 24, 2026**
Added: **2 days ago**
Viewed: 45.2K
"#;
let detail = SextbProvider::parse_detail(markdown).expect("detail should parse");
assert_eq!(detail.director.as_deref(), Some("Himurokku")); assert_eq!(detail.director.as_deref(), Some("Himurokku"));
assert_eq!(detail.label.as_deref(), Some("Madonna")); assert_eq!(detail.label.as_deref(), Some("Madonna"));
assert_eq!(detail.studio.as_ref().map(|value| value.0.as_str()), Some("Madonna")); assert_eq!(
assert_eq!(detail.cast.len(), 1); detail.studio,
Some(("Madonna".to_string(), "madonna-abc123".to_string()))
);
assert_eq!(detail.cast, vec![("Miyuu Imai".to_string(), "miyuu-imai".to_string())]);
assert_eq!(detail.genres.len(), 2); assert_eq!(detail.genres.len(), 2);
assert_eq!(detail.genres[0].1, "big-tits-aa11");
assert_eq!(detail.views, Some(45200)); assert_eq!(detail.views, Some(45200));
assert!(detail.rating.unwrap_or_default() > 70.0);
assert_eq!(detail.quality.as_deref(), Some("HD")); assert_eq!(detail.quality.as_deref(), Some("HD"));
assert!(detail.release_date.is_some()); assert!(detail.release_date.is_some());
assert!(detail.added_at.is_some()); assert!(detail.added_at.is_some());
} }
#[test]
fn parses_listing_cards() {
let html = r#"<div class="tray-content"><div class="tray-item"> <a href="/beb-036"> <img class="tray-item-thumbnail lazy" src="/images/default-cover.jpg" data-src="https://cdn001.imggle.net/webp/poster/a.webp" alt="Alt"> <div class="tray-item-description"> <div class="tray-item-title">Hara &amp; Museum</div> <div class="tray-item-meta-info"> <div class="tray-item-quality"><span class="hd">HD</span></div> <div class="tray-item-runtime">120 min</div> <div class="tray-item-code">BEB-036</div> </div> </div> </a> <div class="tray-item-trailer" data-url="x" data-link="/beb-036" data-poster="https://cdn001.imggle.net/thumb/a.jpg"></div> </div>
<div class="tray-item tray-item-uncensored"> <a href="/akdl-234"> <img data-src="https://cdn001.imggle.net/webp/poster/b.webp" alt="B"> <div class="tray-item-title">Second</div><div class="tray-item-runtime">150 min</div><div class="tray-item-code">AKDL-234</div> </a> </div></div>"#;
let items = SextbProvider::parse_list_items(html).expect("listing should parse");
assert_eq!(items.len(), 2);
assert_eq!(items[0].id, "beb-036");
assert_eq!(items[0].url, "https://sextb.net/beb-036");
assert_eq!(items[0].title, "Hara & Museum");
assert_eq!(items[0].duration, 7200);
assert_eq!(items[0].quality.as_deref(), Some("HD"));
assert_eq!(items[0].code.as_deref(), Some("BEB-036"));
assert_eq!(items[0].thumb, "https://cdn001.imggle.net/webp/poster/a.webp");
assert_eq!(items[1].duration, 9000);
}
#[test]
fn empty_search_result_ignores_recommendation_tray() {
let html = r#"<div class="tray-content"><h3 class="notice msg-result">Sorry, No Video were found that matched your search query.</h3>
<div class="tray-item"> <a href="/abc-001"> <img data-src="https://x/a.webp"> <div class="tray-item-title">Rec</div></a></div></div>"#;
assert!(SextbProvider::parse_list_items(html).unwrap().is_empty());
}
#[test]
fn parses_filter_option_lists() {
let genres = r#"<li><i class="fas fa-folder" style="x"></i> <a href="/genre/30-years-old-7c1gtr1u" title="Genre 30 Years Old">30 Years Old</a> <span>(1138)</span></li>"#;
let g = SextbProvider::parse_genre_options(genres).unwrap();
assert_eq!(g[0].id, "30-years-old-7c1gtr1u");
assert_eq!(g[0].title, "30 Years Old");
let studios = r##"<div class="tray-item tray-item-list"> <a href="/studio/only-amateurs-29p3k0ax" title="Label"><i class="fas fa-folder-open"></i> #only amateurs</a> <span class="total">13</span> </div>"##;
let s = SextbProvider::parse_studio_options(studios).unwrap();
assert_eq!(s[0].id, "only-amateurs-29p3k0ax");
assert_eq!(s[0].title, "only amateurs");
let actresses = r#"<div class="tray-item-actress"> <a href="/actress/th1bdtq81"> <img alt="Yumi"> <div class="tray-item-actress-description"> <div class="tray-item-actress-title">Yumi Kazama</div> </div></a></div>"#;
let a = SextbProvider::parse_actress_options(actresses).unwrap();
assert_eq!(a[0].id, "th1bdtq81");
assert_eq!(a[0].title, "Yumi Kazama");
assert_eq!(
SextbProvider::pagination_last_page("<a href='/list-actress/pg-2'>2</a><a href='/list-actress/pg-73'>73</a>"),
Some(73)
);
}
#[test] #[test]
fn parses_dd_player_buttons_from_detail_html() { fn parses_dd_player_buttons_from_detail_html() {
let html = r#" let html = r#"