simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add get_probe/probe_first_archive; fullporner and fullpornerorg resolve pornstar names without prefix

Browse Source
This commit is contained in:
Simon
2026-10-17 05:34:15 +00:00
parent 665fb2a0ce
commit bdde02fa8c
6 changed files with 161 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
docs/query-resolution-todo.md Normal file
View File

@@ -0,0 +1,19 @@
# Seamless query resolution todo
Goal: a free-form query ("gal ritchie", "family strokes") should reach the matching
pornstar / studio / channel / tag archive **without** a `pornstar:` / `studio:` / `tag:` prefix,
by asking the site (probe) and falling back to keyword search. Prefixes stay as explicit overrides.
Pattern (see `hooporn`, `cumlouder`): after known-category matching, probe page 1 of each entity
archive for the slug with `Requester::get_probe` (404 = instant `None`, Cloudflare challenge goes to
FlareSolverr, 5xx/429 retried once), use the first archive that yields cards (page N stays in that
archive), else keyword search. A probe is only valid after checking what the site answers for an
unknown slug (404 vs soft-404 that lists unrelated videos).
Status: `[ ]` open, `[x]` done and verified live (feed of the archive + unknown name falls back),
`[-]` not applicable (reason).
| | Provider | Prefix kinds today | Plan / evidence |
|---|---|---|---|
| [x] | `hooporn` | cat, pornstar, studio | reference implementation |
| [x] | `cumlouder` | cat, tag, series, girl | reference implementation (girl page, then tag) |

32
src/providers/fullporner.rs
View File

@@ -1,6 +1,8 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error, requester_or_default};
use crate::providers::{
Provider, probe_first_archive, report_provider_error, requester_or_default, slug_dashed,
};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
@@ -134,13 +136,13 @@ impl FullpornerProvider {
}
if let Some(slug) = q.strip_prefix("pornstar:") {
return Target::Pornstar {
slug: slug.trim().to_string(),
slug: slug_dashed(slug),
page,
};
}
if let Some(slug) = q.strip_prefix("star:") {
return Target::Pornstar {
slug: slug.trim().to_string(),
slug: slug_dashed(slug),
page,
};
}
@@ -153,6 +155,25 @@ impl FullpornerProvider {
Target::Latest { page }
}
/// A free-form query that names a pornstar goes to her archive (probed on
/// the site; unknown names 404), otherwise it stays a keyword search.
async fn resolve_keyword(query: &str, page: u32, options: &ServerOptions) -> Option<Target> {
let slug = slug_dashed(query);
if slug.is_empty() {
return None;
}
let probe = Target::Pornstar { slug: slug.clone(), page: 1 };
let mut requester = requester_or_default(options, CHANNEL_ID, "resolve_keyword");
probe_first_archive(
&mut requester,
&[Self::build_listing_url(&probe)],
&format!("{BASE_URL}/"),
|body| body.contains("class=\"video-card\""),
)
.await
.map(|_| Target::Pornstar { slug, page })
}
fn selector(value: &str) -> Result<Selector> {
Selector::parse(value)
.map_err(|e| Error::from(format!("selector `{value}` failed: {e}")))
@@ -394,7 +415,10 @@ impl Provider for FullpornerProvider {
.filter(|q| !q.is_empty())
.map(ToOwned::to_owned);
let target = Self::pick_target(normalized_query.as_deref(), page);
let mut target = Self::pick_target(normalized_query.as_deref(), page);
if let Target::Search { query, .. } = &target {
target = Self::resolve_keyword(query, page, &options).await.unwrap_or(target);
}
match Self::fetch_page(&target, per_page, &options).await {
Ok(items) => items,

30
src/providers/fullpornerorg.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{
use crate::providers::{probe_first_archive, slug_dashed,
Provider, report_provider_error, report_provider_error_background, requester_or_default,
};
use crate::status::*;
@@ -109,12 +109,12 @@ impl FullpornerorgProvider {
if let Some(q) = query.map(str::trim).filter(|q| !q.is_empty()) {
if let Some(slug) = q.strip_prefix("pornstar:") {
return Target::Pornstar {
slug: slug.trim().to_string(),
slug: slug_dashed(slug),
};
}
if let Some(slug) = q.strip_prefix("star:") {
return Target::Pornstar {
slug: slug.trim().to_string(),
slug: slug_dashed(slug),
};
}
return Target::Search {
@@ -153,6 +153,25 @@ impl FullpornerorgProvider {
}
}
/// A free-form query that names a pornstar goes to her archive (probed on
/// the site; unknown names 404), otherwise it stays a keyword search.
async fn resolve_keyword(query: &str, options: &ServerOptions) -> Option<Target> {
let slug = slug_dashed(query);
if slug.is_empty() {
return None;
}
let target = Target::Pornstar { slug };
let mut requester = requester_or_default(options, CHANNEL_ID, "resolve_keyword");
probe_first_archive(
&mut requester,
&[Self::build_listing_url(&target, 1)],
&format!("{BASE_URL}/"),
|body| body.contains("loop-video"),
)
.await
.map(|_| target)
}
fn html_headers(referer: &str) -> Vec<(String, String)> {
vec![
("User-Agent".to_string(), BROWSER_UA.to_string()),
@@ -446,7 +465,10 @@ impl Provider for FullpornerorgProvider {
.filter(|q| !q.is_empty())
.map(ToOwned::to_owned);
let target = Self::pick_target(normalized_query.as_deref());
let mut target = Self::pick_target(normalized_query.as_deref());
if let Target::Search { query } = &target {
target = Self::resolve_keyword(query, &options).await.unwrap_or(target);
}
match Self::fetch_page(target, page, per_page, &options).await {
Ok(items) => items,

35
src/providers/hooporn.rs
View File

@@ -430,40 +430,13 @@ impl HoopornProvider {
.await
}
/// Fetches a listing page. A real 404 (unknown archive, page past the end)
/// is `None` and answers immediately; a Cloudflare challenge is handed to
/// the requester's full chain (FlareSolverr).
/// Fetches a listing page; `None` for a real 404 (see `Requester::get_probe`).
async fn fetch_html(requester: &mut Requester, url: &str) -> Result<Option<String>> {
let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))];
let mut last_err = String::new();
for _ in 0..2 {
match requester.get_raw_with_headers(url, headers.clone()).await {
Ok(response) => {
let status = response.status().as_u16();
let body = response.text().await.unwrap_or_default();
if status == 404 {
return Ok(None);
}
if status == 403
|| status == 503
|| Requester::looks_like_cloudflare_challenge(&body)
{
return requester
.get_with_headers(url, headers, None)
requester
.get_probe(url, headers)
.await
.map(Some)
.map_err(|e| Error::from(format!("fetch failed for {url}: {e}")));
}
if status >= 500 || status == 429 {
last_err = format!("HTTP {status}");
continue;
}
return Ok(Some(body));
}
Err(e) => last_err = e.to_string(),
}
}
Err(Error::from(format!("fetch failed for {url}: {last_err}")))
.map_err(|e| Error::from(format!("fetch failed for {url}: {e}")))
}
async fn fetch(target: &Target, page: u32, options: &ServerOptions) -> Result<Vec<VideoItem>> {

44
src/providers/mod.rs
View File

@@ -612,6 +612,50 @@ pub fn report_provider_error_background(provider_name: &str, context: &str, msg:
});
}
/// `Some(slug)` joined with `-` ("Gal Ritchie" -> "gal-ritchie") for archive URLs.
pub fn slug_dashed(value: &str) -> String {
let mut out = String::new();
let mut last_dash = true;
for c in value.trim().chars() {
if c.is_alphanumeric() {
out.extend(c.to_lowercase());
last_dash = false;
} else if !last_dash {
out.push('-');
last_dash = true;
}
}
out.trim_matches('-').to_string()
}
/// Same as [`slug_dashed`] without separators ("Family Strokes" -> "familystrokes").
pub fn slug_joined(value: &str) -> String {
slug_dashed(value).replace('-', "")
}
/// Probes candidate archive URLs (pornstar / studio / channel / tag pages) in
/// order and returns the index and body of the first that exists and
/// `has_items` accepts. A miss (404, empty page, transport error) moves on to
/// the next candidate; the caller falls back to keyword search on `None`.
/// Uses `Requester::get_probe`, so an unknown archive answers instantly and a
/// Cloudflare challenge is solved through FlareSolverr.
pub async fn probe_first_archive(
requester: &mut Requester,
urls: &[String],
referer: &str,
has_items: impl Fn(&str) -> bool,
) -> Option<(usize, String)> {
for (index, url) in urls.iter().enumerate() {
let headers = vec![("Referer".to_string(), referer.to_string())];
if let Ok(Some(body)) = requester.get_probe(url, headers).await {
if has_items(&body) {
return Some((index, body));
}
}
}
None
}
pub fn requester_or_default(
options: &ServerOptions,
provider_name: &str,

39
src/util/requester.rs
View File

@@ -661,6 +661,45 @@ impl Requester {
Ok(response)
}
/// Existence probe for archive pages (pornstar / studio / tag / category).
///
/// Reads the raw response so a real 404/410 answers immediately with
/// `Ok(None)` instead of escalating through the Jina/FlareSolverr chain
/// (minutes per miss). A Cloudflare challenge (403/503/"Just a moment...")
/// is handed to the full chain so FlareSolverr solves it, and 429/5xx are
/// retried once.
pub async fn get_probe(
&mut self,
url: &str,
headers: Vec<(String, String)>,
) -> Result<Option<String>, AnyErr> {
let mut last_err = String::new();
for _ in 0..2 {
match self.get_raw_with_headers(url, headers.clone()).await {
Ok(response) => {
let status = response.status().as_u16();
let body = response.text().await.unwrap_or_default();
if status == 404 || status == 410 {
return Ok(None);
}
if status == 403
|| status == 503
|| Self::looks_like_cloudflare_challenge(&body)
{
return self.get_with_headers(url, headers, None).await.map(Some);
}
if status >= 500 || status == 429 {
last_err = format!("HTTP {status}");
continue;
}
return Ok(Some(body));
}
Err(e) => last_err = e.to_string(),
}
}
Err(format!("probe failed for {url}: {last_err}").into())
}
pub async fn get(
&mut self,
url: &str,