From bdde02fa8c4ca51e3feef3dbec109f8808484312 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 17 Oct 2026 05:34:15 +0000 Subject: [PATCH] Add get_probe/probe_first_archive; fullporner and fullpornerorg resolve pornstar names without prefix --- docs/query-resolution-todo.md | 19 +++++++++++++++ src/providers/fullporner.rs | 32 +++++++++++++++++++++---- src/providers/fullpornerorg.rs | 30 +++++++++++++++++++---- src/providers/hooporn.rs | 37 ++++------------------------ src/providers/mod.rs | 44 ++++++++++++++++++++++++++++++++++ src/util/requester.rs | 39 ++++++++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 40 deletions(-) create mode 100644 docs/query-resolution-todo.md diff --git a/docs/query-resolution-todo.md b/docs/query-resolution-todo.md new file mode 100644 index 0000000..94eda71 --- /dev/null +++ b/docs/query-resolution-todo.md @@ -0,0 +1,19 @@ +# Seamless query resolution todo + +Goal: a free-form query ("gal ritchie", "family strokes") should reach the matching +pornstar / studio / channel / tag archive **without** a `pornstar:` / `studio:` / `tag:` prefix, +by asking the site (probe) and falling back to keyword search. Prefixes stay as explicit overrides. + +Pattern (see `hooporn`, `cumlouder`): after known-category matching, probe page 1 of each entity +archive for the slug with `Requester::get_probe` (404 = instant `None`, Cloudflare challenge goes to +FlareSolverr, 5xx/429 retried once), use the first archive that yields cards (page N stays in that +archive), else keyword search. A probe is only valid after checking what the site answers for an +unknown slug (404 vs soft-404 that lists unrelated videos). + +Status: `[ ]` open, `[x]` done and verified live (feed of the archive + unknown name falls back), +`[-]` not applicable (reason). + +| | Provider | Prefix kinds today | Plan / evidence | +|---|---|---|---| +| [x] | `hooporn` | cat, pornstar, studio | reference implementation | +| [x] | `cumlouder` | cat, tag, series, girl | reference implementation (girl page, then tag) | diff --git a/src/providers/fullporner.rs b/src/providers/fullporner.rs index aa927d1..1aebcdd 100644 --- a/src/providers/fullporner.rs +++ b/src/providers/fullporner.rs @@ -1,6 +1,8 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{Provider, report_provider_error, requester_or_default}; +use crate::providers::{ + Provider, probe_first_archive, report_provider_error, requester_or_default, slug_dashed, +}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::time::parse_time_to_seconds; @@ -134,13 +136,13 @@ impl FullpornerProvider { } if let Some(slug) = q.strip_prefix("pornstar:") { return Target::Pornstar { - slug: slug.trim().to_string(), + slug: slug_dashed(slug), page, }; } if let Some(slug) = q.strip_prefix("star:") { return Target::Pornstar { - slug: slug.trim().to_string(), + slug: slug_dashed(slug), page, }; } @@ -153,6 +155,25 @@ impl FullpornerProvider { Target::Latest { page } } + /// A free-form query that names a pornstar goes to her archive (probed on + /// the site; unknown names 404), otherwise it stays a keyword search. + async fn resolve_keyword(query: &str, page: u32, options: &ServerOptions) -> Option { + let slug = slug_dashed(query); + if slug.is_empty() { + return None; + } + let probe = Target::Pornstar { slug: slug.clone(), page: 1 }; + let mut requester = requester_or_default(options, CHANNEL_ID, "resolve_keyword"); + probe_first_archive( + &mut requester, + &[Self::build_listing_url(&probe)], + &format!("{BASE_URL}/"), + |body| body.contains("class=\"video-card\""), + ) + .await + .map(|_| Target::Pornstar { slug, page }) + } + fn selector(value: &str) -> Result { Selector::parse(value) .map_err(|e| Error::from(format!("selector `{value}` failed: {e}"))) @@ -394,7 +415,10 @@ impl Provider for FullpornerProvider { .filter(|q| !q.is_empty()) .map(ToOwned::to_owned); - let target = Self::pick_target(normalized_query.as_deref(), page); + let mut target = Self::pick_target(normalized_query.as_deref(), page); + if let Target::Search { query, .. } = &target { + target = Self::resolve_keyword(query, page, &options).await.unwrap_or(target); + } match Self::fetch_page(&target, per_page, &options).await { Ok(items) => items, diff --git a/src/providers/fullpornerorg.rs b/src/providers/fullpornerorg.rs index 65964d1..a3f8701 100644 --- a/src/providers/fullpornerorg.rs +++ b/src/providers/fullpornerorg.rs @@ -1,6 +1,6 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{ +use crate::providers::{probe_first_archive, slug_dashed, Provider, report_provider_error, report_provider_error_background, requester_or_default, }; use crate::status::*; @@ -109,12 +109,12 @@ impl FullpornerorgProvider { if let Some(q) = query.map(str::trim).filter(|q| !q.is_empty()) { if let Some(slug) = q.strip_prefix("pornstar:") { return Target::Pornstar { - slug: slug.trim().to_string(), + slug: slug_dashed(slug), }; } if let Some(slug) = q.strip_prefix("star:") { return Target::Pornstar { - slug: slug.trim().to_string(), + slug: slug_dashed(slug), }; } return Target::Search { @@ -153,6 +153,25 @@ impl FullpornerorgProvider { } } + /// A free-form query that names a pornstar goes to her archive (probed on + /// the site; unknown names 404), otherwise it stays a keyword search. + async fn resolve_keyword(query: &str, options: &ServerOptions) -> Option { + let slug = slug_dashed(query); + if slug.is_empty() { + return None; + } + let target = Target::Pornstar { slug }; + let mut requester = requester_or_default(options, CHANNEL_ID, "resolve_keyword"); + probe_first_archive( + &mut requester, + &[Self::build_listing_url(&target, 1)], + &format!("{BASE_URL}/"), + |body| body.contains("loop-video"), + ) + .await + .map(|_| target) + } + fn html_headers(referer: &str) -> Vec<(String, String)> { vec![ ("User-Agent".to_string(), BROWSER_UA.to_string()), @@ -446,7 +465,10 @@ impl Provider for FullpornerorgProvider { .filter(|q| !q.is_empty()) .map(ToOwned::to_owned); - let target = Self::pick_target(normalized_query.as_deref()); + let mut target = Self::pick_target(normalized_query.as_deref()); + if let Target::Search { query } = &target { + target = Self::resolve_keyword(query, &options).await.unwrap_or(target); + } match Self::fetch_page(target, page, per_page, &options).await { Ok(items) => items, diff --git a/src/providers/hooporn.rs b/src/providers/hooporn.rs index 35575d2..aa1a3a8 100644 --- a/src/providers/hooporn.rs +++ b/src/providers/hooporn.rs @@ -430,40 +430,13 @@ impl HoopornProvider { .await } - /// Fetches a listing page. A real 404 (unknown archive, page past the end) - /// is `None` and answers immediately; a Cloudflare challenge is handed to - /// the requester's full chain (FlareSolverr). + /// Fetches a listing page; `None` for a real 404 (see `Requester::get_probe`). async fn fetch_html(requester: &mut Requester, url: &str) -> Result> { let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))]; - let mut last_err = String::new(); - for _ in 0..2 { - match requester.get_raw_with_headers(url, headers.clone()).await { - Ok(response) => { - let status = response.status().as_u16(); - let body = response.text().await.unwrap_or_default(); - if status == 404 { - return Ok(None); - } - if status == 403 - || status == 503 - || Requester::looks_like_cloudflare_challenge(&body) - { - return requester - .get_with_headers(url, headers, None) - .await - .map(Some) - .map_err(|e| Error::from(format!("fetch failed for {url}: {e}"))); - } - if status >= 500 || status == 429 { - last_err = format!("HTTP {status}"); - continue; - } - return Ok(Some(body)); - } - Err(e) => last_err = e.to_string(), - } - } - Err(Error::from(format!("fetch failed for {url}: {last_err}"))) + requester + .get_probe(url, headers) + .await + .map_err(|e| Error::from(format!("fetch failed for {url}: {e}"))) } async fn fetch(target: &Target, page: u32, options: &ServerOptions) -> Result> { diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 69b6430..df268ab 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -612,6 +612,50 @@ pub fn report_provider_error_background(provider_name: &str, context: &str, msg: }); } +/// `Some(slug)` joined with `-` ("Gal Ritchie" -> "gal-ritchie") for archive URLs. +pub fn slug_dashed(value: &str) -> String { + let mut out = String::new(); + let mut last_dash = true; + for c in value.trim().chars() { + if c.is_alphanumeric() { + out.extend(c.to_lowercase()); + last_dash = false; + } else if !last_dash { + out.push('-'); + last_dash = true; + } + } + out.trim_matches('-').to_string() +} + +/// Same as [`slug_dashed`] without separators ("Family Strokes" -> "familystrokes"). +pub fn slug_joined(value: &str) -> String { + slug_dashed(value).replace('-', "") +} + +/// Probes candidate archive URLs (pornstar / studio / channel / tag pages) in +/// order and returns the index and body of the first that exists and +/// `has_items` accepts. A miss (404, empty page, transport error) moves on to +/// the next candidate; the caller falls back to keyword search on `None`. +/// Uses `Requester::get_probe`, so an unknown archive answers instantly and a +/// Cloudflare challenge is solved through FlareSolverr. +pub async fn probe_first_archive( + requester: &mut Requester, + urls: &[String], + referer: &str, + has_items: impl Fn(&str) -> bool, +) -> Option<(usize, String)> { + for (index, url) in urls.iter().enumerate() { + let headers = vec![("Referer".to_string(), referer.to_string())]; + if let Ok(Some(body)) = requester.get_probe(url, headers).await { + if has_items(&body) { + return Some((index, body)); + } + } + } + None +} + pub fn requester_or_default( options: &ServerOptions, provider_name: &str, diff --git a/src/util/requester.rs b/src/util/requester.rs index ebebe1c..48699d1 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -661,6 +661,45 @@ impl Requester { Ok(response) } + /// Existence probe for archive pages (pornstar / studio / tag / category). + /// + /// Reads the raw response so a real 404/410 answers immediately with + /// `Ok(None)` instead of escalating through the Jina/FlareSolverr chain + /// (minutes per miss). A Cloudflare challenge (403/503/"Just a moment...") + /// is handed to the full chain so FlareSolverr solves it, and 429/5xx are + /// retried once. + pub async fn get_probe( + &mut self, + url: &str, + headers: Vec<(String, String)>, + ) -> Result, AnyErr> { + let mut last_err = String::new(); + for _ in 0..2 { + match self.get_raw_with_headers(url, headers.clone()).await { + Ok(response) => { + let status = response.status().as_u16(); + let body = response.text().await.unwrap_or_default(); + if status == 404 || status == 410 { + return Ok(None); + } + if status == 403 + || status == 503 + || Self::looks_like_cloudflare_challenge(&body) + { + return self.get_with_headers(url, headers, None).await.map(Some); + } + if status >= 500 || status == 429 { + last_err = format!("HTTP {status}"); + continue; + } + return Ok(Some(body)); + } + Err(e) => last_err = e.to_string(), + } + } + Err(format!("probe failed for {url}: {last_err}").into()) + } + pub async fn get( &mut self, url: &str,