allpornstreamd done and sxyprn updated
This commit is contained in:
Simon
ForgeCode
598
src/providers/allpornstream.rs
Normal file
598
src/providers/allpornstream.rs
Normal file
@@ -0,0 +1,598 @@
|
||||
use crate::DbPool;
|
||||
use crate::api::ClientVersion;
|
||||
use crate::providers::{
|
||||
Provider, build_proxy_url, report_provider_error, requester_or_default, strip_url_scheme,
|
||||
};
|
||||
use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
||||
use async_trait::async_trait;
|
||||
use chrono::DateTime;
|
||||
use error_chain::error_chain;
|
||||
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
|
||||
use regex::Regex;
|
||||
use scraper::{Html, Selector};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||
crate::providers::ProviderChannelMetadata {
|
||||
group_id: "mainstream-tube",
|
||||
tags: &["tube", "aggregator", "mixed"],
|
||||
};
|
||||
|
||||
const BASE_URL: &str = "https://allpornstream.com";
|
||||
const CHANNEL_ID: &str = "allpornstream";
|
||||
const BROWSER_UA: &str =
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
||||
|
||||
error_chain! {
|
||||
foreign_links {
|
||||
Io(std::io::Error);
|
||||
HttpRequest(wreq::Error);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum Target {
|
||||
Latest { page: u32 },
|
||||
Search { query: String, page: u32 },
|
||||
Category { slug: String, page: u32 },
|
||||
Producer { slug: String, page: u32 },
|
||||
Actor { slug: String, page: u32 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AllPornStreamProvider {}
|
||||
|
||||
impl AllPornStreamProvider {
|
||||
pub fn new() -> Self {
|
||||
Self {}
|
||||
}
|
||||
|
||||
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||
Channel {
|
||||
id: CHANNEL_ID.to_string(),
|
||||
name: "All Porn Stream".to_string(),
|
||||
description: "Free HD porn videos aggregated from major studios and independent creators.".to_string(),
|
||||
premium: false,
|
||||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=allpornstream.com".to_string(),
|
||||
status: "active".to_string(),
|
||||
categories: vec![],
|
||||
options: vec![
|
||||
ChannelOption {
|
||||
id: "sort".to_string(),
|
||||
title: "Sort".to_string(),
|
||||
description: "Browse the latest feed.".to_string(),
|
||||
systemImage: "list.number".to_string(),
|
||||
colorName: "blue".to_string(),
|
||||
options: vec![FilterOption {
|
||||
id: "new".to_string(),
|
||||
title: "Latest".to_string(),
|
||||
}],
|
||||
multiSelect: false,
|
||||
},
|
||||
ChannelOption {
|
||||
id: "sites".to_string(),
|
||||
title: "Producer".to_string(),
|
||||
description: "Jump directly to a studio or producer page. Use the slug from the URL (e.g. brazzers).".to_string(),
|
||||
systemImage: "building.2".to_string(),
|
||||
colorName: "purple".to_string(),
|
||||
options: vec![FilterOption {
|
||||
id: "all".to_string(),
|
||||
title: "All".to_string(),
|
||||
}],
|
||||
multiSelect: false,
|
||||
},
|
||||
],
|
||||
nsfw: true,
|
||||
cacheDuration: Some(1800),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_url(target: &Target) -> String {
|
||||
match target {
|
||||
Target::Latest { page } => {
|
||||
if *page > 1 {
|
||||
format!("{BASE_URL}/?page={page}")
|
||||
} else {
|
||||
BASE_URL.to_string()
|
||||
}
|
||||
}
|
||||
Target::Search { query, page } => {
|
||||
let encoded =
|
||||
utf8_percent_encode(query, NON_ALPHANUMERIC).to_string();
|
||||
if *page > 1 {
|
||||
format!("{BASE_URL}/?search={encoded}&page={page}")
|
||||
} else {
|
||||
format!("{BASE_URL}/?search={encoded}")
|
||||
}
|
||||
}
|
||||
Target::Category { slug, page } => {
|
||||
if *page > 1 {
|
||||
format!("{BASE_URL}/categories/{slug}?page={page}")
|
||||
} else {
|
||||
format!("{BASE_URL}/categories/{slug}")
|
||||
}
|
||||
}
|
||||
Target::Producer { slug, page } => {
|
||||
if *page > 1 {
|
||||
format!("{BASE_URL}/producers/{slug}?page={page}")
|
||||
} else {
|
||||
format!("{BASE_URL}/producers/{slug}")
|
||||
}
|
||||
}
|
||||
Target::Actor { slug, page } => {
|
||||
if *page > 1 {
|
||||
format!("{BASE_URL}/actors/{slug}?page={page}")
|
||||
} else {
|
||||
format!("{BASE_URL}/actors/{slug}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_duration(text: &str) -> u32 {
|
||||
let parts: Vec<u32> = text
|
||||
.trim()
|
||||
.split(':')
|
||||
.filter_map(|p| p.parse::<u32>().ok())
|
||||
.collect();
|
||||
match parts.as_slice() {
|
||||
[m, s] => m * 60 + s,
|
||||
[h, m, s] => h * 3600 + m * 60 + s,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_uploaded_at(dt: &str) -> Option<u64> {
|
||||
DateTime::parse_from_rfc3339(dt)
|
||||
.ok()
|
||||
.map(|d| d.timestamp() as u64)
|
||||
}
|
||||
|
||||
fn extract_first_image(data_images: &str) -> String {
|
||||
// data_images is a JSON array, already HTML-decoded by scraper's parser.
|
||||
// Find the first https:// URL in it.
|
||||
if let Some(start) = data_images.find("https://") {
|
||||
let rest = &data_images[start..];
|
||||
if let Some(end) = rest.find('"') {
|
||||
return rest[..end].to_string();
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn slug_to_title(slug: &str) -> String {
|
||||
slug.split(['-', '_'])
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| {
|
||||
let mut chars = s.chars();
|
||||
match chars.next() {
|
||||
None => String::new(),
|
||||
Some(f) => format!("{}{}", f.to_uppercase(), chars.collect::<String>()),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
fn parse_listing(&self, html: &str, options: &ServerOptions) -> Vec<VideoItem> {
|
||||
let document = Html::parse_document(html);
|
||||
|
||||
let card_sel = match Selector::parse("[data-thumb-id][data-href][data-title][data-images]")
|
||||
{
|
||||
Ok(s) => s,
|
||||
Err(_) => return vec![],
|
||||
};
|
||||
let time_sel = match Selector::parse("time[datetime]") {
|
||||
Ok(s) => s,
|
||||
Err(_) => return vec![],
|
||||
};
|
||||
let studio_sel = match Selector::parse("[data-ga-category='thumbnail_studio']") {
|
||||
Ok(s) => s,
|
||||
Err(_) => return vec![],
|
||||
};
|
||||
let actor_sel = match Selector::parse("[data-ga-category='thumbnail_actor']") {
|
||||
Ok(s) => s,
|
||||
Err(_) => return vec![],
|
||||
};
|
||||
|
||||
// Duration: span with class starting "absolute bottom-2" containing a time string
|
||||
let dur_re = match Regex::new(
|
||||
r#"<span[^>]*class="absolute[^"]*"[^>]*>(\d+:\d{2}(?::\d{2})?)</span>"#,
|
||||
) {
|
||||
Ok(r) => r,
|
||||
Err(_) => return vec![],
|
||||
};
|
||||
|
||||
// Views: number directly after the eye-icon SVG closing tag
|
||||
let views_re =
|
||||
match Regex::new(r"</svg>\s*(\d+)\s*</div>") {
|
||||
Ok(r) => r,
|
||||
Err(_) => return vec![],
|
||||
};
|
||||
|
||||
let mut items = Vec::new();
|
||||
|
||||
for card in document.select(&card_sel) {
|
||||
let uuid = match card.value().attr("data-thumb-id") {
|
||||
Some(v) if !v.is_empty() => v.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let href = match card.value().attr("data-href") {
|
||||
Some(v) if v.starts_with('/') => v.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let title = match card.value().attr("data-title") {
|
||||
Some(v) if !v.is_empty() => v.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let images_raw = card.value().attr("data-images").unwrap_or_default();
|
||||
let thumb = Self::extract_first_image(images_raw);
|
||||
|
||||
let card_html = card.html();
|
||||
|
||||
// Duration from the overlay span
|
||||
let duration = dur_re
|
||||
.captures(&card_html)
|
||||
.and_then(|c| c.get(1))
|
||||
.map(|m| Self::parse_duration(m.as_str()))
|
||||
.unwrap_or(0);
|
||||
|
||||
// Views from after the eye icon SVG
|
||||
let views = views_re
|
||||
.captures(&card_html)
|
||||
.and_then(|c| c.get(1))
|
||||
.and_then(|m| m.as_str().parse::<u32>().ok());
|
||||
|
||||
// video.url is the stable page URL; the proxy URL goes into formats so the
|
||||
// client can supply the required Referer header alongside the stream request.
|
||||
let detail_url = format!("{BASE_URL}{href}");
|
||||
let proxy_target = strip_url_scheme(&detail_url);
|
||||
let proxy_url = build_proxy_url(options, CHANNEL_ID, &proxy_target);
|
||||
|
||||
let mut item = VideoItem::new(
|
||||
uuid,
|
||||
title,
|
||||
detail_url.clone(),
|
||||
CHANNEL_ID.to_string(),
|
||||
thumb,
|
||||
duration,
|
||||
);
|
||||
|
||||
if !proxy_url.is_empty() {
|
||||
let mut format =
|
||||
VideoFormat::new(proxy_url, "auto".to_string(), "video/mp4".to_string());
|
||||
format.add_http_header("Referer".to_string(), detail_url.clone());
|
||||
item = item.formats(vec![format]);
|
||||
}
|
||||
|
||||
// Upload date
|
||||
if let Some(time_el) = card.select(&time_sel).next() {
|
||||
if let Some(dt) = time_el.value().attr("datetime") {
|
||||
if let Some(ts) = Self::parse_uploaded_at(dt) {
|
||||
item = item.uploaded_at(ts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(v) = views {
|
||||
item = item.views(v);
|
||||
}
|
||||
|
||||
// Producer / studio
|
||||
if let Some(studio_link) = card.select(&studio_sel).next() {
|
||||
let label = studio_link
|
||||
.value()
|
||||
.attr("aria-label")
|
||||
.unwrap_or_default();
|
||||
// "producer: ONLY FANS" → "ONLY FANS"
|
||||
let raw_name = label
|
||||
.strip_prefix("producer: ")
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string();
|
||||
if !raw_name.is_empty() {
|
||||
let producer_href =
|
||||
studio_link.value().attr("href").unwrap_or_default();
|
||||
let slug = producer_href
|
||||
.trim_start_matches("/producers/")
|
||||
.to_string();
|
||||
let display = Self::slug_to_title(&raw_name.to_lowercase().replace(' ', "-"));
|
||||
item = item.uploader(display.clone());
|
||||
if !slug.is_empty() {
|
||||
item = item.uploader_url(format!("{BASE_URL}/producers/{slug}"));
|
||||
item.uploaderId = Some(format!("{CHANNEL_ID}:{slug}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Actors as tags – deduplicate by href, keep the longest text per actor
|
||||
let mut actor_map: HashMap<String, String> = HashMap::new();
|
||||
for actor_link in card.select(&actor_sel) {
|
||||
let actor_href = actor_link
|
||||
.value()
|
||||
.attr("href")
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let text = actor_link
|
||||
.text()
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.to_string();
|
||||
if !actor_href.is_empty() && !text.is_empty() {
|
||||
actor_map
|
||||
.entry(actor_href)
|
||||
.and_modify(|v| {
|
||||
if text.len() > v.len() {
|
||||
*v = text.clone();
|
||||
}
|
||||
})
|
||||
.or_insert(text);
|
||||
}
|
||||
}
|
||||
let mut actors: Vec<String> = actor_map.into_values().collect();
|
||||
actors.sort();
|
||||
if !actors.is_empty() {
|
||||
item = item.tags(actors);
|
||||
}
|
||||
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
items
|
||||
}
|
||||
|
||||
fn resolve_target(query: &str, _sort: &str, page: u32, options: &ServerOptions) -> Target {
|
||||
// Explicit shortcuts: "actor:slug", "producer:slug", "category:slug"
|
||||
if let Some(slug) = query.strip_prefix("actor:") {
|
||||
return Target::Actor {
|
||||
slug: slug.to_string(),
|
||||
page,
|
||||
};
|
||||
}
|
||||
if let Some(slug) = query.strip_prefix("producer:") {
|
||||
return Target::Producer {
|
||||
slug: slug.to_string(),
|
||||
page,
|
||||
};
|
||||
}
|
||||
if let Some(slug) = query.strip_prefix("category:") {
|
||||
return Target::Category {
|
||||
slug: slug.to_string(),
|
||||
page,
|
||||
};
|
||||
}
|
||||
|
||||
// Keyword search
|
||||
if !query.is_empty() {
|
||||
return Target::Search {
|
||||
query: query.to_string(),
|
||||
page,
|
||||
};
|
||||
}
|
||||
|
||||
// Producer filter from options.sites
|
||||
if let Some(sites) = &options.sites {
|
||||
let sites = sites.trim();
|
||||
if !sites.is_empty() && sites != "all" {
|
||||
return Target::Producer {
|
||||
slug: sites.to_string(),
|
||||
page,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Target::Latest { page }
|
||||
}
|
||||
|
||||
async fn fetch_and_parse(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
target: Target,
|
||||
options: ServerOptions,
|
||||
) -> Result<Vec<VideoItem>> {
|
||||
let url = Self::build_url(&target);
|
||||
|
||||
if let Some((time, items)) = cache.get(&url) {
|
||||
if time.elapsed().unwrap_or_default().as_secs() < 300 {
|
||||
return Ok(items.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let mut requester = requester_or_default(&options, CHANNEL_ID, "fetch_and_parse");
|
||||
|
||||
let html = requester
|
||||
.get_with_headers(
|
||||
&url,
|
||||
vec![
|
||||
("user-agent".to_string(), BROWSER_UA.to_string()),
|
||||
("accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string()),
|
||||
("accept-language".to_string(), "en-US,en;q=0.5".to_string()),
|
||||
],
|
||||
Some(wreq::Version::HTTP_11),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::from(format!("request failed url={url}: {e}")))?;
|
||||
|
||||
if html.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let items = self.parse_listing(&html, &options);
|
||||
|
||||
if !items.is_empty() {
|
||||
cache.insert(url, items.clone());
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for AllPornStreamProvider {
|
||||
async fn get_videos(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
pool: DbPool,
|
||||
sort: String,
|
||||
query: Option<String>,
|
||||
page: String,
|
||||
per_page: String,
|
||||
options: ServerOptions,
|
||||
) -> Vec<VideoItem> {
|
||||
let _ = pool;
|
||||
let _ = per_page;
|
||||
let page = page.parse::<u32>().unwrap_or(1);
|
||||
let query_str = query.unwrap_or_default();
|
||||
let target = Self::resolve_target(&query_str, &sort, page, &options);
|
||||
|
||||
match self.fetch_and_parse(cache, target, options).await {
|
||||
Ok(items) => items,
|
||||
Err(e) => {
|
||||
report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await;
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::AllPornStreamProvider;
|
||||
use crate::videos::ServerOptions;
|
||||
|
||||
fn make_options() -> ServerOptions {
|
||||
ServerOptions {
|
||||
featured: None,
|
||||
category: None,
|
||||
sites: None,
|
||||
filter: None,
|
||||
language: None,
|
||||
public_url_base: Some("http://127.0.0.1:18080".to_string()),
|
||||
requester: None,
|
||||
network: None,
|
||||
stars: None,
|
||||
categories: None,
|
||||
duration: None,
|
||||
sort: None,
|
||||
sexuality: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_latest_urls() {
|
||||
assert_eq!(
|
||||
AllPornStreamProvider::build_url(&super::Target::Latest { page: 1 }),
|
||||
"https://allpornstream.com"
|
||||
);
|
||||
assert_eq!(
|
||||
AllPornStreamProvider::build_url(&super::Target::Latest { page: 2 }),
|
||||
"https://allpornstream.com/?page=2"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_search_urls() {
|
||||
assert_eq!(
|
||||
AllPornStreamProvider::build_url(&super::Target::Search {
|
||||
query: "brazzers".to_string(),
|
||||
page: 1
|
||||
}),
|
||||
"https://allpornstream.com/?search=brazzers"
|
||||
);
|
||||
assert_eq!(
|
||||
AllPornStreamProvider::build_url(&super::Target::Search {
|
||||
query: "big tits".to_string(),
|
||||
page: 2
|
||||
}),
|
||||
"https://allpornstream.com/?search=big%20tits&page=2"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_producer_urls() {
|
||||
assert_eq!(
|
||||
AllPornStreamProvider::build_url(&super::Target::Producer {
|
||||
slug: "brazzers".to_string(),
|
||||
page: 1
|
||||
}),
|
||||
"https://allpornstream.com/producers/brazzers"
|
||||
);
|
||||
assert_eq!(
|
||||
AllPornStreamProvider::build_url(&super::Target::Producer {
|
||||
slug: "brazzers".to_string(),
|
||||
page: 2
|
||||
}),
|
||||
"https://allpornstream.com/producers/brazzers?page=2"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_duration() {
|
||||
assert_eq!(AllPornStreamProvider::parse_duration("18:42"), 1122);
|
||||
assert_eq!(AllPornStreamProvider::parse_duration("1:23:45"), 5025);
|
||||
assert_eq!(AllPornStreamProvider::parse_duration("00:59"), 59);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_first_image() {
|
||||
let input =
|
||||
r#"["https://example.com/thumb1.jpg","https://example.com/thumb2.jpg"]"#;
|
||||
assert_eq!(
|
||||
AllPornStreamProvider::extract_first_image(input),
|
||||
"https://example.com/thumb1.jpg"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_cards_from_html() {
|
||||
let provider = AllPornStreamProvider::new();
|
||||
let options = make_options();
|
||||
let html = r#"<!DOCTYPE html><html><body>
|
||||
<div
|
||||
data-thumb-id="34a7e37d-7fca-4f30-ad0b-3ab134a00f9f"
|
||||
data-href="/post/34a7e37d-7fca-4f30-ad0b-3ab134a00f9f/test-video"
|
||||
data-slug="/post/34a7e37d-7fca-4f30-ad0b-3ab134a00f9f/test-video"
|
||||
data-title="Test Video Title"
|
||||
data-images='["https://example.com/thumb.jpg"]'
|
||||
>
|
||||
<span class="absolute bottom-2 right-2 z-10">18:42</span>
|
||||
<a data-ga-category="thumbnail_studio" aria-label="producer: ONLY FANS" href="/producers/only-fans">OF</a>
|
||||
<a data-ga-category="thumbnail_actor" href="/actors/bonnie-blue">BB</a>
|
||||
<a data-ga-category="thumbnail_actor" href="/actors/bonnie-blue">Bonnie Blue</a>
|
||||
<time datetime="2026-05-18T19:06:53.000Z">1 hour ago</time>
|
||||
<div class="flex items-center gap-1"><svg><path/><path fill-rule="evenodd" clip-rule="evenodd"></path></svg>416</div>
|
||||
</div>
|
||||
</body></html>"#;
|
||||
|
||||
let items = provider.parse_listing(html, &options);
|
||||
assert_eq!(items.len(), 1);
|
||||
let item = &items[0];
|
||||
assert_eq!(item.id, "34a7e37d-7fca-4f30-ad0b-3ab134a00f9f");
|
||||
assert_eq!(item.title, "Test Video Title");
|
||||
assert_eq!(item.duration, 1122);
|
||||
assert_eq!(item.views, Some(416));
|
||||
assert!(item.thumb.contains("example.com/thumb.jpg"));
|
||||
assert_eq!(item.uploader.as_deref(), Some("Only Fans"));
|
||||
assert!(item.tags.as_ref().unwrap().contains(&"Bonnie Blue".to_string()));
|
||||
assert!(item.uploadedAt.is_some());
|
||||
// video.url is the page URL; proxy URL is in formats[0]
|
||||
assert_eq!(
|
||||
item.url,
|
||||
"https://allpornstream.com/post/34a7e37d-7fca-4f30-ad0b-3ab134a00f9f/test-video"
|
||||
);
|
||||
let format = &item.formats.as_ref().unwrap()[0];
|
||||
assert!(format.url.contains("/proxy/allpornstream/"));
|
||||
assert!(format.url.contains("allpornstream.com/post/34a7e37d"));
|
||||
let headers = format.http_headers_pairs();
|
||||
assert!(headers.iter().any(|(k, _)| k.to_lowercase() == "referer"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user