Compare commits
3 Commits
7becc3305d
...
cf9701611a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf9701611a | ||
|
|
316c317c81 | ||
|
|
7fbf6cfd15 |
5
build.rs
5
build.rs
@@ -356,6 +356,11 @@ const PROVIDERS: &[ProviderDef] = &[
|
|||||||
module: "jable",
|
module: "jable",
|
||||||
ty: "JableProvider",
|
ty: "JableProvider",
|
||||||
},
|
},
|
||||||
|
ProviderDef {
|
||||||
|
id: "cumlouder",
|
||||||
|
module: "cumlouder",
|
||||||
|
ty: "CumlouderProvider",
|
||||||
|
},
|
||||||
ProviderDef {
|
ProviderDef {
|
||||||
id: "fullporner",
|
id: "fullporner",
|
||||||
module: "fullporner",
|
module: "fullporner",
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
|
|||||||
| `jable` | `jav` | no | yes | HTML JAV archive scraper; extracts `var hlsUrl` from detail pages; m3u8 format requires Referer + browser User-Agent; proxy route handles HEAD (200 OK) and GET (redirect to watch page) since yt-dlp blocks jable.tv; tag/category/model shortcut queries. |
|
| `jable` | `jav` | no | yes | HTML JAV archive scraper; extracts `var hlsUrl` from detail pages; m3u8 format requires Referer + browser User-Agent; proxy route handles HEAD (200 OK) and GET (redirect to watch page) since yt-dlp blocks jable.tv; tag/category/model shortcut queries. |
|
||||||
| `kwiky` | `tiktok` | no | no | Short-form cam clip provider for kwiky.com (ICF/StreamMate platform). Two-step fetch: ID list then metadata batch. Auth: XSRF token extracted from `meta.content = '<TOKEN>'` injected by `/api/state/v1/preload`, cached in an `Arc<RwLock<Option<String>>>` with double-checked locking (same pattern as `xxxtik` bearer token); all API calls also require `x-gateway: 76d59e04-97fa-4ced-aa99-b86ffaf756a2` (platform-identifying fixed UUID observed in browser JS, not stored in cookies or localStorage). Default feed: `GET /api/v1/gateway/v1/quickies/recommended?source=<base64-hotlist>` (cursor-paged, walks up to 5 cursors for higher page numbers). All tag/keyword queries go through `keywordSearch?keyword=...` (offset-paged: `pageToken=(page-1)*perPage`) — the recommended endpoint's `&tag=` param does not reliably filter, so even curated-tag requests use keywordSearch. Media: `media.url` = direct `media.icfcdn.com/*.mp4` (publicly accessible, no Referer/auth), `media.thumbUrl` thumbnail (same CDN, direct), `media.previewUrl` preview clip; both kwiky page URLs (`kwiky.com/quickies/{id}`) and the media CDN URLs pass direct `curl -I` health checks but the page URL is NOT yt-dlp-resolvable (React SPA, generic extractor fails), so `formats` are populated with the direct MP4 URL and `video.url` is the page. Aspect ratio set from `media.width`/`media.height`; `uploadedAt` from RFC3339 `created` field; uploader name/URL/ID from `creator.name` and `creator.id`. 48 curated tags from `/api/v1/gateway/v1/keywords/populartags?gender=f` exposed via `categories` filter option. No proxy needed. |
|
| `kwiky` | `tiktok` | no | no | Short-form cam clip provider for kwiky.com (ICF/StreamMate platform). Two-step fetch: ID list then metadata batch. Auth: XSRF token extracted from `meta.content = '<TOKEN>'` injected by `/api/state/v1/preload`, cached in an `Arc<RwLock<Option<String>>>` with double-checked locking (same pattern as `xxxtik` bearer token); all API calls also require `x-gateway: 76d59e04-97fa-4ced-aa99-b86ffaf756a2` (platform-identifying fixed UUID observed in browser JS, not stored in cookies or localStorage). Default feed: `GET /api/v1/gateway/v1/quickies/recommended?source=<base64-hotlist>` (cursor-paged, walks up to 5 cursors for higher page numbers). All tag/keyword queries go through `keywordSearch?keyword=...` (offset-paged: `pageToken=(page-1)*perPage`) — the recommended endpoint's `&tag=` param does not reliably filter, so even curated-tag requests use keywordSearch. Media: `media.url` = direct `media.icfcdn.com/*.mp4` (publicly accessible, no Referer/auth), `media.thumbUrl` thumbnail (same CDN, direct), `media.previewUrl` preview clip; both kwiky page URLs (`kwiky.com/quickies/{id}`) and the media CDN URLs pass direct `curl -I` health checks but the page URL is NOT yt-dlp-resolvable (React SPA, generic extractor fails), so `formats` are populated with the direct MP4 URL and `video.url` is the page. Aspect ratio set from `media.width`/`media.height`; `uploadedAt` from RFC3339 `created` field; uploader name/URL/ID from `creator.name` and `creator.id`. 48 curated tags from `/api/v1/gateway/v1/keywords/populartags?gender=f` exposed via `categories` filter option. No proxy needed. |
|
||||||
| `fullporner` | `mainstream-tube` | no | no | HTML scraper for fullporner.com; thumbnail IDs derived from `/thumb/{id}.jpg` URLs and used to build direct `xiaoshenke.net/vid/{id}/720` media redirect URLs (Referer + User-Agent headers required); supports cat:/category:/pornstar:/star: shortcut queries; no proxy needed. |
|
| `fullporner` | `mainstream-tube` | no | no | HTML scraper for fullporner.com; thumbnail IDs derived from `/thumb/{id}.jpg` URLs and used to build direct `xiaoshenke.net/vid/{id}/720` media redirect URLs (Referer + User-Agent headers required); supports cat:/category:/pornstar:/star: shortcut queries; no proxy needed. |
|
||||||
|
| `cumlouder` | `mainstream-tube` | no | no | HTML scraper for cumlouder.com (Spanish/amateur tube, same platform family as superporn). Cards are `a.muestra-escena[href^="/porn-video/"]` with lazy thumb in `img[data-src]` (protocol-relative `//th0cdn.cumlouder.com/...`, direct, no Referer/proxy), `h2` title, `.vistas` views, `.minutos` duration (`06:14 m`); relative date only ("2 years ago"), so `uploadedAt` is not set. Default feed `/porn/` (page N≥2 is the bare `/N/` path, not `/porn/N/`); category `/porn-videos/{slug}/{N}/`; tag `/tag/{slug}/{N}/`; series `/series/{slug}/{N}/`. **The site's own `/search/` (POST `form[search]` or `?q=`) is behind a Cloudflare managed challenge — 403 even in a real browser — so keyword queries are routed instead: a query matching a hardcoded category (72, scraped from `/categories/`) goes to the category archive, anything else to `/tag/{slug}/` (unknown tags 404 → empty list).** Supports `cat:`/`category:`, `tag:`, `series:` prefixes and the `categories` option. `video.url` is the detail page, which yt-dlp resolves natively via the generic html5 extractor (the page embeds a signed `m5cdnst.cumlouder.com/*.mp4?secure=…` `<source>`, no Referer needed), so no `formats`/enrichment. The origin intermittently answers a first, uncached request with a Cloudflare 522 (a retry succeeds) — the listing fetch retries up to 3×, and `check.py`/yt-dlp may show a one-off `HTTP=-1`/522 on a cold detail page that is not a provider fault. No proxy needed. |
|
||||||
| `thepornbunny` | `mainstream-tube` | no | yes | KVS-style HTML scraper for thepornbunny.com; 24 items per site page; thumbnails at `https://www.thepornbunny.com/images/thumb/{id}.webp` from `data-original` attribute (no proxy needed); studio exposed as uploader; pornstar names in tags; `/proxy/thepornbunny/{slug}` fetches the video page, extracts `generate_mp4(enc_data, key, rnd, video_id)` args, decrypts `enc_data` via PBKDF2-HMAC-SHA512+AES-256-CBC to get an OK.ru session key, calls `api.ok.ru/fb.do?method=video.get&session_key=KEY&vids=RND` to get signed CDN URLs, and returns 302 to the best-quality okcdn.ru/vkuser.net MP4 URL (no special client headers needed); supports sort: new/popular/rated, 20 hardcoded categories via `categories` option, and tag:/category:/studio:/pornstar: query shortcuts. |
|
| `thepornbunny` | `mainstream-tube` | no | yes | KVS-style HTML scraper for thepornbunny.com; 24 items per site page; thumbnails at `https://www.thepornbunny.com/images/thumb/{id}.webp` from `data-original` attribute (no proxy needed); studio exposed as uploader; pornstar names in tags; `/proxy/thepornbunny/{slug}` fetches the video page, extracts `generate_mp4(enc_data, key, rnd, video_id)` args, decrypts `enc_data` via PBKDF2-HMAC-SHA512+AES-256-CBC to get an OK.ru session key, calls `api.ok.ru/fb.do?method=video.get&session_key=KEY&vids=RND` to get signed CDN URLs, and returns 302 to the best-quality okcdn.ru/vkuser.net MP4 URL (no special client headers needed); supports sort: new/popular/rated, 20 hardcoded categories via `categories` option, and tag:/category:/studio:/pornstar: query shortcuts. |
|
||||||
| `eporner` | `mainstream-tube` | no | no | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); supports sort: new/popular/rated/best; 65 hardcoded categories, plus `cat:`/`category:`, `tag:`, `pornstar:`/`star:`, `uploader:`/`profile:` query-prefix routing to the matching archive, a bare-keyword category-name match, and a background-loaded pornstar name→URL map from `/pornstar-list/` (falls back to `/tag/{slug}` keyword search otherwise). `video.url` is the detail page — **not** yt-dlp-resolvable and not directly playable on its own: opening it bare serves a small restricted/preview clip. Real playback needs per-card enrichment (bounded `buffer_unordered(6)`): fetch the detail page for its 32-hex `hash`, derive the site's own hashed token (`calc_hash` — split into four 8-hex chunks, each parsed as u32 and base36-encoded, concatenated; reverse-engineered from the player JS), then call the site's own `GET /xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false` JSON API for direct per-resolution mp4 URLs into `formats[]`. Separately from the hash dance, the resolved CDN mp4 URLs enforce their own Referer check: a request with no Referer or one pointing at eporner.com gets the real file, but any other non-empty Referer gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the exact "this video is only available at eporner.com"-style short clip a client sees if its default (or app-injected) Referer doesn't match, so every format explicitly declares `http_headers: {Referer: "https://www.eporner.com/"}` rather than relying on the client's default. No proxy needed (formats are direct CDN URLs, just header-gated). **2026-09-21:** the `calc_hash`/hash-regex/`alnum_id_from_href`/`resolve_formats` logic moved to `src/util/eporner_resolve.rs` so `melonstube` (a meta-search aggregator that redirects out to eporner.com for ~34% of its own cards) can reuse the exact same resolution without a cross-provider `use`, which would break either provider's `HOT_TUB_PROVIDER=<id>` single-provider build (only the one selected provider module gets compiled; `src/util/*` always does). `eporner.rs` itself now just calls `crate::util::eporner_resolve::resolve_formats(...)`. |
|
| `eporner` | `mainstream-tube` | no | no | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); supports sort: new/popular/rated/best; 65 hardcoded categories, plus `cat:`/`category:`, `tag:`, `pornstar:`/`star:`, `uploader:`/`profile:` query-prefix routing to the matching archive, a bare-keyword category-name match, and a background-loaded pornstar name→URL map from `/pornstar-list/` (falls back to `/tag/{slug}` keyword search otherwise). `video.url` is the detail page — **not** yt-dlp-resolvable and not directly playable on its own: opening it bare serves a small restricted/preview clip. Real playback needs per-card enrichment (bounded `buffer_unordered(6)`): fetch the detail page for its 32-hex `hash`, derive the site's own hashed token (`calc_hash` — split into four 8-hex chunks, each parsed as u32 and base36-encoded, concatenated; reverse-engineered from the player JS), then call the site's own `GET /xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false` JSON API for direct per-resolution mp4 URLs into `formats[]`. Separately from the hash dance, the resolved CDN mp4 URLs enforce their own Referer check: a request with no Referer or one pointing at eporner.com gets the real file, but any other non-empty Referer gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the exact "this video is only available at eporner.com"-style short clip a client sees if its default (or app-injected) Referer doesn't match, so every format explicitly declares `http_headers: {Referer: "https://www.eporner.com/"}` rather than relying on the client's default. No proxy needed (formats are direct CDN URLs, just header-gated). **2026-09-21:** the `calc_hash`/hash-regex/`alnum_id_from_href`/`resolve_formats` logic moved to `src/util/eporner_resolve.rs` so `melonstube` (a meta-search aggregator that redirects out to eporner.com for ~34% of its own cards) can reuse the exact same resolution without a cross-provider `use`, which would break either provider's `HOT_TUB_PROVIDER=<id>` single-provider build (only the one selected provider module gets compiled; `src/util/*` always does). `eporner.rs` itself now just calls `crate::util::eporner_resolve::resolve_formats(...)`. |
|
||||||
| `xnxx` | `mainstream-tube` | no | no | HTML scraper for xnxx.com (10M+ videos); unified card parser handles two formats: `div.thumb-block[data-eid]` (search) and `div.thumb-block.video[data-video='{"id":...}']` (hits); eid extracted from `/video-{eid}/{slug}` URL path; thumbnails at `thumb-cdn77.xnxx-cdn.com` and `thumbs-gcore.xnxx-cdn.com` (no proxy, no Referer needed); 0-indexed pagination (page 1 = `/hits`, page N = `/hits/{N-1}`); default feed is `/hits` (most-viewed — xnxx has no chronological listing); search via `/search/{slug}` (works for keywords and tags); supports `tag:`, `cat:`, `category:` query shortcuts; yt-dlp resolves `video.url` natively (XNXX extractor, returns 4-7 HLS formats); no proxy needed. |
|
| `xnxx` | `mainstream-tube` | no | no | HTML scraper for xnxx.com (10M+ videos); unified card parser handles two formats: `div.thumb-block[data-eid]` (search) and `div.thumb-block.video[data-video='{"id":...}']` (hits); eid extracted from `/video-{eid}/{slug}` URL path; thumbnails at `thumb-cdn77.xnxx-cdn.com` and `thumbs-gcore.xnxx-cdn.com` (no proxy, no Referer needed); 0-indexed pagination (page 1 = `/hits`, page N = `/hits/{N-1}`); default feed is `/hits` (most-viewed — xnxx has no chronological listing); search via `/search/{slug}` (works for keywords and tags); supports `tag:`, `cat:`, `category:` query shortcuts; yt-dlp resolves `video.url` natively (XNXX extractor, returns 4-7 HLS formats); no proxy needed. |
|
||||||
|
|||||||
508
src/providers/cumlouder.rs
Normal file
508
src/providers/cumlouder.rs
Normal file
@@ -0,0 +1,508 @@
|
|||||||
|
use crate::DbPool;
|
||||||
|
use crate::api::ClientVersion;
|
||||||
|
use crate::providers::{Provider, report_provider_error, requester_or_default};
|
||||||
|
use crate::status::*;
|
||||||
|
use crate::util::cache::VideoCache;
|
||||||
|
use crate::util::time::parse_time_to_seconds;
|
||||||
|
use crate::videos::{ServerOptions, VideoItem};
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use error_chain::error_chain;
|
||||||
|
use scraper::{Html, Selector};
|
||||||
|
|
||||||
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
|
crate::providers::ProviderChannelMetadata {
|
||||||
|
group_id: "mainstream-tube",
|
||||||
|
tags: &["amateur", "spanish", "hd", "free"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const BASE_URL: &str = "https://www.cumlouder.com";
|
||||||
|
const CHANNEL_ID: &str = "cumlouder";
|
||||||
|
|
||||||
|
// Scraped from /categories/ (`a.muestra-categoria[tag-url]`). The site's own
|
||||||
|
// /search/ endpoint sits behind a Cloudflare managed challenge, so keyword
|
||||||
|
// queries are routed to these archives or to `/tag/{slug}/` instead.
|
||||||
|
const CATEGORIES: &[(&str, &str)] = &[
|
||||||
|
("anal", "Anal"),
|
||||||
|
("big-ass", "Big ass"),
|
||||||
|
("spain", "Spain"),
|
||||||
|
("latina", "Latina"),
|
||||||
|
("mature", "Mature"),
|
||||||
|
("teen", "Teen"),
|
||||||
|
("black", "Black"),
|
||||||
|
("squirting", "Squirting"),
|
||||||
|
("hardcore", "Hardcore"),
|
||||||
|
("big-dick", "Big dick"),
|
||||||
|
("natural-tits", "Natural tits"),
|
||||||
|
("cowgirl", "Cowgirl"),
|
||||||
|
("busty", "Busty"),
|
||||||
|
("ass", "Ass"),
|
||||||
|
("facial", "Facial"),
|
||||||
|
("reality", "Reality"),
|
||||||
|
("orgasm", "Orgasm"),
|
||||||
|
("casting", "Casting"),
|
||||||
|
("milf", "Milf"),
|
||||||
|
("tattoo", "Tattoo"),
|
||||||
|
("huge-boobs", "Huge boobs"),
|
||||||
|
("fake-tits", "Fake tits"),
|
||||||
|
("brunette", "Brunette"),
|
||||||
|
("public", "Public"),
|
||||||
|
("cumshot", "Cumshot"),
|
||||||
|
("deep-throat", "Deep throat"),
|
||||||
|
("big-tits", "Big tits"),
|
||||||
|
("party", "Party"),
|
||||||
|
("interracial", "Interracial"),
|
||||||
|
("parody", "Parody"),
|
||||||
|
("amateur", "Amateur"),
|
||||||
|
("pornstar", "Pornstar"),
|
||||||
|
("petite", "Petite"),
|
||||||
|
("rough-sex", "Rough sex"),
|
||||||
|
("blonde", "Blonde"),
|
||||||
|
("doggystyle", "Doggystyle"),
|
||||||
|
("schoolgirl", "Schoolgirl"),
|
||||||
|
("double-penetration", "Double penetration"),
|
||||||
|
("van", "Van"),
|
||||||
|
("creampie", "Creampie"),
|
||||||
|
("barcelona", "Barcelona"),
|
||||||
|
("exhibitionist", "Exhibitionist"),
|
||||||
|
("threesome", "Threesome"),
|
||||||
|
("blowjob", "Blowjob"),
|
||||||
|
("mulatta", "Mulatta"),
|
||||||
|
("money", "Money"),
|
||||||
|
("masturbation", "Masturbation"),
|
||||||
|
("tits", "Tits"),
|
||||||
|
("handjob", "Handjob"),
|
||||||
|
("comedy", "Comedy"),
|
||||||
|
("european", "European"),
|
||||||
|
("beach", "Beach"),
|
||||||
|
("redhead", "Redhead"),
|
||||||
|
("scandal", "Scandal"),
|
||||||
|
("orgy", "Orgy"),
|
||||||
|
("freak", "Freak"),
|
||||||
|
("pov", "POV"),
|
||||||
|
("lesbian", "Lesbian"),
|
||||||
|
("russian", "Russian"),
|
||||||
|
("ass-to-mouth", "Ass to mouth"),
|
||||||
|
("los-angeles", "Los Angeles"),
|
||||||
|
("fuck", "Fuck"),
|
||||||
|
("slapping", "Slapping"),
|
||||||
|
("nice-eyes", "Nice eyes"),
|
||||||
|
("asian", "Asian"),
|
||||||
|
("asslicking", "Asslicking"),
|
||||||
|
("mixer", "Mixer"),
|
||||||
|
("sport", "Sport"),
|
||||||
|
("busted", "Busted"),
|
||||||
|
("piercing", "Piercing"),
|
||||||
|
("gape", "Gape"),
|
||||||
|
("first-time", "First time"),
|
||||||
|
];
|
||||||
|
|
||||||
|
error_chain! {
|
||||||
|
foreign_links {
|
||||||
|
Io(std::io::Error);
|
||||||
|
Url(url::ParseError);
|
||||||
|
}
|
||||||
|
errors {
|
||||||
|
Parse(msg: String) {
|
||||||
|
description("parse error")
|
||||||
|
display("parse error: {}", msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct CumlouderProvider;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
enum Target {
|
||||||
|
Latest,
|
||||||
|
Category(String),
|
||||||
|
Tag(String),
|
||||||
|
Series(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CumlouderProvider {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||||
|
let mut category_options = vec![FilterOption {
|
||||||
|
id: "all".to_string(),
|
||||||
|
title: "All".to_string(),
|
||||||
|
}];
|
||||||
|
category_options.extend(CATEGORIES.iter().map(|(slug, title)| FilterOption {
|
||||||
|
id: (*slug).to_string(),
|
||||||
|
title: (*title).to_string(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
Channel {
|
||||||
|
id: CHANNEL_ID.to_string(),
|
||||||
|
name: "Cumlouder".to_string(),
|
||||||
|
description: "Free amateur and Spanish-scene porn videos from Cumlouder, with category and tag archives.".to_string(),
|
||||||
|
premium: false,
|
||||||
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=cumlouder.com".to_string(),
|
||||||
|
status: "active".to_string(),
|
||||||
|
categories: vec![],
|
||||||
|
options: vec![
|
||||||
|
ChannelOption {
|
||||||
|
id: "sort".to_string(),
|
||||||
|
title: "Sort".to_string(),
|
||||||
|
description: "Browse Cumlouder by newest videos.".to_string(),
|
||||||
|
systemImage: "arrow.up.arrow.down".to_string(),
|
||||||
|
colorName: "blue".to_string(),
|
||||||
|
options: vec![FilterOption {
|
||||||
|
id: "new".to_string(),
|
||||||
|
title: "Latest".to_string(),
|
||||||
|
}],
|
||||||
|
multiSelect: false,
|
||||||
|
},
|
||||||
|
ChannelOption {
|
||||||
|
id: "categories".to_string(),
|
||||||
|
title: "Categories".to_string(),
|
||||||
|
description: "Browse a Cumlouder category archive.".to_string(),
|
||||||
|
systemImage: "square.grid.2x2".to_string(),
|
||||||
|
colorName: "orange".to_string(),
|
||||||
|
options: category_options,
|
||||||
|
multiSelect: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
nsfw: true,
|
||||||
|
cacheDuration: Some(1800),
|
||||||
|
ytdlpCommand: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn slugify(value: &str) -> String {
|
||||||
|
let mut out = String::new();
|
||||||
|
let mut last_dash = true;
|
||||||
|
for c in value.trim().chars() {
|
||||||
|
if c.is_alphanumeric() {
|
||||||
|
out.extend(c.to_lowercase());
|
||||||
|
last_dash = false;
|
||||||
|
} else if !last_dash {
|
||||||
|
out.push('-');
|
||||||
|
last_dash = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.trim_matches('-').to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn category_slug(value: &str) -> Option<&'static str> {
|
||||||
|
let slug = Self::slugify(value);
|
||||||
|
CATEGORIES.iter().find_map(|(s, title)| {
|
||||||
|
(*s == slug || Self::slugify(title) == slug).then_some(*s)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pick_target(query: Option<&str>, options: &ServerOptions) -> Target {
|
||||||
|
if let Some(q) = query.map(str::trim).filter(|q| !q.is_empty()) {
|
||||||
|
let lower = q.to_ascii_lowercase();
|
||||||
|
for prefix in ["cat:", "category:"] {
|
||||||
|
if let Some(rest) = lower.strip_prefix(prefix) {
|
||||||
|
let slug = Self::category_slug(rest).map(str::to_string);
|
||||||
|
return Target::Category(slug.unwrap_or_else(|| Self::slugify(rest)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(rest) = lower.strip_prefix("tag:") {
|
||||||
|
return Target::Tag(Self::slugify(rest));
|
||||||
|
}
|
||||||
|
if let Some(rest) = lower.strip_prefix("series:") {
|
||||||
|
return Target::Series(Self::slugify(rest));
|
||||||
|
}
|
||||||
|
// The site search is behind a Cloudflare challenge: fall back to a
|
||||||
|
// category archive, then to the free-form tag archive.
|
||||||
|
if let Some(slug) = Self::category_slug(q) {
|
||||||
|
return Target::Category(slug.to_string());
|
||||||
|
}
|
||||||
|
return Target::Tag(Self::slugify(q));
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(cat) = options.categories.as_deref() {
|
||||||
|
if cat != "all" && !cat.is_empty() {
|
||||||
|
return Target::Category(Self::slugify(cat));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Target::Latest
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_listing_url(target: &Target, page: u32) -> String {
|
||||||
|
let suffix = if page > 1 {
|
||||||
|
format!("{page}/")
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
match target {
|
||||||
|
// Page 1 is /porn/, page N is the bare /N/ path.
|
||||||
|
Target::Latest => {
|
||||||
|
if page > 1 {
|
||||||
|
format!("{BASE_URL}/{page}/")
|
||||||
|
} else {
|
||||||
|
format!("{BASE_URL}/porn/")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Target::Category(slug) => format!("{BASE_URL}/porn-videos/{slug}/{suffix}"),
|
||||||
|
Target::Tag(slug) => format!("{BASE_URL}/tag/{slug}/{suffix}"),
|
||||||
|
Target::Series(slug) => format!("{BASE_URL}/series/{slug}/{suffix}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn selector(value: &str) -> Result<Selector> {
|
||||||
|
Selector::parse(value).map_err(|e| Error::from(format!("selector `{value}` failed: {e}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn absolute(url: &str) -> String {
|
||||||
|
if let Some(rest) = url.strip_prefix("//") {
|
||||||
|
format!("https://{rest}")
|
||||||
|
} else if url.starts_with('/') {
|
||||||
|
format!("{BASE_URL}{url}")
|
||||||
|
} else {
|
||||||
|
url.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_views(text: &str) -> Option<u32> {
|
||||||
|
let token = text.split_whitespace().next()?.replace(',', "");
|
||||||
|
let (num, mult) = match token.chars().last()? {
|
||||||
|
'K' | 'k' => (&token[..token.len() - 1], 1_000.0),
|
||||||
|
'M' | 'm' => (&token[..token.len() - 1], 1_000_000.0),
|
||||||
|
_ => (token.as_str(), 1.0),
|
||||||
|
};
|
||||||
|
num.parse::<f64>().ok().map(|n| (n * mult) as u32)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_listing_page(html: &str) -> Result<Vec<VideoItem>> {
|
||||||
|
let document = Html::parse_document(html);
|
||||||
|
let card_sel = Self::selector("a.muestra-escena[href]")?;
|
||||||
|
let img_sel = Self::selector("img")?;
|
||||||
|
let title_sel = Self::selector("h2")?;
|
||||||
|
let views_sel = Self::selector(".vistas")?;
|
||||||
|
let minutes_sel = Self::selector(".minutos")?;
|
||||||
|
|
||||||
|
let mut items = Vec::new();
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
for card in document.select(&card_sel) {
|
||||||
|
let href = card.value().attr("href").unwrap_or_default();
|
||||||
|
let Some(slug) = href
|
||||||
|
.strip_prefix("/porn-video/")
|
||||||
|
.map(|s| s.trim_matches('/').to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !seen.insert(slug.clone()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let img = card.select(&img_sel).next();
|
||||||
|
let thumb = img
|
||||||
|
.and_then(|i| i.value().attr("data-src"))
|
||||||
|
.map(Self::absolute)
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let title = card
|
||||||
|
.select(&title_sel)
|
||||||
|
.next()
|
||||||
|
.map(|h| h.text().collect::<String>().trim().to_string())
|
||||||
|
.filter(|t| !t.is_empty())
|
||||||
|
.or_else(|| {
|
||||||
|
img.and_then(|i| i.value().attr("alt"))
|
||||||
|
.map(|t| t.trim().to_string())
|
||||||
|
.filter(|t| !t.is_empty())
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| slug.replace('-', " "));
|
||||||
|
|
||||||
|
let duration = card
|
||||||
|
.select(&minutes_sel)
|
||||||
|
.next()
|
||||||
|
.map(|m| m.text().collect::<String>())
|
||||||
|
.and_then(|t| {
|
||||||
|
let t = t.trim().trim_end_matches('m').trim().to_string();
|
||||||
|
parse_time_to_seconds(&t)
|
||||||
|
})
|
||||||
|
.and_then(|s| u32::try_from(s).ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
let views = card
|
||||||
|
.select(&views_sel)
|
||||||
|
.next()
|
||||||
|
.map(|v| v.text().collect::<String>())
|
||||||
|
.and_then(|t| Self::parse_views(t.trim()));
|
||||||
|
|
||||||
|
let url = format!("{BASE_URL}/porn-video/{slug}/");
|
||||||
|
let mut item = VideoItem::new(
|
||||||
|
slug,
|
||||||
|
title,
|
||||||
|
url,
|
||||||
|
CHANNEL_ID.to_string(),
|
||||||
|
thumb,
|
||||||
|
duration,
|
||||||
|
);
|
||||||
|
item.views = views;
|
||||||
|
item.aspectRatio = Some(16.0 / 9.0);
|
||||||
|
items.push(item);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(items)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_listing(
|
||||||
|
target: &Target,
|
||||||
|
page: u32,
|
||||||
|
per_page: usize,
|
||||||
|
options: &ServerOptions,
|
||||||
|
) -> Result<Vec<VideoItem>> {
|
||||||
|
let url = Self::build_listing_url(target, page);
|
||||||
|
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_listing");
|
||||||
|
let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))];
|
||||||
|
|
||||||
|
// The origin sporadically answers its first uncached request with a
|
||||||
|
// Cloudflare 522; a single retry succeeds.
|
||||||
|
let mut last_err = String::new();
|
||||||
|
let mut html = None;
|
||||||
|
for _ in 0..3 {
|
||||||
|
match requester
|
||||||
|
.get_with_headers(&url, headers.clone(), None)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(body) if body.contains("muestra-escena") => {
|
||||||
|
html = Some(body);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Ok(body) => {
|
||||||
|
last_err = format!("no cards in {} bytes", body.len());
|
||||||
|
// A page with no cards is a real empty archive (404 body
|
||||||
|
// included); only keep retrying on transport errors.
|
||||||
|
html = Some(body);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(e) => last_err = e.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let html = html.ok_or_else(|| Error::from(format!("fetch failed for {url}: {last_err}")))?;
|
||||||
|
|
||||||
|
let mut items = Self::parse_listing_page(&html)?;
|
||||||
|
items.truncate(per_page.max(1));
|
||||||
|
Ok(items)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Provider for CumlouderProvider {
|
||||||
|
async fn get_videos(
|
||||||
|
&self,
|
||||||
|
_cache: VideoCache,
|
||||||
|
_pool: DbPool,
|
||||||
|
_sort: String,
|
||||||
|
query: Option<String>,
|
||||||
|
page: String,
|
||||||
|
per_page: String,
|
||||||
|
options: ServerOptions,
|
||||||
|
) -> Vec<VideoItem> {
|
||||||
|
let page = page.parse::<u32>().unwrap_or(1).max(1);
|
||||||
|
let per_page = per_page.parse::<usize>().unwrap_or(30).clamp(1, 60);
|
||||||
|
let target = Self::pick_target(query.as_deref(), &options);
|
||||||
|
|
||||||
|
match Self::fetch_listing(&target, page, per_page, &options).await {
|
||||||
|
Ok(items) => items,
|
||||||
|
Err(e) => {
|
||||||
|
report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await;
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||||
|
Some(self.build_channel(clientversion))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn opts() -> ServerOptions {
|
||||||
|
ServerOptions {
|
||||||
|
featured: None,
|
||||||
|
category: None,
|
||||||
|
sites: None,
|
||||||
|
filter: None,
|
||||||
|
language: None,
|
||||||
|
public_url_base: None,
|
||||||
|
requester: None,
|
||||||
|
network: None,
|
||||||
|
stars: None,
|
||||||
|
categories: None,
|
||||||
|
duration: None,
|
||||||
|
sort: None,
|
||||||
|
sexuality: None,
|
||||||
|
client_version: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_urls() {
|
||||||
|
assert_eq!(
|
||||||
|
CumlouderProvider::build_listing_url(&Target::Latest, 1),
|
||||||
|
"https://www.cumlouder.com/porn/"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
CumlouderProvider::build_listing_url(&Target::Latest, 3),
|
||||||
|
"https://www.cumlouder.com/3/"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
CumlouderProvider::build_listing_url(&Target::Tag("deepthroat".into()), 2),
|
||||||
|
"https://www.cumlouder.com/tag/deepthroat/2/"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
CumlouderProvider::build_listing_url(&Target::Category("anal".into()), 1),
|
||||||
|
"https://www.cumlouder.com/porn-videos/anal/"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn routes_queries() {
|
||||||
|
let o = opts();
|
||||||
|
assert_eq!(
|
||||||
|
CumlouderProvider::pick_target(Some("Big Tits"), &o),
|
||||||
|
Target::Category("big-tits".into())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
CumlouderProvider::pick_target(Some("deepthroat"), &o),
|
||||||
|
Target::Tag("deepthroat".into())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
CumlouderProvider::pick_target(Some("series:Nacho Vidal"), &o),
|
||||||
|
Target::Series("nacho-vidal".into())
|
||||||
|
);
|
||||||
|
assert_eq!(CumlouderProvider::pick_target(None, &o), Target::Latest);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_cards() {
|
||||||
|
let html = r#"<a class="muestra-escena" href="/porn-video/some-scene/">
|
||||||
|
<img class="thumb" src="data:image/gif;base64,AA" data-src="//th0cdn.cumlouder.com/abc/thumbnail_001.jpg" alt="Some scene">
|
||||||
|
<h2> <span class="ico-h2 sprite"></span> Some scene</h2>
|
||||||
|
<span class="box-fecha-mins">
|
||||||
|
<span class="vistas"> <span class="ico-vistas sprite"></span> 1,154 views</span>
|
||||||
|
<span class="fecha"> <span class="ico-fecha sprite"></span> 2 years ago</span>
|
||||||
|
</span>
|
||||||
|
<span class="minutos"> <span class="ico-minutos sprite"></span> 06:14 m</span>
|
||||||
|
</a>
|
||||||
|
<a class="muestra-escena" href="/porn-videos/anal/"><h2>Category</h2></a>"#;
|
||||||
|
let items = CumlouderProvider::parse_listing_page(html).unwrap();
|
||||||
|
assert_eq!(items.len(), 1);
|
||||||
|
assert_eq!(items[0].id, "some-scene");
|
||||||
|
assert_eq!(items[0].title, "Some scene");
|
||||||
|
assert_eq!(items[0].duration, 374);
|
||||||
|
assert_eq!(items[0].views, Some(1154));
|
||||||
|
assert_eq!(
|
||||||
|
items[0].thumb,
|
||||||
|
"https://th0cdn.cumlouder.com/abc/thumbnail_001.jpg"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
items[0].url,
|
||||||
|
"https://www.cumlouder.com/porn-video/some-scene/"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -498,13 +498,26 @@ impl NoodlemagazineProvider {
|
|||||||
|
|
||||||
let video_url = format!("{}{}", self.url, href);
|
let video_url = format!("{}{}", self.url, href);
|
||||||
|
|
||||||
|
// The title div now carries extra attributes
|
||||||
|
// (`<div class="title" data-video-title-text>`), so match the class
|
||||||
|
// and skip to the tag end; fall back to the thumbnail `alt`, which
|
||||||
|
// holds the same text.
|
||||||
let mut title = video_segment
|
let mut title = video_segment
|
||||||
.split("<div class=\"title\">")
|
.split("<div class=\"title\"")
|
||||||
.nth(1)
|
.nth(1)
|
||||||
.and_then(|s| s.split('<').next())
|
.and_then(|s| s.split_once('>'))
|
||||||
.unwrap_or("")
|
.and_then(|(_, rest)| rest.split('<').next())
|
||||||
.trim()
|
.map(|t| t.trim().to_string())
|
||||||
.to_string();
|
.filter(|t| !t.is_empty())
|
||||||
|
.or_else(|| {
|
||||||
|
video_segment
|
||||||
|
.split("alt=\"")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split('"').next())
|
||||||
|
.map(|t| t.trim().to_string())
|
||||||
|
.filter(|t| !t.is_empty())
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
title = decode(title.as_bytes())
|
title = decode(title.as_bytes())
|
||||||
.to_string()
|
.to_string()
|
||||||
@@ -671,6 +684,30 @@ mod tests {
|
|||||||
assert_eq!(items[0].formats.as_ref().map(|f| f.len()), Some(1));
|
assert_eq!(items[0].formats.as_ref().map(|f| f.len()), Some(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_title_when_title_div_has_extra_attributes() {
|
||||||
|
let provider = NoodlemagazineProvider::new();
|
||||||
|
let options = options();
|
||||||
|
let html = r##"
|
||||||
|
<div class="list_videos" id="list_videos">
|
||||||
|
<div class="item" data-video-title-base="x">
|
||||||
|
<a href="/watch/-1_2" class="item_link">
|
||||||
|
<div class="i_img"><img src="data:image/gif;base64,R0lG" data-src="https://img.pvvstream.pro/preview/a/b.jpg" class="lazyload" alt="Alt text">
|
||||||
|
<div class="m_views"><svg class="icon icon-eye"><use xlink:href="#eye"></use></svg> 133</div>
|
||||||
|
<div class="m_time"><svg class="icon icon-clock-o"><use xlink:href="#clock-o"></use></svg> 04:55</div></div>
|
||||||
|
<div class="i_info"><div class="title" data-video-title-text>The girl & her lover</div></div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
>Show more</div>
|
||||||
|
"##;
|
||||||
|
|
||||||
|
let items = provider.get_video_items_from_html(html.to_string(), &options);
|
||||||
|
|
||||||
|
assert_eq!(items.len(), 1);
|
||||||
|
assert_eq!(items[0].title, "The Girl & Her Lover");
|
||||||
|
assert_eq!(items[0].duration, 295);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn keeps_https_cdn_thumbs_but_drops_non_images() {
|
fn keeps_https_cdn_thumbs_but_drops_non_images() {
|
||||||
let provider = NoodlemagazineProvider::new();
|
let provider = NoodlemagazineProvider::new();
|
||||||
|
|||||||
@@ -5,14 +5,18 @@ use crate::status::*;
|
|||||||
use crate::util::cache::VideoCache;
|
use crate::util::cache::VideoCache;
|
||||||
use crate::util::discord::send_discord_error_report;
|
use crate::util::discord::send_discord_error_report;
|
||||||
use crate::util::parse_abbreviated_number;
|
use crate::util::parse_abbreviated_number;
|
||||||
|
use crate::util::requester::Requester;
|
||||||
use crate::util::time::parse_time_to_seconds;
|
use crate::util::time::parse_time_to_seconds;
|
||||||
use crate::videos::{ServerOptions, VideoItem};
|
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use error_chain::error_chain;
|
use error_chain::error_chain;
|
||||||
|
use futures::stream::{self, StreamExt};
|
||||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
use std::vec;
|
use std::vec;
|
||||||
|
|
||||||
|
const ENRICH_CONCURRENCY: usize = 6;
|
||||||
|
|
||||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
crate::providers::ProviderChannelMetadata {
|
crate::providers::ProviderChannelMetadata {
|
||||||
group_id: "hentai-animation",
|
group_id: "hentai-animation",
|
||||||
@@ -152,6 +156,7 @@ impl Rule34videoProvider {
|
|||||||
"".to_string()
|
"".to_string()
|
||||||
});
|
});
|
||||||
let video_items = self.get_video_items_from_html(text);
|
let video_items = self.get_video_items_from_html(text);
|
||||||
|
let video_items = self.enrich_items(video_items, requester).await;
|
||||||
|
|
||||||
if !video_items.is_empty() {
|
if !video_items.is_empty() {
|
||||||
cache.insert(index, video_items.clone());
|
cache.insert(index, video_items.clone());
|
||||||
@@ -224,6 +229,7 @@ impl Rule34videoProvider {
|
|||||||
"".to_string()
|
"".to_string()
|
||||||
});
|
});
|
||||||
let video_items = self.get_video_items_from_html(text);
|
let video_items = self.get_video_items_from_html(text);
|
||||||
|
let video_items = self.enrich_items(video_items, requester).await;
|
||||||
|
|
||||||
if !video_items.is_empty() {
|
if !video_items.is_empty() {
|
||||||
cache.insert(index, video_items.clone());
|
cache.insert(index, video_items.clone());
|
||||||
@@ -236,6 +242,103 @@ impl Rule34videoProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reads a single-quoted JS string value (`key: '...'`) from the
|
||||||
|
/// `flashvars` object, honouring `\'` escapes.
|
||||||
|
fn flashvar(block: &str, key: &str) -> Option<String> {
|
||||||
|
let marker = format!("{}: '", key);
|
||||||
|
let start = block.find(&marker)? + marker.len();
|
||||||
|
let mut out = String::new();
|
||||||
|
let mut chars = block[start..].chars();
|
||||||
|
while let Some(c) = chars.next() {
|
||||||
|
match c {
|
||||||
|
'\\' => out.push(chars.next()?),
|
||||||
|
'\'' => return Some(out),
|
||||||
|
other => out.push(other),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The page URL is no longer resolvable by yt-dlp's Rule34Video
|
||||||
|
/// extractor ("No video formats found"), so pull the signed mp4 URLs
|
||||||
|
/// (360p/480p/720p, valid ~12h, no cookie/Referer needed) and the tags
|
||||||
|
/// straight from the detail page's `flashvars`.
|
||||||
|
fn parse_detail(html: &str) -> (Vec<VideoFormat>, Vec<String>) {
|
||||||
|
let Some(start) = html.find("var flashvars") else {
|
||||||
|
return (vec![], vec![]);
|
||||||
|
};
|
||||||
|
let block = &html[start..];
|
||||||
|
let block = &block[..block.len().min(6000)];
|
||||||
|
|
||||||
|
let mut formats: Vec<(u32, VideoFormat)> = Vec::new();
|
||||||
|
for (url_key, text_key) in [
|
||||||
|
("video_url", "video_url_text"),
|
||||||
|
("video_alt_url", "video_alt_url_text"),
|
||||||
|
("video_alt_url2", "video_alt_url2_text"),
|
||||||
|
("video_alt_url3", "video_alt_url3_text"),
|
||||||
|
] {
|
||||||
|
let Some(url) = Self::flashvar(block, url_key) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !url.starts_with("http") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let quality = Self::flashvar(block, text_key).unwrap_or_else(|| "auto".to_string());
|
||||||
|
let height = quality.trim_end_matches('p').parse::<u32>().unwrap_or(0);
|
||||||
|
let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string())
|
||||||
|
.format_id(format!("mp4-{}", quality));
|
||||||
|
if height > 0 {
|
||||||
|
format = format.height(height);
|
||||||
|
}
|
||||||
|
formats.push((height, format));
|
||||||
|
}
|
||||||
|
formats.sort_by(|a, b| b.0.cmp(&a.0));
|
||||||
|
|
||||||
|
let mut tags: Vec<String> = Vec::new();
|
||||||
|
for key in ["video_categories", "video_tags"] {
|
||||||
|
for tag in Self::flashvar(block, key).unwrap_or_default().split(',') {
|
||||||
|
let tag = tag.trim();
|
||||||
|
if !tag.is_empty() && !tags.iter().any(|t| t.eq_ignore_ascii_case(tag)) {
|
||||||
|
tags.push(tag.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
(formats.into_iter().map(|(_, f)| f).collect(), tags)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
|
||||||
|
let referer = format!("{}/", self.url);
|
||||||
|
stream::iter(items.into_iter().map(|mut item| {
|
||||||
|
let mut requester = requester.clone();
|
||||||
|
let referer = referer.clone();
|
||||||
|
async move {
|
||||||
|
if item.url.is_empty() {
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
let html = requester
|
||||||
|
.get_with_headers(
|
||||||
|
&item.url,
|
||||||
|
vec![("Referer".to_string(), referer)],
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
let (formats, tags) = Self::parse_detail(&html);
|
||||||
|
if !formats.is_empty() {
|
||||||
|
item.formats = Some(formats);
|
||||||
|
}
|
||||||
|
if !tags.is_empty() {
|
||||||
|
item.tags = Some(tags);
|
||||||
|
}
|
||||||
|
item
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.buffered(ENRICH_CONCURRENCY)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
|
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
|
||||||
if html.is_empty() {
|
if html.is_empty() {
|
||||||
return vec![];
|
return vec![];
|
||||||
@@ -280,15 +383,37 @@ impl Rule34videoProvider {
|
|||||||
Self::extract_between(video_segment, "<div class=\"time\">", "<").unwrap_or("0:00");
|
Self::extract_between(video_segment, "<div class=\"time\">", "<").unwrap_or("0:00");
|
||||||
let duration = parse_time_to_seconds(raw_duration).unwrap_or(0) as u32;
|
let duration = parse_time_to_seconds(raw_duration).unwrap_or(0) as u32;
|
||||||
|
|
||||||
// Views extraction
|
// Views extraction: `<div class="views video-views-count ..."
|
||||||
let views_segment = Self::extract_between(video_segment, "<div class=\"views\">", "<");
|
// title="Views: 13K">`. The sibling `views` div is the comment
|
||||||
let views_count_str = views_segment
|
// count, so match the title rather than the bare class.
|
||||||
.and_then(|s| s.split("</svg>").nth(1))
|
let views = Self::extract_between(video_segment, "title=\"Views: ", "\"")
|
||||||
.unwrap_or("0");
|
.and_then(|v| parse_abbreviated_number(v.trim()))
|
||||||
let views = parse_abbreviated_number(views_count_str.trim()).unwrap_or(0);
|
.or_else(|| {
|
||||||
|
Self::extract_between(video_segment, "<div class=\"views\">", "</div>")
|
||||||
|
.map(|s| s.rsplit("</svg>").next().unwrap_or(s))
|
||||||
|
.and_then(|v| parse_abbreviated_number(v.trim()))
|
||||||
|
})
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
// Thumbnail extraction
|
// Rating extraction: `72% (47)` inside the rating div.
|
||||||
let thumb = Self::extract_between(video_segment, "data-original=\"", "\"")
|
let rating = Self::extract_between(video_segment, "custom-thumbs-up", "</div>")
|
||||||
|
.and_then(|s| s.rsplit("</svg>").next())
|
||||||
|
.and_then(|s| s.trim().split('%').next())
|
||||||
|
.and_then(|s| s.trim().parse::<f32>().ok());
|
||||||
|
|
||||||
|
// Thumbnail extraction: the site dropped `data-original`. Lazy
|
||||||
|
// cards are `<img class="thumb rd-lz" src="data:image/gif..."
|
||||||
|
// data-rd-src="...">`; eager ones carry the real `src`.
|
||||||
|
let thumb = Self::extract_between(video_segment, "<img class=\"thumb", ">")
|
||||||
|
.and_then(|tag| {
|
||||||
|
Self::extract_between(tag, "data-rd-src=\"", "\"")
|
||||||
|
.or_else(|| {
|
||||||
|
Self::extract_between(tag, " src=\"", "\"")
|
||||||
|
.filter(|src| src.starts_with("http"))
|
||||||
|
})
|
||||||
|
.or_else(|| Self::extract_between(tag, "data-rd-jpg=\"", "\""))
|
||||||
|
})
|
||||||
|
.or_else(|| Self::extract_between(video_segment, "data-original=\"", "\""))
|
||||||
.unwrap_or("")
|
.unwrap_or("")
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
@@ -298,10 +423,13 @@ impl Rule34videoProvider {
|
|||||||
.unwrap_or("")
|
.unwrap_or("")
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
items.push(
|
let mut item =
|
||||||
VideoItem::new(id, title, url, "Rule34video".to_string(), thumb, duration)
|
VideoItem::new(id, title, url, "Rule34video".to_string(), thumb, duration)
|
||||||
.views(views),
|
.views(views);
|
||||||
);
|
if let Some(rating) = rating {
|
||||||
|
item = item.rating(rating);
|
||||||
|
}
|
||||||
|
items.push(item);
|
||||||
}
|
}
|
||||||
items
|
items
|
||||||
}
|
}
|
||||||
@@ -339,3 +467,54 @@ impl Provider for Rule34videoProvider {
|
|||||||
Some(self.build_channel(clientversion))
|
Some(self.build_channel(clientversion))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_card_thumb_from_img_src_and_skips_ads() {
|
||||||
|
let html = r#"<div id="custom_list_videos_x"><div class="item thumb video_1 " data-video-card-id="42">
|
||||||
|
<a class="th js-open-popup" href="https://rule34video.com/video/42/some-title/" title="Some Title">
|
||||||
|
<div class="img wrap_image"><img class="thumb" src="https://rule34video.com/contents/videos_screenshots/0/42/336x189/3.jpg" data-rd-jpg="x" alt="a">
|
||||||
|
<div class="time">1:28</div></div><div class="thumb_title">Some Title</div>
|
||||||
|
<div class="rating"><svg class="custom-svg custom-thumbs-up"></svg> 72% (47) </div>
|
||||||
|
<div class="views video-comments-count" title="Comments: 0"><svg></svg> 0 </div>
|
||||||
|
<div class="views video-views-count" title="Views: 6.9K"><svg></svg> 6.9K </div></a></div>
|
||||||
|
<div class="item thumb video_3"><a class="th js-open-popup" href="https://rule34video.com/video/43/lazy/" title="Lazy">
|
||||||
|
<div class="img wrap_image"><img class="thumb rd-lz" src="data:image/gif;base64,R0lG" data-rd-src="https://rule34video.com/contents/videos_screenshots/0/43/336x189/3.jpg" alt="b">
|
||||||
|
<div class="time">0:30</div></div><div class="thumb_title">Lazy</div></a></div>
|
||||||
|
<div class="item thumb video_2"><a class="th" href="https://ads" title="Advertisement"></a></div>
|
||||||
|
<div class="pagination" id="p">"#;
|
||||||
|
let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string());
|
||||||
|
assert_eq!(items.len(), 2);
|
||||||
|
assert_eq!(items[0].id, "42");
|
||||||
|
assert_eq!(items[0].views, Some(6900));
|
||||||
|
assert_eq!(items[0].rating, Some(72.0));
|
||||||
|
assert_eq!(
|
||||||
|
items[1].thumb,
|
||||||
|
"https://rule34video.com/contents/videos_screenshots/0/43/336x189/3.jpg"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
items[0].thumb,
|
||||||
|
"https://rule34video.com/contents/videos_screenshots/0/42/336x189/3.jpg"
|
||||||
|
);
|
||||||
|
assert_eq!(items[0].duration, 88);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_flashvars_formats_and_tags() {
|
||||||
|
let html = r#"var flashvars = { video_id: '1', video_categories: '3D, Cyberpunk 2077', video_tags: 'bondage, panam palmer (cyberpunk), 3d', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#;
|
||||||
|
let (formats, tags) = Rule34videoProvider::parse_detail(html);
|
||||||
|
assert_eq!(formats.len(), 3);
|
||||||
|
assert!(formats[0].url.contains("a_720p"), "best quality first");
|
||||||
|
assert!(formats[2].url.contains("a_360"));
|
||||||
|
assert_eq!(tags, vec!["3D", "Cyberpunk 2077", "bondage", "panam palmer (cyberpunk)"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detail_without_flashvars_yields_nothing() {
|
||||||
|
let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
|
||||||
|
assert!(formats.is_empty() && tags.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user