xlecx
This commit is contained in:
5
build.rs
5
build.rs
@@ -316,6 +316,11 @@ const PROVIDERS: &[ProviderDef] = &[
|
|||||||
module: "thaiporntv",
|
module: "thaiporntv",
|
||||||
ty: "ThaipornTvProvider",
|
ty: "ThaipornTvProvider",
|
||||||
},
|
},
|
||||||
|
ProviderDef {
|
||||||
|
id: "xlecx",
|
||||||
|
module: "xlecx",
|
||||||
|
ty: "XlecxProvider",
|
||||||
|
},
|
||||||
ProviderDef {
|
ProviderDef {
|
||||||
id: "aps",
|
id: "aps",
|
||||||
module: "allpornstream",
|
module: "allpornstream",
|
||||||
|
|||||||
@@ -83,6 +83,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
|
|||||||
| `camsoda` | `live-cams` | no | no | Live-cam provider for camsoda.com (chaturbate-style — `live` performers streaming now, `video.url` = the room page, `is_live=true`, no `formats`). camsoda.com is hard Cloudflare-protected: direct requests and yt-dlp both get HTTP 403, and FlareSolverr was unreliable during development, so the live-browse API is reached through the shared requester's Jina mirror fallback (`r.jina.ai/http://...`, `X-Return-Format: html`); Jina rate-limits per IP, so the provider caches each fetched feed URL for 60s (and serves stale items on a 429 rather than emptying the feed), and a single-provider build (`HOT_TUB_PROVIDER=camsoda`) validates most cleanly (one fetch at a time). Endpoint (found in the non-CF static `main.js` bundle): `GET https://www.camsoda.com/api/v1/browse/react{route}?p=N` returning a body with a top-level `userList` array (Jina wraps it in `<pre>`, so the provider slices out the `{...}` and parses it with `serde_json::Value`, like the chaturbate provider). Per-cam fields: `username`→id + room URL (`/{username}`), `subjectText`→title (html-decoded, falls back to `displayName`), `displayName`→uploader, `connectionCount`→views (string or number tolerated), `thumbUrl`→thumb (direct `media.livemediahost.com` CDN, no proxy/referer), `status` (skip `offline`), `vr`/`private` surfaced as tags. Category option `category` uses verified `browse/react` routes — `all`(featured)/`girls`/`trans`/`couples`/`voyeur-cams`/`new` (`/male` is NOT a path route, camsoda gates male via `gender-hide`); `cat:`/`category:` prefixes and a bare keyword matching a category id route there too. Search: `GET browse/react/search/{dashed-query}?sortByConnection=1` (single connection-sorted result set, no real paging). Playback: `video.url` is the live room page; the room and the token-gated edge HLS (`*.livemediahost.com`) are both Cloudflare-protected, so HLS can't be resolved server-side and no `formats` are populated — yt-dlp has a `Camsoda` live extractor that resolves the room on a non-CF-blocked client, and `check.py` reports the sandbox's CF 403s as expected warnings (`www.camsoda.com` is in its CF allowlist), not errors. The earlier recorded-`/media` JSON scrape was replaced because clips were token-gated/non-playable; live cams are the site's actual product. No proxy needed. |
|
| `camsoda` | `live-cams` | no | no | Live-cam provider for camsoda.com (chaturbate-style — `live` performers streaming now, `video.url` = the room page, `is_live=true`, no `formats`). camsoda.com is hard Cloudflare-protected: direct requests and yt-dlp both get HTTP 403, and FlareSolverr was unreliable during development, so the live-browse API is reached through the shared requester's Jina mirror fallback (`r.jina.ai/http://...`, `X-Return-Format: html`); Jina rate-limits per IP, so the provider caches each fetched feed URL for 60s (and serves stale items on a 429 rather than emptying the feed), and a single-provider build (`HOT_TUB_PROVIDER=camsoda`) validates most cleanly (one fetch at a time). Endpoint (found in the non-CF static `main.js` bundle): `GET https://www.camsoda.com/api/v1/browse/react{route}?p=N` returning a body with a top-level `userList` array (Jina wraps it in `<pre>`, so the provider slices out the `{...}` and parses it with `serde_json::Value`, like the chaturbate provider). Per-cam fields: `username`→id + room URL (`/{username}`), `subjectText`→title (html-decoded, falls back to `displayName`), `displayName`→uploader, `connectionCount`→views (string or number tolerated), `thumbUrl`→thumb (direct `media.livemediahost.com` CDN, no proxy/referer), `status` (skip `offline`), `vr`/`private` surfaced as tags. Category option `category` uses verified `browse/react` routes — `all`(featured)/`girls`/`trans`/`couples`/`voyeur-cams`/`new` (`/male` is NOT a path route, camsoda gates male via `gender-hide`); `cat:`/`category:` prefixes and a bare keyword matching a category id route there too. Search: `GET browse/react/search/{dashed-query}?sortByConnection=1` (single connection-sorted result set, no real paging). Playback: `video.url` is the live room page; the room and the token-gated edge HLS (`*.livemediahost.com`) are both Cloudflare-protected, so HLS can't be resolved server-side and no `formats` are populated — yt-dlp has a `Camsoda` live extractor that resolves the room on a non-CF-blocked client, and `check.py` reports the sandbox's CF 403s as expected warnings (`www.camsoda.com` is in its CF allowlist), not errors. The earlier recorded-`/media` JSON scrape was replaced because clips were token-gated/non-playable; live cams are the site's actual product. No proxy needed. |
|
||||||
| `xvideos` | `mainstream-tube` | no | no | HTML scraper for xvideos.com; handles two card formats: homepage (`div.thumb-block[data-id][data-eid]`) uses `p.title a[title]` + `data-pvv` on img, best-of-month page uses `div.thumb-block.video[data-video=JSON]` with `div.title a` text + `previewVideo` JSON key; thumbnails at `thumb-cdn77.xvideos-cdn.com` / `thumbs-gcore.xvideos-cdn.com` (no proxy needed); latest: `/` (page 1) / `/new/{N-1}` (page N≥2); best-of-month: `/best/{YYYY-MM}` (previous calendar month), page N: `/best/{YYYY-MM}/{N-1}`; search: `/?k={query}` / `/?k={query}&p={N-1}` (0-indexed); tag shortcuts: `/tags/{slug}/{N-1}`; category shortcuts: `/c/{Name}-{ID}/{N-1}` (38 hardcoded categories); `cat:`, `tag:`, `uploader:` query prefix routing; yt-dlp resolves `video.url` natively (XVideos extractor → HLS formats); CDN preview mp4 in `preview` field; no proxy needed. |
|
| `xvideos` | `mainstream-tube` | no | no | HTML scraper for xvideos.com; handles two card formats: homepage (`div.thumb-block[data-id][data-eid]`) uses `p.title a[title]` + `data-pvv` on img, best-of-month page uses `div.thumb-block.video[data-video=JSON]` with `div.title a` text + `previewVideo` JSON key; thumbnails at `thumb-cdn77.xvideos-cdn.com` / `thumbs-gcore.xvideos-cdn.com` (no proxy needed); latest: `/` (page 1) / `/new/{N-1}` (page N≥2); best-of-month: `/best/{YYYY-MM}` (previous calendar month), page N: `/best/{YYYY-MM}/{N-1}`; search: `/?k={query}` / `/?k={query}&p={N-1}` (0-indexed); tag shortcuts: `/tags/{slug}/{N-1}`; category shortcuts: `/c/{Name}-{ID}/{N-1}` (38 hardcoded categories); `cat:`, `tag:`, `uploader:` query prefix routing; yt-dlp resolves `video.url` natively (XVideos extractor → HLS formats); CDN preview mp4 in `preview` field; no proxy needed. |
|
||||||
| `wowxxx` | `studio-network` | no | no | HTML scraper for wow.xxx premium aggregator; default feed `/latest-updates/`, page 2 `/{N}/` suffix (for example `/latest-updates/2/`), search `/search/{query}/relevance/` with the same page suffix; supports `site:`/`studio:`/`network:`/`model:`/`pornstar:`/`tag:`/`cat:` query shortcuts to direct archive routes; list cards expose preview clips (`cast.wow.xxx/preview/*.mp4`), thumbnails (`img.wow.xxx/.../medium@2x/1.jpg`), duration, rating, views, site (as uploader), and model tags; `video.url` is the detail page URL and yt-dlp resolves HTML5 MP4 formats dynamically; no proxy needed. |
|
| `wowxxx` | `studio-network` | no | no | HTML scraper for wow.xxx premium aggregator; default feed `/latest-updates/`, page 2 `/{N}/` suffix (for example `/latest-updates/2/`), search `/search/{query}/relevance/` with the same page suffix; supports `site:`/`studio:`/`network:`/`model:`/`pornstar:`/`tag:`/`cat:` query shortcuts to direct archive routes; list cards expose preview clips (`cast.wow.xxx/preview/*.mp4`), thumbnails (`img.wow.xxx/.../medium@2x/1.jpg`), duration, rating, views, site (as uploader), and model tags; `video.url` is the detail page URL and yt-dlp resolves HTML5 MP4 formats dynamically; no proxy needed. |
|
||||||
|
| `xlecx` | `hentai-animation` | no | no | DLE (DataLife Engine)-style HTML scraper for xlecx.one, following the `thaiporntv` template. Latest feed `/f/cat={id}/sort={mode}/order=desc/` (page N≥2 appends `page/{N}/`); `sort` options new/popular/commented/rated map to DLE's `date`/`news_read`/`comm_num`/`rating` sort params; `category` option is the 2 hardcoded top-level archives (`videos`, `hentai-video`). Search via `/index.php?do=search&subaction=search&story={query}` (page N adds `search_start={N}`). Tag shortcut (`#slug` query) routes to `/tags/{slug}/` (paginated via `/tags/{slug}/page/{N}/`); the ~1,044-entry tag catalog is background-loaded once at startup from `/tags/` (`.tags-list1 a[href*='/tags/']`, absolute-URL anchors) into an `Arc<RwLock<Vec<FilterOption>>>` and exposed as the `filter`/"Tags" `ChannelOption` — note this option (and any `categories`) is deliberately stripped from the live `/api/status` response by `should_hide_status_option`/`decorate_channel` in `providers/mod.rs` (any `id=="filter"` option titled "Tags" is hidden site-wide), so its absence there is expected, not a bug; it is still honored by `/api/videos`. Cards parsed from `h3.thumb__title` + `img` inside standard DLE thumb blocks; id extracted from the `/{id}-{slug}.html` URL suffix. Thumbnails are direct `xlecx.one/uploads/posts/...` webp (no proxy/Referer needed). `video.url` is the detail page URL, which yt-dlp resolves natively via the generic HTML5 embed extractor; no `formats`/proxy needed. No `/api/uploaders` (no stable uploader identity on the site). |
|
||||||
| `xxxtik` | `tiktok` | yes | no | JSON-API short-form aggregator for xxxtik.com — every post is a moderated repost of a RedGifs clip, so the real media backend is the public **RedGifs v2 API**, not xxxtik itself (new pattern; no other provider currently resolves through a third-party media API). Listing API (`xxxtik-api-iw98m.ondigitalocean.app`, found by grepping the Angular `main-es2015.js` bundle): `GET /post/new`, `/post/top/{week,month,year,all}`, `/post/tag/{name}`, `/post/creator/{username}` — all **cursor**-paginated (`?cursor={lastItemId}&limit=N`; the `cursor` is the numeric `id` of the last item from the previous batch, not an offset/count — reaching page N walks N sequential requests, mirroring `fikfap`'s `fetch_cursor_page`). `GET /search?query=Q` returns tag/profile autocomplete suggestions only (no posts), so free-text search is routed through it to resolve a `Tag`/`Creator` target before a second listing call; `tag:`/`category:`/`cat:` and `user:`/`uploader:`/`creator:` query prefixes skip that lookup. Each post's `source` field is a `redgifs.com/watch/{id}` URL; the provider fetches a short-lived anonymous bearer token from `POST api.redgifs.com/v2/auth/temporary` (cached, refreshed once on a 401) and resolves `GET api.redgifs.com/v2/gifs/{id}` (bounded to 8-way concurrency via `buffer_unordered`) for the real `media.redgifs.com/*.mp4` + poster, both fetchable with zero auth/Referer. `video.url` is the xxxtik page (`https://xxxtik.com/feed/{uuid}`, not yt-dlp-resolvable — Angular SPA, generic extractor fails), with `formats` populated from the resolved redgifs mp4; tags merge xxxtik's own tags with RedGifs' tags. 20 curated tags exposed via `categories` (xxxtik has ~62k tags total, too many to background-load). `/api/uploaders` works via `GET /user/by-username/{name}` + `/post/creator/{name}`, but xxxtik's "creator" accounts are inconsistent: some (e.g. `dlhoodninja`, `besttits`) return real posts matching their profile `_count.posts`; others with a nonzero `_count.posts` (e.g. `bigboobsgw`, count 472) return an empty `/post/creator/` list — likely synthetic curation accounts (`name@default` emails) rather than real uploaders. The provider degrades gracefully (returns the profile with `videos: []`, no error) rather than guessing which accounts are "real". No proxy needed — all media/thumb URLs are publicly fetchable with no Referer or auth. |
|
| `xxxtik` | `tiktok` | yes | no | JSON-API short-form aggregator for xxxtik.com — every post is a moderated repost of a RedGifs clip, so the real media backend is the public **RedGifs v2 API**, not xxxtik itself (new pattern; no other provider currently resolves through a third-party media API). Listing API (`xxxtik-api-iw98m.ondigitalocean.app`, found by grepping the Angular `main-es2015.js` bundle): `GET /post/new`, `/post/top/{week,month,year,all}`, `/post/tag/{name}`, `/post/creator/{username}` — all **cursor**-paginated (`?cursor={lastItemId}&limit=N`; the `cursor` is the numeric `id` of the last item from the previous batch, not an offset/count — reaching page N walks N sequential requests, mirroring `fikfap`'s `fetch_cursor_page`). `GET /search?query=Q` returns tag/profile autocomplete suggestions only (no posts), so free-text search is routed through it to resolve a `Tag`/`Creator` target before a second listing call; `tag:`/`category:`/`cat:` and `user:`/`uploader:`/`creator:` query prefixes skip that lookup. Each post's `source` field is a `redgifs.com/watch/{id}` URL; the provider fetches a short-lived anonymous bearer token from `POST api.redgifs.com/v2/auth/temporary` (cached, refreshed once on a 401) and resolves `GET api.redgifs.com/v2/gifs/{id}` (bounded to 8-way concurrency via `buffer_unordered`) for the real `media.redgifs.com/*.mp4` + poster, both fetchable with zero auth/Referer. `video.url` is the xxxtik page (`https://xxxtik.com/feed/{uuid}`, not yt-dlp-resolvable — Angular SPA, generic extractor fails), with `formats` populated from the resolved redgifs mp4; tags merge xxxtik's own tags with RedGifs' tags. 20 curated tags exposed via `categories` (xxxtik has ~62k tags total, too many to background-load). `/api/uploaders` works via `GET /user/by-username/{name}` + `/post/creator/{name}`, but xxxtik's "creator" accounts are inconsistent: some (e.g. `dlhoodninja`, `besttits`) return real posts matching their profile `_count.posts`; others with a nonzero `_count.posts` (e.g. `bigboobsgw`, count 472) return an empty `/post/creator/` list — likely synthetic curation accounts (`name@default` emails) rather than real uploaders. The provider degrades gracefully (returns the profile with `videos: []`, no error) rather than guessing which accounts are "real". No proxy needed — all media/thumb URLs are publicly fetchable with no Referer or auth. |
|
||||||
|
|
||||||
## Proxy Routes
|
## Proxy Routes
|
||||||
|
|||||||
678
src/providers/xlecx.rs
Normal file
678
src/providers/xlecx.rs
Normal file
@@ -0,0 +1,678 @@
|
|||||||
|
use crate::DbPool;
|
||||||
|
use crate::api::ClientVersion;
|
||||||
|
use crate::providers::{Provider, report_provider_error, report_provider_error_background, requester_or_default};
|
||||||
|
use crate::status::*;
|
||||||
|
use crate::util::cache::VideoCache;
|
||||||
|
use crate::util::requester::Requester;
|
||||||
|
use crate::videos::{ServerOptions, VideoItem};
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use error_chain::error_chain;
|
||||||
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||||
|
use percent_encoding::{NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
|
||||||
|
use regex::Regex;
|
||||||
|
use scraper::{ElementRef, Html, Selector};
|
||||||
|
use std::sync::{Arc, RwLock};
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration as StdDuration;
|
||||||
|
use tokio::time::timeout;
|
||||||
|
use wreq::Version;
|
||||||
|
|
||||||
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
|
crate::providers::ProviderChannelMetadata {
|
||||||
|
group_id: "hentai-animation",
|
||||||
|
tags: &["hentai", "anime", "3d", "parody"],
|
||||||
|
};
|
||||||
|
|
||||||
|
error_chain! {
|
||||||
|
foreign_links {
|
||||||
|
Io(std::io::Error);
|
||||||
|
}
|
||||||
|
errors {
|
||||||
|
Parse(msg: String) {
|
||||||
|
description("parse error")
|
||||||
|
display("parse error: {}", msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const BASE_URL: &str = "https://xlecx.one";
|
||||||
|
const CHANNEL_ID: &str = "xlecx";
|
||||||
|
const USER_AGENT: &str =
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36";
|
||||||
|
const HTML_ACCEPT: &str =
|
||||||
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
|
||||||
|
const RESULTS_PER_PAGE: u32 = 18;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct XlecxProvider {
|
||||||
|
url: String,
|
||||||
|
tags: Arc<RwLock<Vec<FilterOption>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum Category {
|
||||||
|
Videos,
|
||||||
|
HentaiVideo,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Category {
|
||||||
|
fn from_option(value: Option<&str>) -> Self {
|
||||||
|
match value {
|
||||||
|
Some("hentai-video") => Category::HentaiVideo,
|
||||||
|
_ => Category::Videos,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cat_param(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Category::Videos => "18",
|
||||||
|
Category::HentaiVideo => "20",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum SortMode {
|
||||||
|
Date,
|
||||||
|
Title,
|
||||||
|
Comments,
|
||||||
|
Views,
|
||||||
|
Rating,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SortMode {
|
||||||
|
fn from_sort(sort: &str) -> Self {
|
||||||
|
match sort {
|
||||||
|
"title" => SortMode::Title,
|
||||||
|
"commented" => SortMode::Comments,
|
||||||
|
"popular" => SortMode::Views,
|
||||||
|
"rated" => SortMode::Rating,
|
||||||
|
_ => SortMode::Date,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sort_param(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
SortMode::Date => "date",
|
||||||
|
SortMode::Title => "title",
|
||||||
|
SortMode::Comments => "comm_num",
|
||||||
|
SortMode::Views => "news_read",
|
||||||
|
SortMode::Rating => "rating",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
enum Target {
|
||||||
|
Archive { category: Category, sort: SortMode },
|
||||||
|
Tag { slug: String },
|
||||||
|
Search { query: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl XlecxProvider {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
let provider = Self {
|
||||||
|
url: BASE_URL.to_string(),
|
||||||
|
tags: Arc::new(RwLock::new(Vec::new())),
|
||||||
|
};
|
||||||
|
provider.spawn_initial_load();
|
||||||
|
provider
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spawn_initial_load(&self) {
|
||||||
|
let url = self.url.clone();
|
||||||
|
let tags = Arc::clone(&self.tags);
|
||||||
|
|
||||||
|
thread::spawn(move || {
|
||||||
|
let runtime = match tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
{
|
||||||
|
Ok(runtime) => runtime,
|
||||||
|
Err(error) => {
|
||||||
|
report_provider_error_background(
|
||||||
|
CHANNEL_ID,
|
||||||
|
"spawn_initial_load.runtime_build",
|
||||||
|
&error.to_string(),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
runtime.block_on(async move {
|
||||||
|
if let Err(error) = Self::load_tags(&url, Arc::clone(&tags)).await {
|
||||||
|
report_provider_error_background(CHANNEL_ID, "load_tags", &error.to_string());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||||
|
let tags = self.tags.read().map(|value| value.clone()).unwrap_or_default();
|
||||||
|
|
||||||
|
Channel {
|
||||||
|
id: CHANNEL_ID.to_string(),
|
||||||
|
name: "XlecX".to_string(),
|
||||||
|
description: "Hentai videos and parody animations.".to_string(),
|
||||||
|
premium: false,
|
||||||
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=xlecx.one".to_string(),
|
||||||
|
status: "active".to_string(),
|
||||||
|
categories: tags.iter().map(|value| value.title.clone()).collect(),
|
||||||
|
options: vec![
|
||||||
|
ChannelOption {
|
||||||
|
id: "sort".to_string(),
|
||||||
|
title: "Sort".to_string(),
|
||||||
|
description: "Sort the videos".to_string(),
|
||||||
|
systemImage: "list.number".to_string(),
|
||||||
|
colorName: "blue".to_string(),
|
||||||
|
options: vec![
|
||||||
|
FilterOption {
|
||||||
|
id: "new".to_string(),
|
||||||
|
title: "Most Recent".to_string(),
|
||||||
|
},
|
||||||
|
FilterOption {
|
||||||
|
id: "popular".to_string(),
|
||||||
|
title: "Most Viewed".to_string(),
|
||||||
|
},
|
||||||
|
FilterOption {
|
||||||
|
id: "commented".to_string(),
|
||||||
|
title: "Most Commented".to_string(),
|
||||||
|
},
|
||||||
|
FilterOption {
|
||||||
|
id: "rated".to_string(),
|
||||||
|
title: "Top Rated".to_string(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
multiSelect: false,
|
||||||
|
},
|
||||||
|
ChannelOption {
|
||||||
|
id: "category".to_string(),
|
||||||
|
title: "Category".to_string(),
|
||||||
|
description: "Filter by category".to_string(),
|
||||||
|
systemImage: "square.grid.2x2".to_string(),
|
||||||
|
colorName: "orange".to_string(),
|
||||||
|
options: vec![
|
||||||
|
FilterOption {
|
||||||
|
id: "videos".to_string(),
|
||||||
|
title: "Videos".to_string(),
|
||||||
|
},
|
||||||
|
FilterOption {
|
||||||
|
id: "hentai-video".to_string(),
|
||||||
|
title: "Hentai Video".to_string(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
multiSelect: false,
|
||||||
|
},
|
||||||
|
ChannelOption {
|
||||||
|
id: "filter".to_string(),
|
||||||
|
title: "Tags".to_string(),
|
||||||
|
description: "Video tags.".to_string(),
|
||||||
|
systemImage: "tag.fill".to_string(),
|
||||||
|
colorName: "green".to_string(),
|
||||||
|
options: tags,
|
||||||
|
multiSelect: false,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
nsfw: true,
|
||||||
|
cacheDuration: Some(1800),
|
||||||
|
ytdlpCommand: Some("yt-dlp".to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn selector(value: &str) -> Result<Selector> {
|
||||||
|
Selector::parse(value)
|
||||||
|
.map_err(|error| Error::from(format!("selector `{value}` parse failed: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collapse_whitespace(text: &str) -> String {
|
||||||
|
text.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_html_entities(text: &str) -> String {
|
||||||
|
decode(text.as_bytes())
|
||||||
|
.to_string()
|
||||||
|
.unwrap_or_else(|_| text.to_string())
|
||||||
|
.replace('\u{a0}', " ")
|
||||||
|
.trim()
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn absolute_url(&self, value: &str) -> String {
|
||||||
|
if value.starts_with("http://") || value.starts_with("https://") {
|
||||||
|
return value.to_string();
|
||||||
|
}
|
||||||
|
format!(
|
||||||
|
"{}/{}",
|
||||||
|
self.url.trim_end_matches('/'),
|
||||||
|
value.trim_start_matches('/')
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_headers(&self, referer: &str) -> Vec<(String, String)> {
|
||||||
|
vec![
|
||||||
|
("Referer".to_string(), referer.to_string()),
|
||||||
|
("User-Agent".to_string(), USER_AGENT.to_string()),
|
||||||
|
("Accept".to_string(), HTML_ACCEPT.to_string()),
|
||||||
|
("Accept-Language".to_string(), "en-US,en;q=0.9".to_string()),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_html(
|
||||||
|
&self,
|
||||||
|
requester: &mut Requester,
|
||||||
|
url: &str,
|
||||||
|
referer: &str,
|
||||||
|
) -> Result<String> {
|
||||||
|
requester
|
||||||
|
.get_with_headers(url, self.html_headers(referer), Some(Version::HTTP_11))
|
||||||
|
.await
|
||||||
|
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
|
||||||
|
if item.id.is_empty() || item.title.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Ok(mut values) = target.write() {
|
||||||
|
if !values
|
||||||
|
.iter()
|
||||||
|
.any(|existing| existing.id == item.id || existing.title == item.title)
|
||||||
|
{
|
||||||
|
values.push(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_tags(base_url: &str, tags: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
|
||||||
|
let mut requester = Requester::new();
|
||||||
|
let provider = Self {
|
||||||
|
url: base_url.to_string(),
|
||||||
|
tags: Arc::clone(&tags),
|
||||||
|
};
|
||||||
|
let html = provider
|
||||||
|
.fetch_html(
|
||||||
|
&mut requester,
|
||||||
|
&format!("{}/tags/", base_url),
|
||||||
|
&format!("{}/", base_url),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let document = Html::parse_document(&html);
|
||||||
|
// Tag cloud entries are plain <a href="/tags/{slug}/">{title}</a> links
|
||||||
|
let a_selector = Self::selector(".tags-list1 a[href*='/tags/']")?;
|
||||||
|
for element in document.select(&a_selector) {
|
||||||
|
let Some(href) = element.value().attr("href") else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let raw_slug = href.trim_end_matches('/').rsplit('/').next().unwrap_or("");
|
||||||
|
if raw_slug.is_empty() || raw_slug.eq_ignore_ascii_case("tags") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let slug = percent_decode_str(raw_slug)
|
||||||
|
.decode_utf8()
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|_| raw_slug.to_string());
|
||||||
|
let title = Self::decode_html_entities(&Self::collapse_whitespace(
|
||||||
|
&element.text().collect::<String>(),
|
||||||
|
));
|
||||||
|
if title.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Self::push_unique(&tags, FilterOption { id: slug, title });
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_target(&self, options: &ServerOptions, sort: &str, query: Option<&str>) -> Target {
|
||||||
|
if let Some(q) = query {
|
||||||
|
if let Some(target) = self.find_tag_target_in_options(q) {
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
return Target::Search {
|
||||||
|
query: q.trim().to_string(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if let Some(value) = options.filter.as_deref() {
|
||||||
|
if let Some(target) = self.find_tag_target_in_options(value) {
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let category = Category::from_option(options.category.as_deref());
|
||||||
|
Target::Archive {
|
||||||
|
category,
|
||||||
|
sort: SortMode::from_sort(sort),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find_tag_target_in_options(&self, value: &str) -> Option<Target> {
|
||||||
|
let normalized = value.trim().to_lowercase();
|
||||||
|
let tags = self.tags.read().ok()?;
|
||||||
|
let option = tags.iter().find(|item| {
|
||||||
|
item.id.eq_ignore_ascii_case(value) || item.title.trim().to_lowercase() == normalized
|
||||||
|
})?;
|
||||||
|
Some(Target::Tag {
|
||||||
|
slug: option.id.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_url_for_target(&self, target: &Target, page: u32) -> String {
|
||||||
|
match target {
|
||||||
|
Target::Archive { category, sort } => self.build_archive_url(*category, *sort, page),
|
||||||
|
Target::Tag { slug } => self.build_tag_url(slug, page),
|
||||||
|
Target::Search { query } => self.build_search_url(query, page),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_archive_url(&self, category: Category, sort: SortMode, page: u32) -> String {
|
||||||
|
let base = format!(
|
||||||
|
"{}/f/cat={}/sort={}/order=desc/",
|
||||||
|
self.url,
|
||||||
|
category.cat_param(),
|
||||||
|
sort.sort_param()
|
||||||
|
);
|
||||||
|
if page <= 1 {
|
||||||
|
base
|
||||||
|
} else {
|
||||||
|
format!("{base}page/{page}/")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_tag_url(&self, slug: &str, page: u32) -> String {
|
||||||
|
let encoded_slug = utf8_percent_encode(slug, NON_ALPHANUMERIC).to_string();
|
||||||
|
if page <= 1 {
|
||||||
|
format!("{}/tags/{}/", self.url, encoded_slug)
|
||||||
|
} else {
|
||||||
|
format!("{}/tags/{}/page/{}/", self.url, encoded_slug, page)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_search_url(&self, query: &str, page: u32) -> String {
|
||||||
|
let encoded_query = utf8_percent_encode(query, NON_ALPHANUMERIC).to_string();
|
||||||
|
if page <= 1 {
|
||||||
|
format!(
|
||||||
|
"{}/index.php?do=search&subaction=search&story={}",
|
||||||
|
self.url, encoded_query
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"{}/index.php?do=search&subaction=search&search_start={}&full_search=0&story={}",
|
||||||
|
self.url, page, encoded_query
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_card(&self, card: ElementRef<'_>) -> Option<VideoItem> {
|
||||||
|
let title_sel = Self::selector("h3.thumb__title").ok()?;
|
||||||
|
let img_sel = Self::selector("img").ok()?;
|
||||||
|
|
||||||
|
let href = card.value().attr("href")?;
|
||||||
|
let id_re = Regex::new(r"/(\d+)-[^/]+\.html$").ok()?;
|
||||||
|
let id = id_re.captures(href)?.get(1)?.as_str().to_string();
|
||||||
|
|
||||||
|
let url = if href.starts_with("http") {
|
||||||
|
href.to_string()
|
||||||
|
} else {
|
||||||
|
self.absolute_url(href)
|
||||||
|
};
|
||||||
|
|
||||||
|
let title = card
|
||||||
|
.select(&title_sel)
|
||||||
|
.next()
|
||||||
|
.map(|e| Self::decode_html_entities(&Self::collapse_whitespace(&e.text().collect::<String>())))
|
||||||
|
.filter(|t| !t.is_empty())?;
|
||||||
|
|
||||||
|
let thumb = card
|
||||||
|
.select(&img_sel)
|
||||||
|
.next()
|
||||||
|
.and_then(|e| e.value().attr("src"))
|
||||||
|
.map(|s| {
|
||||||
|
if s.starts_with("http") {
|
||||||
|
s.to_string()
|
||||||
|
} else {
|
||||||
|
self.absolute_url(s)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.filter(|s| !s.is_empty())?;
|
||||||
|
|
||||||
|
// Gallery entries show a "N img" label instead of a duration; only keep real videos.
|
||||||
|
let card_html = card.html();
|
||||||
|
let dur_re = Regex::new(r#"thumb__label--time[^>]*>(\d{1,2}:\d{2}(?::\d{2})?)<"#).ok()?;
|
||||||
|
let duration_text = dur_re.captures(&card_html)?.get(1)?.as_str().to_string();
|
||||||
|
let duration = parse_duration_mm_ss(&duration_text);
|
||||||
|
|
||||||
|
Some(VideoItem::new(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
url,
|
||||||
|
CHANNEL_ID.to_string(),
|
||||||
|
thumb,
|
||||||
|
duration,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_video_items_from_html(&self, html: String) -> Result<Vec<VideoItem>> {
|
||||||
|
let document = Html::parse_document(&html);
|
||||||
|
let card_selector = Self::selector("a.thumb")?;
|
||||||
|
|
||||||
|
let mut items = Vec::new();
|
||||||
|
for card in document.select(&card_selector) {
|
||||||
|
if let Some(item) = self.parse_card(card) {
|
||||||
|
items.push(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(items)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_items_for_url(
|
||||||
|
&self,
|
||||||
|
cache: VideoCache,
|
||||||
|
url: String,
|
||||||
|
per_page_limit: usize,
|
||||||
|
options: &ServerOptions,
|
||||||
|
) -> Result<Vec<VideoItem>> {
|
||||||
|
if let Some((time, items)) = cache.get(&url) {
|
||||||
|
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
||||||
|
return Ok(items.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_items_for_url");
|
||||||
|
let html = timeout(
|
||||||
|
StdDuration::from_secs(10),
|
||||||
|
self.fetch_html(&mut requester, &url, &url),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| Error::from(format!("list request timed out for {url}")))??;
|
||||||
|
|
||||||
|
let list_items = self.get_video_items_from_html(html)?;
|
||||||
|
let limited_items = list_items
|
||||||
|
.into_iter()
|
||||||
|
.take(per_page_limit.max(1))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
if !limited_items.is_empty() {
|
||||||
|
cache.insert(url, limited_items.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(limited_items)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get(
|
||||||
|
&self,
|
||||||
|
cache: VideoCache,
|
||||||
|
page: u32,
|
||||||
|
sort: &str,
|
||||||
|
query: Option<&str>,
|
||||||
|
per_page_limit: usize,
|
||||||
|
options: ServerOptions,
|
||||||
|
) -> Result<Vec<VideoItem>> {
|
||||||
|
let target = self.resolve_target(&options, sort, query);
|
||||||
|
let url = self.build_url_for_target(&target, page);
|
||||||
|
self.fetch_items_for_url(cache, url, per_page_limit, &options)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse "MM:SS" or "HH:MM:SS" into total seconds.
|
||||||
|
fn parse_duration_mm_ss(text: &str) -> u32 {
|
||||||
|
let parts: Vec<u32> = text
|
||||||
|
.split(':')
|
||||||
|
.filter_map(|p| p.trim().parse().ok())
|
||||||
|
.collect();
|
||||||
|
match parts.as_slice() {
|
||||||
|
[h, m, s] => h * 3600 + m * 60 + s,
|
||||||
|
[m, s] => m * 60 + s,
|
||||||
|
[s] => *s,
|
||||||
|
_ => 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Provider for XlecxProvider {
|
||||||
|
async fn get_videos(
|
||||||
|
&self,
|
||||||
|
cache: VideoCache,
|
||||||
|
pool: DbPool,
|
||||||
|
sort: String,
|
||||||
|
query: Option<String>,
|
||||||
|
page: String,
|
||||||
|
per_page: String,
|
||||||
|
options: ServerOptions,
|
||||||
|
) -> Vec<VideoItem> {
|
||||||
|
let _ = pool;
|
||||||
|
let page = page.parse::<u32>().unwrap_or(1);
|
||||||
|
let per_page_limit = per_page.parse::<usize>().unwrap_or(RESULTS_PER_PAGE as usize);
|
||||||
|
let query_ref = query.as_deref().filter(|q| !q.trim().is_empty());
|
||||||
|
|
||||||
|
let result = self.get(cache, page, &sort, query_ref, per_page_limit, options).await;
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(videos) => videos,
|
||||||
|
Err(error) => {
|
||||||
|
report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await;
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||||
|
Some(self.build_channel(clientversion))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::util::cache::VideoCache;
|
||||||
|
use crate::util::requester::Requester;
|
||||||
|
|
||||||
|
fn provider() -> XlecxProvider {
|
||||||
|
XlecxProvider::new()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_archive_urls() {
|
||||||
|
let provider = provider();
|
||||||
|
assert_eq!(
|
||||||
|
provider.build_archive_url(Category::Videos, SortMode::Date, 1),
|
||||||
|
"https://xlecx.one/f/cat=18/sort=date/order=desc/"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
provider.build_archive_url(Category::Videos, SortMode::Date, 2),
|
||||||
|
"https://xlecx.one/f/cat=18/sort=date/order=desc/page/2/"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
provider.build_archive_url(Category::HentaiVideo, SortMode::Views, 1),
|
||||||
|
"https://xlecx.one/f/cat=20/sort=news_read/order=desc/"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_tag_urls() {
|
||||||
|
let provider = provider();
|
||||||
|
assert_eq!(
|
||||||
|
provider.build_tag_url("yuri", 1),
|
||||||
|
"https://xlecx.one/tags/yuri/"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
provider.build_tag_url("2d animation", 2),
|
||||||
|
"https://xlecx.one/tags/2d%20animation/page/2/"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_search_urls() {
|
||||||
|
let provider = provider();
|
||||||
|
assert_eq!(
|
||||||
|
provider.build_search_url("panty", 1),
|
||||||
|
"https://xlecx.one/index.php?do=search&subaction=search&story=panty"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
provider.build_search_url("panty", 2),
|
||||||
|
"https://xlecx.one/index.php?do=search&subaction=search&search_start=2&full_search=0&story=panty"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_duration() {
|
||||||
|
assert_eq!(parse_duration_mm_ss("02:25"), 145);
|
||||||
|
assert_eq!(parse_duration_mm_ss("0:29"), 29);
|
||||||
|
assert_eq!(parse_duration_mm_ss("1:05:30"), 3930);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_card_and_skips_galleries() {
|
||||||
|
let provider = provider();
|
||||||
|
let html = r#"
|
||||||
|
<div id="dle-content">
|
||||||
|
<a class="thumb d-flex fd-column grid-item thumb--model" href="https://xlecx.one/62329-panty-and-stocking.html">
|
||||||
|
<div class="thumb__desc">
|
||||||
|
<h3 class="thumb__title ws-nowrap">Panty and Stocking</h3>
|
||||||
|
</div>
|
||||||
|
<div class="thumb__img">
|
||||||
|
<img src="/uploads/posts/2026-07/weurwe8fhuif9wef.webp" alt="Panty and Stocking">
|
||||||
|
<div class="thumb__label thumb__label--time flex-grow-1">02:25</div>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
<a class="thumb d-flex fd-column grid-item thumb--model" href="https://xlecx.one/9999-some-comic.html">
|
||||||
|
<div class="thumb__desc">
|
||||||
|
<h3 class="thumb__title ws-nowrap">Some Comic</h3>
|
||||||
|
</div>
|
||||||
|
<div class="thumb__img">
|
||||||
|
<img src="/uploads/posts/2026-07/othercover.webp" alt="Some Comic">
|
||||||
|
<div class="thumb__label thumb__label--time flex-grow-1">57 img</div>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
"#;
|
||||||
|
let items = provider.get_video_items_from_html(html.to_string()).unwrap();
|
||||||
|
assert_eq!(items.len(), 1);
|
||||||
|
assert_eq!(items[0].id, "62329");
|
||||||
|
assert_eq!(items[0].title, "Panty and Stocking");
|
||||||
|
assert_eq!(items[0].duration, 145);
|
||||||
|
assert_eq!(items[0].url, "https://xlecx.one/62329-panty-and-stocking.html");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore]
|
||||||
|
async fn fetches_and_parses_archive() {
|
||||||
|
let provider = provider();
|
||||||
|
let options = ServerOptions {
|
||||||
|
client_version: None,
|
||||||
|
featured: None,
|
||||||
|
category: None,
|
||||||
|
sites: None,
|
||||||
|
filter: None,
|
||||||
|
language: None,
|
||||||
|
public_url_base: Some("http://127.0.0.1:18080".to_string()),
|
||||||
|
requester: Some(Requester::new()),
|
||||||
|
network: None,
|
||||||
|
stars: None,
|
||||||
|
categories: None,
|
||||||
|
duration: None,
|
||||||
|
sort: Some("new".to_string()),
|
||||||
|
sexuality: None,
|
||||||
|
};
|
||||||
|
let videos = provider.get(VideoCache::new(), 1, "new", None, 10, options).await.unwrap();
|
||||||
|
assert!(!videos.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user