Compare commits
12 Commits
edf6732f8e
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf0ae15f93 | ||
|
|
aa00abdd5b | ||
|
|
e5a2afcc70 | ||
|
|
6ba65f1457 | ||
|
|
b131910ea2 | ||
|
|
921ad93149 | ||
|
|
ad2abe0057 | ||
|
|
0aa8929f29 | ||
|
|
6c69b46395 | ||
|
|
f900987434 | ||
|
|
30271b9b7c | ||
|
|
a4364e5be3 |
@@ -59,7 +59,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
|
|||||||
| `redtube` | `mainstream-tube` | no | no | Mainstream archive. |
|
| `redtube` | `mainstream-tube` | no | no | Mainstream archive. |
|
||||||
| `ripnsfw` | `onlyfans` | no | yes | Leaked-creator aggregator for ripnsfw.com — the entire catalogue (~2.5k rows) is a single client-side-rendered page whose data source is a published Google Sheet exported as CSV (`GET /data.csv`, no auth/CF gate); the provider fetches and parses that CSV directly instead of scraping any HTML, replicating the site's own JS `parseCSV`/`csvSplit` column layout (`THUMB,NEW,NOME,DOODSTREAM,LULUSTREAM,<link3>,DOOD,<guests>,SITE(bunkr links),TITLE,ID,#(tags),POST,,DATE,SIZE`). Feed/search/pagination/sort are all done in-memory over the fetched rows (no native site pagination exists — `perPage`-sized pages are sliced locally), with a 180s in-process cache to avoid re-fetching the CSV on every request. `tag:`/`category:`/`cat:` and `model:`/`uploader:` query prefixes filter by the parsed tag list / model name; bare keywords substring-match model name, guests, and tags. `video.url` is the site's client-rendered `https://ripnsfw.com/model/{slug}/{id}` route — the host has no server-side SPA fallback, so this 404s to any plain HTTP client (its custom 404 page stashes the path and JS-redirects to `/`, which restores the route — works in a real browser/webview only); `check.py`'s `_CF_PROTECTED_HOSTS` allowlist includes `ripnsfw.com` for this reason. Media is never hosted on ripnsfw.com itself: each row carries a Doodstream and/or Lulustream embed link, resolved into `formats[]` (Lulustream first, matching the site's own default-player order) via the **existing, previously-unused** `src/proxies/doodstream.rs` / `src/proxies/lulustream.rs` redirect proxies (`build_proxy_url(&options, "doodstream"/"lulustream", &strip_url_scheme(link))`) — no new proxy code was written. Both proxies resolve correctly to real signed CDN URLs (verified against live data), but neither CDN's final edge is reachable from this repo's dev sandbox: Lulustream's `*.tnmr.org` edge 403s datacenter/proxy IPs regardless of UA or TLS impersonation, and Doodstream's final `*.dood.video` edge resolves to a loopback/refused address from non-residential egress — both are IP-reputation-style CDN blocks, not code defects (same class of sandbox limitation already documented for `camsoda`/`hentaihaven`/`animeidhentai`), so `check.py` allowlists both by suffix (`_CF_PROTECTED_SUFFIXES`). No thumbnail proxy needed (`cdn.ripnsfw.com/thumbs/...` is directly hotlinkable, no CF/referer gate). No `/api/uploaders` (model pages exist but have no dedicated profile/stats schema confirmed). |
|
| `ripnsfw` | `onlyfans` | no | yes | Leaked-creator aggregator for ripnsfw.com — the entire catalogue (~2.5k rows) is a single client-side-rendered page whose data source is a published Google Sheet exported as CSV (`GET /data.csv`, no auth/CF gate); the provider fetches and parses that CSV directly instead of scraping any HTML, replicating the site's own JS `parseCSV`/`csvSplit` column layout (`THUMB,NEW,NOME,DOODSTREAM,LULUSTREAM,<link3>,DOOD,<guests>,SITE(bunkr links),TITLE,ID,#(tags),POST,,DATE,SIZE`). Feed/search/pagination/sort are all done in-memory over the fetched rows (no native site pagination exists — `perPage`-sized pages are sliced locally), with a 180s in-process cache to avoid re-fetching the CSV on every request. `tag:`/`category:`/`cat:` and `model:`/`uploader:` query prefixes filter by the parsed tag list / model name; bare keywords substring-match model name, guests, and tags. `video.url` is the site's client-rendered `https://ripnsfw.com/model/{slug}/{id}` route — the host has no server-side SPA fallback, so this 404s to any plain HTTP client (its custom 404 page stashes the path and JS-redirects to `/`, which restores the route — works in a real browser/webview only); `check.py`'s `_CF_PROTECTED_HOSTS` allowlist includes `ripnsfw.com` for this reason. Media is never hosted on ripnsfw.com itself: each row carries a Doodstream and/or Lulustream embed link, resolved into `formats[]` (Lulustream first, matching the site's own default-player order) via the **existing, previously-unused** `src/proxies/doodstream.rs` / `src/proxies/lulustream.rs` redirect proxies (`build_proxy_url(&options, "doodstream"/"lulustream", &strip_url_scheme(link))`) — no new proxy code was written. Both proxies resolve correctly to real signed CDN URLs (verified against live data), but neither CDN's final edge is reachable from this repo's dev sandbox: Lulustream's `*.tnmr.org` edge 403s datacenter/proxy IPs regardless of UA or TLS impersonation, and Doodstream's final `*.dood.video` edge resolves to a loopback/refused address from non-residential egress — both are IP-reputation-style CDN blocks, not code defects (same class of sandbox limitation already documented for `camsoda`/`hentaihaven`/`animeidhentai`), so `check.py` allowlists both by suffix (`_CF_PROTECTED_SUFFIXES`). No thumbnail proxy needed (`cdn.ripnsfw.com/thumbs/...` is directly hotlinkable, no CF/referer gate). No `/api/uploaders` (model pages exist but have no dedicated profile/stats schema confirmed). |
|
||||||
| `rule34gen` | `ai` | no | no | AI group example. |
|
| `rule34gen` | `ai` | no | no | AI group example. |
|
||||||
| `rule34video` | `hentai-animation` | no | no | Hentai group example. |
|
| `rule34video` | `hentai-animation` | no | no | Hentai group example. **Archives:** artists are `/models/{slug}/`, franchises/genres `/categories/{slug}/`, characters and everything else `/tags/{slug}/`; each is paged and sorted through `?mode=async&function=get_block&block_id=custom_list_videos_common_videos&sort_by={sort}&from={page}` (unknown slugs 404). Prefix queries `artist:`/`model:`/`creator:`, `category:`/`franchise:`/`series:`, `tag:`/`character:`; a bare query is probed as artist, then category, then tag, then resolved through `/search_ajax.php?tag=` (characters are tagged `name (franchise)`; an unambiguous exact base-name match wins), else keyword search. Cards are enriched from the detail page, which also yields the credited artist as `uploader` (`rule34video:{slug}`) and artist names as tags. |
|
||||||
| `sextb` | `jav` | no | no | JAV family provider. sextb.net is Cloudflare-gated, so listing/pagination markdown is fetched through a hand-built Jina mirror URL (`mirror_url()`); the target is already an absolute `https://sextb.net/...` URL, so the scheme must be stripped before prepending the `https://r.jina.ai/http://` prefix — concatenating them directly (the pre-fix bug) produces a malformed `.../http://https://...` URL that Jina rejects outright. |
|
| `sextb` | `jav` | no | no | JAV family provider. sextb.net is Cloudflare-gated, so listing/pagination markdown is fetched through a hand-built Jina mirror URL (`mirror_url()`); the target is already an absolute `https://sextb.net/...` URL, so the scheme must be stripped before prepending the `https://r.jina.ai/http://` prefix — concatenating them directly (the pre-fix bug) produces a malformed `.../http://https://...` URL that Jina rejects outright. |
|
||||||
| `shooshtime` | `onlyfans` | no | yes | Redirect proxy plus dedicated media route. Listing cards are `div.card` (card→`a.title` for the link/title, `a.image img` for thumb/preview, `div.duration`, `div.list div.box` for views+date, `div.name a.text` for uploader) — the site redesigned from an older `div.item.thumb`/`span.description` layout, so re-verify these selectors against `{BASE_URL}/videos/` (the real default-sort URL, not the bare homepage) if the feed ever goes empty again. |
|
| `shooshtime` | `onlyfans` | no | yes | Redirect proxy plus dedicated media route. Listing cards are `div.card` (card→`a.title` for the link/title, `a.image img` for thumb/preview, `div.duration`, `div.list div.box` for views+date, `div.name a.text` for uploader) — the site redesigned from an older `div.item.thumb`/`span.description` layout, so re-verify these selectors against `{BASE_URL}/videos/` (the real default-sort URL, not the bare homepage) if the feed ever goes empty again. |
|
||||||
| `spankbang` | `mainstream-tube` | no | yes | Best template for redirect proxy plus anti-bot fetches. |
|
| `spankbang` | `mainstream-tube` | no | yes | Best template for redirect proxy plus anti-bot fetches. |
|
||||||
@@ -83,7 +83,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
|
|||||||
| `fullporner` | `mainstream-tube` | no | no | HTML scraper for fullporner.com; thumbnail IDs derived from `/thumb/{id}.jpg` URLs and used to build direct `xiaoshenke.net/vid/{id}/720` media redirect URLs (Referer + User-Agent headers required); supports cat:/category:/pornstar:/star: shortcut queries; no proxy needed. |
|
| `fullporner` | `mainstream-tube` | no | no | HTML scraper for fullporner.com; thumbnail IDs derived from `/thumb/{id}.jpg` URLs and used to build direct `xiaoshenke.net/vid/{id}/720` media redirect URLs (Referer + User-Agent headers required); supports cat:/category:/pornstar:/star: shortcut queries; no proxy needed. |
|
||||||
| `cumlouder` | `mainstream-tube` | no | no | HTML scraper for cumlouder.com (Spanish/amateur tube, same platform family as superporn). Cards are `a.muestra-escena[href^="/porn-video/"]` with lazy thumb in `img[data-src]` (protocol-relative `//th0cdn.cumlouder.com/...`, direct, no Referer/proxy), `h2` title, `.vistas` views, `.minutos` duration (`06:14 m`); relative date only ("2 years ago"), so `uploadedAt` is not set. Default feed `/porn/` (page N≥2 is the bare `/N/` path, not `/porn/N/`); category `/porn-videos/{slug}/{N}/`; tag `/tag/{slug}/{N}/`; series `/series/{slug}/{N}/`. **The site's own `/search/` (POST `form[search]` or `?q=`) is behind a Cloudflare managed challenge — 403 even in a real browser — so keyword queries are routed instead: a query matching a hardcoded category (72, scraped from `/categories/`) goes to the category archive, anything else to `/tag/{slug}/` (unknown tags 404 → empty list).** Pornstar pages live at `/girl/{slug}/{N}/` (same cards/pagination; unknown slug = fast 404). A free-form keyword that is not a category is tried as a pornstar page first, then as a tag archive; `girl:`/`pornstar:`/`star:`/`model:` force the pornstar page. Supports `cat:`/`category:`, `tag:`, `series:` prefixes and the `categories` option. A Cloudflare challenge on the raw listing fetch is handed to the requester's FlareSolverr chain. `video.url` is the detail page, which yt-dlp resolves natively via the generic html5 extractor (the page embeds a signed `m5cdnst.cumlouder.com/*.mp4?secure=…` `<source>`, no Referer needed), so no `formats`/enrichment. The origin intermittently answers a first, uncached request with a Cloudflare 522 (a retry succeeds) — the listing fetch retries up to 3×, and `check.py`/yt-dlp may show a one-off `HTTP=-1`/522 on a cold detail page that is not a provider fault. No proxy needed. |
|
| `cumlouder` | `mainstream-tube` | no | no | HTML scraper for cumlouder.com (Spanish/amateur tube, same platform family as superporn). Cards are `a.muestra-escena[href^="/porn-video/"]` with lazy thumb in `img[data-src]` (protocol-relative `//th0cdn.cumlouder.com/...`, direct, no Referer/proxy), `h2` title, `.vistas` views, `.minutos` duration (`06:14 m`); relative date only ("2 years ago"), so `uploadedAt` is not set. Default feed `/porn/` (page N≥2 is the bare `/N/` path, not `/porn/N/`); category `/porn-videos/{slug}/{N}/`; tag `/tag/{slug}/{N}/`; series `/series/{slug}/{N}/`. **The site's own `/search/` (POST `form[search]` or `?q=`) is behind a Cloudflare managed challenge — 403 even in a real browser — so keyword queries are routed instead: a query matching a hardcoded category (72, scraped from `/categories/`) goes to the category archive, anything else to `/tag/{slug}/` (unknown tags 404 → empty list).** Pornstar pages live at `/girl/{slug}/{N}/` (same cards/pagination; unknown slug = fast 404). A free-form keyword that is not a category is tried as a pornstar page first, then as a tag archive; `girl:`/`pornstar:`/`star:`/`model:` force the pornstar page. Supports `cat:`/`category:`, `tag:`, `series:` prefixes and the `categories` option. A Cloudflare challenge on the raw listing fetch is handed to the requester's FlareSolverr chain. `video.url` is the detail page, which yt-dlp resolves natively via the generic html5 extractor (the page embeds a signed `m5cdnst.cumlouder.com/*.mp4?secure=…` `<source>`, no Referer needed), so no `formats`/enrichment. The origin intermittently answers a first, uncached request with a Cloudflare 522 (a retry succeeds) — the listing fetch retries up to 3×, and `check.py`/yt-dlp may show a one-off `HTTP=-1`/522 on a cold detail page that is not a provider fault. No proxy needed. |
|
||||||
| `thepornbunny` | `mainstream-tube` | no | yes | KVS-style HTML scraper for thepornbunny.com; 24 items per site page; thumbnails at `https://www.thepornbunny.com/images/thumb/{id}.webp` from `data-original` attribute (no proxy needed); studio exposed as uploader; pornstar names in tags; `/proxy/thepornbunny/{slug}` fetches the video page, extracts `generate_mp4(enc_data, key, rnd, video_id)` args, decrypts `enc_data` via PBKDF2-HMAC-SHA512+AES-256-CBC to get an OK.ru session key, calls `api.ok.ru/fb.do?method=video.get&session_key=KEY&vids=RND` to get signed CDN URLs, and returns 302 to the best-quality okcdn.ru/vkuser.net MP4 URL (no special client headers needed); supports sort: new/popular/rated, 20 hardcoded categories via `categories` option, and tag:/category:/studio:/pornstar: query shortcuts. |
|
| `thepornbunny` | `mainstream-tube` | no | yes | KVS-style HTML scraper for thepornbunny.com; 24 items per site page; thumbnails at `https://www.thepornbunny.com/images/thumb/{id}.webp` from `data-original` attribute (no proxy needed); studio exposed as uploader; pornstar names in tags; `/proxy/thepornbunny/{slug}` fetches the video page, extracts `generate_mp4(enc_data, key, rnd, video_id)` args, decrypts `enc_data` via PBKDF2-HMAC-SHA512+AES-256-CBC to get an OK.ru session key, calls `api.ok.ru/fb.do?method=video.get&session_key=KEY&vids=RND` to get signed CDN URLs, and returns 302 to the best-quality okcdn.ru/vkuser.net MP4 URL (no special client headers needed); supports sort: new/popular/rated, 20 hardcoded categories via `categories` option, and tag:/category:/studio:/pornstar: query shortcuts. |
|
||||||
| `eporner` | `mainstream-tube` | no | no | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); supports sort: new/popular/rated/best; 65 hardcoded categories, plus `cat:`/`category:`, `tag:`, `pornstar:`/`star:`, `uploader:`/`profile:` query-prefix routing to the matching archive, a bare-keyword category-name match, and a background-loaded pornstar name→URL map from `/pornstar-list/` (falls back to `/tag/{slug}` keyword search otherwise). `video.url` is the detail page — **not** yt-dlp-resolvable and not directly playable on its own: opening it bare serves a small restricted/preview clip. Real playback needs per-card enrichment (bounded `buffer_unordered(6)`): fetch the detail page for its 32-hex `hash`, derive the site's own hashed token (`calc_hash` — split into four 8-hex chunks, each parsed as u32 and base36-encoded, concatenated; reverse-engineered from the player JS), then call the site's own `GET /xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false` JSON API for direct per-resolution mp4 URLs into `formats[]`. Separately from the hash dance, the resolved CDN mp4 URLs enforce their own Referer check: a request with no Referer or one pointing at eporner.com gets the real file, but any other non-empty Referer gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the exact "this video is only available at eporner.com"-style short clip a client sees if its default (or app-injected) Referer doesn't match, so every format explicitly declares `http_headers: {Referer: "https://www.eporner.com/"}` rather than relying on the client's default. No proxy needed (formats are direct CDN URLs, just header-gated). **2026-09-21:** the `calc_hash`/hash-regex/`alnum_id_from_href`/`resolve_formats` logic moved to `src/util/eporner_resolve.rs` so `melonstube` (a meta-search aggregator that redirects out to eporner.com for ~34% of its own cards) can reuse the exact same resolution without a cross-provider `use`, which would break either provider's `HOT_TUB_PROVIDER=<id>` single-provider build (only the one selected provider module gets compiled; `src/util/*` always does). `eporner.rs` itself now just calls `crate::util::eporner_resolve::resolve_formats(...)`. |
|
| `eporner` | `mainstream-tube` | no | yes | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); supports sort: new/popular/rated/best; 65 hardcoded categories, plus `cat:`/`category:`, `tag:`, `pornstar:`/`star:`, `uploader:`/`profile:` query-prefix routing to the matching archive, a bare-keyword category-name match, and a background-loaded pornstar name→URL map from `/pornstar-list/` (falls back to `/tag/{slug}` keyword search otherwise). `video.url` is the detail page — **not** yt-dlp-resolvable and not directly playable on its own: opening it bare serves a small restricted/preview clip. Real playback needs per-card enrichment (bounded `buffer_unordered(6)`): fetch the detail page for its 32-hex `hash`, derive the site's own hashed token (`calc_hash` — split into four 8-hex chunks, each parsed as u32 and base36-encoded, concatenated; reverse-engineered from the player JS), then call the site's own `GET /xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false` JSON API for direct per-resolution mp4 URLs into `formats[]`. Separately from the hash dance, the resolved CDN mp4 URLs enforce their own Referer check: a request with no Referer or one pointing at eporner.com gets the real file, but any other non-empty Referer gets a 302 to a static `.../na.mp4` ("not available") decoy clip — the exact "this video is only available at eporner.com"-style short clip a client sees if its default (or app-injected) Referer doesn't match, so every format explicitly declares `http_headers: {Referer: "https://www.eporner.com/"}` rather than relying on the client's default. No proxy needed (formats are direct CDN URLs, just header-gated). **2026-09-21:** the `calc_hash`/hash-regex/`alnum_id_from_href`/`resolve_formats` logic moved to `src/util/eporner_resolve.rs` so `melonstube` (a meta-search aggregator that redirects out to eporner.com for ~34% of its own cards) can reuse the exact same resolution without a cross-provider `use`, which would break either provider's `HOT_TUB_PROVIDER=<id>` single-provider build (only the one selected provider module gets compiled; `src/util/*` always does). `eporner.rs` itself now just calls `crate::util::eporner_resolve::resolve_formats(...)`. **2026-10-22:** the signed CDN URLs expire exactly two hours after the `/xhr/video` call (the leading number in `/{expiry}_{ip}_{n}/`) and an expired link gets the same `na.mp4` decoy, so favourites, history and long-open lists played the one-minute clip. The links are also **IP-locked**: checked from foreign networks (check-host.net nodes) a fresh link answers 302 → `na.mp4`, while the resolving IP gets the file, so a redirect only works for a player behind the server's own public IP (a Jina fetch had misleadingly suggested otherwise). Formats therefore point at the streaming proxy `/proxy/eporner/{alnum_id}/{height}.mp4`: it resolves (cached 90 min per id/height, re-resolved when the CDN answers with a redirect), forwards `Range`, and returns a sized body so `206` responses keep `Content-Length` for AVPlayer. Not a redirect proxy, so app clients keep these in `formats`. All eporner playback bandwidth now flows through the server. melonstube's eporner destinations use the same `resolve_proxied_formats`. |
|
||||||
| `xnxx` | `mainstream-tube` | no | no | HTML scraper for xnxx.com (10M+ videos); unified card parser handles two formats: `div.thumb-block[data-eid]` (search) and `div.thumb-block.video[data-video='{"id":...}']` (hits); eid extracted from `/video-{eid}/{slug}` URL path; thumbnails at `thumb-cdn77.xnxx-cdn.com` and `thumbs-gcore.xnxx-cdn.com` (no proxy, no Referer needed); 0-indexed pagination (page 1 = `/hits`, page N = `/hits/{N-1}`); default feed is `/hits` (most-viewed — xnxx has no chronological listing); search via `/search/{slug}` (works for keywords and tags); supports `tag:`, `cat:`, `category:` query shortcuts; yt-dlp resolves `video.url` natively (XNXX extractor, returns 4-7 HLS formats); no proxy needed. |
|
| `xnxx` | `mainstream-tube` | no | no | HTML scraper for xnxx.com (10M+ videos); unified card parser handles two formats: `div.thumb-block[data-eid]` (search) and `div.thumb-block.video[data-video='{"id":...}']` (hits); eid extracted from `/video-{eid}/{slug}` URL path; thumbnails at `thumb-cdn77.xnxx-cdn.com` and `thumbs-gcore.xnxx-cdn.com` (no proxy, no Referer needed); 0-indexed pagination (page 1 = `/hits`, page N = `/hits/{N-1}`); default feed is `/hits` (most-viewed — xnxx has no chronological listing); search via `/search/{slug}` (works for keywords and tags); supports `tag:`, `cat:`, `category:` query shortcuts; yt-dlp resolves `video.url` natively (XNXX extractor, returns 4-7 HLS formats); no proxy needed. |
|
||||||
| `xgroovy` | `mainstream-tube` | no | no | KVS (Kernel Video Sharing)-style HTML scraper for xgroovy.com; card selector `div.item[data-video-id]` with `data-views`/`data-rating` as clean numeric attributes directly on the card (no text-parsing needed, unlike xhamster); title/url from `a.popito[href] > strong.title`; thumbnails at `i.xgroovy.com` (no proxy, no Referer needed) from `img.thumb[data-jpg|src]`, preview clip from `img[data-preview]`; duration rendered as free text ("N min"/"N sec", not colon-separated) via a bespoke `parse_duration` helper; path-segment pagination (`/new/` = page 1, `/new/2/` = page 2, unlike xhamster's `?page=N` query string); feeds: `/new/` (default), categories via `/categories/{slug}/` (73 hardcoded categories as `categories` option), pornstars via `/pornstars/{slug}/`, channels via `/channels/{slug}/`; search via `/search/{dashed-query}/` (multi-word queries are lowercased and dash-joined); uploader kind inferred from URL path (`/pornstars/` vs `/channels/`) via `div.author-link > a`; supports `cat:`/`category:` query shortcuts plus static category name matching; yt-dlp resolves `video.url` via the generic html5-embed extractor (no dedicated xgroovy extractor, but resolves cleanly to a 1080p format); no proxy needed. |
|
| `xgroovy` | `mainstream-tube` | no | no | KVS (Kernel Video Sharing)-style HTML scraper for xgroovy.com; card selector `div.item[data-video-id]` with `data-views`/`data-rating` as clean numeric attributes directly on the card (no text-parsing needed, unlike xhamster); title/url from `a.popito[href] > strong.title`; thumbnails at `i.xgroovy.com` (no proxy, no Referer needed) from `img.thumb[data-jpg|src]`, preview clip from `img[data-preview]`; duration rendered as free text ("N min"/"N sec", not colon-separated) via a bespoke `parse_duration` helper; path-segment pagination (`/new/` = page 1, `/new/2/` = page 2, unlike xhamster's `?page=N` query string); feeds: `/new/` (default), categories via `/categories/{slug}/` (73 hardcoded categories as `categories` option), pornstars via `/pornstars/{slug}/`, channels via `/channels/{slug}/`; search via `/search/{dashed-query}/` (multi-word queries are lowercased and dash-joined); uploader kind inferred from URL path (`/pornstars/` vs `/channels/`) via `div.author-link > a`; supports `cat:`/`category:` query shortcuts plus static category name matching; yt-dlp resolves `video.url` via the generic html5-embed extractor (no dedicated xgroovy extractor, but resolves cleanly to a 1080p format); no proxy needed. |
|
||||||
| `xhamster` | `mainstream-tube` | no | no | HTML scraper for xhamster.com; card selector `div[data-video-type="video"]` with `data-video-id`; thumbnails via `img[data-role="thumb-preview-img"]` at `ic-vt-nss.xhcdn.com` (no proxy, no Referer needed); pagination via `?page=N` query param (browse feeds use infinite-scroll so only search reliably returns different content per page); feeds: `/newest` (default), `/most-viewed`, `/best`; categories via `/categories/{slug}`; channels via `/channels/{slug}`; 43 hardcoded categories as `categories` option; uploader type inferred from URL path (`/channels/` → channel, `/creators/` → creator, `/pornstars/` → pornstar); supports `cat:`/`category:` and `channel:` query shortcuts, plus static category name matching; preview mp4 clips from `data-previewvideo` attribute; yt-dlp resolves `video.url` natively (xHamster extractor, 28 formats); no proxy needed. |
|
| `xhamster` | `mainstream-tube` | no | no | HTML scraper for xhamster.com; card selector `div[data-video-type="video"]` with `data-video-id`; thumbnails via `img[data-role="thumb-preview-img"]` at `ic-vt-nss.xhcdn.com` (no proxy, no Referer needed); pagination via `?page=N` query param (browse feeds use infinite-scroll so only search reliably returns different content per page); feeds: `/newest` (default), `/most-viewed`, `/best`; categories via `/categories/{slug}`; channels via `/channels/{slug}`; 43 hardcoded categories as `categories` option; uploader type inferred from URL path (`/channels/` → channel, `/creators/` → creator, `/pornstars/` → pornstar); supports `cat:`/`category:` and `channel:` query shortcuts, plus static category name matching; preview mp4 clips from `data-previewvideo` attribute; yt-dlp resolves `video.url` natively (xHamster extractor, 28 formats); no proxy needed. |
|
||||||
@@ -126,6 +126,7 @@ These resolve a provider-specific input into a `302 Location`.
|
|||||||
These return binary media or images, sometimes rewriting manifests or forwarding cookies/referers.
|
These return binary media or images, sometimes rewriting manifests or forwarding cookies/referers.
|
||||||
|
|
||||||
- `/proxy/shooshtime-media/{endpoint}*`
|
- `/proxy/shooshtime-media/{endpoint}*`
|
||||||
|
- `/proxy/eporner/{alnum_id}/{height}.mp4` (streams the IP-locked signed mp4 with Range passthrough; link cached 90 min)
|
||||||
- `/proxy/noodlemagazine/{endpoint}*`
|
- `/proxy/noodlemagazine/{endpoint}*`
|
||||||
- `/proxy/noodlemagazine-thumb/{endpoint}*`
|
- `/proxy/noodlemagazine-thumb/{endpoint}*`
|
||||||
- `/proxy/hanime-cdn/{endpoint}*`
|
- `/proxy/hanime-cdn/{endpoint}*`
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ These rules cut across every step below. They are intentionally grouped at the t
|
|||||||
|
|
||||||
### Output Shape
|
### Output Shape
|
||||||
|
|
||||||
|
- **Verified Hot Tub app clients cannot follow redirects in `formats[].url`.** Redirect proxies (`REDIRECT_PROXIES` in `src/proxy.rs`) are for `video.url` in the app response; `api.rs` moves such formats to `url` for verified app clients as a safety net. Other clients keep the redirect-proxy URLs in `formats`.
|
||||||
- Use `formats` for real media URLs (HLS, multi-quality, signed token), and keep `video.url` on a stable page URL. Never set `video.url` to a tokenized stream that needs a `Referer` to play — the `url` field has no header contract.
|
- Use `formats` for real media URLs (HLS, multi-quality, signed token), and keep `video.url` on a stable page URL. Never set `video.url` to a tokenized stream that needs a `Referer` to play — the `url` field has no header contract.
|
||||||
- If thumbnails need a `Referer` and the player has no per-thumb header mechanism, proxy them via `/proxy/<id>-thumb/...`. The same applies to manifests with relative URIs.
|
- If thumbnails need a `Referer` and the player has no per-thumb header mechanism, proxy them via `/proxy/<id>-thumb/...`. The same applies to manifests with relative URIs.
|
||||||
- Keep filter `id` values stable and machine-targeted. Display `title`s change; ids route.
|
- Keep filter `id` values stable and machine-targeted. Display `title`s change; ids route.
|
||||||
|
|||||||
82
src/api.rs
82
src/api.rs
@@ -714,6 +714,15 @@ async fn videos_post(
|
|||||||
.collect();
|
.collect();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The Hot Tub app cannot follow a redirect in `formats[].url`: for verified
|
||||||
|
// app clients, formats that point at a redirecting `/proxy/...` become the
|
||||||
|
// item's `url` instead.
|
||||||
|
if clientversion.is_verified_hottub() {
|
||||||
|
for video in video_items.iter_mut() {
|
||||||
|
move_redirect_formats_to_url(video);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(literal_query) = literal_query.as_deref() {
|
if let Some(literal_query) = literal_query.as_deref() {
|
||||||
#[cfg(feature = "debug")]
|
#[cfg(feature = "debug")]
|
||||||
let before = video_items.len();
|
let before = video_items.len();
|
||||||
@@ -794,6 +803,30 @@ async fn videos_post(
|
|||||||
Ok(web::HttpResponse::Ok().json(&videos))
|
Ok(web::HttpResponse::Ok().json(&videos))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Moves formats that point at a redirecting proxy to `video.url`.
|
||||||
|
///
|
||||||
|
/// If direct (non-redirecting) formats remain they are kept and `url` is left
|
||||||
|
/// alone; otherwise `url` becomes the first (best-listed) redirecting proxy URL
|
||||||
|
/// and `formats` is dropped entirely.
|
||||||
|
pub(crate) fn move_redirect_formats_to_url(video: &mut VideoItem) {
|
||||||
|
let Some(formats) = video.formats.take() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let (redirecting, direct): (Vec<_>, Vec<_>) = formats
|
||||||
|
.into_iter()
|
||||||
|
.partition(|format| crate::proxy::is_redirect_proxy_url(&format.url));
|
||||||
|
if redirecting.is_empty() {
|
||||||
|
video.formats = Some(direct);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if direct.is_empty() {
|
||||||
|
video.url = redirecting[0].url.clone();
|
||||||
|
video.formats = None;
|
||||||
|
} else {
|
||||||
|
video.formats = Some(direct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn uploaders_post(
|
async fn uploaders_post(
|
||||||
uploader_request: web::types::Json<UploadersRequest>,
|
uploader_request: web::types::Json<UploadersRequest>,
|
||||||
cache: web::types::State<VideoCache>,
|
cache: web::types::State<VideoCache>,
|
||||||
@@ -1082,4 +1115,53 @@ mod tests {
|
|||||||
assert_eq!(item.uploader.as_deref(), Some("FreeusePorn"));
|
assert_eq!(item.uploader.as_deref(), Some("FreeusePorn"));
|
||||||
assert_eq!(item.formats.as_ref().map(|formats| formats.len()), Some(2));
|
assert_eq!(item.formats.as_ref().map(|formats| formats.len()), Some(2));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn item_with_formats(urls: &[&str]) -> VideoItem {
|
||||||
|
let mut item = VideoItem::new(
|
||||||
|
"id".to_string(),
|
||||||
|
"t".to_string(),
|
||||||
|
"https://site.example/watch/1".to_string(),
|
||||||
|
"c".to_string(),
|
||||||
|
"https://thumb".to_string(),
|
||||||
|
10,
|
||||||
|
);
|
||||||
|
item.formats = Some(
|
||||||
|
urls.iter()
|
||||||
|
.map(|u| crate::videos::VideoFormat::new(u.to_string(), "q".to_string(), "mp4".to_string()))
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
item
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redirect_only_formats_move_to_url() {
|
||||||
|
let mut item = item_with_formats(&[
|
||||||
|
"http://h/proxy/hqporner/1080",
|
||||||
|
"http://h/proxy/hqporner/720",
|
||||||
|
]);
|
||||||
|
move_redirect_formats_to_url(&mut item);
|
||||||
|
assert_eq!(item.url, "http://h/proxy/hqporner/1080");
|
||||||
|
assert!(item.formats.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn direct_formats_are_kept_and_redirect_ones_dropped() {
|
||||||
|
let mut item = item_with_formats(&[
|
||||||
|
"http://h/proxy/hqporner/1080",
|
||||||
|
"https://cdn.example/v.mp4",
|
||||||
|
]);
|
||||||
|
move_redirect_formats_to_url(&mut item);
|
||||||
|
assert_eq!(item.url, "https://site.example/watch/1");
|
||||||
|
let formats = item.formats.expect("direct format kept");
|
||||||
|
assert_eq!(formats.len(), 1);
|
||||||
|
assert_eq!(formats[0].url, "https://cdn.example/v.mp4");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_proxy_formats_are_untouched() {
|
||||||
|
let mut item = item_with_formats(&["http://h/proxy/noodlemagazine/x.mp4"]);
|
||||||
|
move_redirect_formats_to_url(&mut item);
|
||||||
|
assert_eq!(item.url, "https://site.example/watch/1");
|
||||||
|
assert_eq!(item.formats.map(|f| f.len()), Some(1));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -799,6 +799,39 @@ impl ArchivebateProvider {
|
|||||||
.map(|value| value.to_string())
|
.map(|value| value.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The proxied format needs no network, so an item always plays through
|
||||||
|
/// it even when the best-effort header enrichment below times out.
|
||||||
|
fn with_default_format(item: VideoItem, options: &ServerOptions) -> VideoItem {
|
||||||
|
let format_url = Self::proxied_video(options, &item.url);
|
||||||
|
if format_url.is_empty() || item.formats.is_some() {
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
let mut item = item;
|
||||||
|
item.formats = Some(vec![VideoFormat::new(
|
||||||
|
format_url,
|
||||||
|
"source".to_string(),
|
||||||
|
"mp4".to_string(),
|
||||||
|
)]);
|
||||||
|
item
|
||||||
|
}
|
||||||
|
|
||||||
|
/// For verified Hot Tub app clients: no `formats` at all. The item's `url`
|
||||||
|
/// is the lazy `/proxy/archivebate` redirect, which the app plays directly
|
||||||
|
/// (it resolves the media at playback time).
|
||||||
|
fn proxy_as_item_url(videos: Vec<VideoItem>, options: &ServerOptions) -> Vec<VideoItem> {
|
||||||
|
videos
|
||||||
|
.into_iter()
|
||||||
|
.map(|mut video| {
|
||||||
|
let proxy_url = Self::proxied_video(options, &video.url);
|
||||||
|
if !proxy_url.is_empty() {
|
||||||
|
video.url = proxy_url;
|
||||||
|
}
|
||||||
|
video.formats = None;
|
||||||
|
video
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
async fn enrich_video(&self, item: VideoItem, options: &ServerOptions) -> VideoItem {
|
async fn enrich_video(&self, item: VideoItem, options: &ServerOptions) -> VideoItem {
|
||||||
let page_url = item.url.clone();
|
let page_url = item.url.clone();
|
||||||
let format_url = Self::proxied_video(options, &page_url);
|
let format_url = Self::proxied_video(options, &page_url);
|
||||||
@@ -1148,11 +1181,15 @@ impl ArchivebateProvider {
|
|||||||
platform: candidate.platform.clone(),
|
platform: candidate.platform.clone(),
|
||||||
gender: candidate.gender.clone(),
|
gender: candidate.gender.clone(),
|
||||||
};
|
};
|
||||||
let items = self
|
// A candidate whose profile page is gone (the search index lists
|
||||||
|
// deleted profiles) must not abort the whole search: try the next.
|
||||||
|
match self
|
||||||
.fetch_items_for_target(cache.clone(), &target, 1, per_page, &options)
|
.fetch_items_for_target(cache.clone(), &target, 1, per_page, &options)
|
||||||
.await?;
|
.await
|
||||||
if !items.is_empty() {
|
{
|
||||||
return Ok(items);
|
Ok(items) if !items.is_empty() => return Ok(items),
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(_) => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1188,6 +1225,14 @@ impl Provider for ArchivebateProvider {
|
|||||||
if videos.is_empty() {
|
if videos.is_empty() {
|
||||||
return videos;
|
return videos;
|
||||||
}
|
}
|
||||||
|
let is_app_client = options
|
||||||
|
.client_version
|
||||||
|
.as_ref()
|
||||||
|
.map(ClientVersion::is_verified_hottub)
|
||||||
|
.unwrap_or(false);
|
||||||
|
if is_app_client {
|
||||||
|
return Self::proxy_as_item_url(videos, &options);
|
||||||
|
}
|
||||||
stream::iter(videos.into_iter().map(|video| {
|
stream::iter(videos.into_iter().map(|video| {
|
||||||
let provider = self.clone();
|
let provider = self.clone();
|
||||||
let options = options.clone();
|
let options = options.clone();
|
||||||
@@ -1199,11 +1244,11 @@ impl Provider for ArchivebateProvider {
|
|||||||
.await;
|
.await;
|
||||||
match timeout_result {
|
match timeout_result {
|
||||||
Ok(enriched) => enriched,
|
Ok(enriched) => enriched,
|
||||||
Err(_) => video,
|
Err(_) => Self::with_default_format(video, &options),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
.buffer_unordered(4)
|
.buffered(4)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -1218,3 +1263,52 @@ impl Provider for ArchivebateProvider {
|
|||||||
Some(self.build_channel(clientversion))
|
Some(self.build_channel(clientversion))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn options() -> ServerOptions {
|
||||||
|
ServerOptions {
|
||||||
|
client_version: None,
|
||||||
|
featured: None,
|
||||||
|
category: None,
|
||||||
|
sites: None,
|
||||||
|
filter: None,
|
||||||
|
language: None,
|
||||||
|
public_url_base: Some("http://hottub.example".to_string()),
|
||||||
|
requester: None,
|
||||||
|
network: None,
|
||||||
|
stars: None,
|
||||||
|
categories: None,
|
||||||
|
duration: None,
|
||||||
|
sort: None,
|
||||||
|
sexuality: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn app_clients_get_the_proxy_url_and_no_formats() {
|
||||||
|
let mut item = VideoItem::new(
|
||||||
|
"archivebate:1".to_string(),
|
||||||
|
"t".to_string(),
|
||||||
|
"https://archivebate.com/watch/1".to_string(),
|
||||||
|
CHANNEL_ID.to_string(),
|
||||||
|
"https://cdn.example/t.jpg".to_string(),
|
||||||
|
10,
|
||||||
|
);
|
||||||
|
item.formats = Some(vec![VideoFormat::new(
|
||||||
|
"x".to_string(),
|
||||||
|
"source".to_string(),
|
||||||
|
"mp4".to_string(),
|
||||||
|
)]);
|
||||||
|
|
||||||
|
let out = ArchivebateProvider::proxy_as_item_url(vec![item], &options());
|
||||||
|
assert_eq!(out.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
out[0].url,
|
||||||
|
"http://hottub.example/proxy/archivebate/archivebate.com/watch/1"
|
||||||
|
);
|
||||||
|
assert!(out[0].formats.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -440,14 +440,25 @@ impl EpornerProvider {
|
|||||||
Ok(cards)
|
Ok(cards)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn build_item(requester: &mut Requester, card: EpornerCard) -> VideoItem {
|
async fn build_item(
|
||||||
|
requester: &mut Requester,
|
||||||
|
options: &ServerOptions,
|
||||||
|
card: EpornerCard,
|
||||||
|
) -> VideoItem {
|
||||||
// The hash/xhr resolution + Referer handling live in
|
// The hash/xhr resolution + Referer handling live in
|
||||||
// `crate::util::eporner_resolve` (shared with other providers that
|
// `crate::util::eporner_resolve` (shared with other providers that
|
||||||
// redirect out to eporner.com, e.g. `melonstube`) rather than here,
|
// redirect out to eporner.com, e.g. `melonstube`) rather than here,
|
||||||
// since single-provider builds can't depend on another provider's
|
// since single-provider builds can't depend on another provider's
|
||||||
// module — see that module's doc comment.
|
// module — see that module's doc comment.
|
||||||
let formats =
|
// Formats point at the eporner streaming proxy: the CDN links expire
|
||||||
crate::util::eporner_resolve::resolve_formats(requester, &card.alnum_id, &card.page_url)
|
// after two hours and only serve the server's IP; anything else plays
|
||||||
|
// a one-minute "watch on eporner.com" clip.
|
||||||
|
let formats = crate::util::eporner_resolve::resolve_proxied_formats(
|
||||||
|
requester,
|
||||||
|
options,
|
||||||
|
&card.alnum_id,
|
||||||
|
&card.page_url,
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let mut item = VideoItem::new(
|
let mut item = VideoItem::new(
|
||||||
@@ -648,7 +659,8 @@ impl EpornerProvider {
|
|||||||
let cards = Self::parse_list_cards(&html)?;
|
let cards = Self::parse_list_cards(&html)?;
|
||||||
let items = stream::iter(cards.into_iter().take(per_page.max(1)).map(|card| {
|
let items = stream::iter(cards.into_iter().take(per_page.max(1)).map(|card| {
|
||||||
let mut req = requester.clone();
|
let mut req = requester.clone();
|
||||||
async move { Self::build_item(&mut req, card).await }
|
let options = &options;
|
||||||
|
async move { Self::build_item(&mut req, options, card).await }
|
||||||
}))
|
}))
|
||||||
.buffer_unordered(ENRICH_CONCURRENCY)
|
.buffer_unordered(ENRICH_CONCURRENCY)
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
|
|||||||
@@ -1,13 +1,8 @@
|
|||||||
use aes_gcm::aead::{Aead, KeyInit, Payload};
|
|
||||||
use aes_gcm::{Aes256Gcm, Key, Nonce};
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use base64::Engine;
|
|
||||||
use error_chain::error_chain;
|
use error_chain::error_chain;
|
||||||
use futures::future::join_all;
|
use futures::future::join_all;
|
||||||
use serde_json::json;
|
|
||||||
use sha2::{Digest, Sha256};
|
|
||||||
use std::sync::{Arc, Mutex, OnceLock};
|
use std::sync::{Arc, Mutex, OnceLock};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::SystemTime;
|
||||||
use std::vec;
|
use std::vec;
|
||||||
|
|
||||||
use crate::DbPool;
|
use crate::DbPool;
|
||||||
@@ -16,6 +11,7 @@ use crate::db;
|
|||||||
use crate::providers::{Provider, report_provider_error, report_provider_error_background};
|
use crate::providers::{Provider, report_provider_error, report_provider_error_background};
|
||||||
use crate::status::*;
|
use crate::status::*;
|
||||||
use crate::util::cache::VideoCache;
|
use crate::util::cache::VideoCache;
|
||||||
|
use crate::util::hanime_resolve::{self as resolve, HanimeSource, SITE_ORIGIN};
|
||||||
use crate::videos::{self, ServerOptions, VideoItem};
|
use crate::videos::{self, ServerOptions, VideoItem};
|
||||||
|
|
||||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
@@ -34,14 +30,6 @@ error_chain! {
|
|||||||
/// Full catalogue dump. Ignores every query parameter, so filtering, sorting and
|
/// Full catalogue dump. Ignores every query parameter, so filtering, sorting and
|
||||||
/// pagination all happen client side. ~860 KB compressed, CDN cached for an hour.
|
/// pagination all happen client side. ~860 KB compressed, CDN cached for an hour.
|
||||||
const INDEX_URL: &str = "https://guest.freeanimehentai.net/api/v11/search_hvs";
|
const INDEX_URL: &str = "https://guest.freeanimehentai.net/api/v11/search_hvs";
|
||||||
const HANDSHAKE_URL: &str = "https://auth.hanime.tv/api/v11/handshake";
|
|
||||||
const SITE_ORIGIN: &str = "https://hanime.tv";
|
|
||||||
/// Envelope encryption used for the handshake request/response bodies.
|
|
||||||
const HANDSHAKE_KEY_SEED: &[u8] = b"htv-insecure-handshake-v1";
|
|
||||||
const HANDSHAKE_AAD: &[u8] = b"htv-insecure-v1";
|
|
||||||
/// Pieces of the `x-signature` pre-image, as baked into the site's wasm module.
|
|
||||||
const SIGNATURE_SECRET: &str = "Xkdi29";
|
|
||||||
const SIGNATURE_SALT: &str = "mn2";
|
|
||||||
const INDEX_TTL_SECS: u64 = 3600;
|
const INDEX_TTL_SECS: u64 = 3600;
|
||||||
|
|
||||||
type IndexCache = OnceLock<Mutex<Option<(SystemTime, Arc<Vec<HanimeSearchResult>>)>>>;
|
type IndexCache = OnceLock<Mutex<Option<(SystemTime, Arc<Vec<HanimeSearchResult>>)>>>;
|
||||||
@@ -76,99 +64,6 @@ struct HanimeSearchResult {
|
|||||||
released_at_unix: u64,
|
released_at_unix: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(serde::Deserialize, Debug)]
|
|
||||||
struct HanimeHandshakePayload {
|
|
||||||
#[serde(default)]
|
|
||||||
sources: Vec<HanimeSource>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Deserialize, Debug, Clone)]
|
|
||||||
struct HanimeSource {
|
|
||||||
#[serde(default)]
|
|
||||||
src: String,
|
|
||||||
#[serde(default)]
|
|
||||||
height: u32,
|
|
||||||
#[serde(default)]
|
|
||||||
label: String,
|
|
||||||
/// `"normal"` for the free streams, `"promotion"` for the premium-only teaser
|
|
||||||
/// entries which always carry an empty `src`.
|
|
||||||
#[serde(default)]
|
|
||||||
kind: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn b64() -> base64::engine::general_purpose::GeneralPurpose {
|
|
||||||
base64::engine::general_purpose::URL_SAFE_NO_PAD
|
|
||||||
}
|
|
||||||
|
|
||||||
fn b64_decode(value: &str) -> Result<Vec<u8>> {
|
|
||||||
b64()
|
|
||||||
.decode(value.trim_end_matches('='))
|
|
||||||
.map_err(|e| Error::from(format!("base64url decode failed: {e}")))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn handshake_cipher() -> Aes256Gcm {
|
|
||||||
let digest = Sha256::digest(HANDSHAKE_KEY_SEED);
|
|
||||||
let key = Key::<Aes256Gcm>::from_slice(digest.as_slice());
|
|
||||||
Aes256Gcm::new(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `{"v":1,"alg":"AES-256-GCM","iv":..,"tag":..,"data":..}`, base64url encoded.
|
|
||||||
fn seal_envelope(plaintext: &[u8]) -> Result<String> {
|
|
||||||
let mut iv = [0u8; 12];
|
|
||||||
rand::fill(&mut iv);
|
|
||||||
let sealed = handshake_cipher()
|
|
||||||
.encrypt(
|
|
||||||
Nonce::from_slice(&iv),
|
|
||||||
Payload {
|
|
||||||
msg: plaintext,
|
|
||||||
aad: HANDSHAKE_AAD,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.map_err(|e| Error::from(format!("handshake encrypt failed: {e}")))?;
|
|
||||||
if sealed.len() < 16 {
|
|
||||||
return Err(Error::from("handshake ciphertext too short"));
|
|
||||||
}
|
|
||||||
let (data, tag) = sealed.split_at(sealed.len() - 16);
|
|
||||||
let envelope = json!({
|
|
||||||
"v": 1,
|
|
||||||
"alg": "AES-256-GCM",
|
|
||||||
"iv": b64().encode(iv),
|
|
||||||
"tag": b64().encode(tag),
|
|
||||||
"data": b64().encode(data),
|
|
||||||
});
|
|
||||||
Ok(b64().encode(envelope.to_string().as_bytes()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn open_envelope(token: &str) -> Result<Vec<u8>> {
|
|
||||||
let envelope: serde_json::Value = serde_json::from_slice(&b64_decode(token)?)
|
|
||||||
.map_err(|e| Error::from(format!("handshake envelope is not JSON: {e}")))?;
|
|
||||||
let field = |name: &str| -> Result<Vec<u8>> {
|
|
||||||
let raw = envelope
|
|
||||||
.get(name)
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| Error::from(format!("handshake envelope missing {name}")))?;
|
|
||||||
b64_decode(raw)
|
|
||||||
};
|
|
||||||
let iv = field("iv")?;
|
|
||||||
let mut ciphertext = field("data")?;
|
|
||||||
ciphertext.extend_from_slice(&field("tag")?);
|
|
||||||
handshake_cipher()
|
|
||||||
.decrypt(
|
|
||||||
Nonce::from_slice(&iv),
|
|
||||||
Payload {
|
|
||||||
msg: &ciphertext,
|
|
||||||
aad: HANDSHAKE_AAD,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.map_err(|e| Error::from(format!("handshake decrypt failed: {e}")))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn signature_for(timestamp: u64) -> String {
|
|
||||||
let pre_image =
|
|
||||||
format!("{timestamp},{SIGNATURE_SECRET},{SITE_ORIGIN},{SIGNATURE_SALT},{timestamp}");
|
|
||||||
hex::encode(Sha256::digest(pre_image.as_bytes()))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct HanimeProvider;
|
pub struct HanimeProvider;
|
||||||
|
|
||||||
@@ -242,6 +137,25 @@ impl HanimeProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fresh free sources for a slug, best quality first.
|
||||||
|
async fn fetch_sources(&self, slug: &str, options: &ServerOptions) -> Result<Vec<HanimeSource>> {
|
||||||
|
let mut requester =
|
||||||
|
crate::providers::requester_or_default(options, module_path!(), "missing_requester");
|
||||||
|
resolve::fetch_sources(&mut requester, slug)
|
||||||
|
.await
|
||||||
|
.map_err(Error::from)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The stream URLs expire, so players get a stable redirect that does a fresh
|
||||||
|
/// handshake on every play instead of the signed link itself.
|
||||||
|
fn proxied_url(options: &ServerOptions, slug: &str, source: &HanimeSource) -> String {
|
||||||
|
crate::providers::build_proxy_url(
|
||||||
|
options,
|
||||||
|
"hanime",
|
||||||
|
&format!("{slug}/{}.m3u8", source.height),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn db_key(slug: &str) -> String {
|
fn db_key(slug: &str) -> String {
|
||||||
format!("https://hanime.tv/videos/hentai/{slug}")
|
format!("https://hanime.tv/videos/hentai/{slug}")
|
||||||
}
|
}
|
||||||
@@ -319,86 +233,6 @@ impl HanimeProvider {
|
|||||||
Ok(items)
|
Ok(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the free `sources` for a slug, best quality first. The auth host
|
|
||||||
/// occasionally refuses one of a burst of parallel connections, so retry once.
|
|
||||||
async fn fetch_sources(&self, slug: &str, options: &ServerOptions) -> Result<Vec<HanimeSource>> {
|
|
||||||
match self.handshake(slug, options).await {
|
|
||||||
Ok(sources) => Ok(sources),
|
|
||||||
Err(_) => {
|
|
||||||
ntex::time::sleep(ntex::time::Millis(500)).await;
|
|
||||||
self.handshake(slug, options).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handshake(&self, slug: &str, options: &ServerOptions) -> Result<Vec<HanimeSource>> {
|
|
||||||
let timestamp = SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_secs();
|
|
||||||
let token = seal_envelope(
|
|
||||||
json!({
|
|
||||||
"timestamp_unix": timestamp,
|
|
||||||
"directive": "htv_player_handshake",
|
|
||||||
"slug": slug,
|
|
||||||
})
|
|
||||||
.to_string()
|
|
||||||
.as_bytes(),
|
|
||||||
)?;
|
|
||||||
|
|
||||||
let mut requester =
|
|
||||||
crate::providers::requester_or_default(options, module_path!(), "missing_requester");
|
|
||||||
let response = requester
|
|
||||||
.post_json(
|
|
||||||
HANDSHAKE_URL,
|
|
||||||
&json!({ "token": token }),
|
|
||||||
vec![
|
|
||||||
("Origin".to_string(), SITE_ORIGIN.to_string()),
|
|
||||||
("Referer".to_string(), format!("{SITE_ORIGIN}/")),
|
|
||||||
("Accept".to_string(), "application/json".to_string()),
|
|
||||||
("x-signature-version".to_string(), "web2".to_string()),
|
|
||||||
("x-signature".to_string(), signature_for(timestamp)),
|
|
||||||
("x-time".to_string(), timestamp.to_string()),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(|e| Error::from(format!("handshake request failed: {e}")))?;
|
|
||||||
|
|
||||||
let status = response.status();
|
|
||||||
if !status.is_success() {
|
|
||||||
return Err(Error::from(format!("handshake returned HTTP {status}")));
|
|
||||||
}
|
|
||||||
|
|
||||||
// The interesting part of the answer travels in a response header, not the body.
|
|
||||||
let encrypted = response
|
|
||||||
.headers()
|
|
||||||
.get("x-token")
|
|
||||||
.and_then(|v| v.to_str().ok())
|
|
||||||
.map(|v| v.to_string())
|
|
||||||
.ok_or_else(|| Error::from("handshake response is missing x-token"))?;
|
|
||||||
|
|
||||||
let payload: HanimeHandshakePayload = serde_json::from_slice(&open_envelope(&encrypted)?)
|
|
||||||
.map_err(|e| Error::from(format!("Failed to parse handshake payload: {e}")))?;
|
|
||||||
|
|
||||||
let mut sources: Vec<HanimeSource> = payload
|
|
||||||
.sources
|
|
||||||
.into_iter()
|
|
||||||
.filter(|s| s.kind == "normal" && !s.src.is_empty())
|
|
||||||
.collect();
|
|
||||||
sources.sort_by(|a, b| b.height.cmp(&a.height));
|
|
||||||
match sources.is_empty() {
|
|
||||||
true => Err(Error::from("handshake returned no playable sources")),
|
|
||||||
false => Ok(sources),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn source_url(source: &HanimeSource) -> String {
|
|
||||||
match source.src.starts_with("http") {
|
|
||||||
true => source.src.clone(),
|
|
||||||
false => format!("{SITE_ORIGIN}{}", source.src),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn quality_label(source: &HanimeSource) -> String {
|
fn quality_label(source: &HanimeSource) -> String {
|
||||||
match source.label.trim_end_matches('p') {
|
match source.label.trim_end_matches('p') {
|
||||||
"" => source.height.to_string(),
|
"" => source.height.to_string(),
|
||||||
@@ -452,15 +286,15 @@ impl HanimeProvider {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|source| {
|
.map(|source| {
|
||||||
videos::VideoFormat::new(
|
videos::VideoFormat::new(
|
||||||
Self::source_url(source),
|
Self::proxied_url(&options, &hit.slug, source),
|
||||||
Self::quality_label(source),
|
Self::quality_label(source),
|
||||||
"m3u8".to_string(),
|
"m3u8".to_string(),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
let stream_url = Self::source_url(&sources[0]);
|
let stream_url = Self::proxied_url(&options, &hit.slug, &sources[0]);
|
||||||
let duration = self
|
let duration = self
|
||||||
.fetch_duration(&stream_url, &options)
|
.fetch_duration(&resolve::source_url(&sources[0]), &options)
|
||||||
.await
|
.await
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
if let Ok(mut conn) = pool.get() {
|
if let Ok(mut conn) = pool.get() {
|
||||||
@@ -490,11 +324,8 @@ impl HanimeProvider {
|
|||||||
.and_then(|mut conn| db::get_video(&mut conn, db_key.clone()).ok().flatten());
|
.and_then(|mut conn| db::get_video(&mut conn, db_key.clone()).ok().flatten());
|
||||||
|
|
||||||
match db_result {
|
match db_result {
|
||||||
Some(video_url) if video_url.contains("m3u8") || video_url.contains("/hls/") => {
|
Some(video_url) if video_url.contains("/proxy/hanime/") => {
|
||||||
let duration = self
|
let duration = 0;
|
||||||
.fetch_duration(&video_url, &options)
|
|
||||||
.await
|
|
||||||
.unwrap_or_default();
|
|
||||||
let formats = vec![videos::VideoFormat::new(
|
let formats = vec![videos::VideoFormat::new(
|
||||||
video_url.clone(),
|
video_url.clone(),
|
||||||
"720".to_string(),
|
"720".to_string(),
|
||||||
|
|||||||
@@ -494,11 +494,18 @@ impl MelonstubeProvider {
|
|||||||
Self::parse_kvs_formats(&html, &origin)
|
Self::parse_kvs_formats(&html, &origin)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Through the eporner streaming proxy: its CDN links expire after two
|
||||||
|
/// hours and only serve the server's IP (else a "watch on eporner.com" clip).
|
||||||
async fn resolve_eporner(
|
async fn resolve_eporner(
|
||||||
requester: &mut Requester,
|
requester: &mut Requester,
|
||||||
|
options: &ServerOptions,
|
||||||
detail_url: &str,
|
detail_url: &str,
|
||||||
) -> Option<Vec<VideoFormat>> {
|
) -> Option<Vec<VideoFormat>> {
|
||||||
crate::util::eporner_resolve::resolve_public_formats(requester, detail_url).await
|
let alnum_id = crate::util::eporner_resolve::alnum_id_from_href(detail_url)?;
|
||||||
|
crate::util::eporner_resolve::resolve_proxied_formats(
|
||||||
|
requester, options, &alnum_id, detail_url,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
fn xhamster_mirror_mp4_regex() -> &'static Regex {
|
fn xhamster_mirror_mp4_regex() -> &'static Regex {
|
||||||
@@ -632,10 +639,11 @@ impl MelonstubeProvider {
|
|||||||
async fn resolve_source_formats(
|
async fn resolve_source_formats(
|
||||||
kind: SourceKind,
|
kind: SourceKind,
|
||||||
requester: &mut Requester,
|
requester: &mut Requester,
|
||||||
|
options: &ServerOptions,
|
||||||
detail_url: &str,
|
detail_url: &str,
|
||||||
) -> Option<Vec<VideoFormat>> {
|
) -> Option<Vec<VideoFormat>> {
|
||||||
match kind {
|
match kind {
|
||||||
SourceKind::Eporner => Self::resolve_eporner(requester, detail_url).await,
|
SourceKind::Eporner => Self::resolve_eporner(requester, options, detail_url).await,
|
||||||
SourceKind::Kvs => Self::resolve_kvs(requester, detail_url, false).await,
|
SourceKind::Kvs => Self::resolve_kvs(requester, detail_url, false).await,
|
||||||
SourceKind::KvsCfGated => Self::resolve_kvs(requester, detail_url, true).await,
|
SourceKind::KvsCfGated => Self::resolve_kvs(requester, detail_url, true).await,
|
||||||
SourceKind::XhamsterMirror => Self::resolve_xhamster_mirror(requester, detail_url).await,
|
SourceKind::XhamsterMirror => Self::resolve_xhamster_mirror(requester, detail_url).await,
|
||||||
@@ -904,6 +912,7 @@ impl MelonstubeProvider {
|
|||||||
if !pending.is_empty() {
|
if !pending.is_empty() {
|
||||||
let resolved = stream::iter(pending.into_iter().map(|(idx, kind, dest_url)| {
|
let resolved = stream::iter(pending.into_iter().map(|(idx, kind, dest_url)| {
|
||||||
let mut req = requester.clone();
|
let mut req = requester.clone();
|
||||||
|
let options = &options;
|
||||||
async move {
|
async move {
|
||||||
// Bounded so one slow/hung third-party destination (e.g.
|
// Bounded so one slow/hung third-party destination (e.g.
|
||||||
// an unreachable FlareSolverr fallback) can't stall the
|
// an unreachable FlareSolverr fallback) can't stall the
|
||||||
@@ -911,7 +920,7 @@ impl MelonstubeProvider {
|
|||||||
// its plain destination URL with no formats.
|
// its plain destination URL with no formats.
|
||||||
let formats = tokio::time::timeout(
|
let formats = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(ENRICH_TIMEOUT_SECS),
|
std::time::Duration::from_secs(ENRICH_TIMEOUT_SECS),
|
||||||
Self::resolve_source_formats(kind, &mut req, &dest_url),
|
Self::resolve_source_formats(kind, &mut req, options, &dest_url),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use crate::DbPool;
|
use crate::DbPool;
|
||||||
use crate::api::ClientVersion;
|
use crate::api::ClientVersion;
|
||||||
use crate::providers::Provider;
|
use crate::providers::{Provider, requester_or_default, slug_dashed};
|
||||||
use crate::status::*;
|
use crate::status::*;
|
||||||
use crate::util::cache::VideoCache;
|
use crate::util::cache::VideoCache;
|
||||||
use crate::util::discord::send_discord_error_report;
|
use crate::util::discord::send_discord_error_report;
|
||||||
@@ -12,10 +12,23 @@ use async_trait::async_trait;
|
|||||||
use error_chain::error_chain;
|
use error_chain::error_chain;
|
||||||
use futures::stream::{self, StreamExt};
|
use futures::stream::{self, StreamExt};
|
||||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use scraper::{Html, Selector};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
|
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
|
||||||
use std::vec;
|
use std::vec;
|
||||||
|
|
||||||
const ENRICH_CONCURRENCY: usize = 6;
|
const ENRICH_CONCURRENCY: usize = 4;
|
||||||
|
/// The signed `v-acctoken` format URLs last ~12h; keep details well inside that.
|
||||||
|
const DETAIL_TTL: Duration = Duration::from_secs(2 * 60 * 60);
|
||||||
|
const DETAIL_CACHE_SIZE: usize = 5000;
|
||||||
|
const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60);
|
||||||
|
const TARGET_CACHE_SIZE: usize = 2000;
|
||||||
|
/// A probed archive smaller than this loses to keyword search: "elf girl" is a
|
||||||
|
/// one-video tag, while the search finds hundreds.
|
||||||
|
const MIN_PROBED_ARCHIVE_ITEMS: usize = 4;
|
||||||
|
|
||||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
crate::providers::ProviderChannelMetadata {
|
crate::providers::ProviderChannelMetadata {
|
||||||
@@ -36,15 +49,66 @@ error_chain! {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
/// What a request resolves to. Artists live under `/models/`, franchises and
|
||||||
|
/// genres under `/categories/`, characters and everything else under `/tags/`.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
enum Target {
|
||||||
|
Latest,
|
||||||
|
Search(String),
|
||||||
|
Artist(String),
|
||||||
|
Category(String),
|
||||||
|
Tag(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Target {
|
||||||
|
/// Archive path (`models/{slug}`, ...) for the archive targets.
|
||||||
|
fn archive_path(&self) -> Option<String> {
|
||||||
|
match self {
|
||||||
|
Target::Artist(slug) => Some(format!("models/{slug}")),
|
||||||
|
Target::Category(slug) => Some(format!("categories/{slug}")),
|
||||||
|
Target::Tag(slug) => Some(format!("tags/{slug}")),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct Rule34videoProvider {
|
pub struct Rule34videoProvider {
|
||||||
url: String,
|
url: String,
|
||||||
|
/// Enriched items (formats, tags, artist) by video URL. Every listing costs
|
||||||
|
/// ~23 detail pages, again for the prefetched next page; refetching them
|
||||||
|
/// on each request is what runs into the site's 429 rate limit.
|
||||||
|
details: VideoCache,
|
||||||
|
/// Bare query -> resolved target, so later pages skip the archive probes.
|
||||||
|
targets: Arc<Mutex<HashMap<String, (Instant, Target)>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Rule34videoProvider {
|
impl Rule34videoProvider {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
|
let mut details = VideoCache::new();
|
||||||
|
details.max_size(DETAIL_CACHE_SIZE);
|
||||||
Rule34videoProvider {
|
Rule34videoProvider {
|
||||||
url: "https://rule34video.com".to_string(),
|
url: "https://rule34video.com".to_string(),
|
||||||
|
details,
|
||||||
|
targets: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remembered_target(&self, query: &str) -> Option<Target> {
|
||||||
|
let targets = self.targets.lock().ok()?;
|
||||||
|
let (time, target) = targets.get(&slug_dashed(query))?;
|
||||||
|
(time.elapsed() < TARGET_TTL).then(|| target.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remember_target(&self, query: &str, target: &Target) {
|
||||||
|
if let Ok(mut targets) = self.targets.lock() {
|
||||||
|
if targets.len() >= TARGET_CACHE_SIZE {
|
||||||
|
targets.retain(|_, (time, _)| time.elapsed() < TARGET_TTL);
|
||||||
|
if targets.len() >= TARGET_CACHE_SIZE {
|
||||||
|
targets.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
targets.insert(slug_dashed(query), (Instant::now(), target.clone()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,6 +157,192 @@ impl Rule34videoProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Explicit `artist:` / `category:` / `tag:` style prefixes (the server has
|
||||||
|
/// already stripped a leading `#`).
|
||||||
|
fn parse_prefixed(query: &str) -> Option<Target> {
|
||||||
|
let (kind, value) = query.trim().split_once(':')?;
|
||||||
|
let slug = slug_dashed(value);
|
||||||
|
if slug.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
match kind.trim().to_ascii_lowercase().as_str() {
|
||||||
|
"artist" | "artists" | "model" | "models" | "author" | "creator" | "uploader" => {
|
||||||
|
Some(Target::Artist(slug))
|
||||||
|
}
|
||||||
|
"cat" | "category" | "categories" | "franchise" | "series" | "game" => {
|
||||||
|
Some(Target::Category(slug))
|
||||||
|
}
|
||||||
|
"tag" | "tags" | "character" | "char" => Some(Target::Tag(slug)),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Titles of the tags offered by `/search_ajax.php?tag=`.
|
||||||
|
fn parse_tag_suggestions(html: &str) -> Vec<String> {
|
||||||
|
let document = Html::parse_fragment(html);
|
||||||
|
let Ok(selector) = Selector::parse("input[data-title]") else {
|
||||||
|
return vec![];
|
||||||
|
};
|
||||||
|
document
|
||||||
|
.select(&selector)
|
||||||
|
.filter_map(|input| input.value().attr("data-title"))
|
||||||
|
.map(|title| title.trim().to_string())
|
||||||
|
.filter(|title| !title.is_empty())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The suggested tag whose name (ignoring a trailing "(franchise)") equals
|
||||||
|
/// the query, e.g. "samus aran" -> "samus aran (metroid)".
|
||||||
|
fn best_tag_title(titles: &[String], query: &str) -> Option<String> {
|
||||||
|
let wanted = slug_dashed(query);
|
||||||
|
if wanted.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut exact = titles.iter().filter(|title| {
|
||||||
|
let base = title.split('(').next().unwrap_or(title);
|
||||||
|
slug_dashed(base) == wanted || slug_dashed(title) == wanted
|
||||||
|
});
|
||||||
|
let first = exact.next()?;
|
||||||
|
// Several franchises share a character name: ambiguous, leave it to search.
|
||||||
|
exact.next().is_none().then(|| first.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A free-form query that names an artist, category or tag goes to that
|
||||||
|
/// archive (probed on the site; unknown names 404), else keyword search.
|
||||||
|
/// The second value is false when a probe failed (rate limit, outage) and
|
||||||
|
/// the answer must not be remembered.
|
||||||
|
async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> (Target, bool) {
|
||||||
|
let slug = slug_dashed(query);
|
||||||
|
if slug.is_empty() {
|
||||||
|
return (Target::Search(query.trim().to_string()), true);
|
||||||
|
}
|
||||||
|
// Artist slugs are inconsistent on the site: "general-butch" but "biggreen".
|
||||||
|
let mut candidates = vec![Target::Artist(slug.clone())];
|
||||||
|
if slug.contains('-') {
|
||||||
|
candidates.push(Target::Artist(slug.replace('-', "")));
|
||||||
|
}
|
||||||
|
candidates.push(Target::Category(slug.clone()));
|
||||||
|
candidates.push(Target::Tag(slug));
|
||||||
|
let mut requester = requester_or_default(options, "rule34video", "resolve_keyword");
|
||||||
|
let referer = format!("{}/", self.url);
|
||||||
|
let mut conclusive = true;
|
||||||
|
for candidate in &candidates {
|
||||||
|
let Some(path) = candidate.archive_path() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let url = format!("{}/{path}/", self.url);
|
||||||
|
let headers = vec![("Referer".to_string(), referer.clone())];
|
||||||
|
match requester.get_probe(&url, headers).await {
|
||||||
|
Ok(Some(body)) => {
|
||||||
|
let items = self.get_video_items_from_html(body).len();
|
||||||
|
if items >= MIN_PROBED_ARCHIVE_ITEMS {
|
||||||
|
return (candidate.clone(), true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(_) => conclusive = false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Characters are tagged "name (franchise)": ask the site's own tag search.
|
||||||
|
let suggest_url = format!(
|
||||||
|
"{}/search_ajax.php?tag={}",
|
||||||
|
self.url,
|
||||||
|
utf8_percent_encode(query.trim(), NON_ALPHANUMERIC)
|
||||||
|
);
|
||||||
|
let headers = vec![
|
||||||
|
("Referer".to_string(), referer),
|
||||||
|
("X-Requested-With".to_string(), "XMLHttpRequest".to_string()),
|
||||||
|
];
|
||||||
|
match requester.get_probe(&suggest_url, headers).await {
|
||||||
|
Ok(Some(body)) => {
|
||||||
|
let titles = Self::parse_tag_suggestions(&body);
|
||||||
|
if let Some(title) = Self::best_tag_title(&titles, query) {
|
||||||
|
return (Target::Tag(slug_dashed(&title)), conclusive);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(_) => conclusive = false,
|
||||||
|
}
|
||||||
|
(Target::Search(query.trim().to_string()), conclusive)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_target(&self, query: Option<&str>, options: &ServerOptions) -> Target {
|
||||||
|
let Some(query) = query.map(str::trim).filter(|q| !q.is_empty()) else {
|
||||||
|
return Target::Latest;
|
||||||
|
};
|
||||||
|
if let Some(target) = Self::parse_prefixed(query) {
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
if let Some(target) = self.remembered_target(query) {
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
let (target, conclusive) = self.resolve_keyword(query, options).await;
|
||||||
|
if conclusive {
|
||||||
|
self.remember_target(query, &target);
|
||||||
|
}
|
||||||
|
target
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn archive(
|
||||||
|
&self,
|
||||||
|
cache: VideoCache,
|
||||||
|
target: &Target,
|
||||||
|
page: u8,
|
||||||
|
sort: &str,
|
||||||
|
options: ServerOptions,
|
||||||
|
) -> Result<Vec<VideoItem>> {
|
||||||
|
let Some(path) = target.archive_path() else {
|
||||||
|
return Ok(vec![]);
|
||||||
|
};
|
||||||
|
let sort_val = match sort {
|
||||||
|
"post_date" | "video_viewed" | "rating" | "duration" | "pseudo_random" => sort,
|
||||||
|
_ => "post_date",
|
||||||
|
};
|
||||||
|
let index = format!("rule34video:{path}:{page}:{sort_val}");
|
||||||
|
if sort_val != "pseudo_random" {
|
||||||
|
if let Some((time, items)) = cache.get(&index) {
|
||||||
|
if time.elapsed().unwrap_or_default().as_secs() < 300 {
|
||||||
|
return Ok(items.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut requester = options.requester.clone().ok_or("Requester missing")?;
|
||||||
|
let url = format!(
|
||||||
|
"{}/{path}/?mode=async&function=get_block&block_id=custom_list_videos_common_videos&sort_by={sort_val}&from={page}",
|
||||||
|
self.url
|
||||||
|
);
|
||||||
|
let headers = vec![("Referer".to_string(), format!("{}/{path}/", self.url))];
|
||||||
|
let stale = || cache.get(&index).map(|(_, items)| items).unwrap_or_default();
|
||||||
|
let text = match requester.get_probe(&url, headers).await {
|
||||||
|
Ok(Some(text)) => text,
|
||||||
|
// A 404 is an unknown archive (or a page past the end).
|
||||||
|
Ok(None) => return Ok(vec![]),
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("rule34video archive fetch failed for {url}: {e}");
|
||||||
|
return Ok(stale());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let video_items = self.get_video_items_from_html(text);
|
||||||
|
let video_items = self.enrich_items(video_items, requester).await;
|
||||||
|
if video_items.is_empty() {
|
||||||
|
return Ok(stale());
|
||||||
|
}
|
||||||
|
cache.insert(index, video_items.clone());
|
||||||
|
Ok(video_items)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `/search/{q}/` takes dashes for spaces. A raw `/` splits the path (404),
|
||||||
|
/// `%20` is refused (403) and `&` cuts the query short, so keep only words.
|
||||||
|
fn search_path_segment(query: &str) -> String {
|
||||||
|
slug_dashed(query)
|
||||||
|
.split('-')
|
||||||
|
.map(|word| utf8_percent_encode(word, NON_ALPHANUMERIC).to_string())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("-")
|
||||||
|
}
|
||||||
|
|
||||||
/// Helper to safely extract a string between two delimiters
|
/// Helper to safely extract a string between two delimiters
|
||||||
fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> {
|
fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> {
|
||||||
let start_idx = content.find(start_pat)? + start_pat.len();
|
let start_idx = content.find(start_pat)? + start_pat.len();
|
||||||
@@ -208,7 +458,7 @@ impl Rule34videoProvider {
|
|||||||
let url = format!(
|
let url = format!(
|
||||||
"{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}",
|
"{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}",
|
||||||
self.url,
|
self.url,
|
||||||
query.replace(" ", "-"),
|
Self::search_path_segment(query),
|
||||||
sort_val,
|
sort_val,
|
||||||
page,
|
page,
|
||||||
page,
|
page,
|
||||||
@@ -307,30 +557,106 @@ impl Rule34videoProvider {
|
|||||||
(formats.into_iter().map(|(_, f)| f).collect(), tags)
|
(formats.into_iter().map(|(_, f)| f).collect(), tags)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Artists credited on a video page: `(name, slug)` from the "Artist" group.
|
||||||
|
fn parse_artists(html: &str) -> Vec<(String, String)> {
|
||||||
|
let document = Html::parse_document(html);
|
||||||
|
let Ok(group_sel) = Selector::parse("[data-suggest-type=\"model\"] a[href*=\"/models/\"]")
|
||||||
|
else {
|
||||||
|
return vec![];
|
||||||
|
};
|
||||||
|
let name_sel = Selector::parse(".name").ok();
|
||||||
|
let mut artists: Vec<(String, String)> = Vec::new();
|
||||||
|
for link in document.select(&group_sel) {
|
||||||
|
let href = link.value().attr("href").unwrap_or_default();
|
||||||
|
let slug = href
|
||||||
|
.split("/models/")
|
||||||
|
.nth(1)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.trim_matches('/')
|
||||||
|
.to_string();
|
||||||
|
let name = name_sel
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|sel| link.select(sel).next())
|
||||||
|
.map(|n| n.text().collect::<String>())
|
||||||
|
.unwrap_or_else(|| link.text().collect::<String>());
|
||||||
|
let name = name.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||||
|
if slug.is_empty() || name.is_empty() || artists.iter().any(|(_, s)| *s == slug) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
artists.push((name, slug));
|
||||||
|
}
|
||||||
|
artists
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cached_detail(&self, item: &mut VideoItem) -> bool {
|
||||||
|
let Some((time, cached)) = self.details.get(&item.url) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Some(cached) = cached.into_iter().next() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if time.elapsed().unwrap_or(DETAIL_TTL) >= DETAIL_TTL {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
item.formats = cached.formats;
|
||||||
|
item.tags = cached.tags;
|
||||||
|
item.uploader = cached.uploader;
|
||||||
|
item.uploaderUrl = cached.uploaderUrl;
|
||||||
|
item.uploaderId = cached.uploaderId;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
|
async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
|
||||||
let referer = format!("{}/", self.url);
|
let referer = format!("{}/", self.url);
|
||||||
|
let base = self.url.clone();
|
||||||
|
// Once the site rate-limits a detail page, the rest of the batch keeps
|
||||||
|
// the listing data instead of digging the hole deeper.
|
||||||
|
let throttled = Arc::new(AtomicBool::new(false));
|
||||||
stream::iter(items.into_iter().map(|mut item| {
|
stream::iter(items.into_iter().map(|mut item| {
|
||||||
let mut requester = requester.clone();
|
let mut requester = requester.clone();
|
||||||
let referer = referer.clone();
|
let referer = referer.clone();
|
||||||
|
let base = base.clone();
|
||||||
|
let throttled = throttled.clone();
|
||||||
async move {
|
async move {
|
||||||
if item.url.is_empty() {
|
if item.url.is_empty() || self.cached_detail(&mut item) {
|
||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
let html = requester
|
if throttled.load(Ordering::Relaxed) {
|
||||||
.get_with_headers(
|
return item;
|
||||||
&item.url,
|
}
|
||||||
vec![("Referer".to_string(), referer)],
|
// `get_probe` hands challenges to the full fallback chain but
|
||||||
None,
|
// reports a lasting 429 as an error rather than escalating every
|
||||||
)
|
// detail page to FlareSolverr.
|
||||||
.await
|
let headers = vec![("Referer".to_string(), referer)];
|
||||||
.unwrap_or_default();
|
let html = match requester.get_probe(&item.url, headers).await {
|
||||||
let (formats, tags) = Self::parse_detail(&html);
|
Ok(Some(html)) => html,
|
||||||
|
Ok(None) => return item,
|
||||||
|
Err(_) => {
|
||||||
|
throttled.store(true, Ordering::Relaxed);
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let (formats, mut tags) = Self::parse_detail(&html);
|
||||||
if !formats.is_empty() {
|
if !formats.is_empty() {
|
||||||
item.formats = Some(formats);
|
item.formats = Some(formats);
|
||||||
}
|
}
|
||||||
|
let artists = Self::parse_artists(&html);
|
||||||
|
if let Some((name, slug)) = artists.first() {
|
||||||
|
item.uploader = Some(name.clone());
|
||||||
|
item.uploaderUrl = Some(format!("{base}/models/{slug}/"));
|
||||||
|
item.uploaderId = Some(format!("rule34video:{slug}"));
|
||||||
|
}
|
||||||
|
for (name, _) in &artists {
|
||||||
|
if !tags.iter().any(|t| t.eq_ignore_ascii_case(name)) {
|
||||||
|
tags.push(name.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
if !tags.is_empty() {
|
if !tags.is_empty() {
|
||||||
item.tags = Some(tags);
|
item.tags = Some(tags);
|
||||||
}
|
}
|
||||||
|
if item.formats.is_some() {
|
||||||
|
self.details.insert(item.url.clone(), vec![item.clone()]);
|
||||||
|
}
|
||||||
item
|
item
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
@@ -344,15 +670,15 @@ impl Rule34videoProvider {
|
|||||||
return vec![];
|
return vec![];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Safely isolate the video listing section
|
// Isolate the video listing section. A result that fits on one page has
|
||||||
let video_listing = match Self::extract_between(
|
// no pagination block, so the listing then runs to the end.
|
||||||
&html,
|
let Some(start) = html.find("id=\"custom_list_videos") else {
|
||||||
"id=\"custom_list_videos",
|
return vec![];
|
||||||
"<div class=\"pagination\"",
|
|
||||||
) {
|
|
||||||
Some(content) => content,
|
|
||||||
None => return vec![],
|
|
||||||
};
|
};
|
||||||
|
let video_listing = &html[start..];
|
||||||
|
let video_listing = video_listing
|
||||||
|
.find("<div class=\"pagination\"")
|
||||||
|
.map_or(video_listing, |end| &video_listing[..end]);
|
||||||
|
|
||||||
let mut items = Vec::new();
|
let mut items = Vec::new();
|
||||||
// Skip the first split result as it's the preamble
|
// Skip the first split result as it's the preamble
|
||||||
@@ -449,9 +775,11 @@ impl Provider for Rule34videoProvider {
|
|||||||
) -> Vec<VideoItem> {
|
) -> Vec<VideoItem> {
|
||||||
let page_num = page.parse::<u8>().unwrap_or(1);
|
let page_num = page.parse::<u8>().unwrap_or(1);
|
||||||
|
|
||||||
let result = match query {
|
let target = self.resolve_target(query.as_deref(), &options).await;
|
||||||
Some(q) => self.query(cache, page_num, &q, &sort, options).await,
|
let result = match &target {
|
||||||
None => self.get(cache, page_num, &sort, options).await,
|
Target::Latest => self.get(cache, page_num, &sort, options).await,
|
||||||
|
Target::Search(q) => self.query(cache, page_num, q, &sort, options).await,
|
||||||
|
archive => self.archive(cache, archive, page_num, &sort, options).await,
|
||||||
};
|
};
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
@@ -517,4 +845,81 @@ mod tests {
|
|||||||
let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
|
let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
|
||||||
assert!(formats.is_empty() && tags.is_empty());
|
assert!(formats.is_empty() && tags.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_prefixed_queries() {
|
||||||
|
assert_eq!(
|
||||||
|
Rule34videoProvider::parse_prefixed("artist:General Butch"),
|
||||||
|
Some(Target::Artist("general-butch".into()))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
Rule34videoProvider::parse_prefixed("category:Super Smash Bros"),
|
||||||
|
Some(Target::Category("super-smash-bros".into()))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
Rule34videoProvider::parse_prefixed("tag:black widow (marvel)"),
|
||||||
|
Some(Target::Tag("black-widow-marvel".into()))
|
||||||
|
);
|
||||||
|
assert_eq!(Rule34videoProvider::parse_prefixed("samus aran"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn picks_the_unambiguous_character_tag() {
|
||||||
|
let titles = vec![
|
||||||
|
"samus aran (metroid)".to_string(),
|
||||||
|
"samus aran's mother".to_string(),
|
||||||
|
];
|
||||||
|
assert_eq!(
|
||||||
|
Rule34videoProvider::best_tag_title(&titles, "samus aran").as_deref(),
|
||||||
|
Some("samus aran (metroid)")
|
||||||
|
);
|
||||||
|
let ambiguous = vec!["peach (mario)".to_string(), "peach (pokemon)".to_string()];
|
||||||
|
assert_eq!(Rule34videoProvider::best_tag_title(&ambiguous, "peach"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_tag_suggestions_and_artists() {
|
||||||
|
let suggest = r#"<div class="item checkbox-container"><input id="tag_143" type="checkbox" value="143" data-title="samus aran (metroid)"/></div>"#;
|
||||||
|
assert_eq!(
|
||||||
|
Rule34videoProvider::parse_tag_suggestions(suggest),
|
||||||
|
vec!["samus aran (metroid)".to_string()]
|
||||||
|
);
|
||||||
|
let page = r#"<div class="col vp-group" data-suggest-type="model"><div class="label">Artist</div>
|
||||||
|
<span class="chip"><a class="item btn_link" href="https://rule34video.com/models/joigang-va/">
|
||||||
|
<span class="name">JOIGang (VA)</span></a></span></div>"#;
|
||||||
|
assert_eq!(
|
||||||
|
Rule34videoProvider::parse_artists(page),
|
||||||
|
vec![("JOIGang (VA)".to_string(), "joigang-va".to_string())]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_a_single_page_listing_without_pagination() {
|
||||||
|
let html = r#"<div id="custom_list_videos_common_videos_items">
|
||||||
|
<div class="item thumb video_7"><a class="th js-open-popup" href="https://rule34video.com/video/7/elf/">
|
||||||
|
<img class="thumb" src="https://rule34video.com/t/7.jpg"><div class="time">0:30</div>
|
||||||
|
<div class="thumb_title">Elf</div></a></div></div></div>"#;
|
||||||
|
let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string());
|
||||||
|
assert_eq!(items.len(), 1);
|
||||||
|
assert_eq!(items[0].id, "7");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn search_segment_uses_dashes_and_drops_path_breaking_punctuation() {
|
||||||
|
assert_eq!(Rule34videoProvider::search_path_segment("elf girl"), "elf-girl");
|
||||||
|
assert_eq!(Rule34videoProvider::search_path_segment("2b/9s"), "2b-9s");
|
||||||
|
assert_eq!(Rule34videoProvider::search_path_segment("Tifa & Aerith"), "tifa-aerith");
|
||||||
|
assert_eq!(Rule34videoProvider::search_path_segment("Pokémon"), "pok%C3%A9mon");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remembers_resolved_targets_by_slug() {
|
||||||
|
let provider = Rule34videoProvider::new();
|
||||||
|
provider.remember_target("Samus Aran", &Target::Tag("samus-aran-metroid".into()));
|
||||||
|
assert_eq!(
|
||||||
|
provider.remembered_target("samus aran"),
|
||||||
|
Some(Target::Tag("samus-aran-metroid".into()))
|
||||||
|
);
|
||||||
|
assert_eq!(provider.remembered_target("samus"), None);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -229,21 +229,78 @@ impl ArchivebateProxy {
|
|||||||
embed_url: &str,
|
embed_url: &str,
|
||||||
requester: &mut Requester,
|
requester: &mut Requester,
|
||||||
) -> Option<String> {
|
) -> Option<String> {
|
||||||
let response = requester
|
// The embed host is sometimes slow on a first (cold) request and fast on
|
||||||
|
// the next, so one timed-out attempt is retried.
|
||||||
|
for _ in 0..2 {
|
||||||
|
let Ok(response) = requester
|
||||||
.get_raw_with_headers_timeout(
|
.get_raw_with_headers_timeout(
|
||||||
embed_url,
|
embed_url,
|
||||||
Self::html_headers(detail_url),
|
Self::html_headers(detail_url),
|
||||||
Some(StdDuration::from_secs(8)),
|
Some(StdDuration::from_secs(10)),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.ok()?;
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let html = response.text().await.ok()?;
|
let html = response.text().await.ok()?;
|
||||||
Self::extract_mixdrop_media_url(&html)
|
return Self::extract_mixdrop_media_url(&html);
|
||||||
}
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A resolved Mixdrop media file plus the embed page it came from (the CDN is
|
||||||
|
/// played with that page as `Referer`).
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ResolvedMedia {
|
||||||
|
pub media_url: String,
|
||||||
|
pub embed_url: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves an archivebate watch page (`https://archivebate.com/watch/{id}`)
|
||||||
|
/// to the real media URL: watch page -> Mixdrop embed -> unpacked
|
||||||
|
/// `MDCore.wurl`. Shared by the lazy `/proxy/archivebate` redirect and by the
|
||||||
|
/// provider, which hands the app the media URL directly.
|
||||||
|
pub async fn resolve_media(requester: &mut Requester, detail_url: &str) -> Option<ResolvedMedia> {
|
||||||
|
let detail_html = requester
|
||||||
|
.get_with_headers(
|
||||||
|
detail_url,
|
||||||
|
ArchivebateProxy::html_headers("https://archivebate.com/"),
|
||||||
|
Some(Version::HTTP_11),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
if detail_html.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut embeds: Vec<String> = Vec::new();
|
||||||
|
if let Some(iframe_url) = ArchivebateProxy::first_iframe_source_from_html(&detail_html)
|
||||||
|
.map(|value| ArchivebateProxy::absolute_url(&value))
|
||||||
|
{
|
||||||
|
if ArchivebateProxy::is_mixdrop_host(&iframe_url) {
|
||||||
|
embeds.push(iframe_url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(embed_url) = ArchivebateProxy::download_fid_from_detail_html(&detail_html)
|
||||||
|
.map(|value| ArchivebateProxy::absolute_url(&value))
|
||||||
|
.and_then(|value| ArchivebateProxy::mixdrop_embed_url_from_download_url(&value))
|
||||||
|
{
|
||||||
|
embeds.push(embed_url);
|
||||||
|
}
|
||||||
|
|
||||||
|
for embed_url in embeds {
|
||||||
|
if let Some(media_url) =
|
||||||
|
ArchivebateProxy::resolve_mixdrop_media_from_embed(detail_url, &embed_url, requester).await
|
||||||
|
{
|
||||||
|
return Some(ResolvedMedia { media_url, embed_url });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
impl crate::proxies::Proxy for ArchivebateProxy {
|
impl crate::proxies::Proxy for ArchivebateProxy {
|
||||||
@@ -251,41 +308,11 @@ impl crate::proxies::Proxy for ArchivebateProxy {
|
|||||||
let Some(detail_url) = Self::normalize_detail_request(&url) else {
|
let Some(detail_url) = Self::normalize_detail_request(&url) else {
|
||||||
return String::new();
|
return String::new();
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut requester = requester.get_ref().clone();
|
let mut requester = requester.get_ref().clone();
|
||||||
let detail_html = requester
|
resolve_media(&mut requester, &detail_url)
|
||||||
.get_with_headers(
|
|
||||||
&detail_url,
|
|
||||||
Self::html_headers("https://archivebate.com/"),
|
|
||||||
Some(Version::HTTP_11),
|
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
.unwrap_or_default();
|
.map(|resolved| resolved.media_url)
|
||||||
if detail_html.is_empty() {
|
.unwrap_or_default()
|
||||||
return String::new();
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(iframe_url) = Self::first_iframe_source_from_html(&detail_html).map(|value| Self::absolute_url(&value)) {
|
|
||||||
if Self::is_mixdrop_host(&iframe_url) {
|
|
||||||
if let Some(media_url) =
|
|
||||||
Self::resolve_mixdrop_media_from_embed(&detail_url, &iframe_url, &mut requester).await
|
|
||||||
{
|
|
||||||
return media_url;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(download_fid) = Self::download_fid_from_detail_html(&detail_html).map(|value| Self::absolute_url(&value)) {
|
|
||||||
if let Some(embed_url) = Self::mixdrop_embed_url_from_download_url(&download_fid) {
|
|
||||||
if let Some(media_url) =
|
|
||||||
Self::resolve_mixdrop_media_from_embed(&detail_url, &embed_url, &mut requester).await
|
|
||||||
{
|
|
||||||
return media_url;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
String::new()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
196
src/proxies/eporner.rs
Normal file
196
src/proxies/eporner.rs
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::rc::Rc;
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use futures::StreamExt;
|
||||||
|
use http_body_util::BodyExt;
|
||||||
|
use ntex::http::Method;
|
||||||
|
use ntex::http::body::{Body, SizedStream};
|
||||||
|
use ntex::http::header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, RANGE};
|
||||||
|
use ntex::util::Bytes;
|
||||||
|
use ntex::web::{self, HttpRequest};
|
||||||
|
use wreq::Client;
|
||||||
|
use wreq::redirect::Policy;
|
||||||
|
use wreq_util::Emulation;
|
||||||
|
|
||||||
|
use crate::util::eporner_resolve::resolve_ranked;
|
||||||
|
use crate::util::requester::Requester;
|
||||||
|
|
||||||
|
/// The signed CDN links expire two hours after resolving; renew well before.
|
||||||
|
const LINK_TTL: Duration = Duration::from_secs(90 * 60);
|
||||||
|
const LINK_CACHE_SIZE: usize = 2000;
|
||||||
|
|
||||||
|
/// `{alnum_id}/{height}` -> (resolved at, signed CDN URL).
|
||||||
|
fn links() -> &'static Mutex<HashMap<String, (Instant, String)>> {
|
||||||
|
static LINKS: OnceLock<Mutex<HashMap<String, (Instant, String)>>> = OnceLock::new();
|
||||||
|
LINKS.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Redirects are not followed so the CDN's `na.mp4` decoy shows up as a 302.
|
||||||
|
fn cdn_client() -> &'static Client {
|
||||||
|
static CLIENT: OnceLock<Client> = OnceLock::new();
|
||||||
|
CLIENT.get_or_init(|| {
|
||||||
|
Client::builder()
|
||||||
|
.tls_cert_verification(false)
|
||||||
|
.emulation(Emulation::Firefox151)
|
||||||
|
.redirect(Policy::none())
|
||||||
|
.build()
|
||||||
|
.expect("eporner CDN client")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `{alnum_id}[/{height}][.mp4]`.
|
||||||
|
fn parse_endpoint(endpoint: &str) -> Option<(String, Option<u32>)> {
|
||||||
|
let endpoint = endpoint.trim_matches('/').trim_end_matches(".mp4");
|
||||||
|
let mut parts = endpoint.split('/');
|
||||||
|
let alnum_id = parts
|
||||||
|
.next()
|
||||||
|
.filter(|id| !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric()))?;
|
||||||
|
let height = parts.next().and_then(|h| h.parse::<u32>().ok());
|
||||||
|
Some((alnum_id.to_string(), height))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_link(
|
||||||
|
requester: &web::types::State<Requester>,
|
||||||
|
alnum_id: &str,
|
||||||
|
height: Option<u32>,
|
||||||
|
) -> Option<String> {
|
||||||
|
let page_url = format!("https://www.eporner.com/video-{alnum_id}/");
|
||||||
|
let mut requester = requester.get_ref().clone();
|
||||||
|
let ranked = resolve_ranked(&mut requester, alnum_id, &page_url).await?;
|
||||||
|
height
|
||||||
|
.and_then(|h| ranked.iter().find(|(height, _)| *height == h))
|
||||||
|
.or_else(|| ranked.first())
|
||||||
|
.map(|(_, format)| format.url.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cached_link(
|
||||||
|
requester: &web::types::State<Requester>,
|
||||||
|
alnum_id: &str,
|
||||||
|
height: Option<u32>,
|
||||||
|
fresh: bool,
|
||||||
|
) -> Option<String> {
|
||||||
|
let key = format!("{alnum_id}/{}", height.unwrap_or(0));
|
||||||
|
if !fresh {
|
||||||
|
if let Ok(links) = links().lock() {
|
||||||
|
if let Some((time, url)) = links.get(&key) {
|
||||||
|
if time.elapsed() < LINK_TTL {
|
||||||
|
return Some(url.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let url = resolve_link(requester, alnum_id, height).await?;
|
||||||
|
if let Ok(mut links) = links().lock() {
|
||||||
|
if links.len() >= LINK_CACHE_SIZE {
|
||||||
|
links.retain(|_, (time, _)| time.elapsed() < LINK_TTL);
|
||||||
|
if links.len() >= LINK_CACHE_SIZE {
|
||||||
|
links.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
links.insert(key, (Instant::now(), url.clone()));
|
||||||
|
}
|
||||||
|
Some(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Streams the mp4 through this server. The signed CDN links only serve the
|
||||||
|
/// IP that resolved them (checked from other networks: they answer every
|
||||||
|
/// other client with a 302 to `static.eporner.com/na.mp4`, a one-minute
|
||||||
|
/// "watch it on eporner.com" clip), so a redirect to them only ever works for
|
||||||
|
/// a player that shares the server's public IP.
|
||||||
|
pub async fn serve_media(
|
||||||
|
req: HttpRequest,
|
||||||
|
requester: web::types::State<Requester>,
|
||||||
|
) -> Result<impl web::Responder, web::Error> {
|
||||||
|
let endpoint = req.match_info().query("endpoint").to_string();
|
||||||
|
let Some((alnum_id, height)) = parse_endpoint(&endpoint) else {
|
||||||
|
return Ok(web::HttpResponse::BadRequest().finish());
|
||||||
|
};
|
||||||
|
let range = req
|
||||||
|
.headers()
|
||||||
|
.get(RANGE)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.map(str::to_string);
|
||||||
|
let head = req.method() == Method::HEAD;
|
||||||
|
|
||||||
|
let mut upstream = None;
|
||||||
|
for fresh in [false, true] {
|
||||||
|
let Some(url) = cached_link(&requester, &alnum_id, height, fresh).await else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let mut request = if head {
|
||||||
|
cdn_client().head(&url)
|
||||||
|
} else {
|
||||||
|
cdn_client().get(&url)
|
||||||
|
};
|
||||||
|
request = request.header("Referer", "https://www.eporner.com/");
|
||||||
|
if let Some(range) = &range {
|
||||||
|
request = request.header("Range", range);
|
||||||
|
}
|
||||||
|
match request.send().await {
|
||||||
|
// A redirect is the decoy for an expired or foreign link.
|
||||||
|
Ok(response) if !response.status().is_redirection() => {
|
||||||
|
upstream = Some(response);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_ => continue,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let Some(upstream) = upstream else {
|
||||||
|
return Ok(web::HttpResponse::BadGateway().finish());
|
||||||
|
};
|
||||||
|
|
||||||
|
let status = upstream.status();
|
||||||
|
let headers = upstream.headers().clone();
|
||||||
|
let mut builder = web::HttpResponse::build(status);
|
||||||
|
builder.header(
|
||||||
|
CONTENT_TYPE,
|
||||||
|
headers
|
||||||
|
.get(CONTENT_TYPE)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.unwrap_or("video/mp4"),
|
||||||
|
);
|
||||||
|
builder.header(ACCEPT_RANGES, "bytes");
|
||||||
|
if let Some(value) = headers.get(CONTENT_RANGE.as_str()).and_then(|v| v.to_str().ok()) {
|
||||||
|
builder.header(CONTENT_RANGE, value);
|
||||||
|
}
|
||||||
|
if head {
|
||||||
|
// `finish()` would send `Content-Length: 0` over the upstream length.
|
||||||
|
return Ok(builder.body(Body::None));
|
||||||
|
}
|
||||||
|
let length = headers
|
||||||
|
.get(CONTENT_LENGTH.as_str())
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.and_then(|v| v.parse::<u64>().ok());
|
||||||
|
let stream = Box::pin(upstream.into_data_stream().map(|chunk| {
|
||||||
|
chunk
|
||||||
|
.map(|bytes| Bytes::copy_from_slice(bytes.as_ref()))
|
||||||
|
.map_err(|e| std::io::Error::other(e.to_string()))
|
||||||
|
}));
|
||||||
|
let Some(length) = length else {
|
||||||
|
return Ok(builder.streaming(stream));
|
||||||
|
};
|
||||||
|
// A sized body keeps `Content-Length` on the (206) responses; a chunked
|
||||||
|
// one is what AVPlayer refuses to seek in.
|
||||||
|
let stream = stream.map(|chunk| chunk.map_err(|e| Rc::new(e) as Rc<dyn std::error::Error>));
|
||||||
|
Ok(builder.body(Body::from_message(SizedStream::new(length, stream))))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::parse_endpoint;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_id_and_optional_height() {
|
||||||
|
assert_eq!(
|
||||||
|
parse_endpoint("exfeTXHuL1x/720.mp4"),
|
||||||
|
Some(("exfeTXHuL1x".to_string(), Some(720)))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse_endpoint("/exfeTXHuL1x/"),
|
||||||
|
Some(("exfeTXHuL1x".to_string(), None))
|
||||||
|
);
|
||||||
|
assert_eq!(parse_endpoint("../etc"), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
71
src/proxies/hanime.rs
Normal file
71
src/proxies/hanime.rs
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
use ntex::http::Method;
|
||||||
|
use ntex::http::header::LOCATION;
|
||||||
|
use ntex::web::{self, HttpRequest};
|
||||||
|
|
||||||
|
use crate::util::hanime_resolve::{fetch_sources, pick_source, source_url};
|
||||||
|
use crate::util::requester::Requester;
|
||||||
|
|
||||||
|
/// `{slug}[/{height}][.m3u8]`.
|
||||||
|
fn parse_endpoint(endpoint: &str) -> Option<(String, Option<u32>)> {
|
||||||
|
let endpoint = endpoint.trim_matches('/').trim_end_matches(".m3u8");
|
||||||
|
let mut parts = endpoint.split('/');
|
||||||
|
let slug = parts
|
||||||
|
.next()
|
||||||
|
.filter(|slug| {
|
||||||
|
!slug.is_empty()
|
||||||
|
&& slug
|
||||||
|
.chars()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
|
||||||
|
})?;
|
||||||
|
let height = parts.next().and_then(|h| h.parse::<u32>().ok());
|
||||||
|
Some((slug.to_string(), height))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// hanime's stream links expire, so the channel hands out this stable URL and a
|
||||||
|
/// fresh handshake happens on every play: redirects to the current playlist.
|
||||||
|
pub async fn redirect_to_media(
|
||||||
|
req: HttpRequest,
|
||||||
|
requester: web::types::State<Requester>,
|
||||||
|
) -> Result<impl web::Responder, web::Error> {
|
||||||
|
let endpoint = req.match_info().query("endpoint").to_string();
|
||||||
|
let Some((slug, height)) = parse_endpoint(&endpoint) else {
|
||||||
|
return Ok(web::HttpResponse::BadRequest().finish());
|
||||||
|
};
|
||||||
|
if req.method() == Method::HEAD {
|
||||||
|
return Ok(web::HttpResponse::Ok().finish());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut requester = requester.get_ref().clone();
|
||||||
|
let Ok(sources) = fetch_sources(&mut requester, &slug).await else {
|
||||||
|
return Ok(web::HttpResponse::BadGateway().finish());
|
||||||
|
};
|
||||||
|
let Some(source) = pick_source(&sources, height) else {
|
||||||
|
return Ok(web::HttpResponse::NotFound().finish());
|
||||||
|
};
|
||||||
|
Ok(web::HttpResponse::Found()
|
||||||
|
.header(LOCATION, source_url(source))
|
||||||
|
.finish())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::parse_endpoint;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_slug_and_height() {
|
||||||
|
assert_eq!(
|
||||||
|
parse_endpoint("natsu-zuma-2/720.m3u8"),
|
||||||
|
Some(("natsu-zuma-2".to_string(), Some(720)))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse_endpoint("/natsu-zuma-2"),
|
||||||
|
Some(("natsu-zuma-2".to_string(), None))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_unsafe_slugs() {
|
||||||
|
assert_eq!(parse_endpoint("../etc/720.m3u8"), None);
|
||||||
|
assert_eq!(parse_endpoint(""), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,7 +24,9 @@ pub mod animeidhentai;
|
|||||||
pub mod archivebate;
|
pub mod archivebate;
|
||||||
pub mod clapdat;
|
pub mod clapdat;
|
||||||
pub mod doodstream;
|
pub mod doodstream;
|
||||||
|
pub mod eporner;
|
||||||
pub mod fikfapthumb;
|
pub mod fikfapthumb;
|
||||||
|
pub mod hanime;
|
||||||
pub mod hanimecdn;
|
pub mod hanimecdn;
|
||||||
pub mod hanimethumb;
|
pub mod hanimethumb;
|
||||||
pub mod heavyfetish;
|
pub mod heavyfetish;
|
||||||
|
|||||||
80
src/proxy.rs
80
src/proxy.rs
@@ -20,6 +20,40 @@ use crate::proxies::thaiporntv::ThaipornTvProxy;
|
|||||||
use crate::proxies::*;
|
use crate::proxies::*;
|
||||||
use crate::util::requester::Requester;
|
use crate::util::requester::Requester;
|
||||||
|
|
||||||
|
/// Proxies that answer with a `302` to the real media instead of streaming it.
|
||||||
|
/// The Hot Tub app cannot follow a redirect inside `formats[].url`, so for app
|
||||||
|
/// clients a format pointing at one of these is moved to `video.url` (see
|
||||||
|
/// `move_redirect_formats_to_url`). Keep in sync with the `proxy2redirect`
|
||||||
|
/// routes below; `redirect_proxy_list_matches_routes` enforces it.
|
||||||
|
pub const REDIRECT_PROXIES: &[&str] = &[
|
||||||
|
"archivebate",
|
||||||
|
"clapdat",
|
||||||
|
"doodstream",
|
||||||
|
"lulustream",
|
||||||
|
"sxyprn",
|
||||||
|
"javtiful",
|
||||||
|
"spankbang",
|
||||||
|
"porndish",
|
||||||
|
"hqporner",
|
||||||
|
"heavyfetish",
|
||||||
|
"vjav",
|
||||||
|
"pornhd3x",
|
||||||
|
"vidara",
|
||||||
|
"pimpbunny",
|
||||||
|
"thaiporntv",
|
||||||
|
"tube8",
|
||||||
|
"supjav",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// True for a `/proxy/<redirect proxy>/...` URL (absolute or host-relative).
|
||||||
|
pub fn is_redirect_proxy_url(url: &str) -> bool {
|
||||||
|
let Some(rest) = url.split_once("/proxy/").map(|(_, rest)| rest) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let name = rest.split('/').next().unwrap_or_default();
|
||||||
|
REDIRECT_PROXIES.contains(&name)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn config(cfg: &mut web::ServiceConfig) {
|
pub fn config(cfg: &mut web::ServiceConfig) {
|
||||||
cfg.service(
|
cfg.service(
|
||||||
web::resource("/archivebate/{endpoint}*")
|
web::resource("/archivebate/{endpoint}*")
|
||||||
@@ -123,6 +157,11 @@ pub fn config(cfg: &mut web::ServiceConfig) {
|
|||||||
.route(web::post().to(crate::proxies::fikfapthumb::get_image))
|
.route(web::post().to(crate::proxies::fikfapthumb::get_image))
|
||||||
.route(web::get().to(crate::proxies::fikfapthumb::get_image)),
|
.route(web::get().to(crate::proxies::fikfapthumb::get_image)),
|
||||||
)
|
)
|
||||||
|
.service(
|
||||||
|
web::resource("/hanime/{endpoint}*")
|
||||||
|
.route(web::get().to(crate::proxies::hanime::redirect_to_media))
|
||||||
|
.route(web::head().to(crate::proxies::hanime::redirect_to_media)),
|
||||||
|
)
|
||||||
.service(
|
.service(
|
||||||
web::resource("/hanime-cdn/{endpoint}*")
|
web::resource("/hanime-cdn/{endpoint}*")
|
||||||
.route(web::post().to(crate::proxies::hanimecdn::get_image))
|
.route(web::post().to(crate::proxies::hanimecdn::get_image))
|
||||||
@@ -194,6 +233,11 @@ pub fn config(cfg: &mut web::ServiceConfig) {
|
|||||||
.route(web::post().to(crate::proxies::melonstube::serve))
|
.route(web::post().to(crate::proxies::melonstube::serve))
|
||||||
.route(web::get().to(crate::proxies::melonstube::serve)),
|
.route(web::get().to(crate::proxies::melonstube::serve)),
|
||||||
);
|
);
|
||||||
|
cfg.service(
|
||||||
|
web::resource("/eporner/{endpoint}*")
|
||||||
|
.route(web::get().to(crate::proxies::eporner::serve_media))
|
||||||
|
.route(web::head().to(crate::proxies::eporner::serve_media)),
|
||||||
|
);
|
||||||
cfg.service(
|
cfg.service(
|
||||||
web::resource("/xiaoshenke/{endpoint}*")
|
web::resource("/xiaoshenke/{endpoint}*")
|
||||||
.route(web::post().to(crate::proxies::xiaoshenke::serve_media))
|
.route(web::post().to(crate::proxies::xiaoshenke::serve_media))
|
||||||
@@ -239,3 +283,39 @@ fn get_proxy(proxy: &str) -> Option<AnyProxy> {
|
|||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod redirect_proxy_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recognises_redirect_proxy_urls() {
|
||||||
|
assert!(is_redirect_proxy_url("http://h/proxy/archivebate/archivebate.com/watch/1"));
|
||||||
|
assert!(is_redirect_proxy_url("/proxy/hqporner/x"));
|
||||||
|
assert!(!is_redirect_proxy_url("http://h/proxy/noodlemagazine/x"));
|
||||||
|
assert!(!is_redirect_proxy_url("https://cdn.example/video.mp4"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redirect_proxy_list_matches_routes() {
|
||||||
|
let source = include_str!("proxy.rs");
|
||||||
|
let config_body = source
|
||||||
|
.split("pub fn config(")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|rest| rest.split("async fn proxy2redirect(").next())
|
||||||
|
.expect("config body");
|
||||||
|
let mut from_routes: Vec<String> = Vec::new();
|
||||||
|
for block in config_body.split("web::resource(\"/").skip(1) {
|
||||||
|
let name = block.split("/{endpoint}").next().unwrap_or_default();
|
||||||
|
let until_next = block.split("web::resource(").next().unwrap_or(block);
|
||||||
|
if until_next.contains("proxy2redirect") {
|
||||||
|
from_routes.push(name.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut listed: Vec<String> = REDIRECT_PROXIES.iter().map(|s| s.to_string()).collect();
|
||||||
|
from_routes.sort();
|
||||||
|
from_routes.dedup();
|
||||||
|
listed.sort();
|
||||||
|
assert_eq!(listed, from_routes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
//! always compiled regardless of provider selection.
|
//! always compiled regardless of provider selection.
|
||||||
|
|
||||||
use crate::util::requester::Requester;
|
use crate::util::requester::Requester;
|
||||||
use crate::videos::VideoFormat;
|
use crate::videos::{ServerOptions, VideoFormat};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
use wreq::Version;
|
use wreq::Version;
|
||||||
@@ -101,11 +101,18 @@ fn calc_hash(hash: &str) -> Option<String> {
|
|||||||
/// when playback doesn't carry the header the CDN wants. Each format
|
/// when playback doesn't carry the header the CDN wants. Each format
|
||||||
/// therefore declares an explicit `Referer` so the client's own default (or
|
/// therefore declares an explicit `Referer` so the client's own default (or
|
||||||
/// lack thereof) can't accidentally trip this check.
|
/// lack thereof) can't accidentally trip this check.
|
||||||
pub async fn resolve_formats(
|
///
|
||||||
|
/// The signed CDN URLs also expire two hours after this call (the leading
|
||||||
|
/// number in `/{expiry}_{ip}_{n}/` is that deadline) and only serve the IP
|
||||||
|
/// embedded there; anything else gets the same `na.mp4` decoy.
|
||||||
|
/// Anything handed to a client should therefore be a
|
||||||
|
/// [`resolve_proxied_formats`] URL, streamed through this server.
|
||||||
|
/// Returns `(height, format)`, best first.
|
||||||
|
pub async fn resolve_ranked(
|
||||||
requester: &mut Requester,
|
requester: &mut Requester,
|
||||||
alnum_id: &str,
|
alnum_id: &str,
|
||||||
page_url: &str,
|
page_url: &str,
|
||||||
) -> Option<Vec<VideoFormat>> {
|
) -> Option<Vec<(u32, VideoFormat)>> {
|
||||||
let detail_html = requester
|
let detail_html = requester
|
||||||
.get_with_headers(page_url, html_headers(page_url), Some(Version::HTTP_11))
|
.get_with_headers(page_url, html_headers(page_url), Some(Version::HTTP_11))
|
||||||
.await
|
.await
|
||||||
@@ -151,25 +158,56 @@ pub async fn resolve_formats(
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
ranked.sort_by(|a, b| b.0.cmp(&a.0));
|
ranked.sort_by(|a, b| b.0.cmp(&a.0));
|
||||||
Some(ranked.into_iter().map(|(_, f)| f).collect())
|
Some(ranked)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolves eporner.com direct mp4 formats for an arbitrary eporner detail
|
/// Resolves the qualities a video offers, but points each one at the
|
||||||
/// page URL, extracting the alnum id from the URL itself. For use by
|
/// `/proxy/eporner/{alnum_id}/{height}.mp4` streaming proxy: the CDN URLs
|
||||||
/// providers that only have a raw destination URL (e.g. a meta-search
|
/// expire after two hours and only serve the IP that resolved them.
|
||||||
/// aggregator's out-link), not a pre-parsed listing card.
|
pub async fn resolve_proxied_formats(
|
||||||
pub async fn resolve_public_formats(
|
|
||||||
requester: &mut Requester,
|
requester: &mut Requester,
|
||||||
|
options: &ServerOptions,
|
||||||
|
alnum_id: &str,
|
||||||
page_url: &str,
|
page_url: &str,
|
||||||
) -> Option<Vec<VideoFormat>> {
|
) -> Option<Vec<VideoFormat>> {
|
||||||
let alnum_id = alnum_id_from_href(page_url)?;
|
let ranked = resolve_ranked(requester, alnum_id, page_url).await?;
|
||||||
resolve_formats(requester, &alnum_id, page_url).await
|
let base = options.public_url_base.as_deref().unwrap_or("");
|
||||||
|
Some(proxied_formats(base, alnum_id, ranked))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds the URLs like `providers::build_proxy_url`, which this always
|
||||||
|
/// compiled module cannot rely on in single-provider builds.
|
||||||
|
fn proxied_formats(
|
||||||
|
public_url_base: &str,
|
||||||
|
alnum_id: &str,
|
||||||
|
ranked: Vec<(u32, VideoFormat)>,
|
||||||
|
) -> Vec<VideoFormat> {
|
||||||
|
let base = public_url_base.trim_end_matches('/');
|
||||||
|
ranked
|
||||||
|
.into_iter()
|
||||||
|
.map(|(height, mut format)| {
|
||||||
|
format.url = format!("{base}/proxy/eporner/{alnum_id}/{height}.mp4");
|
||||||
|
format
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn proxied_formats_point_at_the_streaming_proxy() {
|
||||||
|
let ranked = vec![(
|
||||||
|
720,
|
||||||
|
VideoFormat::new("https://vid.cdn/x-720p.mp4".into(), "720p".into(), "mp4".into()),
|
||||||
|
)];
|
||||||
|
let formats = proxied_formats("https://hottub.example/", "exfeTXHuL1x", ranked);
|
||||||
|
assert_eq!(formats[0].url, "https://hottub.example/proxy/eporner/exfeTXHuL1x/720.mp4");
|
||||||
|
// Streamed, not redirected: app clients must keep it in `formats`.
|
||||||
|
assert!(!crate::proxy::is_redirect_proxy_url(&formats[0].url));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn calc_hash_matches_site_algorithm() {
|
fn calc_hash_matches_site_algorithm() {
|
||||||
// Reverse-engineered from vjs.js; verified against a live page/xhr pair.
|
// Reverse-engineered from vjs.js; verified against a live page/xhr pair.
|
||||||
|
|||||||
208
src/util/hanime_resolve.rs
Normal file
208
src/util/hanime_resolve.rs
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
//! hanime.tv stream resolution, shared by the `hanime` channel and its
|
||||||
|
//! `/proxy/hanime/` redirect proxy.
|
||||||
|
//!
|
||||||
|
//! Lives here (always compiled) rather than in `providers/hanime.rs` because
|
||||||
|
//! `HOT_TUB_PROVIDER=<id>` single-provider builds only compile the selected
|
||||||
|
//! provider, while the proxy routes are always registered.
|
||||||
|
|
||||||
|
use aes_gcm::aead::{Aead, KeyInit, Payload};
|
||||||
|
use aes_gcm::{Aes256Gcm, Key, Nonce};
|
||||||
|
use base64::Engine;
|
||||||
|
use serde_json::json;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use crate::util::requester::Requester;
|
||||||
|
|
||||||
|
const HANDSHAKE_URL: &str = "https://auth.hanime.tv/api/v11/handshake";
|
||||||
|
pub const SITE_ORIGIN: &str = "https://hanime.tv";
|
||||||
|
/// Envelope encryption used for the handshake request/response bodies.
|
||||||
|
const HANDSHAKE_KEY_SEED: &[u8] = b"htv-insecure-handshake-v1";
|
||||||
|
const HANDSHAKE_AAD: &[u8] = b"htv-insecure-v1";
|
||||||
|
/// Pieces of the `x-signature` pre-image, as baked into the site's wasm module.
|
||||||
|
const SIGNATURE_SECRET: &str = "Xkdi29";
|
||||||
|
const SIGNATURE_SALT: &str = "mn2";
|
||||||
|
|
||||||
|
type Result<T> = std::result::Result<T, String>;
|
||||||
|
|
||||||
|
fn err(message: impl Into<String>) -> String {
|
||||||
|
message.into()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(serde::Deserialize, Debug)]
|
||||||
|
pub struct HanimeHandshakePayload {
|
||||||
|
#[serde(default)]
|
||||||
|
pub sources: Vec<HanimeSource>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(serde::Deserialize, Debug, Clone)]
|
||||||
|
pub struct HanimeSource {
|
||||||
|
#[serde(default)]
|
||||||
|
pub src: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub height: u32,
|
||||||
|
#[serde(default)]
|
||||||
|
pub label: String,
|
||||||
|
/// `"normal"` for the free streams, `"promotion"` for the premium-only teaser
|
||||||
|
/// entries which always carry an empty `src`.
|
||||||
|
#[serde(default)]
|
||||||
|
pub kind: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn b64() -> base64::engine::general_purpose::GeneralPurpose {
|
||||||
|
base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||||
|
}
|
||||||
|
|
||||||
|
fn b64_decode(value: &str) -> Result<Vec<u8>> {
|
||||||
|
b64()
|
||||||
|
.decode(value.trim_end_matches('='))
|
||||||
|
.map_err(|e| err(format!("base64url decode failed: {e}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handshake_cipher() -> Aes256Gcm {
|
||||||
|
let digest = Sha256::digest(HANDSHAKE_KEY_SEED);
|
||||||
|
let key = Key::<Aes256Gcm>::from_slice(digest.as_slice());
|
||||||
|
Aes256Gcm::new(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `{"v":1,"alg":"AES-256-GCM","iv":..,"tag":..,"data":..}`, base64url encoded.
|
||||||
|
fn seal_envelope(plaintext: &[u8]) -> Result<String> {
|
||||||
|
let mut iv = [0u8; 12];
|
||||||
|
rand::fill(&mut iv);
|
||||||
|
let sealed = handshake_cipher()
|
||||||
|
.encrypt(
|
||||||
|
Nonce::from_slice(&iv),
|
||||||
|
Payload {
|
||||||
|
msg: plaintext,
|
||||||
|
aad: HANDSHAKE_AAD,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|e| err(format!("handshake encrypt failed: {e}")))?;
|
||||||
|
if sealed.len() < 16 {
|
||||||
|
return Err(err("handshake ciphertext too short"));
|
||||||
|
}
|
||||||
|
let (data, tag) = sealed.split_at(sealed.len() - 16);
|
||||||
|
let envelope = json!({
|
||||||
|
"v": 1,
|
||||||
|
"alg": "AES-256-GCM",
|
||||||
|
"iv": b64().encode(iv),
|
||||||
|
"tag": b64().encode(tag),
|
||||||
|
"data": b64().encode(data),
|
||||||
|
});
|
||||||
|
Ok(b64().encode(envelope.to_string().as_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_envelope(token: &str) -> Result<Vec<u8>> {
|
||||||
|
let envelope: serde_json::Value = serde_json::from_slice(&b64_decode(token)?)
|
||||||
|
.map_err(|e| err(format!("handshake envelope is not JSON: {e}")))?;
|
||||||
|
let field = |name: &str| -> Result<Vec<u8>> {
|
||||||
|
let raw = envelope
|
||||||
|
.get(name)
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.ok_or_else(|| err(format!("handshake envelope missing {name}")))?;
|
||||||
|
b64_decode(raw)
|
||||||
|
};
|
||||||
|
let iv = field("iv")?;
|
||||||
|
let mut ciphertext = field("data")?;
|
||||||
|
ciphertext.extend_from_slice(&field("tag")?);
|
||||||
|
handshake_cipher()
|
||||||
|
.decrypt(
|
||||||
|
Nonce::from_slice(&iv),
|
||||||
|
Payload {
|
||||||
|
msg: &ciphertext,
|
||||||
|
aad: HANDSHAKE_AAD,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.map_err(|e| err(format!("handshake decrypt failed: {e}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn signature_for(timestamp: u64) -> String {
|
||||||
|
let pre_image =
|
||||||
|
format!("{timestamp},{SIGNATURE_SECRET},{SITE_ORIGIN},{SIGNATURE_SALT},{timestamp}");
|
||||||
|
hex::encode(Sha256::digest(pre_image.as_bytes()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the free `sources` for a slug, best quality first. The auth host
|
||||||
|
/// occasionally refuses one of a burst of parallel connections, so retry once.
|
||||||
|
pub async fn fetch_sources(requester: &mut Requester, slug: &str) -> Result<Vec<HanimeSource>> {
|
||||||
|
match handshake(requester, slug).await {
|
||||||
|
Ok(sources) => Ok(sources),
|
||||||
|
Err(_) => {
|
||||||
|
ntex::time::sleep(ntex::time::Millis(500)).await;
|
||||||
|
handshake(requester, slug).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handshake(requester: &mut Requester, slug: &str) -> Result<Vec<HanimeSource>> {
|
||||||
|
let timestamp = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs();
|
||||||
|
let token = seal_envelope(
|
||||||
|
json!({
|
||||||
|
"timestamp_unix": timestamp,
|
||||||
|
"directive": "htv_player_handshake",
|
||||||
|
"slug": slug,
|
||||||
|
})
|
||||||
|
.to_string()
|
||||||
|
.as_bytes(),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let response = requester
|
||||||
|
.post_json(
|
||||||
|
HANDSHAKE_URL,
|
||||||
|
&json!({ "token": token }),
|
||||||
|
vec![
|
||||||
|
("Origin".to_string(), SITE_ORIGIN.to_string()),
|
||||||
|
("Referer".to_string(), format!("{SITE_ORIGIN}/")),
|
||||||
|
("Accept".to_string(), "application/json".to_string()),
|
||||||
|
("x-signature-version".to_string(), "web2".to_string()),
|
||||||
|
("x-signature".to_string(), signature_for(timestamp)),
|
||||||
|
("x-time".to_string(), timestamp.to_string()),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| err(format!("handshake request failed: {e}")))?;
|
||||||
|
|
||||||
|
let status = response.status();
|
||||||
|
if !status.is_success() {
|
||||||
|
return Err(err(format!("handshake returned HTTP {status}")));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The interesting part of the answer travels in a response header, not the body.
|
||||||
|
let encrypted = response
|
||||||
|
.headers()
|
||||||
|
.get("x-token")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.map(|v| v.to_string())
|
||||||
|
.ok_or_else(|| err("handshake response is missing x-token"))?;
|
||||||
|
|
||||||
|
let payload: HanimeHandshakePayload = serde_json::from_slice(&open_envelope(&encrypted)?)
|
||||||
|
.map_err(|e| err(format!("Failed to parse handshake payload: {e}")))?;
|
||||||
|
|
||||||
|
let mut sources: Vec<HanimeSource> = payload
|
||||||
|
.sources
|
||||||
|
.into_iter()
|
||||||
|
.filter(|s| s.kind == "normal" && !s.src.is_empty())
|
||||||
|
.collect();
|
||||||
|
sources.sort_by(|a, b| b.height.cmp(&a.height));
|
||||||
|
match sources.is_empty() {
|
||||||
|
true => Err(err("handshake returned no playable sources")),
|
||||||
|
false => Ok(sources),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn source_url(source: &HanimeSource) -> String {
|
||||||
|
match source.src.starts_with("http") {
|
||||||
|
true => source.src.clone(),
|
||||||
|
false => format!("{SITE_ORIGIN}{}", source.src),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The source of the requested height, else the best one.
|
||||||
|
pub fn pick_source(sources: &[HanimeSource], height: Option<u32>) -> Option<&HanimeSource> {
|
||||||
|
height
|
||||||
|
.and_then(|h| sources.iter().find(|s| s.height == h))
|
||||||
|
.or_else(|| sources.first())
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ pub mod discord;
|
|||||||
pub mod eporner_resolve;
|
pub mod eporner_resolve;
|
||||||
pub mod flaresolverr;
|
pub mod flaresolverr;
|
||||||
pub mod flow_debug;
|
pub mod flow_debug;
|
||||||
|
pub mod hanime_resolve;
|
||||||
pub mod hoster_proxy;
|
pub mod hoster_proxy;
|
||||||
pub mod proxy;
|
pub mod proxy;
|
||||||
pub mod requester;
|
pub mod requester;
|
||||||
|
|||||||
@@ -222,6 +222,15 @@ impl Requester {
|
|||||||
body.contains("Just a moment...")
|
body.contains("Just a moment...")
|
||||||
|| body.contains("cf-turnstile-response")
|
|| body.contains("cf-turnstile-response")
|
||||||
|| body.contains("Enable JavaScript and cookies to continue")
|
|| body.contains("Enable JavaScript and cookies to continue")
|
||||||
|
|| Self::looks_like_ddos_guard_challenge(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DDoS-Guard interstitials (rule34video) are short JS-check pages, never real content.
|
||||||
|
fn looks_like_ddos_guard_challenge(body: &str) -> bool {
|
||||||
|
body.len() < 20_000
|
||||||
|
&& (body.contains("/.well-known/ddos-guard/")
|
||||||
|
|| body.contains("check.ddos-guard.net")
|
||||||
|
|| body.contains("DDoS-Guard</title>"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn jina_mirror_url(url: &str) -> Option<String> {
|
fn jina_mirror_url(url: &str) -> Option<String> {
|
||||||
@@ -665,6 +674,18 @@ impl Requester {
|
|||||||
url::Url::parse(url).ok().map(|u| u.path().to_string())
|
url::Url::parse(url).ok().map(|u| u.path().to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `Url` percent-encodes a non-ASCII path ("/tags/pok%C3%A9mon/") while the
|
||||||
|
/// response URI keeps it raw ("/tags/pokémon/"): compare them decoded.
|
||||||
|
fn same_path(asked: &str, landed: &str) -> bool {
|
||||||
|
let normalize = |path: &str| {
|
||||||
|
percent_encoding::percent_decode_str(path)
|
||||||
|
.decode_utf8_lossy()
|
||||||
|
.trim_end_matches('/')
|
||||||
|
.to_lowercase()
|
||||||
|
};
|
||||||
|
normalize(asked) == normalize(landed)
|
||||||
|
}
|
||||||
|
|
||||||
/// Existence probe for archive pages (pornstar / studio / tag / category).
|
/// Existence probe for archive pages (pornstar / studio / tag / category).
|
||||||
///
|
///
|
||||||
/// Reads the raw response so a real 404/410 answers immediately with
|
/// Reads the raw response so a real 404/410 answers immediately with
|
||||||
@@ -678,7 +699,11 @@ impl Requester {
|
|||||||
headers: Vec<(String, String)>,
|
headers: Vec<(String, String)>,
|
||||||
) -> Result<Option<String>, AnyErr> {
|
) -> Result<Option<String>, AnyErr> {
|
||||||
let mut last_err = String::new();
|
let mut last_err = String::new();
|
||||||
for _ in 0..2 {
|
for attempt in 0..2 {
|
||||||
|
if attempt > 0 {
|
||||||
|
// An immediate retry of a 429 is all but certain to hit it again.
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
|
||||||
|
}
|
||||||
match self.get_raw_with_headers(url, headers.clone()).await {
|
match self.get_raw_with_headers(url, headers.clone()).await {
|
||||||
Ok(response) => {
|
Ok(response) => {
|
||||||
let status = response.status().as_u16();
|
let status = response.status().as_u16();
|
||||||
@@ -687,9 +712,7 @@ impl Requester {
|
|||||||
// lands on a different path is a miss.
|
// lands on a different path is a miss.
|
||||||
let redirected_away = Self::path_of(url)
|
let redirected_away = Self::path_of(url)
|
||||||
.zip(Some(response.uri().path().to_string()))
|
.zip(Some(response.uri().path().to_string()))
|
||||||
.is_some_and(|(asked, landed)| {
|
.is_some_and(|(asked, landed)| !Self::same_path(&asked, &landed));
|
||||||
!asked.trim_end_matches('/').eq_ignore_ascii_case(landed.trim_end_matches('/'))
|
|
||||||
});
|
|
||||||
let body = response.text().await.unwrap_or_default();
|
let body = response.text().await.unwrap_or_default();
|
||||||
if status == 404 || status == 410 {
|
if status == 404 || status == 410 {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -884,7 +907,9 @@ impl Requester {
|
|||||||
rate_limit_retries,
|
rate_limit_retries,
|
||||||
MAX_RATE_LIMIT_RETRIES
|
MAX_RATE_LIMIT_RETRIES
|
||||||
);
|
);
|
||||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
// 1s, 2s, 4s: a site-wide limit needs longer than a second to lift.
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(1 << (rate_limit_retries - 1)))
|
||||||
|
.await;
|
||||||
continue;
|
continue;
|
||||||
} else {
|
} else {
|
||||||
crate::flow_debug!(
|
crate::flow_debug!(
|
||||||
@@ -1018,6 +1043,14 @@ impl Requester {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::Requester;
|
use super::Requester;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn probe_paths_compare_percent_decoded() {
|
||||||
|
let asked = Requester::path_of("https://rule34video.com/tags/pokémon-species/").unwrap();
|
||||||
|
assert!(Requester::same_path(&asked, "/tags/pokémon-species/"));
|
||||||
|
assert!(Requester::same_path("/Models/X", "/models/x/"));
|
||||||
|
assert!(!Requester::same_path(&asked, "/search/"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn new_requesters_share_cookie_jar() {
|
fn new_requesters_share_cookie_jar() {
|
||||||
let a = Requester::new();
|
let a = Requester::new();
|
||||||
@@ -1033,4 +1066,88 @@ mod tests {
|
|||||||
|
|
||||||
assert!(cookie_header.contains("shared_cookie=1"));
|
assert!(cookie_header.contains("shared_cookie=1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn serve(listener: tokio::net::TcpListener, reply: impl Fn(&str) -> String + Send + 'static) {
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
loop {
|
||||||
|
let Ok((mut sock, _)) = listener.accept().await else { return };
|
||||||
|
let mut buf = vec![0u8; 16384];
|
||||||
|
let mut len = 0;
|
||||||
|
// read headers, then the declared body
|
||||||
|
loop {
|
||||||
|
let Ok(n) = sock.read(&mut buf[len..]).await else { break };
|
||||||
|
if n == 0 { break; }
|
||||||
|
len += n;
|
||||||
|
let text = String::from_utf8_lossy(&buf[..len]).to_string();
|
||||||
|
if let Some(head_end) = text.find("\r\n\r\n") {
|
||||||
|
let want = text
|
||||||
|
.lines()
|
||||||
|
.find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse::<usize>().unwrap_or(0)))
|
||||||
|
.unwrap_or(0);
|
||||||
|
if len >= head_end + 4 + want { break; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let req = String::from_utf8_lossy(&buf[..len]).to_string();
|
||||||
|
let _ = sock.write_all(reply(&req).as_bytes()).await;
|
||||||
|
let _ = sock.shutdown().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn http(status: &str, ctype: &str, extra: &str, body: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"HTTP/1.1 {status}\r\ncontent-type: {ctype}\r\n{extra}content-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||||
|
body.len()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// DDoS-Guard answers either a bare `403 Access denied.` (blocked fingerprint/IP) or a
|
||||||
|
/// short JS-check page. Both must end up at FlareSolverr and return the real listing.
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
async fn ddos_guard_blocks_are_solved_through_flaresolverr() {
|
||||||
|
// distinct hosts: the cookie jar is process-wide, so a solved cookie would leak across passes
|
||||||
|
for (interstitial, host) in [(false, "127.0.0.1"), (true, "localhost")] {
|
||||||
|
let origin = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let flare = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let origin_port = origin.local_addr().unwrap().port();
|
||||||
|
let flare_port = flare.local_addr().unwrap().port();
|
||||||
|
|
||||||
|
tokio::spawn(serve(origin, move |req| {
|
||||||
|
if req.contains("ddg_ok=1") {
|
||||||
|
http("200 OK", "text/html", "", "<div class=\"item thumb video_1\">real listing</div>")
|
||||||
|
} else if interstitial {
|
||||||
|
http(
|
||||||
|
"200 OK",
|
||||||
|
"text/html",
|
||||||
|
"server: ddos-guard\r\n",
|
||||||
|
"<html><head><title>DDoS-Guard</title><script src=\"/.well-known/ddos-guard/check?context=free_splash\"></script></head></html>",
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
http("403 Forbidden", "text/plain", "server: ddos-guard\r\n", "Access denied.")
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
tokio::spawn(serve(flare, move |req| {
|
||||||
|
let body = if req.contains("sessions.create") {
|
||||||
|
r#"{"status":"ok","message":"","session":"mock"}"#.to_string()
|
||||||
|
} else if req.contains("sessions.destroy") {
|
||||||
|
r#"{"status":"ok","message":""}"#.to_string()
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
r#"{{"status":"ok","message":"","startTimestamp":0,"endTimestamp":1,"version":"mock","solution":{{"url":"http://127.0.0.1:{origin_port}/","status":200,"response":"<div class=\"item thumb video_1\">real listing</div>","headers":{{}},"userAgent":"Mozilla/5.0 mock","cookies":[{{"name":"ddg_ok","value":"1","domain":"","path":"/"}}]}}}}"#
|
||||||
|
)
|
||||||
|
};
|
||||||
|
http("200 OK", "application/json", "", &body)
|
||||||
|
}));
|
||||||
|
|
||||||
|
unsafe { std::env::set_var("FLARE_URL", format!("http://127.0.0.1:{flare_port}/v1")) };
|
||||||
|
let mut requester = Requester::new();
|
||||||
|
let body = requester
|
||||||
|
.get(&format!("http://{host}:{origin_port}/?mode=async"), None)
|
||||||
|
.await
|
||||||
|
.expect("requester should fall through to FlareSolverr");
|
||||||
|
assert!(
|
||||||
|
body.contains("real listing"),
|
||||||
|
"interstitial={interstitial}: got {body:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user