Compare commits
19 Commits
cf9701611a
...
2addcc2e7c
| Author | SHA1 | Date | |
|---|---|---|---|
|
2addcc2e7c | ||
|
|
40eab41b2b | ||
|
|
5a0fb649ee | ||
|
|
2d35f29f44 | ||
|
|
5176178607 | ||
|
|
0824941c23 | ||
|
|
2fccea0fc2 | ||
|
|
e4c5cb2f0a | ||
|
|
ae516aba91 | ||
|
|
95700c67a5 | ||
|
|
765fcb5579 | ||
|
|
a1e6021766 | ||
|
|
78acac2150 | ||
|
|
a20a4ab642 | ||
|
|
5098d142f9 | ||
|
|
aff5df31d6 | ||
|
|
9bc8cf9b6a | ||
|
|
39314c9a53 | ||
|
|
5b62023d99 |
@@ -20,7 +20,7 @@ ntex-files = "3"
|
||||
serde = "1.0.228"
|
||||
serde_json = "1.0.145"
|
||||
tokio = { version = "1.49", features = ["full"] }
|
||||
wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json"] }
|
||||
wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json", "gzip", "brotli", "deflate", "zstd"] }
|
||||
wreq-util = "3.0.0-rc.14"
|
||||
percent-encoding = "2.3.2"
|
||||
capitalize = "0.3.4"
|
||||
|
||||
62
check.py
62
check.py
@@ -18,6 +18,8 @@ Checks per channel:
|
||||
- yt-dlp -J on each format.url that is not a direct media file
|
||||
"""
|
||||
|
||||
import os
|
||||
import shlex
|
||||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
@@ -149,13 +151,15 @@ def _is_cf_protected(url: str) -> bool:
|
||||
|
||||
|
||||
def http_ok(url: str, headers: dict | None = None) -> tuple[bool, int]:
|
||||
"""Return (ok, http_status). Tries HEAD then ranged GET on 405."""
|
||||
"""Return (ok, http_status). Tries HEAD, then a ranged GET when HEAD is refused."""
|
||||
h = {"User-Agent": _BROWSER_UA, **(headers or {})}
|
||||
try:
|
||||
r = requests.head(url, headers=h, timeout=HTTP_TIMEOUT, allow_redirects=True)
|
||||
if r.status_code in (200, 206):
|
||||
return True, r.status_code
|
||||
if r.status_code == 405:
|
||||
# Some servers mishandle HEAD (405/501, or even 404/403 while GET is 200);
|
||||
# confirm with a ranged GET before reporting the URL as broken.
|
||||
if r.status_code in (403, 404, 405, 501):
|
||||
r2 = requests.get(
|
||||
url,
|
||||
headers={**h, "Range": "bytes=0-1023"},
|
||||
@@ -176,6 +180,29 @@ def http_ok(url: str, headers: dict | None = None) -> tuple[bool, int]:
|
||||
return False, -3
|
||||
|
||||
|
||||
# Per-channel yt-dlp options taken from the channel's declared `ytdlpCommand`
|
||||
# (e.g. "yt-dlp --impersonate chrome-120"), which real clients apply. Probing
|
||||
# without them reports failures (TLS-fingerprint 403/502) no client sees.
|
||||
_CHANNEL_YTDLP_ARGS: dict[str, list[str]] = {}
|
||||
|
||||
|
||||
def register_ytdlp_command(channel_id: str, command: Optional[str]) -> None:
|
||||
if not command:
|
||||
return
|
||||
try:
|
||||
parts = shlex.split(command)
|
||||
except ValueError:
|
||||
return
|
||||
if parts and os.path.basename(parts[0]) in ("yt-dlp", "yt_dlp"):
|
||||
parts = parts[1:]
|
||||
if parts:
|
||||
_CHANNEL_YTDLP_ARGS[channel_id] = parts
|
||||
|
||||
|
||||
def ytdlp_args_for(channel_id: str) -> list[str]:
|
||||
return list(_CHANNEL_YTDLP_ARGS.get(channel_id, []))
|
||||
|
||||
|
||||
def ytdlp_extract(url: str, extra_args: list[str] | None = None) -> tuple[Optional[dict], str]:
|
||||
"""Run yt-dlp -j and return (parsed_info_or_None, stderr_text).
|
||||
|
||||
@@ -327,7 +354,29 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
||||
if not vurl:
|
||||
results.err(channel_id, f"{label}: missing url")
|
||||
elif "127.0.0.1" in vurl or "localhost" in vurl:
|
||||
results.err(channel_id, f"{label}: url points to hottub server: {vurl}")
|
||||
# A proxied video.url is legitimate when the page itself cannot be
|
||||
# played directly (see the hottub skill: "if a video URL requires a
|
||||
# server-side proxy, set it as the video URL"). Verify that it
|
||||
# actually resolves instead of rejecting it outright.
|
||||
resolved_vurl = follow_proxy_redirect(vurl)
|
||||
if resolved_vurl and resolved_vurl != vurl:
|
||||
ok, code = http_ok(resolved_vurl)
|
||||
target = resolved_vurl
|
||||
else:
|
||||
ok, code = http_ok(vurl)
|
||||
target = vurl
|
||||
if ok:
|
||||
results.info(channel_id, f"{label}: proxied url OK (HTTP {code})")
|
||||
elif _is_cf_protected(target):
|
||||
results.warn(
|
||||
channel_id,
|
||||
f"{label}: proxied url target unreachable HTTP={code} (CF-protected host, expected): {vurl}",
|
||||
)
|
||||
else:
|
||||
results.err(
|
||||
channel_id,
|
||||
f"{label}: proxied url does not resolve (HTTP={code}, target {target[:120]}): {vurl}",
|
||||
)
|
||||
else:
|
||||
ok, code = http_ok(vurl)
|
||||
if not ok:
|
||||
@@ -387,7 +436,7 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
||||
ytdlp_url = vurl if vurl and "127.0.0.1" not in vurl and "localhost" not in vurl else ""
|
||||
if not formats and ytdlp_url and is_page_url(ytdlp_url):
|
||||
results.info(channel_id, f"{label}: yt-dlp extract {ytdlp_url}")
|
||||
yt, stderr = ytdlp_extract(ytdlp_url)
|
||||
yt, stderr = ytdlp_extract(ytdlp_url, extra_args=ytdlp_args_for(channel_id))
|
||||
if yt is None:
|
||||
if _is_cf_protected(ytdlp_url):
|
||||
results.warn(
|
||||
@@ -455,7 +504,7 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
||||
for hk, hv in fheaders.items():
|
||||
extra_args += ["--add-header", f"{hk}:{hv}"]
|
||||
results.info(channel_id, f"{label} format[{j}]: yt-dlp extract {furl}")
|
||||
yt, stderr = ytdlp_extract(furl, extra_args=extra_args)
|
||||
yt, stderr = ytdlp_extract(furl, extra_args=ytdlp_args_for(channel_id) + extra_args)
|
||||
if yt is None:
|
||||
resolved = follow_proxy_redirect(furl)
|
||||
if _is_cf_protected(furl) or _is_cf_protected(resolved):
|
||||
@@ -579,6 +628,9 @@ def main():
|
||||
print("ERROR: no channels in /api/status response")
|
||||
sys.exit(1)
|
||||
|
||||
for ch in channels:
|
||||
register_ytdlp_command(ch["id"], ch.get("ytdlpCommand"))
|
||||
|
||||
print(f"status OK — {len(channels)} channels")
|
||||
|
||||
# --- per-channel checks ---
|
||||
|
||||
86
docs/provider-fix-todo.md
Normal file
86
docs/provider-fix-todo.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# Provider fix todo
|
||||
|
||||
Source: `check.py` + search probe audit, 2026-10-10. Serial re-run of every flagged channel,
|
||||
so timeouts below are not load artifacts. Update the status column as each item is fixed and
|
||||
verified (`HOT_TUB_PROVIDER=<id>` build, `/api/videos` feed + search + page 2, `yt-dlp` on
|
||||
`video.url` or `formats[0].url`, `python3 check.py <id>`).
|
||||
|
||||
Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not reproducible / not a provider bug (reason noted).
|
||||
|
||||
## 1. Channels returning no items
|
||||
|
||||
| | Channel | Symptom | Notes |
|
||||
|---|---|---|---|
|
||||
| [x] | `javtiful` | feed times out (150s) then 0 items; search 0 | site redesign (`article.video-card`); rewrote parser, page URL as `video.url` (yt-dlp-native), no per-item detail fetches. 23 items, p2/search OK |
|
||||
| [x] | `sextb` | 0 items on feed, search and page 2 | Jina mirror now 403/429; site is plain HTML reachable with the emulating client. Rewrote listing/detail/option parsers for HTML, removed mirror. Remaining check.py errors are tooling items (proxy URL policy; imggle.net thumbs are TLS-gated for `requests`) |
|
||||
| [-] | `camsoda` | 0 items on every request | not a provider bug here: `www.camsoda.com` is DNS-sinkholed by this host's resolver (0.0.0.0; public resolvers return 74.122.168.1), local FlareSolverr gets ERR_CONNECTION_REFUSED, Jina is 403/429, and Cloudflare serves a managed challenge to non-browser clients. Needs the production network to verify. |
|
||||
| [-] | `hentaimama` | 0 items or 85-150s timeouts | not fixable in code from this host: site now serves a Cloudflare managed challenge (`cf-mitigated: challenge`) on every path; browser-TLS impersonation gets 403 and the local FlareSolverr (:8191, process outside this sandbox) hangs even on a non-CF page (javtiful, 60s). Latency is the requester waiting on FlareSolverr. Needs a healthy FlareSolverr to verify. |
|
||||
| [-] | `hotbunny` | 0 items; first request 78s | same cause: `hotbunny.ai` (including `/api/post/*`) now answers a Cloudflare managed challenge; previously the JSON API was reachable directly. Needs a healthy FlareSolverr. |
|
||||
|
||||
## 2. Playback: page URL not resolvable by yt-dlp / formats not playable
|
||||
|
||||
| | Channel | Symptom | Notes |
|
||||
|---|---|---|---|
|
||||
| [x] | `thaiporntv` | page URL "Unsupported URL", no `formats` | player attribute renamed `data-enc` -> `data-video-sources` (reversed base64 JSON `{quality,url}`); decoder handles both. 3 formats/item, yt-dlp OK, check.py passes |
|
||||
| [x] | `pornmz` | page URL "Unsupported URL", no `formats` | site changed `itemprop="contentURL"` -> `contentUrl`; selector now case-insensitive. Enrichment concurrency 10 + 25s per-item timeout. Origin is slow (15s+/detail) so first feed 25-50s. check.py passes |
|
||||
| [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end |
|
||||
| [x] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | sponsored `channel` cards carry a stale `data-video-id` (404) while the card link is the live page; id/URL now come from the link. Signed `pix-cdn77` thumbs 403 without a Referer: new host-restricted `/proxy/aylo-thumb` (also fixes ph-shorties). check.py passes |
|
||||
| [x] | `hypnotube` | page URLs 404 on all sampled items | the 404 was a check.py artifact (server answers HEAD 404, GET 200); check.py now confirms with a ranged GET. Real bug found on the way: the site returns ONE results page per searchgate POST, so any 2nd search on the shared session (and every page 2) came back empty. Each (query, page) now uses its own isolated session. 3 consecutive searches + page 2 verified |
|
||||
| [x] | `fullpornerorg` | `lulustream` proxy 403 / 502 | three separate causes: (1) hoster files deleted ("File is no longer available") but still listed -> now probed and dropped; (2) `myvidplay.com` embeds lost their host (bare ID sent to lulustream, wrong namespace) -> routed to the doodstream proxy with host kept, which now allows myvidplay/playmogo; (3) tnmr.org m3u8 is TLS-fingerprint gated (works with chrome TLS = the channel's `--impersonate chrome-120`; plain yt-dlp 403s are the checker's expected CF warnings). myvidplay playback could not be verified: the embed now returns a Turnstile captcha shell to this client |
|
||||
| [-] | `ripnsfw` | `doodstream` format 502 | lulustream format on the same items resolves (302). The doodstream format cannot be verified here: `*.dood.video` resolves to 127.0.0.1 on this host's resolver, so the CDN is unreachable. Needs the production network |
|
||||
| [-] | `freepornvideosxxx` | yt-dlp 502 on page URL | listing/search/p2 fine (24 items). Page URL is yt-dlp-native (html5 `<source>` 480/720/2160). The 502 is the site's edge answering plain yt-dlp's TLS fingerprint (3 of 4 items 502; 4/4 fine with `--impersonate chrome-120`; curl always 200). The channel declares that command but see "Open decision" below: status never delivers it |
|
||||
| [-] | `hdporn92` | m3u8 formats time out (curl + yt-dlp) | provider works (embed deobfuscated, signed m3u8 returned, yt-dlp resolves the first item). Every morencius.com request (embed page AND playlist) carries a constant ~20.1s server-side delay from this egress (detail page on hdporn92.com is 0.25s), which exceeds check.py's 15s HEAD / 20s yt-dlp socket timeouts. Raising enrichment concurrency 4 -> 10 made it worse (108s, then timeouts), so it was reverted. Likely a host tarpit for this network; verify from the production IP |
|
||||
|
||||
## 3. Thumbnails
|
||||
|
||||
| | Channel | Symptom | Notes |
|
||||
|---|---|---|---|
|
||||
| [x] | `xxdbx` | malformed thumbs `https:/7004….jpg` | assets are now site-relative (`/id.jpg`) rather than `//host/..`; new `absolute_url` resolves all three forms. 36 items, p2/search OK, check.py passes |
|
||||
| [x] | `ph-shorties` | thumbs return 403 | same Referer-gated `pix-cdn77.phncdn.com` thumbs; routed through `aylo-thumb` proxy. check.py passes |
|
||||
| [x] | `noodlemagazine` | thumbs unreachable (-1 / 400) in check.py | thumb proxy dropped the preview's query string (`?id&tkn..`), rejected extensionless `getVideoPreview` paths, and warmed up via noodlemagazine.com through the fallback chain on every image (10-28s). All 24 thumbs now 200 in ~0.08s. Titles fix from earlier commit stands |
|
||||
|
||||
## 4. Search
|
||||
|
||||
| | Channel | Symptom | Notes |
|
||||
|---|---|---|---|
|
||||
| [-] | `fapello` | "girl" and "milf" both 0 | by design, documented in the provider: the site's search returns photo models, not videos; no keyword video search exists, so queries are literal-filtered over the feed |
|
||||
| [x] | `chaturbate` | "girl" and "milf" both 0 | format arguments swapped in the search URL (`girl/api/...?keywords=https://chaturbate.com`, a relative-URL builder error); fixed + unit test. Also stale-cache branch never refreshed. 90 results per query, yt-dlp resolves the live stream |
|
||||
| [x] | `cumlouder` | unknown tag takes 146s before returning 0 | a real 404 went through the requester's Jina/FlareSolverr fallback chain; now the raw response is read, 404 returns empty immediately (0s), 5xx/429 still retried. check.py passes |
|
||||
| [x] | `vjav` | "girl" 0, "milf" 10 | not term-specific: "girl" exactly matches a member username with no uploads, so the implicit uploader match hijacked the keyword search. Empty uploader/tag matches now fall back to keyword search (site API returns 34k results for "girl"). check.py passes |
|
||||
| [x] | `xxxtik` | "girl" 0 (not re-run serially) | three things. (1) shared requester: raw requests did not replay the remembered FlareSolverr identity, so a RedGifs token minted under the solved UA was rejected (401) on every gif lookup after one transient connect error (empty feed, 71s); `get_raw_with_headers_timeout` now adopts it. (2) tag/creator pages mixing non-RedGifs posts (erome/onlyfans links) emptied whole pages; now pulls up to 4 raw batches per page. (3) "girl" itself has no RedGifs posts in its first 600 entries, so empty is correct there; "milf" returns 10. Feed/p2 10 items, check.py passes. First request can still take ~50s when api.redgifs.com has a connect error (FlareSolverr detour) |
|
||||
| [x] | `hypnotube` | "milf" 0, "girl" 20 | same root cause as above (second search on a session returned nothing), not term-specific |
|
||||
|
||||
## 5. Slow channels
|
||||
|
||||
| | Channel | Symptom | Notes |
|
||||
|---|---|---|---|
|
||||
| [-] | `aps` | 30-98s per request; check.py 30s timeout | works (57 items, p2/search OK). allpornstream.com answers every client, including browser-TLS impersonation, with a Cloudflare interactive challenge ("Just a moment..."), so each uncached request is a FlareSolverr solve; the local FlareSolverr is slow/wedged. Provider code is not the bottleneck |
|
||||
| [-] | `hdporn92` | feed about 66s per request | consequence of the 20s per-embed host delay above (10 embeds per page); not fixable in the provider without moving to play-time resolution, which would put the 20s on every play and every HLS playlist request |
|
||||
| [-] | `porndish` | 14-23s feed/search | same class: Cloudflare challenge on porndish.com (plain/yt-dlp 403s), resolved through FlareSolverr per uncached request; also its myvidplay playback goes through the `curl_cffi` python subprocess, which is not installed in this sandbox |
|
||||
|
||||
## 6. Tooling
|
||||
|
||||
| | Item | Notes |
|
||||
|---|---|---|
|
||||
| [x] | `check.py` "url points to hottub server" | contradicted the hottub skill | proxied `video.url` is now verified to resolve (redirect target reachable, CF hosts -> warning) instead of hard-failing. Affected: spankbang, hqporner, pornhd3x, pimpbunny, sxyprn, noodlemagazine |
|
||||
|
||||
## Decision: `ytdlpCommand` delivery (agreed: targeted rollout)
|
||||
|
||||
Providers declare `Channel.ytdlpCommand` (~90 channels), but `/api/status` used to send it only for `pimpbunny`.
|
||||
Delivering all of them at once risks breaking working channels if the app's yt-dlp lacks impersonation support, so
|
||||
`decorate_channel` now delivers a provider's declared command **only for channels in `YTDLP_COMMAND_CHANNELS`**
|
||||
(`src/providers/mod.rs`): `freepornvideosxxx`, `missav`. Add a channel there once plain yt-dlp is verified to fail for it.
|
||||
|
||||
Still to do with the app side: confirm the app applies `ytdlpCommand` for non-pimpbunny channels and that its yt-dlp has an
|
||||
impersonation target; if a minimum client version is needed, gate on `ClientVersion` (not done, the minimum is unknown).
|
||||
`check.py` applies whatever `/api/status` exposes, so it verifies each addition.
|
||||
|
||||
## Found while fixing (pre-existing, unrelated)
|
||||
|
||||
- `src/proxy.rs`: the `pornhub-thumb` resource was registered as `/proxy/pornhub-thumb` inside the `/proxy` scope (unreachable); fixed alongside `aylo-thumb`. No provider uses it today.
|
||||
- Failing unit tests, untouched files: `proxies::hanimecdn::tests::prefixes_relative_paths_with_hanime_cdn_host`, `proxies::vidara::tests::normalizes_detail_request_with_path_only`.
|
||||
|
||||
## Environmental / needs the production IP (not worked here)
|
||||
|
||||
`porndish`, `supjav`, `jable`, `fapello`, `missav`, `wowxxx`, `hentaihaven` return Cloudflare 403 to
|
||||
yt-dlp/curl from this machine while the server itself returns items. `hsex` returned 429 (rate limit).
|
||||
@@ -138,6 +138,34 @@ impl ChaturbateProvider {
|
||||
Ok(video_items)
|
||||
}
|
||||
|
||||
/// `keywords=` search on the room list. (The base URL and query used to be
|
||||
/// passed in swapped order, producing `milf/api/...?keywords=https://...`.)
|
||||
fn build_search_url(base: &str, query: &str, page: u8) -> String {
|
||||
let keywords = query
|
||||
.trim()
|
||||
.chars()
|
||||
.map(|c| match c {
|
||||
' ' => "+".to_string(),
|
||||
c if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '~') => {
|
||||
c.to_string()
|
||||
}
|
||||
c => {
|
||||
let mut buf = [0u8; 4];
|
||||
c.encode_utf8(&mut buf)
|
||||
.bytes()
|
||||
.map(|b| format!("%{b:02X}"))
|
||||
.collect::<String>()
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
format!(
|
||||
"{}/api/ts/roomlist/room-list/?keywords={}&limit=90&offset={}",
|
||||
base,
|
||||
keywords,
|
||||
90 * (page.max(1) as u32 - 1)
|
||||
)
|
||||
}
|
||||
|
||||
async fn query(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
@@ -145,22 +173,15 @@ impl ChaturbateProvider {
|
||||
query: &str,
|
||||
options: ServerOptions,
|
||||
) -> Result<Vec<VideoItem>> {
|
||||
let mut video_url = format!(
|
||||
"{}/api/ts/roomlist/room-list/?keywords={}&limit=90&offset={}",
|
||||
query,
|
||||
self.url,
|
||||
90 * (page - 1)
|
||||
);
|
||||
video_url = video_url.replace(" ", "+");
|
||||
let video_url = Self::build_search_url(&self.url, query, page);
|
||||
// Check our Video Cache. If the result is younger than 1 hour, we return it.
|
||||
let old_items = match cache.get(&video_url) {
|
||||
Some((time, items)) => {
|
||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 1 {
|
||||
return Ok(items.clone());
|
||||
} else {
|
||||
let _ = cache.check().await;
|
||||
return Ok(items.clone());
|
||||
}
|
||||
// Expired: refetch, keeping the stale list as the fallback.
|
||||
items.clone()
|
||||
}
|
||||
None => {
|
||||
vec![]
|
||||
@@ -347,3 +368,18 @@ impl Provider for ChaturbateProvider {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ChaturbateProvider;
|
||||
|
||||
#[test]
|
||||
fn builds_search_url_with_base_first() {
|
||||
assert_eq!(
|
||||
ChaturbateProvider::build_search_url("https://chaturbate.com", "big tits", 2),
|
||||
"https://chaturbate.com/api/ts/roomlist/room-list/?keywords=big+tits&limit=90&offset=90"
|
||||
);
|
||||
assert!(ChaturbateProvider::build_search_url("https://chaturbate.com", "a&b", 1)
|
||||
.contains("keywords=a%26b&"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,8 @@ const REFERER: &str = "https://coomer.st/";
|
||||
const JSON_ACCEPT: &str = "text/css,*/*;q=0.1";
|
||||
const CACHE_TTL_SECS: u64 = 60 * 15;
|
||||
const MAX_PAGE: u16 = 50;
|
||||
/// The API pages by offset (`o`), in blocks of this many posts. `page=` is ignored.
|
||||
const API_PAGE_SIZE: u16 = 50;
|
||||
const DEFAULT_PER_PAGE: usize = 30;
|
||||
const USER_POSTS_FETCH_CONCURRENCY: usize = 4;
|
||||
|
||||
@@ -239,19 +241,18 @@ impl CoomerProvider {
|
||||
}
|
||||
}
|
||||
|
||||
fn offset_for(page: u16) -> u32 {
|
||||
(page.max(1).min(MAX_PAGE) as u32 - 1) * API_PAGE_SIZE as u32
|
||||
}
|
||||
|
||||
fn build_main_url(target: &Target, page: u16) -> String {
|
||||
let page = page.max(1).min(MAX_PAGE);
|
||||
let offset = Self::offset_for(page);
|
||||
match target {
|
||||
Target::Latest => format!("{}/api/v1/posts?limit=50&page={page}", BASE_URL),
|
||||
Target::Popular => format!("{}/api/v1/posts/popular?limit=50&page={page}", BASE_URL),
|
||||
// Search is non-paginated: ?page=N is ignored by the server for
|
||||
// q=... requests, but we still emit a `page=1` for consistency.
|
||||
Target::Latest => format!("{}/api/v1/posts?o={offset}", BASE_URL),
|
||||
Target::Popular => format!("{}/api/v1/posts/popular?o={offset}", BASE_URL),
|
||||
Target::Search { query } => {
|
||||
let encoded = percent_encode(query);
|
||||
format!(
|
||||
"{}/api/v1/posts?q={encoded}&limit=50&page=1",
|
||||
BASE_URL
|
||||
)
|
||||
format!("{}/api/v1/posts?q={encoded}&o={offset}", BASE_URL)
|
||||
}
|
||||
// Uploader URL is built separately.
|
||||
Target::Uploader { .. } => String::new(),
|
||||
@@ -259,14 +260,12 @@ impl CoomerProvider {
|
||||
}
|
||||
|
||||
fn build_uploader_url(service: &Service, username: &str, page: u16) -> String {
|
||||
let page = page.max(1).min(MAX_PAGE);
|
||||
// The user-posts endpoint is also non-paginated, but we send page=1 for
|
||||
// clarity. Service comes from the enum so the path is stable.
|
||||
format!(
|
||||
"{}/api/v1/{}/user/{}/posts?limit=50&page={page}",
|
||||
"{}/api/v1/{}/user/{}/posts?o={}",
|
||||
BASE_URL,
|
||||
service.as_str(),
|
||||
percent_encode(username),
|
||||
Self::offset_for(page),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -308,25 +307,21 @@ impl CoomerProvider {
|
||||
per_page_limit: usize,
|
||||
options: &ServerOptions,
|
||||
) -> Result<Vec<VideoItem>> {
|
||||
let cache_key = match &target {
|
||||
Target::Uploader { service, username } => {
|
||||
format!("uploader::{}::{}::{}", service.as_str(), username, page)
|
||||
}
|
||||
_ => format!("main::{}::{}", target_label(&target), page),
|
||||
};
|
||||
|
||||
if let Some((time, items)) = cache.get(&cache_key) {
|
||||
if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS {
|
||||
return Ok(items.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let url = match &target {
|
||||
Target::Uploader { service, username } => {
|
||||
Self::build_uploader_url(service, username, page)
|
||||
}
|
||||
_ => Self::build_main_url(&target, page),
|
||||
};
|
||||
// Key on the full request URL: the search query must be part of it,
|
||||
// otherwise every search shares one cached page.
|
||||
let cache_key = format!("coomer::{url}");
|
||||
|
||||
if let Some((time, items)) = cache.get(&cache_key) {
|
||||
if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS {
|
||||
return Ok(items.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let posts = self.fetch_json(options, "coomer.fetch_items", &url).await?;
|
||||
eprintln!("[coomer] fetched {} posts from {}", posts.len(), url);
|
||||
@@ -389,47 +384,46 @@ impl CoomerProvider {
|
||||
});
|
||||
let page_url = format!("{}/{}/user/{}/post/{}", BASE_URL, service, user, id);
|
||||
|
||||
// Coomer returns the file list in `attachments`; the main image/video
|
||||
// lives in `file` for the first attachment. The `file.path` is the
|
||||
// data-path (without the /data/ prefix); we have to prepend it.
|
||||
let preview_file = post.file.as_ref();
|
||||
let preview_url: Option<String> = preview_file
|
||||
// Coomer returns the main file in `file` and the rest in `attachments`.
|
||||
// Pick the first *video* across both: a video can sit in an attachment
|
||||
// while `file` is a cover image. Image-only posts have nothing
|
||||
// playable (and yt-dlp can't resolve the post page), so they are
|
||||
// dropped from this video channel.
|
||||
let is_video_ext =
|
||||
|ext: &str| matches!(ext.to_ascii_lowercase().as_str(), "mp4" | "webm" | "mov" | "mkv" | "m4v");
|
||||
let mut candidates: Vec<&ApiFile> = Vec::new();
|
||||
if let Some(file) = post.file.as_ref() {
|
||||
candidates.push(file);
|
||||
}
|
||||
if let Some(attachments) = post.attachments.as_ref() {
|
||||
candidates.extend(attachments.iter());
|
||||
}
|
||||
let video_file = candidates.into_iter().find(|file| {
|
||||
file.path.as_deref().is_some_and(|path| !path.is_empty())
|
||||
&& file
|
||||
.name
|
||||
.as_deref()
|
||||
.and_then(extension_from_name)
|
||||
.or_else(|| file.path.as_deref().and_then(extension_from_name))
|
||||
.is_some_and(|ext| is_video_ext(ext))
|
||||
})?;
|
||||
|
||||
let preview_url: Option<String> = post
|
||||
.file
|
||||
.as_ref()
|
||||
.and_then(|file| file.path.as_deref())
|
||||
.filter(|path| !path.is_empty())
|
||||
.map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
|
||||
let mut format_url: Option<String> = None;
|
||||
let mut ext_hint: Option<String> = None;
|
||||
|
||||
if let Some(file) = preview_file {
|
||||
if let Some(name) = file.name.as_deref() {
|
||||
if let Some(ext) = extension_from_name(name) {
|
||||
ext_hint = Some(ext.to_string());
|
||||
}
|
||||
}
|
||||
// The main file is the playable media. For images it's still
|
||||
// served as the media URL; for videos it's the actual .mp4.
|
||||
if let Some(path) = file.path.as_deref() {
|
||||
format_url = Some(format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
|
||||
}
|
||||
}
|
||||
|
||||
if format_url.is_none() {
|
||||
if let Some(attach) = post.attachments.as_ref().and_then(|a| a.first()) {
|
||||
if let Some(path) = attach.path.as_deref() {
|
||||
format_url = Some(format!(
|
||||
"{}/data/{}",
|
||||
BASE_URL,
|
||||
path.trim_start_matches('/')
|
||||
));
|
||||
}
|
||||
if ext_hint.is_none() {
|
||||
if let Some(name) = attach.name.as_deref() {
|
||||
if let Some(ext) = extension_from_name(name) {
|
||||
ext_hint = Some(ext.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let format_url: Option<String> = video_file
|
||||
.path
|
||||
.as_deref()
|
||||
.map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
|
||||
let ext_hint: Option<String> = video_file
|
||||
.name
|
||||
.as_deref()
|
||||
.and_then(extension_from_name)
|
||||
.or_else(|| video_file.path.as_deref().and_then(extension_from_name))
|
||||
.map(|ext| ext.to_ascii_lowercase());
|
||||
|
||||
// Final fallback for the thumbnail: the preview file's data URL, or
|
||||
// the format URL if the file itself is the only image.
|
||||
@@ -461,7 +455,7 @@ impl CoomerProvider {
|
||||
|
||||
if let Some(ref url) = format_url {
|
||||
let ext = ext_hint.unwrap_or_else(|| "mp4".to_string());
|
||||
let is_video = matches!(ext.as_str(), "mp4" | "webm" | "mov" | "mkv");
|
||||
let is_video = is_video_ext(&ext);
|
||||
// An image-only post (very common - many creators post photos,
|
||||
// not just videos) has nothing playable; emitting a "format"
|
||||
// pointing at a static jpg/png just gives players a broken
|
||||
@@ -495,15 +489,6 @@ impl CoomerProvider {
|
||||
}
|
||||
}
|
||||
|
||||
fn target_label(target: &Target) -> &'static str {
|
||||
match target {
|
||||
Target::Latest => "latest",
|
||||
Target::Popular => "popular",
|
||||
Target::Search { .. } => "search",
|
||||
Target::Uploader { .. } => "uploader",
|
||||
}
|
||||
}
|
||||
|
||||
fn percent_encode(value: &str) -> String {
|
||||
let mut out = String::with_capacity(value.len());
|
||||
for byte in value.as_bytes() {
|
||||
@@ -660,11 +645,11 @@ mod tests {
|
||||
fn builds_latest_url() {
|
||||
assert_eq!(
|
||||
CoomerProvider::build_main_url(&Target::Latest, 1),
|
||||
"https://coomer.st/api/v1/posts?limit=50&page=1"
|
||||
"https://coomer.st/api/v1/posts?o=0"
|
||||
);
|
||||
assert_eq!(
|
||||
CoomerProvider::build_main_url(&Target::Latest, 3),
|
||||
"https://coomer.st/api/v1/posts?limit=50&page=3"
|
||||
"https://coomer.st/api/v1/posts?o=100"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -672,7 +657,7 @@ mod tests {
|
||||
fn builds_popular_url() {
|
||||
assert_eq!(
|
||||
CoomerProvider::build_main_url(&Target::Popular, 1),
|
||||
"https://coomer.st/api/v1/posts/popular?limit=50&page=1"
|
||||
"https://coomer.st/api/v1/posts/popular?o=0"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -683,22 +668,24 @@ mod tests {
|
||||
};
|
||||
let url = CoomerProvider::build_main_url(&target, 1);
|
||||
assert!(url.contains("q=milf%20%2B%20solo"));
|
||||
assert!(url.contains("limit=50"));
|
||||
assert!(url.ends_with("&o=0"));
|
||||
let page_three = CoomerProvider::build_main_url(&target, 3);
|
||||
assert!(page_three.ends_with("&o=100"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_uploader_url_per_service() {
|
||||
assert_eq!(
|
||||
CoomerProvider::build_uploader_url(&Service::Onlyfans, "tabycatxoxo", 1),
|
||||
"https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?limit=50&page=1"
|
||||
"https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?o=0"
|
||||
);
|
||||
assert_eq!(
|
||||
CoomerProvider::build_uploader_url(&Service::Fansly, "ruby", 2),
|
||||
"https://coomer.st/api/v1/fansly/user/ruby/posts?limit=50&page=2"
|
||||
"https://coomer.st/api/v1/fansly/user/ruby/posts?o=50"
|
||||
);
|
||||
assert_eq!(
|
||||
CoomerProvider::build_uploader_url(&Service::Candfans, "model", 1),
|
||||
"https://coomer.st/api/v1/candfans/user/model/posts?limit=50&page=1"
|
||||
"https://coomer.st/api/v1/candfans/user/model/posts?o=0"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -835,11 +822,25 @@ mod tests {
|
||||
"file": {"name": "p.jpg", "path": "/x/p.jpg"}
|
||||
}"#;
|
||||
let post: ApiPost = serde_json::from_str(json).expect("parse");
|
||||
// Image-only posts have nothing playable and are dropped.
|
||||
assert!(CoomerProvider::build_video_item(post).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_video_item_uses_video_attachment_when_file_is_image() {
|
||||
let json = r#"{
|
||||
"id": "43",
|
||||
"user": "ruby",
|
||||
"service": "fansly",
|
||||
"title": "clip",
|
||||
"file": {"name": "cover.jpg", "path": "/c/cover.jpg"},
|
||||
"attachments": [{"name": "a.jpg", "path": "/a/a.jpg"}, {"name": "v.MP4", "path": "/v/v.mp4"}]
|
||||
}"#;
|
||||
let post: ApiPost = serde_json::from_str(json).expect("parse");
|
||||
let item = CoomerProvider::build_video_item(post).expect("item");
|
||||
assert_eq!(item.title, "Hello there friends");
|
||||
assert_eq!(item.thumb, "https://coomer.st/data/x/p.jpg");
|
||||
// For an image attachment, no formats are emitted (preview is enough).
|
||||
assert!(item.formats.is_none());
|
||||
let formats = item.formats.expect("video attachment must yield a format");
|
||||
assert_eq!(formats[0].url, "https://coomer.st/data/v/v.mp4");
|
||||
assert_eq!(item.thumb, "https://coomer.st/data/c/cover.jpg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -357,22 +357,33 @@ impl CumlouderProvider {
|
||||
let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))];
|
||||
|
||||
// The origin sporadically answers its first uncached request with a
|
||||
// Cloudflare 522; a single retry succeeds.
|
||||
// Cloudflare 522; a retry succeeds. Use the raw response so a real
|
||||
// 404 (unknown tag/category) answers immediately instead of
|
||||
// escalating through the requester's mirror/FlareSolverr fallbacks,
|
||||
// which cost minutes.
|
||||
let mut last_err = String::new();
|
||||
let mut html = None;
|
||||
for _ in 0..3 {
|
||||
match requester
|
||||
.get_with_headers(&url, headers.clone(), None)
|
||||
.get_raw_with_headers(&url, headers.clone())
|
||||
.await
|
||||
{
|
||||
Ok(body) if body.contains("muestra-escena") => {
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
Ok(body) => {
|
||||
Ok(response) => {
|
||||
let status = response.status().as_u16();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
if status == 404 {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
if body.contains("muestra-escena") {
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
if status >= 500 || status == 429 {
|
||||
last_err = format!("HTTP {status}");
|
||||
continue;
|
||||
}
|
||||
// A page with no cards is a real empty archive.
|
||||
last_err = format!("no cards in {} bytes", body.len());
|
||||
// A page with no cards is a real empty archive (404 body
|
||||
// included); only keep retrying on transport errors.
|
||||
html = Some(body);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ use futures::stream::{self, StreamExt};
|
||||
use regex::Regex;
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
const DELETED_EMBED: &str = "hoster file deleted";
|
||||
|
||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||
crate::providers::ProviderChannelMetadata {
|
||||
group_id: "mainstream-tube",
|
||||
@@ -311,8 +313,28 @@ impl FullpornerorgProvider {
|
||||
None
|
||||
}
|
||||
|
||||
/// Builds the lulustream proxy URL for a given embed ID.
|
||||
fn build_proxy_url(embed_id: &str, options: &ServerOptions) -> String {
|
||||
/// The hoster serves a tiny stub page for removed files. Listings on
|
||||
/// fullporner.org keep pointing at them, so detect it and skip the item.
|
||||
fn embed_is_deleted(html: &str) -> bool {
|
||||
let lower = html.to_ascii_lowercase();
|
||||
lower.contains("no longer available")
|
||||
|| lower.contains("has been deleted")
|
||||
// DoodStream-protocol hosts (myvidplay.com) title the stub this way.
|
||||
|| lower.contains("<title>video not found")
|
||||
}
|
||||
|
||||
/// Builds the proxy URL for an embed. The ID alone is only unique per
|
||||
/// hoster: `myvidplay.com` is a DoodStream-protocol host, so those embeds
|
||||
/// must keep their host and go through the doodstream proxy; resolving the
|
||||
/// same ID on lulustream finds nothing (or a different, deleted file).
|
||||
fn build_proxy_url(embed_url: &str, embed_id: &str, options: &ServerOptions) -> String {
|
||||
if embed_url.contains("myvidplay.com/") {
|
||||
return crate::providers::build_proxy_url(
|
||||
options,
|
||||
"doodstream",
|
||||
&format!("myvidplay.com/e/{embed_id}"),
|
||||
);
|
||||
}
|
||||
crate::providers::build_proxy_url(options, "lulustream", embed_id)
|
||||
}
|
||||
|
||||
@@ -324,9 +346,17 @@ impl FullpornerorgProvider {
|
||||
let embed_id = Self::extract_embed_id(&embed_url)
|
||||
.ok_or_else(|| Error::from(format!("could not parse embed id from {embed_url}")))?;
|
||||
|
||||
// Skip files the hoster has already removed (fail open if the probe
|
||||
// itself is blocked or times out).
|
||||
if let Ok(embed_html) = Self::fetch_html(options, &embed_url, BASE_URL).await {
|
||||
if Self::embed_is_deleted(&embed_html) {
|
||||
return Err(Error::from(DELETED_EMBED));
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: build the proxy URL (will hit /proxy/lulustream/{embed_id} which
|
||||
// resolves the actual HLS URL via the lulustream proxy).
|
||||
let proxy_url = Self::build_proxy_url(&embed_id, options);
|
||||
let proxy_url = Self::build_proxy_url(&embed_url, &embed_id, options);
|
||||
|
||||
// Build the format. The proxy will return a redirect to the actual HLS URL.
|
||||
let mut format = VideoFormat::m3u8(proxy_url, "auto".to_string(), "m3u8".to_string());
|
||||
@@ -371,11 +401,13 @@ impl FullpornerorgProvider {
|
||||
match Self::enrich_card(stub, &options).await {
|
||||
Ok(item) => Some(item),
|
||||
Err(error) => {
|
||||
report_provider_error_background(
|
||||
CHANNEL_ID,
|
||||
"fetch_page.enrich_card",
|
||||
&error.to_string(),
|
||||
);
|
||||
if error.to_string() != DELETED_EMBED {
|
||||
report_provider_error_background(
|
||||
CHANNEL_ID,
|
||||
"fetch_page.enrich_card",
|
||||
&error.to_string(),
|
||||
);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -482,6 +514,36 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn myvidplay_embeds_route_to_doodstream_with_host() {
|
||||
let options = ServerOptions {
|
||||
client_version: None, featured: None, category: None, sites: None, filter: None,
|
||||
language: None, public_url_base: None, requester: None, network: None, stars: None,
|
||||
categories: None, duration: None, sort: None, sexuality: None,
|
||||
};
|
||||
assert_eq!(
|
||||
FullpornerorgProvider::build_proxy_url("https://myvidplay.com/e/abc123", "abc123", &options),
|
||||
"/proxy/doodstream/myvidplay.com/e/abc123"
|
||||
);
|
||||
assert_eq!(
|
||||
FullpornerorgProvider::build_proxy_url("https://luluvdo.com/e/abc123", "abc123", &options),
|
||||
"/proxy/lulustream/abc123"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_deleted_embed_stub() {
|
||||
assert!(FullpornerorgProvider::embed_is_deleted(
|
||||
"<body>File is no longer available as it expired or has been deleted.</body>"
|
||||
));
|
||||
assert!(FullpornerorgProvider::embed_is_deleted(
|
||||
"<html><head><title>Video not found | DoodStream</title></head></html>"
|
||||
));
|
||||
assert!(!FullpornerorgProvider::embed_is_deleted(
|
||||
"<html><title>Payton Preslee - Lulustream.mp4</title><script>eval(function(p,a,c,k,e,d){}</script></html>"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn picks_search_for_query() {
|
||||
match FullpornerorgProvider::pick_target(Some("test")) {
|
||||
|
||||
@@ -14,7 +14,11 @@ use htmlentity::entity::{ICodedDataTrait, decode};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::{thread, vec};
|
||||
use titlecase::Titlecase;
|
||||
use wreq::Client;
|
||||
use wreq::Version;
|
||||
use wreq::cookie::Jar;
|
||||
use wreq::redirect::Policy;
|
||||
use wreq_util::Emulation;
|
||||
|
||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||
crate::providers::ProviderChannelMetadata {
|
||||
@@ -246,6 +250,43 @@ impl HypnotubeProvider {
|
||||
video_items
|
||||
}
|
||||
|
||||
/// Fetches one page of search results.
|
||||
///
|
||||
/// The site hands out exactly one results page per `searchgate.php` POST
|
||||
/// (a second GET of the same search, or any later search on the same
|
||||
/// session, comes back empty), so every (query, page) uses its own
|
||||
/// isolated cookie session instead of the shared requester jar. Page 1 is
|
||||
/// the redirect target of the POST; page N>1 is fetched directly.
|
||||
async fn fetch_search_html(&self, query: &str, page: u8) -> Result<String> {
|
||||
let client = Client::builder()
|
||||
.tls_cert_verification(false)
|
||||
.emulation(Emulation::Firefox136)
|
||||
.cookie_provider(Arc::new(Jar::default()))
|
||||
.redirect(if page <= 1 { Policy::default() } else { Policy::none() })
|
||||
.build()?;
|
||||
|
||||
let body = format!("q={}&type=videos", query.trim().replace(' ', "+"));
|
||||
let gate = client
|
||||
.post(format!("{}/searchgate.php", self.url))
|
||||
.header("Content-Type", "application/x-www-form-urlencoded")
|
||||
.header("Referer", format!("{}/", self.url))
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
if page <= 1 {
|
||||
return Ok(gate.text().await?);
|
||||
}
|
||||
|
||||
let slug = query.trim().replace(' ', "%20");
|
||||
let page_url = format!("{}/search/videos/{}/page{}.html", self.url, slug, page);
|
||||
let response = client
|
||||
.get(page_url)
|
||||
.header("Referer", format!("{}/search/videos/{}/", self.url, slug))
|
||||
.send()
|
||||
.await?;
|
||||
Ok(response.text().await?)
|
||||
}
|
||||
|
||||
async fn query(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
@@ -281,41 +322,18 @@ impl HypnotubeProvider {
|
||||
}
|
||||
};
|
||||
|
||||
let mut requester =
|
||||
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
|
||||
let post_response = match requester
|
||||
.post(
|
||||
format!("{}/searchgate.php", self.url).as_str(),
|
||||
format!("q={}&type=videos", query.replace(" ", "+")).as_str(),
|
||||
vec![("Content-Type", "application/x-www-form-urlencoded")],
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(response) => response,
|
||||
let text = match self.fetch_search_html(query, page).await {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
crate::providers::report_provider_error(
|
||||
"hypnotube",
|
||||
"query.search_post",
|
||||
"query.search",
|
||||
&format!("url={video_url}; error={e}"),
|
||||
)
|
||||
.await;
|
||||
return old_items;
|
||||
}
|
||||
};
|
||||
let text = match post_response.text().await {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
eprint!("Hypnotube search POST request failed: {}", e);
|
||||
crate::providers::report_provider_error_background(
|
||||
"hypnotube",
|
||||
"query.search_post.text",
|
||||
&e.to_string(),
|
||||
);
|
||||
return vec![];
|
||||
}
|
||||
};
|
||||
// println!("Hypnotube search POST response status: {}", p.text().await.unwrap_or_default());
|
||||
// let text = requester.get(&video_url, Some(Version::HTTP_11)).await.unwrap();
|
||||
if text.contains("Sorry, no results were found.") {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
@@ -1,20 +1,16 @@
|
||||
use crate::DbPool;
|
||||
use crate::api::ClientVersion;
|
||||
use crate::providers::{Provider, build_proxy_url, strip_url_scheme};
|
||||
use crate::providers::Provider;
|
||||
use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::discord::{format_error_chain, send_discord_error_report};
|
||||
use crate::util::requester::Requester;
|
||||
use crate::util::parse_abbreviated_number;
|
||||
use crate::util::time::parse_time_to_seconds;
|
||||
use crate::videos::{ServerOptions, VideoItem};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use error_chain::error_chain;
|
||||
use futures::future::join_all;
|
||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::vec;
|
||||
use titlecase::Titlecase;
|
||||
use wreq::Version;
|
||||
|
||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||
@@ -40,16 +36,13 @@ error_chain! {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JavtifulProvider {
|
||||
url: String,
|
||||
categories: Arc<RwLock<Vec<FilterOption>>>,
|
||||
}
|
||||
|
||||
impl JavtifulProvider {
|
||||
pub fn new() -> Self {
|
||||
let provider = Self {
|
||||
Self {
|
||||
url: "https://javtiful.com".to_string(),
|
||||
categories: Arc::new(RwLock::new(vec![])),
|
||||
};
|
||||
provider
|
||||
}
|
||||
}
|
||||
|
||||
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
|
||||
@@ -61,18 +54,7 @@ impl JavtifulProvider {
|
||||
premium: false,
|
||||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=javtiful.com".to_string(),
|
||||
status: "active".to_string(),
|
||||
categories: self
|
||||
.categories
|
||||
.read()
|
||||
.map(|categories| categories.iter().map(|c| c.title.clone()).collect())
|
||||
.unwrap_or_else(|e| {
|
||||
crate::providers::report_provider_error_background(
|
||||
"javtiful",
|
||||
"build_channel.categories_read",
|
||||
&e.to_string(),
|
||||
);
|
||||
vec![]
|
||||
}),
|
||||
categories: vec![],
|
||||
options: vec![ChannelOption {
|
||||
id: "sort".to_string(),
|
||||
title: "Sort".to_string(),
|
||||
@@ -80,35 +62,85 @@ impl JavtifulProvider {
|
||||
systemImage: "list.number".to_string(),
|
||||
colorName: "blue".to_string(),
|
||||
options: vec![
|
||||
FilterOption {
|
||||
id: "relevance".into(),
|
||||
title: "Relevance".into(),
|
||||
},
|
||||
FilterOption {
|
||||
id: "latest".into(),
|
||||
title: "Latest".into(),
|
||||
},
|
||||
FilterOption {
|
||||
id: "popular".into(),
|
||||
title: "Popular".into(),
|
||||
title: "Popular This Week".into(),
|
||||
},
|
||||
FilterOption {
|
||||
id: "most_viewed".into(),
|
||||
title: "Most Viewed".into(),
|
||||
},
|
||||
FilterOption {
|
||||
id: "most_liked".into(),
|
||||
title: "Most Liked".into(),
|
||||
},
|
||||
],
|
||||
multiSelect: false,
|
||||
}],
|
||||
nsfw: true,
|
||||
cacheDuration: Some(1800),
|
||||
ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string())
|
||||
ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
|
||||
if let Ok(mut vec) = target.write() {
|
||||
if !vec.iter().any(|x| x.id == item.id) {
|
||||
vec.push(item);
|
||||
}
|
||||
/// Maps the app's sort ids to the site's `?sort=` values. The default
|
||||
/// listing (no parameter) is newest first.
|
||||
fn sort_param(sort: &str) -> &'static str {
|
||||
match sort {
|
||||
"popular" => "sort=popular_week&",
|
||||
"most_viewed" => "sort=most_viewed&",
|
||||
"most_liked" => "sort=most_liked&",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_listing(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
video_url: String,
|
||||
page: u8,
|
||||
options: &ServerOptions,
|
||||
) -> Result<Vec<VideoItem>> {
|
||||
let old_items = match cache.get(&video_url) {
|
||||
Some((time, items)) if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 => {
|
||||
return Ok(items.clone());
|
||||
}
|
||||
Some((_, items)) => items.clone(),
|
||||
None => vec![],
|
||||
};
|
||||
|
||||
let mut requester =
|
||||
crate::providers::requester_or_default(options, module_path!(), "missing_requester");
|
||||
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
crate::providers::report_provider_error(
|
||||
"javtiful",
|
||||
"fetch_listing.request",
|
||||
&format!("url={video_url}; error={e}"),
|
||||
)
|
||||
.await;
|
||||
return Ok(old_items);
|
||||
}
|
||||
};
|
||||
// Out-of-range pages are served as page 1 again; only trust the grid
|
||||
// when the pager marks the requested page as current.
|
||||
if page > 1 && !text.contains(&format!("aria-current=\"page\">{}<", page)) {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let video_items = self.get_video_items_from_html(&text);
|
||||
if video_items.is_empty() {
|
||||
return Ok(old_items);
|
||||
}
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url, video_items.clone());
|
||||
Ok(video_items)
|
||||
}
|
||||
|
||||
async fn get(
|
||||
&self,
|
||||
cache: VideoCache,
|
||||
@@ -116,58 +148,13 @@ impl JavtifulProvider {
|
||||
sort: &str,
|
||||
options: ServerOptions,
|
||||
) -> Result<Vec<VideoItem>> {
|
||||
let sort_string = match sort {
|
||||
"latest" => "sort=latest&",
|
||||
"popular" => "sort=popular&",
|
||||
_ => "",
|
||||
};
|
||||
let video_url = format!("{}/videos?{}page={}", self.url, sort_string, page);
|
||||
println!("Fetching URL: {}", video_url);
|
||||
let old_items = match cache.get(&video_url) {
|
||||
Some((time, items)) => {
|
||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
||||
return Ok(items.clone());
|
||||
} else {
|
||||
items.clone()
|
||||
}
|
||||
}
|
||||
None => {
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
|
||||
let mut requester =
|
||||
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
|
||||
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
crate::providers::report_provider_error(
|
||||
"javtiful",
|
||||
"get.request",
|
||||
&format!("url={video_url}; error={e}"),
|
||||
)
|
||||
.await;
|
||||
return Ok(old_items);
|
||||
}
|
||||
};
|
||||
if page > 1
|
||||
&& !text.contains(&format!(
|
||||
"aria-current=\"page\">{}</a>",
|
||||
page
|
||||
))
|
||||
{
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let video_items: Vec<VideoItem> = self
|
||||
.get_video_items_from_html(text.clone(), &mut requester, &options)
|
||||
.await;
|
||||
if !video_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), video_items.clone());
|
||||
} else {
|
||||
return Ok(old_items);
|
||||
}
|
||||
Ok(video_items)
|
||||
let video_url = format!(
|
||||
"{}/videos?{}page={}",
|
||||
self.url,
|
||||
Self::sort_param(sort),
|
||||
page
|
||||
);
|
||||
self.fetch_listing(cache, video_url, page, &options).await
|
||||
}
|
||||
|
||||
async fn query(
|
||||
@@ -177,271 +164,83 @@ impl JavtifulProvider {
|
||||
query: &str,
|
||||
options: ServerOptions,
|
||||
) -> Result<Vec<VideoItem>> {
|
||||
let sort_string = match options.sort.as_deref().unwrap_or("") {
|
||||
"latest" => "sort=latest&",
|
||||
"popular" => "sort=popular&",
|
||||
_ => "",
|
||||
};
|
||||
let sort = options.sort.clone().unwrap_or_default();
|
||||
let video_url = format!(
|
||||
"{}/search?{}q={}&page={}",
|
||||
self.url,
|
||||
sort_string,
|
||||
query.replace(" ", "+"),
|
||||
Self::sort_param(&sort),
|
||||
query.trim().replace(' ', "+"),
|
||||
page
|
||||
);
|
||||
// Check our Video Cache. If the result is younger than 1 hour, we return it.
|
||||
let old_items = match cache.get(&video_url) {
|
||||
Some((time, items)) => {
|
||||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
||||
return Ok(items.clone());
|
||||
} else {
|
||||
let _ = cache.check().await;
|
||||
return Ok(items.clone());
|
||||
}
|
||||
}
|
||||
None => {
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
|
||||
let mut requester =
|
||||
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
|
||||
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
|
||||
Ok(text) => text,
|
||||
Err(e) => {
|
||||
crate::providers::report_provider_error(
|
||||
"javtiful",
|
||||
"query.request",
|
||||
&format!("url={video_url}; error={e}"),
|
||||
)
|
||||
.await;
|
||||
return Ok(old_items);
|
||||
}
|
||||
};
|
||||
if page > 1
|
||||
&& !text.contains(&format!(
|
||||
"aria-current=\"page\">{}</a>",
|
||||
page
|
||||
))
|
||||
{
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let video_items: Vec<VideoItem> = self
|
||||
.get_video_items_from_html(text.clone(), &mut requester, &options)
|
||||
.await;
|
||||
if !video_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), video_items.clone());
|
||||
} else {
|
||||
return Ok(old_items);
|
||||
}
|
||||
Ok(video_items)
|
||||
self.fetch_listing(cache, video_url, page, &options).await
|
||||
}
|
||||
|
||||
async fn get_video_items_from_html(
|
||||
&self,
|
||||
html: String,
|
||||
requester: &mut Requester,
|
||||
options: &ServerOptions,
|
||||
) -> Vec<VideoItem> {
|
||||
if html.is_empty() || html.contains("404 Not Found") {
|
||||
return vec![];
|
||||
}
|
||||
fn between<'a>(text: &'a str, start: &str, end: &str) -> Option<&'a str> {
|
||||
let from = text.find(start)? + start.len();
|
||||
let rest = &text[from..];
|
||||
Some(&rest[..rest.find(end)?])
|
||||
}
|
||||
|
||||
let block = match html
|
||||
.split("front-pagination")
|
||||
.next()
|
||||
.and_then(|s| s.split("front-video-grid").nth(1))
|
||||
{
|
||||
Some(b) => b,
|
||||
None => {
|
||||
eprint!("Javtiful Provider: Failed to get block from html");
|
||||
let e = Error::from(ErrorKind::Parse("html".into()));
|
||||
send_discord_error_report(
|
||||
e.to_string(),
|
||||
Some(format_error_chain(&e)),
|
||||
Some("Javtiful Provider"),
|
||||
Some(&format!("Failed to get block from html:\n```{html}\n```")),
|
||||
file!(),
|
||||
line!(),
|
||||
module_path!(),
|
||||
)
|
||||
.await;
|
||||
return vec![];
|
||||
}
|
||||
};
|
||||
|
||||
let futures = block
|
||||
.split("\"front-video-card\"")
|
||||
fn get_video_items_from_html(&self, html: &str) -> Vec<VideoItem> {
|
||||
// Cards are `<article class="video-card">`; sponsored slots add a
|
||||
// `video-card--partner` modifier and link off-site, so skip them.
|
||||
html.split("<article class=\"video-card")
|
||||
.skip(1)
|
||||
.filter(|seg| !seg.contains("front-ad-card"))
|
||||
.map(|el| self.get_video_item(el.to_string(), requester.clone(), options));
|
||||
|
||||
join_all(futures)
|
||||
.await
|
||||
.into_iter()
|
||||
.inspect(|r| {
|
||||
if let Err(e) = r {
|
||||
eprint!("Javtiful Provider: Failed to get video item:{}\n", e);
|
||||
// Prepare data to move into the background task
|
||||
let msg = e.to_string();
|
||||
let chain = format_error_chain(&e);
|
||||
|
||||
// Spawn the report into the background - NO .await here
|
||||
tokio::spawn(async move {
|
||||
let _ = send_discord_error_report(
|
||||
msg,
|
||||
Some(chain),
|
||||
Some("Javtiful Provider"),
|
||||
Some("Failed to get video item"),
|
||||
file!(), // Note: these might report the utility line
|
||||
line!(), // better to hardcode or pass from outside
|
||||
module_path!(),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
})
|
||||
.filter_map(Result::ok)
|
||||
.filter(|seg| !seg.starts_with("video-card--partner") && !seg.starts_with(" video-card--partner"))
|
||||
.filter_map(|seg| self.get_video_item(seg))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn get_video_item(
|
||||
&self,
|
||||
seg: String,
|
||||
mut requester: Requester,
|
||||
options: &ServerOptions,
|
||||
) -> Result<VideoItem> {
|
||||
let video_url = format!(
|
||||
"{}{}",
|
||||
self.url,
|
||||
seg.split(" href=\"")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('"').next())
|
||||
.ok_or_else(|| ErrorKind::Parse(format!("video url\n\n{seg}")))?
|
||||
.to_string()
|
||||
);
|
||||
fn get_video_item(&self, seg: &str) -> Option<VideoItem> {
|
||||
let href = Self::between(seg, "href=\"", "\"")?;
|
||||
if !href.starts_with("/video/") {
|
||||
return None;
|
||||
}
|
||||
let video_url = format!("{}{}", self.url, href);
|
||||
let id = href.trim_start_matches("/video/").split('/').next()?.to_string();
|
||||
if id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut title = match seg.contains("front-video-title") {
|
||||
true => seg
|
||||
.split("front-video-title")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('>').nth(1))
|
||||
.and_then(|s| s.split('<').next())
|
||||
.ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}")))?
|
||||
.trim()
|
||||
.to_string(),
|
||||
false => seg
|
||||
.split("alt=\"")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('\"').next())
|
||||
.ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}")))?
|
||||
.trim()
|
||||
.to_string(),
|
||||
};
|
||||
|
||||
title = decode(title.as_bytes())
|
||||
let title_raw = Self::between(seg, "class=\"video-card__title\" title=\"", "\"")
|
||||
.or_else(|| Self::between(seg, "alt=\"Thumbnail for ", "\""))?
|
||||
.trim();
|
||||
let title = decode(title_raw.as_bytes())
|
||||
.to_string()
|
||||
.unwrap_or(title)
|
||||
.titlecase();
|
||||
let id = video_url
|
||||
.split("/video/")
|
||||
.filter(|s| !s.is_empty())
|
||||
.nth(1)
|
||||
.ok_or_else(|| ErrorKind::Parse(format!("video id\n\n{seg}")))?
|
||||
.to_string();
|
||||
let thumb_block = seg.split("<img ").nth(1);
|
||||
.unwrap_or_else(|_| title_raw.to_string());
|
||||
|
||||
let thumb = match thumb_block {
|
||||
Some(block) => format!("{}{}", self.url,block
|
||||
.split("data-front-lazy-src=\"")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('"').next())
|
||||
.unwrap_or("")
|
||||
.to_string()),
|
||||
None => "".to_string(),
|
||||
};
|
||||
let mut preview = seg
|
||||
.split("data-front-video-preview-src=\"")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('"').next())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let raw_duration = seg
|
||||
.split("class=\"front-duration-tag\">")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('<').next())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
|
||||
let (tags, views) = self.extract_media(&video_url, &mut requester).await?;
|
||||
|
||||
if preview.len() == 0 {
|
||||
preview = format!("https://trailers.jav.si/preview/{id}.mp4");
|
||||
}
|
||||
let proxy_url = build_proxy_url(
|
||||
options,
|
||||
"javtiful",
|
||||
&strip_url_scheme(video_url.clone().as_str()),
|
||||
);
|
||||
let video_item = VideoItem::new(id, title, proxy_url, "javtiful".into(), thumb, duration)
|
||||
.tags(tags)
|
||||
.preview(preview)
|
||||
.views(views);
|
||||
Ok(video_item)
|
||||
}
|
||||
|
||||
async fn extract_media(
|
||||
&self,
|
||||
url: &str,
|
||||
requester: &mut Requester,
|
||||
) -> Result<(Vec<String>, u32)> {
|
||||
let text = requester
|
||||
.get(url, Some(Version::HTTP_2))
|
||||
.await
|
||||
.map_err(|e| Error::from(format!("{}", e)))?;
|
||||
let tags = text
|
||||
.split("related-actress")
|
||||
.next()
|
||||
.and_then(|s| s.split("video-comments").next())
|
||||
.and_then(|s| s.split(">Tags").nth(1))
|
||||
.and_then(|s| s.split("</div>").next())
|
||||
.map(|tag_block| {
|
||||
tag_block
|
||||
.split("<a ")
|
||||
.skip(1)
|
||||
.filter_map(|tag_el| {
|
||||
tag_el
|
||||
.split('>')
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('<').next())
|
||||
.map(|s| {
|
||||
decode(s.as_bytes())
|
||||
.to_string()
|
||||
.unwrap_or(s.to_string())
|
||||
.titlecase()
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
let thumb = Self::between(seg, "data-front-lazy-src=\"", "\"")
|
||||
.map(|t| {
|
||||
if t.starts_with("http") {
|
||||
t.to_string()
|
||||
} else {
|
||||
format!("{}{}", self.url, t)
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| vec![]);
|
||||
for tag in &tags {
|
||||
Self::push_unique(
|
||||
&self.categories,
|
||||
FilterOption {
|
||||
id: tag.to_ascii_lowercase().replace(" ", "+"),
|
||||
title: tag.to_string(),
|
||||
},
|
||||
);
|
||||
}
|
||||
let views = text
|
||||
.split(" views</span>")
|
||||
.next()
|
||||
.and_then(|s| s.split('>').last())
|
||||
.and_then(|s| s.trim().replace(".", "").replace(",", "").parse::<u32>().ok())
|
||||
.unwrap_or_default();
|
||||
let preview = Self::between(seg, "data-front-video-preview-src=\"", "\"")
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
// `<span class="video-card__details"><strong>FHD</strong>02:49:46</span>`
|
||||
let duration = Self::between(seg, "class=\"video-card__details\">", "</span>")
|
||||
.and_then(|d| d.rsplit("</strong>").next())
|
||||
.and_then(|d| parse_time_to_seconds(d.trim()))
|
||||
.unwrap_or(0) as u32;
|
||||
|
||||
// `<div class="video-card__meta">12.9K views <span ..>·</span> 6 hours ago</div>`
|
||||
let views = Self::between(seg, "class=\"video-card__meta\">", "views")
|
||||
.and_then(|v| parse_abbreviated_number(&v.trim().replace(',', "")))
|
||||
.unwrap_or(0);
|
||||
|
||||
Ok((tags, views))
|
||||
// The page URL resolves natively in yt-dlp (html5 `<source>`), so no
|
||||
// proxy and no formats are needed.
|
||||
let mut item = VideoItem::new(id, title, video_url, "javtiful".into(), thumb, duration)
|
||||
.views(views);
|
||||
if !preview.is_empty() {
|
||||
item = item.preview(preview);
|
||||
}
|
||||
Some(item)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -474,3 +273,32 @@ impl Provider for JavtifulProvider {
|
||||
Some(self.build_channel(v))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::JavtifulProvider;
|
||||
|
||||
#[test]
|
||||
fn parses_cards_and_skips_partner_slots() {
|
||||
let html = r#"<div class="video-grid">
|
||||
<article class="video-card"><a href="/video/114451/royd-358" class="video-card__thumbnail" data-front-video-preview-src="https://x.jav.si/p.mp4"><img src="/ph.svg" data-front-lazy-src="/uploads/t.jpg" alt="Thumbnail for ROYD-358 T"><span class="video-card__details"><strong>FHD</strong>02:49:46</span></a>
|
||||
<div class="video-card__copy"><a href="/video/114451/royd-358" class="video-card__title" title="ROYD-358 I’ll & You">ROYD-358</a><div class="video-card__meta">12.9K views <span aria-hidden="true">·</span> 6 hours ago</div></div></article>
|
||||
<article class="video-card video-card--partner"><a href="https://ads.example/c" class="video-card__thumbnail"><img data-front-lazy-src="/a.jpg" alt="Partner thumbnail for Ad"></a></article>
|
||||
</div>"#;
|
||||
let items = JavtifulProvider::new().get_video_items_from_html(html);
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0].id, "114451");
|
||||
assert_eq!(items[0].url, "https://javtiful.com/video/114451/royd-358");
|
||||
assert_eq!(items[0].thumb, "https://javtiful.com/uploads/t.jpg");
|
||||
assert_eq!(items[0].duration, 2 * 3600 + 49 * 60 + 46);
|
||||
assert_eq!(items[0].views, Some(12900));
|
||||
assert!(items[0].formats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_sort_values() {
|
||||
assert_eq!(JavtifulProvider::sort_param("latest"), "");
|
||||
assert_eq!(JavtifulProvider::sort_param("popular"), "sort=popular_week&");
|
||||
assert_eq!(JavtifulProvider::sort_param("most_viewed"), "sort=most_viewed&");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -644,6 +644,25 @@ pub fn requester_or_default(
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// Routes signed Aylo CDN thumbnails (`pix-cdn77.t8cdn.com`,
|
||||
/// `pix-cdn77.phncdn.com`) through the local `aylo-thumb` proxy, because the
|
||||
/// CDN answers 403 unless a matching Referer is sent. Other URLs are returned
|
||||
/// unchanged.
|
||||
pub fn proxy_aylo_thumb(options: &ServerOptions, thumb: &str) -> String {
|
||||
let needs_proxy = thumb
|
||||
.strip_prefix("https://")
|
||||
.or_else(|| thumb.strip_prefix("http://"))
|
||||
.and_then(|rest| rest.split('/').next())
|
||||
.is_some_and(|host| {
|
||||
host.starts_with("pix-cdn77.")
|
||||
&& (host.ends_with(".t8cdn.com") || host.ends_with(".phncdn.com"))
|
||||
});
|
||||
if !needs_proxy {
|
||||
return thumb.to_string();
|
||||
}
|
||||
build_proxy_url(options, "aylo-thumb", &strip_url_scheme(thumb))
|
||||
}
|
||||
|
||||
pub fn strip_url_scheme(url: &str) -> String {
|
||||
url.strip_prefix("https://")
|
||||
.or_else(|| url.strip_prefix("http://"))
|
||||
@@ -759,11 +778,20 @@ fn sanitize_status_options(options: Vec<ChannelOption>) -> Vec<ChannelOption> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Channels whose own declared `ytdlpCommand` is delivered in `/api/status`.
|
||||
/// Verified: `freepornvideosxxx` (plain yt-dlp gets 502 on its pages, fine with
|
||||
/// `--impersonate chrome-120`) and `missav` (Cloudflare 403 without it).
|
||||
const YTDLP_COMMAND_CHANNELS: &[&str] = &["freepornvideosxxx", "missav"];
|
||||
|
||||
pub fn decorate_channel(channel: Channel) -> ChannelView {
|
||||
let metadata = channel_metadata_for(&channel.id);
|
||||
let runtime_status = current_provider_channel_status(&channel.id);
|
||||
let ytdlp_command = match channel.id.as_str() {
|
||||
"pimpbunny" => Some("yt-dlp --compat-options allow-unsafe-ext".to_string()),
|
||||
// Providers declare a `ytdlpCommand`, but it is only sent to clients
|
||||
// for channels where plain yt-dlp was verified to fail (TLS-fingerprint
|
||||
// 403/502 that real clients hit). Widen this list one channel at a time.
|
||||
id if YTDLP_COMMAND_CHANNELS.contains(&id) => channel.ytdlpCommand.clone(),
|
||||
_ => None,
|
||||
};
|
||||
ChannelView {
|
||||
@@ -1353,6 +1381,25 @@ mod tests {
|
||||
assert_eq!(groups[2].id, "jav");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delivers_declared_ytdlp_command_only_for_allowlisted_channels() {
|
||||
for id in ["freepornvideosxxx", "missav"] {
|
||||
PROVIDER_RUNTIME_STATUS.remove(id);
|
||||
assert_eq!(
|
||||
decorate_channel(base_channel(id)).ytdlpCommand.as_deref(),
|
||||
Some("yt-dlp --impersonate chrome-120"),
|
||||
"{id} should deliver its declared command"
|
||||
);
|
||||
}
|
||||
// Declares the same command but is not allowlisted.
|
||||
PROVIDER_RUNTIME_STATUS.remove("hsex");
|
||||
assert_eq!(decorate_channel(base_channel("hsex")).ytdlpCommand, None);
|
||||
// An allowlisted id that declares nothing stays None.
|
||||
let mut undeclared = base_channel("missav");
|
||||
undeclared.ytdlpCommand = None;
|
||||
assert_eq!(decorate_channel(undeclared).ytdlpCommand, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decorates_pimpbunny_with_ytdlp_command() {
|
||||
PROVIDER_RUNTIME_STATUS.remove("pimpbunny");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use crate::DbPool;
|
||||
use crate::api::ClientVersion;
|
||||
use crate::providers::{Provider, report_provider_error, requester_or_default};
|
||||
use crate::providers::{Provider, proxy_aylo_thumb, report_provider_error, requester_or_default};
|
||||
use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::videos::{ServerOptions, VideoItem};
|
||||
@@ -408,6 +408,11 @@ impl PornhubShortiesProvider {
|
||||
let items: Vec<VideoItem> = shorties
|
||||
.into_iter()
|
||||
.filter_map(Self::item_to_video)
|
||||
.map(|mut video| {
|
||||
// The signed pix-cdn77 thumbnails 403 without a Referer.
|
||||
video.thumb = proxy_aylo_thumb(&options, &video.thumb);
|
||||
video
|
||||
})
|
||||
.collect();
|
||||
|
||||
if items.is_empty() {
|
||||
|
||||
@@ -835,7 +835,8 @@ impl PornmzProvider {
|
||||
}
|
||||
|
||||
fn meta_itemprop(document: &Html, itemprop: &str) -> Option<String> {
|
||||
let selector = Self::selector(&format!("meta[itemprop=\"{itemprop}\"]")).ok()?;
|
||||
// The site mixes `contentURL` / `contentUrl`; match the value case-insensitively.
|
||||
let selector = Self::selector(&format!("meta[itemprop=\"{itemprop}\" i]")).ok()?;
|
||||
document
|
||||
.select(&selector)
|
||||
.next()
|
||||
@@ -1016,9 +1017,21 @@ impl PornmzProvider {
|
||||
let enriched = stream::iter(items.into_iter().map(|item| {
|
||||
let provider = self.clone();
|
||||
let options = options.clone();
|
||||
async move { provider.enrich_item(item, &options).await }
|
||||
async move {
|
||||
// Detail pages are slow (~15s); don't let one hung page stall the list.
|
||||
let fallback = item.clone();
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(25),
|
||||
provider.enrich_item(item, &options),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(enriched) => enriched,
|
||||
Err(_) => fallback,
|
||||
}
|
||||
}
|
||||
}))
|
||||
.buffer_unordered(6)
|
||||
.buffer_unordered(10)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
@@ -1239,6 +1252,23 @@ mod tests {
|
||||
assert!(item.uploadedAt.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_lowercase_content_url_meta() {
|
||||
let provider = PornmzProvider::new();
|
||||
let item = VideoItem::new(
|
||||
"pm1".to_string(),
|
||||
"T".to_string(),
|
||||
"https://pornmz.com/video/id=pm1".to_string(),
|
||||
CHANNEL_ID.to_string(),
|
||||
String::new(),
|
||||
0,
|
||||
);
|
||||
let html = r#"<div class="entry-content"><meta itemprop="contentUrl" content="https://video.twimg.com/x/pl/a.m3u8" /></div>"#;
|
||||
let item = provider.apply_detail_video(item, html).expect("detail");
|
||||
assert_eq!(item.url, "https://video.twimg.com/x/pl/a.m3u8");
|
||||
assert_eq!(item.formats.as_ref().map(|f| f.len()), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derives_mp4_media_extension() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -13,6 +13,7 @@ use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc};
|
||||
use error_chain::error_chain;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use htmlentity::entity::ICodedDataTrait;
|
||||
use regex::Regex;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration as StdDuration;
|
||||
@@ -40,7 +41,6 @@ error_chain! {
|
||||
}
|
||||
|
||||
const BASE_URL: &str = "https://sextb.net";
|
||||
const MIRROR_PREFIX: &str = "https://r.jina.ai/http://";
|
||||
const CHANNEL_ID: &str = "sextb";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -242,18 +242,6 @@ impl SextbProvider {
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn mirror_url(&self, url: &str) -> String {
|
||||
// `url` is already an absolute `https://sextb.net/...` URL, and
|
||||
// MIRROR_PREFIX already supplies a scheme — concatenating them
|
||||
// directly produces a malformed `.../http://https://...` target
|
||||
// that Jina rejects, so strip the scheme first.
|
||||
let stripped = url
|
||||
.strip_prefix("https://")
|
||||
.or_else(|| url.strip_prefix("http://"))
|
||||
.unwrap_or(url);
|
||||
format!("{MIRROR_PREFIX}{stripped}")
|
||||
}
|
||||
|
||||
fn slugify(value: &str) -> String {
|
||||
let mut output = String::new();
|
||||
let mut needs_dash = false;
|
||||
@@ -336,35 +324,36 @@ impl SextbProvider {
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_markdown(&self, requester: &mut Requester, url: &str) -> Result<String> {
|
||||
let mirror_url = self.mirror_url(url);
|
||||
/// The site serves plain HTML; fetch it directly. (It used to be read
|
||||
/// through the r.jina.ai markdown mirror, which now answers 403/429.)
|
||||
async fn fetch_html(&self, requester: &mut Requester, url: &str) -> Result<String> {
|
||||
requester
|
||||
.get(&mirror_url, Some(Version::HTTP_11))
|
||||
.get(url, Some(Version::HTTP_2))
|
||||
.await
|
||||
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))
|
||||
}
|
||||
|
||||
fn pagination_last_page(markdown: &str) -> Option<u16> {
|
||||
let regex = Regex::new(r"\[(?P<page>\d+)\]\(https://sextb\.net/[^)]*pg-(?P=page)\)")
|
||||
.ok()?;
|
||||
fn pagination_last_page(html: &str) -> Option<u16> {
|
||||
let regex = Regex::new(r#"/pg-(?P<page>\d+)['"]"#).ok()?;
|
||||
regex
|
||||
.captures_iter(markdown)
|
||||
.captures_iter(html)
|
||||
.filter_map(|captures| captures.name("page")?.as_str().parse::<u16>().ok())
|
||||
.max()
|
||||
}
|
||||
|
||||
fn parse_genre_options(markdown: &str) -> Result<Vec<FilterOption>> {
|
||||
let regex = Self::regex(
|
||||
r#"(?m)^\*\s+\[(?P<title>[^\]]+)\]\(https://sextb\.net/genre/(?P<slug>[^)\s]+)[^)]*\)\((?P<count>\d+)\)$"#,
|
||||
)?;
|
||||
fn collect_options(html: &str, pattern: &str) -> Result<Vec<FilterOption>> {
|
||||
let regex = Self::regex(pattern)?;
|
||||
let mut items = Vec::new();
|
||||
for captures in regex.captures_iter(markdown) {
|
||||
let title = Self::normalize_title(captures.name("title").map(|value| value.as_str()).unwrap_or_default());
|
||||
for captures in regex.captures_iter(html) {
|
||||
let slug = captures
|
||||
.name("slug")
|
||||
.map(|value| value.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
.map(|value| value.as_str().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
let raw = captures.name("title").map(|value| value.as_str()).unwrap_or_default();
|
||||
let decoded = htmlentity::entity::decode(raw.as_bytes())
|
||||
.to_string()
|
||||
.unwrap_or_else(|_| raw.to_string());
|
||||
let title = Self::normalize_title(decoded.trim_start_matches('#'));
|
||||
if title.is_empty() || slug.is_empty() {
|
||||
continue;
|
||||
}
|
||||
@@ -373,46 +362,25 @@ impl SextbProvider {
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
fn parse_studio_options(markdown: &str) -> Result<Vec<FilterOption>> {
|
||||
let regex = Self::regex(
|
||||
r#"(?m)^\[(?P<title>[^\]]+)\]\(https://sextb\.net/studio/(?P<slug>[^)\s]+)[^)]*\)"#,
|
||||
)?;
|
||||
let mut items = Vec::new();
|
||||
for captures in regex.captures_iter(markdown) {
|
||||
let title =
|
||||
Self::normalize_title(captures.name("title").map(|value| value.as_str()).unwrap_or_default());
|
||||
let slug = captures
|
||||
.name("slug")
|
||||
.map(|value| value.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
if title.is_empty() || slug.is_empty() {
|
||||
continue;
|
||||
}
|
||||
items.push(FilterOption { id: slug, title });
|
||||
}
|
||||
Ok(items)
|
||||
fn parse_genre_options(html: &str) -> Result<Vec<FilterOption>> {
|
||||
Self::collect_options(
|
||||
html,
|
||||
r#"<li><i class="fas fa-folder"[^>]*></i>\s*<a href="/genre/(?P<slug>[^"?]+)"[^>]*>(?P<title>[^<]+)</a>"#,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_actress_options(markdown: &str) -> Result<Vec<FilterOption>> {
|
||||
let regex = Self::regex(
|
||||
r#"(?m)^\[\!\[Image \d+: (?P<title>[^\]]+)\]\(https://[^)]+\)\s+(?P=title)\s+[0-9.]+[KM]\s+\d{2}/\d{2}/\d{4}\]\(https://sextb\.net/actress/[^)\s]+\)$"#,
|
||||
)?;
|
||||
let mut items = Vec::new();
|
||||
for captures in regex.captures_iter(markdown) {
|
||||
let title =
|
||||
Self::normalize_title(captures.name("title").map(|value| value.as_str()).unwrap_or_default());
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let slug_source = title.split('(').next().unwrap_or(&title).trim();
|
||||
let slug = Self::slugify(slug_source);
|
||||
if slug.is_empty() {
|
||||
continue;
|
||||
}
|
||||
items.push(FilterOption { id: slug, title: slug_source.to_string() });
|
||||
}
|
||||
Ok(items)
|
||||
fn parse_studio_options(html: &str) -> Result<Vec<FilterOption>> {
|
||||
Self::collect_options(
|
||||
html,
|
||||
r#"<div class="tray-item tray-item-list">\s*<a href="/studio/(?P<slug>[^"?]+)"[^>]*>(?:<i[^>]*></i>)?\s*(?P<title>[^<]+)</a>"#,
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_actress_options(html: &str) -> Result<Vec<FilterOption>> {
|
||||
Self::collect_options(
|
||||
html,
|
||||
r#"<div class="tray-item-actress">\s*<a href="/actress/(?P<slug>[^"?]+)">.*?<div class="tray-item-actress-title">(?P<title>[^<]+)</div>"#,
|
||||
)
|
||||
}
|
||||
|
||||
async fn load_genres(base_url: &str, genres: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
|
||||
@@ -423,8 +391,8 @@ impl SextbProvider {
|
||||
actresses: Arc::new(RwLock::new(vec![])),
|
||||
};
|
||||
let mut requester = Requester::new();
|
||||
let markdown = provider.fetch_markdown(&mut requester, &format!("{base_url}/genres")).await?;
|
||||
for item in Self::parse_genre_options(&markdown)? {
|
||||
let html = provider.fetch_html(&mut requester, &format!("{base_url}/genres")).await?;
|
||||
for item in Self::parse_genre_options(&html)? {
|
||||
Self::push_unique(&genres, item);
|
||||
}
|
||||
Ok(())
|
||||
@@ -438,9 +406,9 @@ impl SextbProvider {
|
||||
actresses: Arc::new(RwLock::new(vec![])),
|
||||
};
|
||||
let mut requester = Requester::new();
|
||||
let markdown =
|
||||
provider.fetch_markdown(&mut requester, &format!("{base_url}/list-studios")).await?;
|
||||
for item in Self::parse_studio_options(&markdown)? {
|
||||
let html =
|
||||
provider.fetch_html(&mut requester, &format!("{base_url}/list-studios")).await?;
|
||||
for item in Self::parse_studio_options(&html)? {
|
||||
Self::push_unique(&studios, item);
|
||||
}
|
||||
Ok(())
|
||||
@@ -458,17 +426,17 @@ impl SextbProvider {
|
||||
};
|
||||
let mut requester = Requester::new();
|
||||
let first_page =
|
||||
provider.fetch_markdown(&mut requester, &format!("{base_url}/list-actress")).await?;
|
||||
provider.fetch_html(&mut requester, &format!("{base_url}/list-actress")).await?;
|
||||
for item in Self::parse_actress_options(&first_page)? {
|
||||
Self::push_unique(&actresses, item);
|
||||
}
|
||||
|
||||
let last_page = Self::pagination_last_page(&first_page).unwrap_or(1).min(80);
|
||||
for page in 2..=last_page {
|
||||
let markdown = provider
|
||||
.fetch_markdown(&mut requester, &format!("{base_url}/list-actress/pg-{page}"))
|
||||
let html = provider
|
||||
.fetch_html(&mut requester, &format!("{base_url}/list-actress/pg-{page}"))
|
||||
.await?;
|
||||
for item in Self::parse_actress_options(&markdown)? {
|
||||
for item in Self::parse_actress_options(&html)? {
|
||||
Self::push_unique(&actresses, item);
|
||||
}
|
||||
}
|
||||
@@ -600,120 +568,63 @@ impl SextbProvider {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_list_prefix_tags(prefix: &str) -> Vec<String> {
|
||||
let cleaned = Self::collapse_whitespace(prefix);
|
||||
if cleaned.is_empty() {
|
||||
return vec![];
|
||||
fn parse_list_items(html: &str) -> Result<Vec<ListItemSeed>> {
|
||||
// A no-hit search still renders a "recommended" tray under the notice.
|
||||
if html.contains("No Video were found") {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let bracket_regex = Regex::new(r"\[(?P<tag>[^\]]+)\]").ok();
|
||||
let mut tags = Vec::new();
|
||||
if let Some(regex) = bracket_regex {
|
||||
for captures in regex.captures_iter(&cleaned) {
|
||||
if let Some(tag) = captures.name("tag").map(|value| value.as_str().trim()) {
|
||||
if !tag.is_empty() {
|
||||
tags.push(tag.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let without_brackets = Regex::new(r"\[[^\]]+\]")
|
||||
.ok()
|
||||
.map(|regex| regex.replace_all(&cleaned, " ").to_string())
|
||||
.unwrap_or(cleaned.clone());
|
||||
for token in without_brackets.split(" ") {
|
||||
let value = Self::collapse_whitespace(token);
|
||||
if !value.is_empty() {
|
||||
tags.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
tags.sort();
|
||||
tags.dedup();
|
||||
tags
|
||||
}
|
||||
|
||||
fn parse_list_items(markdown: &str) -> Result<Vec<ListItemSeed>> {
|
||||
let regex = Self::regex(
|
||||
r#"(?s)\[\!\[Image \d+: (?P<alt>.*?)\]\((?P<thumb>https?://[^)]+)\)(?P<body>.*?)\]\((?P<url>https://sextb\.net/[^)]+)\)"#,
|
||||
)?;
|
||||
let meta_regex = Self::regex(
|
||||
r#"(?s)^(?P<head>.*?)\s+(?P<quality>HD|SD)\s+(?P<duration>\d+)\s+min\s+(?P<code>[A-Za-z0-9\-]+)$"#,
|
||||
)?;
|
||||
let mut items = Vec::new();
|
||||
|
||||
for captures in regex.captures_iter(markdown) {
|
||||
let url = captures
|
||||
.name("url")
|
||||
.map(|value| value.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
if !url.starts_with("https://sextb.net/") {
|
||||
// Cards are `<div class="tray-item">`, optionally with modifiers such
|
||||
// as `tray-item tray-item-uncensored`; `tray-item-title` etc. must not match.
|
||||
let card_start = Self::regex(r#"<div class="tray-item(?: [^"]*)?">"#)?;
|
||||
for card in card_start.split(html).skip(1) {
|
||||
|
||||
let Some(href) = Self::attr_between(card, "<a href=\"", "\"") else {
|
||||
continue;
|
||||
};
|
||||
let slug = href.trim_matches('/').to_string();
|
||||
if slug.is_empty() || slug.contains('/') {
|
||||
continue;
|
||||
}
|
||||
let url = format!("{BASE_URL}/{slug}");
|
||||
|
||||
let body = Self::collapse_whitespace(
|
||||
captures.name("body").map(|value| value.as_str()).unwrap_or_default(),
|
||||
let title_raw = Self::attr_between(card, "class=\"tray-item-title\">", "</div>")
|
||||
.or_else(|| Self::attr_between(card, "alt=\"", "\" "))
|
||||
.unwrap_or_default();
|
||||
let title = Self::normalize_title(
|
||||
&htmlentity::entity::decode(title_raw.as_bytes())
|
||||
.to_string()
|
||||
.unwrap_or_else(|_| title_raw.to_string()),
|
||||
);
|
||||
let meta = match meta_regex.captures(&body) {
|
||||
Some(meta) => meta,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let alt_title = Self::normalize_title(
|
||||
captures.name("alt").map(|value| value.as_str()).unwrap_or_default(),
|
||||
);
|
||||
let head = Self::normalize_title(meta.name("head").map(|value| value.as_str()).unwrap_or_default());
|
||||
let title = if !alt_title.is_empty() && head.contains(&alt_title) {
|
||||
alt_title.clone()
|
||||
} else if !alt_title.is_empty() && !alt_title.eq_ignore_ascii_case("Julia") {
|
||||
alt_title.clone()
|
||||
} else {
|
||||
head.clone()
|
||||
};
|
||||
if title.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let prefix = if !alt_title.is_empty() && head.contains(&alt_title) {
|
||||
let prefix = head.split(&alt_title).next().unwrap_or_default();
|
||||
prefix.to_string()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let slug = url
|
||||
.trim_start_matches("https://sextb.net/")
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
if slug.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let thumb = captures
|
||||
.name("thumb")
|
||||
.map(|value| value.as_str())
|
||||
let thumb = Self::attr_between(card, "data-src=\"", "\"")
|
||||
.or_else(|| Self::attr_between(card, "data-poster=\"", "\""))
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
if thumb.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let quality = meta
|
||||
.name("quality")
|
||||
.map(|value| value.as_str().to_string())
|
||||
.filter(|value| !value.is_empty());
|
||||
let code = meta
|
||||
.name("code")
|
||||
.map(|value| value.as_str().to_string())
|
||||
.filter(|value| !value.is_empty());
|
||||
let duration = meta
|
||||
.name("duration")
|
||||
.map(|value| Self::parse_duration(value.as_str()))
|
||||
let quality = if card.contains("class=\"hd\"") {
|
||||
Some("HD".to_string())
|
||||
} else if card.contains("class=\"sd\"") {
|
||||
Some("SD".to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let duration = Self::attr_between(card, "class=\"tray-item-runtime\">", "</div>")
|
||||
.map(|value| value.replace("min", ""))
|
||||
.map(|value| Self::parse_duration(&value))
|
||||
.unwrap_or(0);
|
||||
let code = Self::attr_between(card, "class=\"tray-item-code\">", "</div>")
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty());
|
||||
|
||||
let mut tags = Self::parse_list_prefix_tags(&prefix);
|
||||
let mut tags = Vec::new();
|
||||
if let Some(value) = quality.clone() {
|
||||
tags.push(value);
|
||||
}
|
||||
@@ -739,13 +650,30 @@ impl SextbProvider {
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
fn extract_link_texts(line: &str) -> Result<Vec<(String, String)>> {
|
||||
let regex = Self::regex(r#"\[(?P<title>[^\]]+)\]\((?P<url>https://sextb\.net/[^)\s]+)"#)?;
|
||||
fn attr_between<'a>(text: &'a str, start: &str, end: &str) -> Option<&'a str> {
|
||||
let from = text.find(start)? + start.len();
|
||||
let rest = &text[from..];
|
||||
Some(&rest[..rest.find(end)?])
|
||||
}
|
||||
|
||||
/// Returns `(title, absolute-url)` for every `<a ...><strong>Title</strong></a>`.
|
||||
fn extract_link_texts(fragment: &str) -> Result<Vec<(String, String)>> {
|
||||
let regex = Self::regex(
|
||||
r#"<a href="(?P<url>[^"]+)"[^>]*>\s*<strong>(?P<title>[^<]+)</strong>\s*</a>"#,
|
||||
)?;
|
||||
Ok(regex
|
||||
.captures_iter(line)
|
||||
.captures_iter(fragment)
|
||||
.filter_map(|captures| {
|
||||
let title = captures.name("title")?.as_str().trim().trim_matches('*').to_string();
|
||||
let url = captures.name("url")?.as_str().to_string();
|
||||
let raw = captures.name("title")?.as_str().trim();
|
||||
let title = htmlentity::entity::decode(raw.as_bytes())
|
||||
.to_string()
|
||||
.unwrap_or_else(|_| raw.to_string());
|
||||
let url = captures.name("url")?.as_str();
|
||||
let url = if url.starts_with('/') {
|
||||
format!("{BASE_URL}{url}")
|
||||
} else {
|
||||
url.to_string()
|
||||
};
|
||||
Some((title, url))
|
||||
})
|
||||
.collect())
|
||||
@@ -757,27 +685,30 @@ impl SextbProvider {
|
||||
Some(path.trim_start_matches('/').trim_end_matches('/').to_string())
|
||||
}
|
||||
|
||||
fn parse_detail(markdown: &str) -> Result<DetailMetadata> {
|
||||
fn strong_text(fragment: &str) -> String {
|
||||
Self::attr_between(fragment, "<strong>", "</strong>")
|
||||
.map(|value| value.trim().to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn parse_detail(html: &str) -> Result<DetailMetadata> {
|
||||
let mut metadata = DetailMetadata::default();
|
||||
|
||||
let aliases_regex = Self::regex(r#"(?m)^# \*\*(?P<body>.+?)\*\*$"#)?;
|
||||
if let Some(captures) = aliases_regex.captures(markdown) {
|
||||
let body = captures.name("body").map(|value| value.as_str()).unwrap_or_default();
|
||||
let link_regex = Self::regex(r#"\[(?P<alias>[^\]]+)\]\(https://sextb\.net/search/[^)]+\)"#)?;
|
||||
for alias_match in link_regex.captures_iter(body) {
|
||||
if let Some(alias) = alias_match.name("alias").map(|value| value.as_str().trim()) {
|
||||
if !alias.is_empty() {
|
||||
metadata.aliases.push(alias.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let tag_regex = Self::regex(r"<[^>]+>")?;
|
||||
|
||||
let line_regex = Self::regex(r#"(?m)^\s*(?P<key>Director|Label|Studio|Cast\(s\)|Genre\(s\)|Quality|Release Date|Runtimes|Added|Viewed|Description):\s*(?P<value>.+)$"#)?;
|
||||
for captures in line_regex.captures_iter(markdown) {
|
||||
let key = captures.name("key").map(|value| value.as_str()).unwrap_or_default();
|
||||
let value = captures.name("value").map(|value| value.as_str()).unwrap_or_default().trim();
|
||||
match key {
|
||||
// Info rows are `<div class="description"><i ..></i> Key: <strong>..</strong></div>`.
|
||||
for block in html.split("<div class=\"description").skip(1) {
|
||||
let block = block.split("</div>").next().unwrap_or(block);
|
||||
// Drop the remainder of the opening tag (extra classes, `>`).
|
||||
let Some((_, block)) = block.split_once('>') else {
|
||||
continue;
|
||||
};
|
||||
let Some((key_part, value)) = block.split_once(':') else {
|
||||
continue;
|
||||
};
|
||||
let key = tag_regex.replace_all(key_part, " ").trim().to_string();
|
||||
let value = value.trim();
|
||||
match key.as_str() {
|
||||
"Director" => {
|
||||
metadata.director = Self::extract_link_texts(value)?
|
||||
.first()
|
||||
@@ -796,71 +727,48 @@ impl SextbProvider {
|
||||
}
|
||||
}
|
||||
"Cast(s)" => {
|
||||
let mut cast = Vec::new();
|
||||
for (title, url) in Self::extract_link_texts(value)? {
|
||||
let slug = Self::detail_link_slug(&url, "actress")
|
||||
.unwrap_or_else(|| Self::slugify(&title));
|
||||
cast.push((title, slug));
|
||||
metadata.cast.push((title, slug));
|
||||
}
|
||||
metadata.cast = cast;
|
||||
}
|
||||
"Genre(s)" => {
|
||||
let mut genres = Vec::new();
|
||||
for (title, url) in Self::extract_link_texts(value)? {
|
||||
let slug = Self::detail_link_slug(&url, "genre")
|
||||
.unwrap_or_else(|| Self::slugify(&title));
|
||||
genres.push((title, slug));
|
||||
metadata.genres.push((title, slug));
|
||||
}
|
||||
metadata.genres = genres;
|
||||
}
|
||||
"Quality" => {
|
||||
let quality = value.trim_matches('*').trim().to_string();
|
||||
let quality = Self::strong_text(value);
|
||||
if !quality.is_empty() {
|
||||
metadata.quality = Some(quality);
|
||||
}
|
||||
}
|
||||
"Release Date" => {
|
||||
let cleaned = value.trim_matches('*').trim();
|
||||
metadata.release_date = Self::parse_absolute_date(cleaned);
|
||||
metadata.release_date = Self::parse_absolute_date(&Self::strong_text(value));
|
||||
}
|
||||
"Runtimes" => {}
|
||||
"Added" => {
|
||||
let cleaned = value.trim_matches('*').trim();
|
||||
metadata.added_at = Self::parse_relative_or_date(cleaned);
|
||||
metadata.added_at = Self::parse_relative_or_date(&Self::strong_text(value));
|
||||
}
|
||||
"Viewed" => {
|
||||
metadata.views = Self::parse_abbrev(value.trim());
|
||||
let text = tag_regex.replace_all(value, " ").to_string();
|
||||
metadata.views = Self::parse_abbrev(text.trim());
|
||||
}
|
||||
"Description" => {}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let stats_regex =
|
||||
Self::regex(r#"(?m)(?P<likes>[0-9.]+[KM]?)\s+Liked(?:\s+(?P<dislikes>[0-9.]+[KM]?)\s+Disliked)?(?:\s+(?P<favorites>[0-9.]+[KM]?)\s+Favorited)?\s+(?P<views>[0-9.]+[KM]?)\s+Views"#)?;
|
||||
if let Some(captures) = stats_regex.captures(markdown) {
|
||||
if metadata.views.is_none() {
|
||||
metadata.views = captures
|
||||
.name("views")
|
||||
.and_then(|value| Self::parse_abbrev(value.as_str()));
|
||||
}
|
||||
if let (Some(likes), Some(dislikes)) = (
|
||||
captures
|
||||
.name("likes")
|
||||
.and_then(|value| Self::parse_abbrev(value.as_str())),
|
||||
captures
|
||||
.name("dislikes")
|
||||
.and_then(|value| Self::parse_abbrev(value.as_str())),
|
||||
) {
|
||||
let total = likes + dislikes;
|
||||
if total > 0 {
|
||||
metadata.rating = Some((likes as f32 / total as f32) * 100.0);
|
||||
}
|
||||
}
|
||||
if metadata.views.is_none() {
|
||||
let views_regex = Self::regex(r"(?P<views>[0-9][0-9.,]*[KM]?)\s+Views")?;
|
||||
metadata.views = views_regex
|
||||
.captures(html)
|
||||
.and_then(|captures| captures.name("views"))
|
||||
.and_then(|value| Self::parse_abbrev(value.as_str()));
|
||||
}
|
||||
|
||||
metadata.aliases.sort();
|
||||
metadata.aliases.dedup();
|
||||
// Like/dislike counts are filled in client-side, so no rating here.
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
@@ -1078,12 +986,12 @@ impl SextbProvider {
|
||||
let mut requester = requester_or_default(options, CHANNEL_ID, "enrich_seed");
|
||||
let detail_fetch = timeout(
|
||||
StdDuration::from_secs(10),
|
||||
self.fetch_markdown(&mut requester, &seed.url),
|
||||
self.fetch_html(&mut requester, &seed.url),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut item = match detail_fetch {
|
||||
Ok(Ok(markdown)) => match Self::parse_detail(&markdown) {
|
||||
Ok(Ok(html)) => match Self::parse_detail(&html) {
|
||||
Ok(detail) => self.apply_detail_metadata(seed.clone(), detail),
|
||||
Err(error) => {
|
||||
report_provider_error_background(CHANNEL_ID, "parse_detail", &error.to_string());
|
||||
@@ -1130,14 +1038,14 @@ impl SextbProvider {
|
||||
}
|
||||
|
||||
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_items_for_url");
|
||||
let markdown = timeout(
|
||||
let html = timeout(
|
||||
StdDuration::from_secs(12),
|
||||
self.fetch_markdown(&mut requester, &url),
|
||||
self.fetch_html(&mut requester, &url),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::from(format!("list request timed out for {url}")))??;
|
||||
|
||||
let seeds = Self::parse_list_items(&markdown)?
|
||||
let seeds = Self::parse_list_items(&html)?
|
||||
.into_iter()
|
||||
.take(per_page_limit.max(1))
|
||||
.collect::<Vec<_>>();
|
||||
@@ -1266,15 +1174,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_jina_mirror_url_without_double_scheme() {
|
||||
let url = provider().mirror_url("https://sextb.net/censored?genre=all");
|
||||
assert_eq!(
|
||||
url,
|
||||
"https://r.jina.ai/http://sextb.net/censored?genre=all"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_search_page_two_url() {
|
||||
let url = provider().build_url_for_target(
|
||||
@@ -1308,33 +1207,81 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parses_detail_metadata_block() {
|
||||
let markdown = r#"
|
||||
57 Liked 16 Disliked 88 Favorited 45.2K Views
|
||||
let html = r#"
|
||||
<div class="col-7">
|
||||
<div class="description"> <i class="fa fa-user"></i> Director: <a href="/director/himurokku" title="Himurokku"><strong>Himurokku</strong></a> <i class="fas fa-heart favorite-director" data-id="1"></i> </div>
|
||||
<div class="description"> <i class="fa fa-tag"></i> Label: <a href="/label/madonna-x1" title="Madonna"><strong>Madonna</strong></a> </div>
|
||||
<div class="description"> <i class="fa fa-camera"></i> Studio: <a href="/studio/madonna-abc123" title="MADONNA"><strong>Madonna</strong></a> </div>
|
||||
<div class="description"> <i class="fa fa-users"></i> Cast(s): <a href="/actress/miyuu-imai" title="Miyuu Imai"><strong>Miyuu Imai</strong></a> <i class="fas fa-heart"></i> </div>
|
||||
<div class="description"> <i class="fa fa-list"></i> Genre(s): <a href="/genre/big-tits-aa11" title="Big Tits"><strong>Big Tits</strong></a>, <a href="/genre/creampie-bb22" title="Creampie"><strong>Creampie</strong></a> </div>
|
||||
<div class="description"><i class="fa fa-film"></i> Quality: <strong>HD</strong></div>
|
||||
<div class="description"><i class="fa fa-calendar"></i> Release Date: <strong>Mar. 24, 2026</strong></div>
|
||||
<div class="description"><i class="fa fa-calendar"></i> Added: <strong>2 days ago</strong></div>
|
||||
<div class="description visible-sm"><i class="fa fa-eye"></i> Viewed: 45.2K</div>
|
||||
</div>"#;
|
||||
|
||||
Director: [**Himurokku**](https://sextb.net/director/himurokku "Himurokku")
|
||||
Label: [**Madonna**](https://sextb.net/label/madonna "Madonna")
|
||||
Studio: [**Madonna**](https://sextb.net/studio/madonna "MADONNA")
|
||||
Cast(s): [**Miyuu Imai**](https://sextb.net/actress/miyuu-imai "Miyuu Imai")
|
||||
Genre(s): [**Big Tits**](https://sextb.net/genre/big-tits "Big Tits"), [**Creampie**](https://sextb.net/genre/creampie "Creampie")
|
||||
Quality: **HD**
|
||||
Release Date: **Mar. 24, 2026**
|
||||
Added: **2 days ago**
|
||||
Viewed: 45.2K
|
||||
"#;
|
||||
|
||||
let detail = SextbProvider::parse_detail(markdown).expect("detail should parse");
|
||||
let detail = SextbProvider::parse_detail(html).expect("detail should parse");
|
||||
assert_eq!(detail.director.as_deref(), Some("Himurokku"));
|
||||
assert_eq!(detail.label.as_deref(), Some("Madonna"));
|
||||
assert_eq!(detail.studio.as_ref().map(|value| value.0.as_str()), Some("Madonna"));
|
||||
assert_eq!(detail.cast.len(), 1);
|
||||
assert_eq!(
|
||||
detail.studio,
|
||||
Some(("Madonna".to_string(), "madonna-abc123".to_string()))
|
||||
);
|
||||
assert_eq!(detail.cast, vec![("Miyuu Imai".to_string(), "miyuu-imai".to_string())]);
|
||||
assert_eq!(detail.genres.len(), 2);
|
||||
assert_eq!(detail.genres[0].1, "big-tits-aa11");
|
||||
assert_eq!(detail.views, Some(45200));
|
||||
assert!(detail.rating.unwrap_or_default() > 70.0);
|
||||
assert_eq!(detail.quality.as_deref(), Some("HD"));
|
||||
assert!(detail.release_date.is_some());
|
||||
assert!(detail.added_at.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_listing_cards() {
|
||||
let html = r#"<div class="tray-content"><div class="tray-item"> <a href="/beb-036"> <img class="tray-item-thumbnail lazy" src="/images/default-cover.jpg" data-src="https://cdn001.imggle.net/webp/poster/a.webp" alt="Alt"> <div class="tray-item-description"> <div class="tray-item-title">Hara & Museum</div> <div class="tray-item-meta-info"> <div class="tray-item-quality"><span class="hd">HD</span></div> <div class="tray-item-runtime">120 min</div> <div class="tray-item-code">BEB-036</div> </div> </div> </a> <div class="tray-item-trailer" data-url="x" data-link="/beb-036" data-poster="https://cdn001.imggle.net/thumb/a.jpg"></div> </div>
|
||||
<div class="tray-item tray-item-uncensored"> <a href="/akdl-234"> <img data-src="https://cdn001.imggle.net/webp/poster/b.webp" alt="B"> <div class="tray-item-title">Second</div><div class="tray-item-runtime">150 min</div><div class="tray-item-code">AKDL-234</div> </a> </div></div>"#;
|
||||
let items = SextbProvider::parse_list_items(html).expect("listing should parse");
|
||||
assert_eq!(items.len(), 2);
|
||||
assert_eq!(items[0].id, "beb-036");
|
||||
assert_eq!(items[0].url, "https://sextb.net/beb-036");
|
||||
assert_eq!(items[0].title, "Hara & Museum");
|
||||
assert_eq!(items[0].duration, 7200);
|
||||
assert_eq!(items[0].quality.as_deref(), Some("HD"));
|
||||
assert_eq!(items[0].code.as_deref(), Some("BEB-036"));
|
||||
assert_eq!(items[0].thumb, "https://cdn001.imggle.net/webp/poster/a.webp");
|
||||
assert_eq!(items[1].duration, 9000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_search_result_ignores_recommendation_tray() {
|
||||
let html = r#"<div class="tray-content"><h3 class="notice msg-result">Sorry, No Video were found that matched your search query.</h3>
|
||||
<div class="tray-item"> <a href="/abc-001"> <img data-src="https://x/a.webp"> <div class="tray-item-title">Rec</div></a></div></div>"#;
|
||||
assert!(SextbProvider::parse_list_items(html).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_filter_option_lists() {
|
||||
let genres = r#"<li><i class="fas fa-folder" style="x"></i> <a href="/genre/30-years-old-7c1gtr1u" title="Genre 30 Years Old">30 Years Old</a> <span>(1138)</span></li>"#;
|
||||
let g = SextbProvider::parse_genre_options(genres).unwrap();
|
||||
assert_eq!(g[0].id, "30-years-old-7c1gtr1u");
|
||||
assert_eq!(g[0].title, "30 Years Old");
|
||||
|
||||
let studios = r##"<div class="tray-item tray-item-list"> <a href="/studio/only-amateurs-29p3k0ax" title="Label"><i class="fas fa-folder-open"></i> #only amateurs</a> <span class="total">13</span> </div>"##;
|
||||
let s = SextbProvider::parse_studio_options(studios).unwrap();
|
||||
assert_eq!(s[0].id, "only-amateurs-29p3k0ax");
|
||||
assert_eq!(s[0].title, "only amateurs");
|
||||
|
||||
let actresses = r#"<div class="tray-item-actress"> <a href="/actress/th1bdtq81"> <img alt="Yumi"> <div class="tray-item-actress-description"> <div class="tray-item-actress-title">Yumi Kazama</div> </div></a></div>"#;
|
||||
let a = SextbProvider::parse_actress_options(actresses).unwrap();
|
||||
assert_eq!(a[0].id, "th1bdtq81");
|
||||
assert_eq!(a[0].title, "Yumi Kazama");
|
||||
|
||||
assert_eq!(
|
||||
SextbProvider::pagination_last_page("<a href='/list-actress/pg-2'>2</a><a href='/list-actress/pg-73'>73</a>"),
|
||||
Some(73)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_dd_player_buttons_from_detail_html() {
|
||||
let html = r#"
|
||||
|
||||
@@ -406,6 +406,53 @@ impl ThaipornTvProvider {
|
||||
Ok(formats)
|
||||
}
|
||||
|
||||
/// Current player attribute: `data-video-sources` holds the base64 of a
|
||||
/// JSON array (`[{"quality":"720p","url":"..."}]`) written backwards.
|
||||
/// Falls back to the older XOR-keyed `data-enc` payload.
|
||||
fn decode_video_sources(encoded: &str) -> Result<Vec<VideoFormat>> {
|
||||
let reversed: String = encoded.trim().chars().rev().collect();
|
||||
let padding = (4 - reversed.len() % 4) % 4;
|
||||
let padded = format!("{}{}", reversed, "=".repeat(padding));
|
||||
let parsed = general_purpose::STANDARD
|
||||
.decode(&padded)
|
||||
.ok()
|
||||
.and_then(|bytes| String::from_utf8(bytes).ok())
|
||||
.and_then(|text| serde_json::from_str::<serde_json::Value>(&text).ok())
|
||||
.and_then(|json| json.as_array().cloned());
|
||||
|
||||
let Some(entries) = parsed else {
|
||||
return Self::decode_data_enc(encoded);
|
||||
};
|
||||
|
||||
let mut formats = Vec::new();
|
||||
for entry in entries {
|
||||
let Some(url) = entry
|
||||
.get("url")
|
||||
.or_else(|| entry.get("u"))
|
||||
.and_then(|v| v.as_str())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let quality = entry
|
||||
.get("quality")
|
||||
.or_else(|| entry.get("q"))
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("auto");
|
||||
let mut format = VideoFormat::new(
|
||||
url.to_string(),
|
||||
quality.to_string(),
|
||||
"application/x-mpegURL".to_string(),
|
||||
)
|
||||
.format_id(format!("m3u8-{quality}"));
|
||||
format.add_http_header("Referer".to_string(), BASE_URL.to_string());
|
||||
formats.push(format);
|
||||
}
|
||||
if formats.is_empty() {
|
||||
return Err(Error::from("no formats in video sources"));
|
||||
}
|
||||
Ok(formats)
|
||||
}
|
||||
|
||||
fn parse_card(&self, card: ElementRef<'_>) -> Option<VideoItem> {
|
||||
// Selectors for the Tailwind-based redesign
|
||||
let playthumb_sel = Self::selector("a.playthumb").ok()?;
|
||||
@@ -528,11 +575,15 @@ impl ThaipornTvProvider {
|
||||
.map_err(|_| Error::from(format!("detail request timed out for {}", item.url)))??;
|
||||
|
||||
let document = Html::parse_document(&detail_fetch);
|
||||
let video_element_selector = Self::selector("video[data-enc]")?;
|
||||
let video_element_selector = Self::selector("video[data-video-sources], video[data-enc]")?;
|
||||
|
||||
if let Some(video_element) = document.select(&video_element_selector).next() {
|
||||
if let Some(data_enc) = video_element.value().attr("data-enc") {
|
||||
match Self::decode_data_enc(data_enc) {
|
||||
let attr = video_element
|
||||
.value()
|
||||
.attr("data-video-sources")
|
||||
.or_else(|| video_element.value().attr("data-enc"));
|
||||
if let Some(data_enc) = attr {
|
||||
match Self::decode_video_sources(data_enc) {
|
||||
Ok(formats) => {
|
||||
item.formats = Some(formats);
|
||||
},
|
||||
@@ -731,6 +782,31 @@ mod tests {
|
||||
assert_eq!(json["http_headers"]["Referer"], BASE_URL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_reversed_video_sources() {
|
||||
use base64::Engine as _;
|
||||
let json = r#"[{"hd":false,"quality":"480p","url":"https://web.techvids.top/m3u8/x_480p.m3u8"},{"hd":true,"quality":"720p","url":"https://web.techvids.top/m3u8/x_720p.m3u8"}]"#;
|
||||
let encoded: String = general_purpose::STANDARD
|
||||
.encode(json)
|
||||
.trim_end_matches('=')
|
||||
.chars()
|
||||
.rev()
|
||||
.collect();
|
||||
let formats = ThaipornTvProvider::decode_video_sources(&encoded).unwrap();
|
||||
assert_eq!(formats.len(), 2);
|
||||
assert_eq!(formats[1].url, "https://web.techvids.top/m3u8/x_720p.m3u8");
|
||||
let json = serde_json::to_value(&formats[1]).unwrap();
|
||||
assert_eq!(json["quality"], "720p");
|
||||
assert_eq!(json["http_headers"]["Referer"], BASE_URL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_sources_falls_back_to_legacy_data_enc() {
|
||||
let encoded = "C1BwKX4pMSljf397eDEkJHxuaSV_bmhjfWJveCV_ZHskZjh-MyQ6PT4zVD8zO3slZjh-MyknKXopMSk_Mzt7KScpYykxbWpneG52Vg";
|
||||
let formats = ThaipornTvProvider::decode_video_sources(encoded).unwrap();
|
||||
assert_eq!(formats[0].url, "https://web.techvids.top/m3u8/1658_480p.m3u8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_duration() {
|
||||
assert_eq!(parse_duration_mm_ss("50:47"), 3047);
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use crate::DbPool;
|
||||
use crate::api::ClientVersion;
|
||||
use crate::providers::{Provider, build_proxy_url, report_provider_error, requester_or_default};
|
||||
use crate::providers::{
|
||||
Provider, build_proxy_url, proxy_aylo_thumb, report_provider_error, requester_or_default,
|
||||
};
|
||||
use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::parse_abbreviated_number;
|
||||
@@ -257,11 +259,24 @@ impl Tube8Provider {
|
||||
let mut items = Vec::new();
|
||||
|
||||
for card in document.select(&card_sel) {
|
||||
let id = card
|
||||
.value()
|
||||
.attr("data-video-id")
|
||||
.map(|v| v.to_string())
|
||||
.filter(|v| !v.is_empty())
|
||||
// The card's own link is authoritative: sponsored `channel` cards
|
||||
// carry a `data-video-id` that 404s while the link target
|
||||
// (`/porn-video/<other id>/`) is the live page.
|
||||
let link_id = link_sel
|
||||
.as_ref()
|
||||
.and_then(|s| card.select(s).next())
|
||||
.and_then(|v| v.value().attr("href"))
|
||||
.and_then(|href| href.split("/porn-video/").nth(1))
|
||||
.and_then(|rest| rest.split('/').next())
|
||||
.filter(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_digit()))
|
||||
.map(|v| v.to_string());
|
||||
let id = link_id
|
||||
.or_else(|| {
|
||||
card.value()
|
||||
.attr("data-video-id")
|
||||
.map(|v| v.to_string())
|
||||
.filter(|v| !v.is_empty())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if id.is_empty() {
|
||||
continue;
|
||||
@@ -296,6 +311,7 @@ impl Tube8Provider {
|
||||
.or_else(|| v.value().attr("src"))
|
||||
})
|
||||
.map(|v| self.normalize_url(v))
|
||||
.map(|v| proxy_aylo_thumb(options, &v))
|
||||
.unwrap_or_default();
|
||||
|
||||
// Preview flipbook clip from the thumbnail link's data-mediabook
|
||||
@@ -536,6 +552,27 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_link_target_id_when_card_video_id_is_stale() {
|
||||
let p = provider();
|
||||
let html = r#"
|
||||
<article class="video-box pc js_video-box js-pop" data-video-id="170549881"
|
||||
data-uploader-type="channel" aria-label="Sponsored clip">
|
||||
<a href="/porn-video/67452091/" data-testid="plw_video_thumbnail_link">
|
||||
<img class="thumb-image js_lazy" data-src="https://ei-ph.t8cdn.com/t.jpg"/>
|
||||
</a>
|
||||
</article>"#;
|
||||
let opts = ServerOptions {
|
||||
client_version: None, featured: None, category: None, sites: None, filter: None,
|
||||
language: None, public_url_base: None, requester: None, network: None, stars: None,
|
||||
categories: None, duration: None, sort: None, sexuality: None,
|
||||
};
|
||||
let items = p.parse_items(html, &opts);
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0].id, "67452091");
|
||||
assert_eq!(items[0].url, "https://www.tube8.com/porn-video/67452091/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_video_cards() {
|
||||
let p = provider();
|
||||
|
||||
@@ -1568,11 +1568,28 @@ impl Provider for VjavProvider {
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
let target = self.pick_target(normalized_query.as_deref(), &options).await;
|
||||
// A generic keyword can exactly match a category or member name
|
||||
// ("girl" is a member with no uploads), which would hijack the search
|
||||
// and return nothing. Remember to fall back to keyword search then.
|
||||
let keyword_fallback = normalized_query
|
||||
.as_ref()
|
||||
.filter(|_| matches!(target, Target::Uploader { .. } | Target::Tag { .. }))
|
||||
.map(|query| Target::Search {
|
||||
query: query.clone(),
|
||||
});
|
||||
|
||||
match self
|
||||
let result = self
|
||||
.fetch_target_items(target, &sort, page, per_page, &options)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
let result = match (result, keyword_fallback) {
|
||||
(Ok(items), Some(fallback)) if items.is_empty() => {
|
||||
self.fetch_target_items(fallback, &sort, page, per_page, &options)
|
||||
.await
|
||||
}
|
||||
(result, _) => result,
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(items) => items,
|
||||
Err(error) => {
|
||||
report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await;
|
||||
|
||||
@@ -52,6 +52,23 @@ impl XxdbxProvider {
|
||||
provider
|
||||
}
|
||||
|
||||
/// Resolves the card's asset URLs, which the site serves protocol-relative
|
||||
/// (`//prev.xxdbx.com/..`) or, now, site-relative (`/1234.jpg`).
|
||||
fn absolute_url(raw: &str) -> String {
|
||||
let raw = raw.trim();
|
||||
if raw.is_empty() {
|
||||
String::new()
|
||||
} else if raw.starts_with("http://") || raw.starts_with("https://") {
|
||||
raw.to_string()
|
||||
} else if let Some(rest) = raw.strip_prefix("//") {
|
||||
format!("https://{rest}")
|
||||
} else if raw.starts_with('/') {
|
||||
format!("https://xxdbx.com{raw}")
|
||||
} else {
|
||||
format!("https://xxdbx.com/{raw}")
|
||||
}
|
||||
}
|
||||
|
||||
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
|
||||
let _ = clientversion;
|
||||
Channel {
|
||||
@@ -272,26 +289,24 @@ impl XxdbxProvider {
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
|
||||
let thumb = format!(
|
||||
"https:{}",
|
||||
video_segment
|
||||
.split("<img ")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("src=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.last()
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
);
|
||||
let thumb_raw = video_segment
|
||||
.split("<img ")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("src=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.last()
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let thumb = Self::absolute_url(&thumb_raw);
|
||||
let raw_duration = video_segment
|
||||
.split("<div class=\"v_dur\">")
|
||||
.collect::<Vec<&str>>()
|
||||
@@ -306,21 +321,19 @@ impl XxdbxProvider {
|
||||
.to_string();
|
||||
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
|
||||
|
||||
let preview = format!(
|
||||
"https:{}",
|
||||
video_segment
|
||||
.split("data-preview=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
);
|
||||
let preview_raw = video_segment
|
||||
.split("data-preview=\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(1)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.split("\"")
|
||||
.collect::<Vec<&str>>()
|
||||
.get(0)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let preview = Self::absolute_url(&preview_raw);
|
||||
let tags = video_segment
|
||||
.split("<div class=\"v_tags\">")
|
||||
.collect::<Vec<&str>>()
|
||||
@@ -439,3 +452,25 @@ impl Provider for XxdbxProvider {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::XxdbxProvider;
|
||||
|
||||
#[test]
|
||||
fn resolves_site_relative_protocol_relative_and_absolute_assets() {
|
||||
assert_eq!(
|
||||
XxdbxProvider::absolute_url("/7004779764736.jpg"),
|
||||
"https://xxdbx.com/7004779764736.jpg"
|
||||
);
|
||||
assert_eq!(
|
||||
XxdbxProvider::absolute_url("//prev.xxdbx.com/700477973230.mp4"),
|
||||
"https://prev.xxdbx.com/700477973230.mp4"
|
||||
);
|
||||
assert_eq!(
|
||||
XxdbxProvider::absolute_url("https://cdn.example/a.jpg"),
|
||||
"https://cdn.example/a.jpg"
|
||||
);
|
||||
assert_eq!(XxdbxProvider::absolute_url(""), "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ const RESOLVE_CONCURRENCY: usize = 8;
|
||||
// xxxtik's listing endpoints are cursor-based (the numeric `id` of the last item
|
||||
// returned), not page-number based, so reaching page N requires walking N
|
||||
// sequential requests from the start of the feed.
|
||||
const MAX_REFILL_FETCHES: usize = 4;
|
||||
const MAX_PAGE_WALK: u16 = 25;
|
||||
|
||||
// xxxtik exposes ~60k tags through /tag/options, far too many to background-load
|
||||
@@ -423,18 +424,43 @@ impl XxxtikProvider {
|
||||
let mut items = Vec::new();
|
||||
|
||||
for _ in 0..page {
|
||||
let batch = self.fetch_one_page(path, cursor, per_page, options).await?;
|
||||
if batch.is_empty() {
|
||||
// Tag and creator pages mix in posts whose source is not RedGifs
|
||||
// (erome links, empty sources); those cannot be resolved here and
|
||||
// used to leave whole pages empty ("girl" returned nothing). Keep
|
||||
// pulling raw batches, bounded, until a page has enough
|
||||
// resolvable posts.
|
||||
let mut collected: Vec<Post> = Vec::new();
|
||||
let mut exhausted = false;
|
||||
for _ in 0..MAX_REFILL_FETCHES {
|
||||
let batch = self.fetch_one_page(path, cursor, per_page, options).await?;
|
||||
if batch.is_empty() {
|
||||
exhausted = true;
|
||||
break;
|
||||
}
|
||||
cursor = batch.last().map(|post| post.id).unwrap_or(cursor);
|
||||
let short_batch = batch.len() < per_page;
|
||||
collected.extend(batch.into_iter().filter(Self::is_redgifs_post));
|
||||
if collected.len() >= per_page || short_batch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if exhausted && collected.is_empty() {
|
||||
items = Vec::new();
|
||||
break;
|
||||
}
|
||||
cursor = batch.last().map(|post| post.id).unwrap_or(cursor);
|
||||
items = batch;
|
||||
items = collected;
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
fn is_redgifs_post(post: &Post) -> bool {
|
||||
post.source
|
||||
.as_deref()
|
||||
.map(|source| source.contains("redgifs.com/"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn fetch_target_posts(
|
||||
&self,
|
||||
target: &Target,
|
||||
@@ -791,6 +817,25 @@ impl Provider for XxxtikProvider {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn only_redgifs_sources_are_resolvable() {
|
||||
let mk = |source: Option<&str>| super::Post {
|
||||
id: 1,
|
||||
uuid: "u".to_string(),
|
||||
description: None,
|
||||
source: source.map(str::to_string),
|
||||
status: None,
|
||||
created_at: String::new(),
|
||||
author: Default::default(),
|
||||
tags: vec![],
|
||||
};
|
||||
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://www.redgifs.com/watch/abc"))));
|
||||
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://redgifs.com/watch/abc"))));
|
||||
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("https://www.erome.com/a/CE5z46ne#39"))));
|
||||
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some(""))));
|
||||
assert!(!XxxtikProvider::is_redgifs_post(&mk(None)));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
|
||||
126
src/proxies/aylothumb.rs
Normal file
126
src/proxies/aylothumb.rs
Normal file
@@ -0,0 +1,126 @@
|
||||
use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
|
||||
use ntex::{
|
||||
http::Response,
|
||||
web::{self, HttpRequest, error},
|
||||
};
|
||||
use url::Url;
|
||||
|
||||
use crate::util::requester::Requester;
|
||||
|
||||
/// Referer the image CDN expects, or `None` when the host is not allowed.
|
||||
///
|
||||
/// The signed `pix-cdn77.*` thumbnail hosts used by tube8 and the pornhub
|
||||
/// shorties feed answer 403 without a matching Referer, so apps that load
|
||||
/// the URL directly get broken images.
|
||||
fn referer_for_host(host: &str) -> Option<&'static str> {
|
||||
let host = host.to_ascii_lowercase();
|
||||
if host == "t8cdn.com" || host.ends_with(".t8cdn.com") {
|
||||
Some("https://www.tube8.com/")
|
||||
} else if host == "phncdn.com" || host.ends_with(".phncdn.com") {
|
||||
Some("https://www.pornhub.com/")
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn is_image_path(path: &str) -> bool {
|
||||
let path = path.to_ascii_lowercase();
|
||||
// The signed thumbnail paths end in processing options such as
|
||||
// `/plain/rs:fit:320:180/vts:3`, not a file extension, so accept those too.
|
||||
[".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"]
|
||||
.iter()
|
||||
.any(|ext| path.ends_with(ext))
|
||||
|| path.contains("/plain/")
|
||||
}
|
||||
|
||||
fn upstream_url(req: &HttpRequest) -> Option<(String, &'static str)> {
|
||||
let endpoint = req.match_info().query("endpoint").trim_start_matches('/');
|
||||
let mut image_url = if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
|
||||
endpoint.to_string()
|
||||
} else {
|
||||
format!("https://{endpoint}")
|
||||
};
|
||||
|
||||
// The signed URL's own query (`hash`, `validto`) arrives as the request query.
|
||||
let query = req.query_string();
|
||||
if !query.is_empty() && !image_url.contains('?') {
|
||||
image_url.push('?');
|
||||
image_url.push_str(query);
|
||||
}
|
||||
|
||||
let parsed = Url::parse(&image_url).ok()?;
|
||||
if parsed.scheme() != "https" {
|
||||
return None;
|
||||
}
|
||||
let referer = referer_for_host(parsed.host_str()?)?;
|
||||
if !is_image_path(parsed.path()) {
|
||||
return None;
|
||||
}
|
||||
Some((image_url, referer))
|
||||
}
|
||||
|
||||
pub async fn get_image(
|
||||
req: HttpRequest,
|
||||
requester: web::types::State<Requester>,
|
||||
) -> Result<impl web::Responder, web::Error> {
|
||||
let Some((image_url, referer)) = upstream_url(&req) else {
|
||||
return Ok(web::HttpResponse::BadRequest().finish());
|
||||
};
|
||||
|
||||
let upstream = match requester
|
||||
.get_ref()
|
||||
.clone()
|
||||
.get_raw_with_headers(
|
||||
image_url.as_str(),
|
||||
vec![("Referer".to_string(), referer.to_string())],
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(response) => response,
|
||||
Err(_) => return Ok(web::HttpResponse::NotFound().finish()),
|
||||
};
|
||||
|
||||
let status = upstream.status();
|
||||
let headers = upstream.headers().clone();
|
||||
let bytes = upstream.bytes().await.map_err(error::ErrorBadGateway)?;
|
||||
|
||||
let mut resp = Response::build(status);
|
||||
if let Some(ct) = headers.get(CONTENT_TYPE) {
|
||||
if let Ok(ct_str) = ct.to_str() {
|
||||
resp.set_header(CONTENT_TYPE, ct_str);
|
||||
}
|
||||
}
|
||||
if let Some(cl) = headers.get(CONTENT_LENGTH) {
|
||||
if let Ok(cl_str) = cl.to_str() {
|
||||
resp.set_header(CONTENT_LENGTH, cl_str);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(resp.body(bytes.to_vec()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn maps_hosts_to_referers_and_rejects_others() {
|
||||
assert_eq!(
|
||||
referer_for_host("pix-cdn77.t8cdn.com"),
|
||||
Some("https://www.tube8.com/")
|
||||
);
|
||||
assert_eq!(
|
||||
referer_for_host("pix-cdn77.phncdn.com"),
|
||||
Some("https://www.pornhub.com/")
|
||||
);
|
||||
assert_eq!(referer_for_host("evil.example.com"), None);
|
||||
assert_eq!(referer_for_host("t8cdn.com.evil.example"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_signed_processing_paths_and_extensions() {
|
||||
assert!(is_image_path("/c6531/videos/1/original.mov/plain/rs:fit:320:180/vts:3"));
|
||||
assert!(is_image_path("/videos/thumb.JPG"));
|
||||
assert!(!is_image_path("/videos/clip.mp4"));
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,10 @@ impl DoodstreamProxy {
|
||||
| "www.streamhg.com"
|
||||
| "doodstream.com"
|
||||
| "www.doodstream.com"
|
||||
| "myvidplay.com"
|
||||
| "www.myvidplay.com"
|
||||
| "playmogo.com"
|
||||
| "www.playmogo.com"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -12,8 +12,16 @@ fn normalize_image_url(endpoint: &str) -> String {
|
||||
endpoint.to_string()
|
||||
} else if endpoint.starts_with("hanime-cdn.com/") || endpoint == "hanime-cdn.com" {
|
||||
format!("https://{endpoint}")
|
||||
} else {
|
||||
} else if endpoint
|
||||
.split('/')
|
||||
.next()
|
||||
.is_some_and(|first| first.contains('.'))
|
||||
{
|
||||
// `other-host.tld/path`: the caller already supplied a host.
|
||||
format!("https://{endpoint}")
|
||||
} else {
|
||||
// Site-relative path such as `images/covers/x.png`.
|
||||
format!("https://hanime-cdn.com/{endpoint}")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::proxies::lulustream::LulustreamProxy;
|
||||
use crate::proxies::thaiporntv::ThaipornTvProxy;
|
||||
|
||||
pub mod allpornstream;
|
||||
pub mod aylothumb;
|
||||
pub mod melonstube;
|
||||
pub mod animeidhentai;
|
||||
pub mod archivebate;
|
||||
|
||||
@@ -105,9 +105,11 @@ impl NoodlemagazineProxy {
|
||||
|
||||
fn has_allowed_image_extension(path: &str) -> bool {
|
||||
let path = path.to_ascii_lowercase();
|
||||
[".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"]
|
||||
.iter()
|
||||
.any(|ext| path.ends_with(ext))
|
||||
// `.../i.mycdn.me/getVideoPreview?id=..` previews have no extension.
|
||||
path.ends_with("/getvideopreview")
|
||||
|| [".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"]
|
||||
.iter()
|
||||
.any(|ext| path.ends_with(ext))
|
||||
}
|
||||
|
||||
fn is_disallowed_thumb_host(host: &str) -> bool {
|
||||
@@ -292,20 +294,23 @@ pub async fn get_image(
|
||||
requester: web::types::State<Requester>,
|
||||
) -> Result<impl web::Responder, web::Error> {
|
||||
let endpoint = req.match_info().query("endpoint").to_string();
|
||||
let image_url = NoodlemagazineProxy::normalize_image_url(&endpoint);
|
||||
let mut image_url = NoodlemagazineProxy::normalize_image_url(&endpoint);
|
||||
// The preview's own query (`id`, `idx`, `type`, `tkn`, `fn`) arrives as the
|
||||
// request query string, not as part of the matched path.
|
||||
let query = req.query_string();
|
||||
if !query.is_empty() && !image_url.contains('?') {
|
||||
image_url.push('?');
|
||||
image_url.push_str(query);
|
||||
}
|
||||
|
||||
if !NoodlemagazineProxy::is_allowed_thumb_url(&image_url) {
|
||||
return Ok(web::HttpResponse::BadRequest().finish());
|
||||
}
|
||||
|
||||
// No site warm-up first: the image CDN answers directly (~0.1s). Warming
|
||||
// via noodlemagazine.com cost 10-30s through the requester's fallbacks;
|
||||
// the retry below still warms up when the first fetch fails.
|
||||
let mut requester = requester.get_ref().clone();
|
||||
let _ = requester
|
||||
.get_with_headers(
|
||||
NoodlemagazineProxy::root_referer(),
|
||||
NoodlemagazineProxy::root_html_headers(),
|
||||
Some(Version::HTTP_11),
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut headers = NoodlemagazineProxy::image_headers(&requester, image_url.as_str());
|
||||
let mut upstream = requester
|
||||
@@ -409,6 +414,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn allows_https_image_thumbs_but_rejects_local_or_non_images() {
|
||||
assert!(NoodlemagazineProxy::is_allowed_thumb_url(
|
||||
"https://img.pvvstream.pro/preview/a/-1_2/240/i.mycdn.me/getVideoPreview?id=1&idx=0&type=39&tkn=x"
|
||||
));
|
||||
assert!(NoodlemagazineProxy::is_allowed_thumb_url(
|
||||
"https://noodlemagazine.com/thumbs/example.webp"
|
||||
));
|
||||
|
||||
@@ -116,9 +116,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn normalizes_detail_request_with_path_only() {
|
||||
let (url, video_id) = VidaraProxy::normalize_detail_request("video/1000/demo")
|
||||
let (url, video_id) = VidaraProxy::normalize_detail_request("v/eJ9O4QqG1Ln2")
|
||||
.expect("detail request should parse");
|
||||
assert_eq!(url, "https://vidara.so/video/1000/demo");
|
||||
assert_eq!(video_id, "1000");
|
||||
assert_eq!(url, "https://vidara.so/v/eJ9O4QqG1Ln2");
|
||||
assert_eq!(video_id, "eJ9O4QqG1Ln2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_paths_outside_the_embed_route() {
|
||||
assert!(VidaraProxy::normalize_detail_request("video/1000/demo").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,7 +149,12 @@ pub fn config(cfg: &mut web::ServiceConfig) {
|
||||
.route(web::get().to(crate::proxies::hotbunnythumb::get_image)),
|
||||
);
|
||||
cfg.service(
|
||||
web::resource("/proxy/pornhub-thumb/{endpoint}*")
|
||||
web::resource("/aylo-thumb/{endpoint}*")
|
||||
.route(web::post().to(crate::proxies::aylothumb::get_image))
|
||||
.route(web::get().to(crate::proxies::aylothumb::get_image)),
|
||||
);
|
||||
cfg.service(
|
||||
web::resource("/pornhub-thumb/{endpoint}*")
|
||||
.route(web::post().to(crate::proxies::pornhubthumb::get_image))
|
||||
.route(web::get().to(crate::proxies::pornhubthumb::get_image)),
|
||||
);
|
||||
|
||||
@@ -496,6 +496,10 @@ impl Requester {
|
||||
timeout: Option<Duration>,
|
||||
http_version: Option<Version>,
|
||||
) -> Result<Response, wreq::Error> {
|
||||
// Replay the solved FlareSolverr identity here too: a token minted
|
||||
// through `get_with_headers` under that user agent is rejected (401)
|
||||
// when the follow-up raw request goes out with the default one.
|
||||
self.adopt_remembered_identity(url);
|
||||
let cookie_preview = self.debug_cookie_preview_from_owned_headers(url, &headers);
|
||||
#[cfg(not(feature = "debug"))]
|
||||
let _ = &cookie_preview;
|
||||
|
||||
Reference in New Issue
Block a user