From 3b591c1950ae23a3f2969de8d50457706fbb61a0 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 6 Jul 2026 17:51:06 +0000 Subject: [PATCH] melonstube fix --- docs/provider-catalog.md | 2 +- src/providers/melonstube.rs | 76 ++++++++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index 4b34789..bee22a6 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -29,7 +29,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `hsex` | `chinese` | yes | no | Strong template for tags, uploaders, and direct HLS formats. | | `hypnotube` | `fetish-kink` | no | no | Fetish/tube hybrid. | | `javtiful` | `jav` | no | no | JAV channel family. | -| `melonstube` | `mainstream-tube` | no | yes | Meta-search aggregator for melonstube.com — every card is an `/out/?l=&c=&v=3` redirect link whose destination (a third-party host) is embedded, zero-network, in a MessagePack blob under the `l` param (mixed binary/string fields: view/click counts, a JSON date-range blob, an array of related-video ids, and the destination URL itself as one msgpack string). Decoding is local: base64-decode `l`, then regex-extract the `https?://...` destination directly out of the raw (lossy-UTF8-converted) msgpack bytes — **the regex must be a positive allowlist of legal URI characters** (`[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`), not a denylist of a few excluded characters: the byte immediately following the msgpack string (a length-prefix marker for the next field, e.g. `\xcd`) lossy-converts to U+FFFD (code point 0xFFFD), which a denylist limited to ASCII control chars (`[^\x00-\x1f\\"']`) fails to exclude, silently appending a corrupt trailing character to the URL. Feeds: `/new` (latest), `/popular`, `/rating` (all three are static curated lists, ~120 items, true pagination via `?page=N`); search via `/search?q=`. Destination hosts fall into three buckets: (1) the common case — yt-dlp (`--impersonate chrome-120`) resolves the destination directly, so `video.url` is just the decoded destination URL, no proxy; (2) `JUNK_HOSTS` (currently `fhgte.com`) — observed dead-end paywall/signup funnels with no free playable video, so cards pointing there are dropped entirely rather than surfaced as false-positive results; (3) `HARD_HOSTS` (`manysex.com`, `videomanysex.com`) — yt-dlp cannot resolve these, so `video.url` is routed through `/proxy/melonstube/{host}/{path}`. The proxy (`src/proxies/melonstube.rs`) ports vjav.rs's Cyrillic-homoglyph-obfuscated base64 decode chain to pull the real `get_file` path/query out of `videofile.php`'s `video_url` field, then makes two manual (non-auto-redirect) hops to the final CDN URL: hop 1 needs the manysex.com/videomanysex.com page as `Referer`; hop 2 (to the signed `ahcdn.com` URL) must be sent with **no** Referer at all, because the CDN's signed URL embeds a literal `referer=none,.manysex.com,.gstatic.com` allow-list that rejects the videomanysex.com Referer used on hop 1 — auto-redirect clients that forward the same Referer to every hop get a 403 at hop 2. Before returning the 302, an anti-false-positive check confirms the resolved URL string contains the requested numeric video id and that a ranged GET (`Range: bytes=0-65535`) returns 200/206 with a `video/*`/`octet-stream` content-type, so a paywall/ad/error page can't masquerade as the real stream. Thumbnails (`ttcache.com`) load directly, no proxy. No `/api/uploaders` (aggregator has no stable uploader identity). | +| `melonstube` | `mainstream-tube` | no | yes | Meta-search aggregator for melonstube.com — every card is an `/out/?l=&c=&v=3` redirect link whose destination (a third-party host) is embedded, zero-network, in a MessagePack blob under the `l` param (mixed binary/string fields: view/click counts, a JSON date-range blob, an array of related-video ids, and the destination URL itself as one msgpack string). Decoding is local: base64-decode `l`, then regex-extract the `https?://...` destination directly out of the raw (lossy-UTF8-converted) msgpack bytes — **the regex must be a positive allowlist of legal URI characters** (`[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`), not a denylist of a few excluded characters: the byte immediately following the msgpack string (a length-prefix marker for the next field, e.g. `\xcd`) lossy-converts to U+FFFD (code point 0xFFFD), which a denylist limited to ASCII control chars (`[^\x00-\x1f\\"']`) fails to exclude, silently appending a corrupt trailing character to the URL. Feeds: `/new` (latest), `/popular`, `/rating` (all three are static curated lists, ~120 items, true pagination via `?page=N`); search is NOT the vestigial `/search?q=` GET endpoint (returns generic filler content behind a fake static result counter regardless of query) — the real mechanism mirrors the site's own search form: POST `search_query[query]=` to `/searching/by-form`, which 303-redirects (cookies + redirect auto-followed by the shared `Requester`) to a resolved, genuinely query-relevant destination depending on entity match — `/pornstar/`, `/category/`, `/search/a/` (studio), or `/search/` (generic free text) — all of which support `?page=N` pagination; a non-matching query resolves to a page with zero result cards, which `parse_listing` naturally returns as empty. Destination hosts fall into three buckets: (1) the common case — yt-dlp (`--impersonate chrome-120`) resolves the destination directly, so `video.url` is just the decoded destination URL, no proxy; (2) `JUNK_HOSTS` (currently `fhgte.com`) — observed dead-end paywall/signup funnels with no free playable video, so cards pointing there are dropped entirely rather than surfaced as false-positive results; (3) `HARD_HOSTS` (`manysex.com`, `videomanysex.com`) — yt-dlp cannot resolve these, so `video.url` is routed through `/proxy/melonstube/{host}/{path}`. The proxy (`src/proxies/melonstube.rs`) ports vjav.rs's Cyrillic-homoglyph-obfuscated base64 decode chain to pull the real `get_file` path/query out of `videofile.php`'s `video_url` field, then makes two manual (non-auto-redirect) hops to the final CDN URL: hop 1 needs the manysex.com/videomanysex.com page as `Referer`; hop 2 (to the signed `ahcdn.com` URL) must be sent with **no** Referer at all, because the CDN's signed URL embeds a literal `referer=none,.manysex.com,.gstatic.com` allow-list that rejects the videomanysex.com Referer used on hop 1 — auto-redirect clients that forward the same Referer to every hop get a 403 at hop 2. Before returning the 302, an anti-false-positive check confirms the resolved URL string contains the requested numeric video id and that a ranged GET (`Range: bytes=0-65535`) returns 200/206 with a `video/*`/`octet-stream` content-type, so a paywall/ad/error page can't masquerade as the real stream. Thumbnails (`ttcache.com`) load directly, no proxy. No `/api/uploaders` (aggregator has no stable uploader identity). | | `missav` | `jav` | no | no | HLS format pattern. | | `noodlemagazine` | `mainstream-tube` | no | yes | Best template for media and thumbnail proxying. | | `okporn` | `mainstream-tube` | no | no | Simple mainstream archive. | diff --git a/src/providers/melonstube.rs b/src/providers/melonstube.rs index 121f95d..22cff4d 100644 --- a/src/providers/melonstube.rs +++ b/src/providers/melonstube.rs @@ -10,9 +10,9 @@ use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD}; use chrono::{Duration as ChronoDuration, Utc}; use error_chain::error_chain; -use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use regex::Regex; use scraper::{Html, Selector}; +use url::form_urlencoded::Serializer; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { @@ -118,14 +118,9 @@ impl MelonstubeProvider { format!("{BASE_URL}/rating") } } - Target::Search { query, page } => { - let encoded = utf8_percent_encode(query, NON_ALPHANUMERIC).to_string(); - if *page > 1 { - format!("{BASE_URL}/search?q={encoded}&page={page}") - } else { - format!("{BASE_URL}/search?q={encoded}") - } - } + Target::Search { .. } => unreachable!( + "Target::Search is resolved asynchronously via resolve_search_base_url" + ), } } @@ -386,13 +381,58 @@ impl MelonstubeProvider { } } + fn build_search_request_body(query: &str) -> String { + let mut serializer = Serializer::new(String::new()); + serializer.append_pair("search_query[query]", query); + serializer.finish() + } + + async fn resolve_search_base_url(query: &str, options: &ServerOptions) -> Result { + let search_url = format!("{BASE_URL}/searching/by-form"); + let body = Self::build_search_request_body(query); + + let mut requester = requester_or_default(options, CHANNEL_ID, "resolve_search_base_url"); + + let response = requester + .post( + &search_url, + &body, + vec![ + ("Content-Type", "application/x-www-form-urlencoded"), + ("Referer", BASE_URL), + ("User-Agent", BROWSER_UA), + ], + ) + .await + .map_err(|e| Error::from(format!("search submit failed url={search_url}: {e}")))?; + + Ok(response.url().to_string().trim_end_matches('/').to_string()) + } + async fn fetch_and_parse( &self, cache: VideoCache, target: Target, options: ServerOptions, ) -> Result> { - let url = Self::build_url(&target); + let url = match &target { + Target::Search { query, page } => { + let base = match Self::resolve_search_base_url(query, &options).await { + Ok(base) => base, + Err(e) => { + report_provider_error(CHANNEL_ID, "fetch_and_parse.search", &e.to_string()) + .await; + return Ok(vec![]); + } + }; + if *page > 1 { + format!("{base}?page={page}") + } else { + base + } + } + _ => Self::build_url(&target), + }; if let Some((time, items)) = cache.get(&url) { if time.elapsed().unwrap_or_default().as_secs() < 300 { @@ -513,20 +553,10 @@ mod tests { } #[test] - fn builds_search_urls() { + fn builds_search_request_body() { assert_eq!( - MelonstubeProvider::build_url(&super::Target::Search { - query: "teen".to_string(), - page: 1 - }), - "https://www.melonstube.com/search?q=teen" - ); - assert_eq!( - MelonstubeProvider::build_url(&super::Target::Search { - query: "teen".to_string(), - page: 2 - }), - "https://www.melonstube.com/search?q=teen&page=2" + MelonstubeProvider::build_search_request_body("adriana chechik"), + "search_query%5Bquery%5D=adriana+chechik" ); }