advanced probing

backend/main.py

@@ -11,6 +11,7 @@ from yt_dlp.networking.impersonate import ImpersonateTarget
 from curl_cffi import requests as impersonate_requests
 import threading
 import io
+import time
 from urllib.parse import urljoin
 
 # Browser to impersonate at the TLS/HTTP layer. Some origins (e.g. the
@@ -163,6 +164,135 @@ def videos_proxy():
     except Exception as e:
         return jsonify({"error": str(e)}), 500
 
+# Cache of yt-dlp metadata resolutions so the on-demand probe endpoint doesn't
+# re-extract the same video on every hover/scroll. Signed media URLs expire, so
+# entries are intentionally short-lived.
+RESOLVE_CACHE_TTL = 300
+_resolve_cache = {}
+_resolve_cache_lock = threading.Lock()
+
+# Per-format fields the frontend needs to rank formats and build stream/probe
+# URLs (see resolveStreamSources/rankFormats in videos.js). Everything else in a
+# yt-dlp format dict is dropped to keep the payload small.
+_RESOLVE_FORMAT_FIELDS = ('url', 'http_headers', 'height', 'width', 'tbr',
+                          'fps', 'vcodec', 'acodec', 'ext', 'video_ext', 'quality')
+
+# Some channels surface pages that yt-dlp can't extract because the video is
+# embedded in a third-party JS player iframe (e.g. the xtremestream family used
+# by tube.perverzija.com). The player page declares its HLS playlist URL as
+# `m3u8_loader_url + video_id`, so we scrape the page for the embed iframe, then
+# read those two variables out of the player to reconstruct the stream URL.
+_EMBED_IFRAME_RE = re.compile(r'''<iframe[^>]+src=["']([^"']+)''', re.I)
+_EMBED_LOADER_RE = re.compile(r'''m3u8_loader_url\s*=\s*[`'"]([^`'"]+)[`'"]''')
+_EMBED_VIDEOID_RE = re.compile(r'''video_id\s*=\s*[`'"]([^`'"]+)[`'"]''')
+
+def resolve_unsupported_embed(page_url):
+    """Best-effort resolver for iframe-embedded JS players yt-dlp can't handle.
+    Returns an info-like dict ({'url','formats','http_headers','is_live'}) whose
+    single format is the embed's HLS playlist, or None if nothing was found."""
+    try:
+        sess = get_impersonate_session()
+        page = sess.get(page_url, headers={'Referer': page_url}, timeout=15)
+        embed_url = None
+        for src in _EMBED_IFRAME_RE.findall(page.text):
+            candidate = urljoin(page_url, src)
+            if '/player/' in candidate or 'index.php?data=' in candidate:
+                embed_url = candidate
+                break
+        if not embed_url:
+            return None
+
+        player = sess.get(embed_url, headers={'Referer': page_url}, timeout=15)
+        loader = _EMBED_LOADER_RE.search(player.text)
+        video_id = _EMBED_VIDEOID_RE.search(player.text)
+        if not (loader and video_id):
+            return None
+        stream_url = loader.group(1) + video_id.group(1)
+
+        parsed = urllib.parse.urlparse(embed_url)
+        referer = f"{parsed.scheme}://{parsed.netloc}/"
+        headers = {'Referer': referer}
+        return {
+            'url': stream_url,
+            'is_live': False,
+            'http_headers': headers,
+            'formats': [{'url': stream_url, 'ext': 'm3u8', 'protocol': 'm3u8', 'http_headers': headers}],
+        }
+    except Exception:
+        return None
+
+@app.route('/api/resolve', methods=['POST', 'GET'])
+def resolve_video():
+    """Resolve a page URL to its playable formats via yt-dlp and return them as
+    JSON. The frontend calls this on demand (when a card is hovered or scrolled
+    into view) to learn the real media URLs so it can background-probe them for
+    direct, proxy-free playability."""
+    if request.method == 'POST':
+        source = request.json or {}
+        video_url = source.get('url')
+    else:
+        source = request.args
+        video_url = request.args.get('url')
+
+    if not video_url:
+        return jsonify({"error": "No URL provided"}), 400
+
+    now = time.time()
+    with _resolve_cache_lock:
+        cached = _resolve_cache.get(video_url)
+        if cached and cached[0] > now:
+            return jsonify(cached[1])
+
+    ydl_opts = {
+        'quiet': True,
+        'no_warnings': True,
+        'skip_download': True,
+        # Match /api/stream so the resolved formats reflect what playback will
+        # actually fetch from fingerprinting origins.
+        'impersonate': ImpersonateTarget.from_str(IMPERSONATE_TARGET),
+    }
+    passthrough_headers = collect_passthrough_headers(source)
+    if passthrough_headers:
+        ydl_opts['http_headers'] = passthrough_headers
+
+    try:
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            info = ydl.extract_info(video_url, download=False)
+    except Exception as e:
+        # Many channels point at sites yt-dlp can't extract ("Unsupported URL").
+        # That's not fatal here -- the embed fallback below may still find a
+        # stream, and otherwise we return empty formats so playback falls back to
+        # the proxy.
+        app.logger.info("[resolve] yt-dlp could not extract %s: %s", video_url, e)
+        info = None
+
+    # Fall back to scraping iframe-embedded JS players yt-dlp doesn't support.
+    if not (info and (info.get('formats') or info.get('url'))):
+        embed = resolve_unsupported_embed(video_url)
+        if embed:
+            info = embed
+
+    formats = []
+    for fmt in ((info.get('formats') if info else None) or []):
+        if not fmt.get('url'):
+            continue
+        formats.append({k: fmt.get(k) for k in _RESOLVE_FORMAT_FIELDS if fmt.get(k) is not None})
+
+    result = {
+        'url': info.get('url') if info else None,
+        'http_headers': (info.get('http_headers') if info else None) or {},
+        'isLive': bool(info.get('is_live')) if info else False,
+        'formats': formats,
+    }
+
+    with _resolve_cache_lock:
+        # Drop expired entries so the cache doesn't grow without bound.
+        for key in [k for k, v in _resolve_cache.items() if v[0] <= now]:
+            _resolve_cache.pop(key, None)
+        _resolve_cache[video_url] = (now + RESOLVE_CACHE_TTL, result)
+
+    return jsonify(result)
+
 @app.route('/api/image', methods=['GET', 'HEAD'])
 def image_proxy():
     image_url = request.args.get('url')
@@ -664,7 +794,19 @@ def stream_video():
 
         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
             # Extract the info
-            info = ydl.extract_info(video_url, download=False)
+            try:
+                info = ydl.extract_info(video_url, download=False)
+            except Exception as ydl_err:
+                # yt-dlp can't extract iframe-embedded JS players; scrape the
+                # embed for its HLS playlist and proxy that directly instead.
+                embed = resolve_unsupported_embed(video_url)
+                if not embed:
+                    raise
+                dbg(f"embed fallback resolved {video_url} -> {embed['url']}")
+                if request.method == 'HEAD':
+                    return Response("", status=200, content_type='application/vnd.apple.mpegurl')
+                return proxy_hls_playlist(embed['url'], embed['http_headers'].get('Referer'),
+                                          upstream_headers=embed['http_headers'])
             dbg(f"yt_dlp extractor={info.get('extractor')} protocol={info.get('protocol')}")
             
             # Try to get the URL from the info dict (works for progressive downloads)

frontend/js/videos.js

@@ -290,9 +290,6 @@ App.videos = App.videos || {};
         items.forEach(v => {
             if (state.renderedVideoIds.has(v.id)) return;
             state.loadedVideos.push(v);
-            // Probe in the background whether this video's best source plays
-            // directly, so playback can bypass the proxy when proven.
-            App.videos.probeVideoSources(v);
 
             const card = document.createElement('div');
             card.className = 'video-card';
@@ -410,6 +407,11 @@ App.videos = App.videos || {};
                 App.player.open(v, { originEl: card });
             };
             grid.appendChild(card);
+            // Resolve formats + probe direct playability on demand: when the
+            // card scrolls near the viewport, or the moment it's hovered.
+            cardVideo.set(card, v);
+            probeObserver.observe(card);
+            card.addEventListener('pointerenter', () => App.videos.resolveAndProbe(v), { once: true });
             state.renderedVideoIds.add(v.id);
         });
 
@@ -650,7 +652,7 @@ App.videos = App.videos || {};
         const isLive = !!(videoOrUrl && typeof videoOrUrl === 'object' &&
             (videoOrUrl.isLive || (videoOrUrl.meta && videoOrUrl.meta.isLive)));
         if (typeof videoOrUrl === 'string') {
-            return videoOrUrl ? [{ url: videoOrUrl, referer: deriveReferer(videoOrUrl), userAgent: '', isLive }] : [];
+            return videoOrUrl ? [{ url: videoOrUrl, referer: deriveReferer(videoOrUrl), userAgent: '', isLive, refererRequired: false }] : [];
         }
         if (!videoOrUrl || typeof videoOrUrl !== 'object') return [];
 
@@ -664,9 +666,14 @@ App.videos = App.videos || {};
         }
 
         const sources = App.videos.rankFormats(meta.formats, preferredHeight).map((fmt) => {
-            const referer = headerValue(fmt.http_headers, 'Referer') || metaReferer || deriveReferer(fmt.url);
+            // An *explicit* Referer (from the extractor) signals the upstream
+            // enforces it; deriveReferer is only a best-effort fallback. The
+            // browser can't set a cross-origin Referer, so refererRequired tells
+            // callers (the probe) that direct playback can't work.
+            const explicitReferer = headerValue(fmt.http_headers, 'Referer') || metaReferer;
+            const referer = explicitReferer || deriveReferer(fmt.url);
             const userAgent = headerValue(fmt.http_headers, 'User-Agent') || metaUserAgent;
-            return { url: fmt.url, referer, userAgent, isLive };
+            return { url: fmt.url, referer, userAgent, isLive, refererRequired: !!explicitReferer };
         });
 
         if (!sources.length) {
@@ -676,7 +683,8 @@ App.videos = App.videos || {};
                     url: fallbackUrl,
                     referer: metaReferer || deriveReferer(fallbackUrl),
                     userAgent: metaUserAgent,
-                    isLive
+                    isLive,
+                    refererRequired: !!metaReferer
                 });
             }
         }
@@ -685,37 +693,41 @@ App.videos = App.videos || {};
 
     App.videos.resolveStreamSource = function(videoOrUrl, options) {
         const sources = App.videos.resolveStreamSources(videoOrUrl, options);
-        return sources.length ? sources[0] : { url: '', referer: '', userAgent: '', isLive: false };
+        return sources.length ? sources[0] : { url: '', referer: '', userAgent: '', isLive: false, refererRequired: false };
     };
 
     // Background "direct playability" probe. The backend proxy exists to work
     // around CORS, hotlink (403) protection, and TLS fingerprinting. When the
     // browser can fetch a media URL cross-origin and actually read the response
     // (CORS allowed, not blocked/403), playing it directly works and the proxy
-    // is pure overhead. We probe each loaded video's best source in the
+    // is pure overhead. CORS is an origin-level policy, so the answer is the
-    // background and, only when proven, let the player skip the proxy.
+    // same for every media URL served by a given host: we probe (and cache)
+    // once per host and let the player skip the proxy for any URL on a host
+    // that's been proven.
     const DIRECT_PROBE_TIMEOUT_MS = 8000;
-    // Only URLs that the player can hand straight to <video>/hls.js are worth
-    // probing; anything else (e.g. a live channel page URL) is resolved
-    // server-side and must keep going through the proxy.
-    const DIRECT_PLAYABLE_RE = /\.(mp4|m4v|m4s|webm|ts|mov|m3u8)($|\?)/i;
 
-    // url -> true (proven directly playable) | false (proven not). Absent means
+    const directHostOf = (url) => {
-    // unknown/unprobed, in which case the proxy is used.
+        try { return new URL(url).host; } catch (err) { return ''; }
+    };
+
+    // host -> true (proven directly playable) | false (proven not). Absent
+    // means unknown/unprobed, in which case the proxy is used.
     App.videos._directStatus = new Map();
     const directPending = new Map();
 
     App.videos.isDirectProven = function(url) {
-        return App.videos._directStatus.get(url) === true;
+        return App.videos._directStatus.get(directHostOf(url)) === true;
     };
 
     App.videos.probeDirect = function(url) {
         if (!url) return Promise.resolve(false);
-        if (App.videos._directStatus.has(url)) {
+        const host = directHostOf(url);
-            return Promise.resolve(App.videos._directStatus.get(url));
+        if (!host) return Promise.resolve(false);
+        if (App.videos._directStatus.has(host)) {
+            return Promise.resolve(App.videos._directStatus.get(host));
         }
-        if (directPending.has(url)) {
+        if (directPending.has(host)) {
-            return directPending.get(url);
+            return directPending.get(host);
         }
         const promise = (async () => {
             const controller = new AbortController();
@@ -742,19 +754,24 @@ App.videos = App.videos || {};
             } finally {
                 clearTimeout(timer);
             }
-            App.videos._directStatus.set(url, ok);
+            App.videos._directStatus.set(host, ok);
-            directPending.delete(url);
+            directPending.delete(host);
-            console.log(`[direct-probe] ${ok ? 'DIRECT' : 'PROXY '} (${detail}) ${url}`);
+            console.log(`[direct-probe] ${ok ? 'DIRECT' : 'PROXY '} (${detail}) ${host}`);
             return ok;
         })();
-        directPending.set(url, promise);
+        directPending.set(host, promise);
         return promise;
     };
 
     // Kicks off a background probe of a video's best (first-played) source so a
-    // later playback can skip the proxy if the direct URL is proven reachable.
+    // later playback can skip the proxy if its host is proven reachable. Only
+    // runs once the video has resolved formats (see resolveAndProbe): those are
+    // real media URLs (or redirects to them), whereas a bare listing item only
+    // carries a page URL that the player can't use directly.
     App.videos.probeVideoSources = function(video) {
         if (!video || typeof video !== 'object') return;
+        const meta = video.meta || video;
+        if (!meta || !Array.isArray(meta.formats) || !meta.formats.length) return;
         let sources;
         try {
             sources = App.videos.resolveStreamSources(video);
@@ -763,10 +780,68 @@ App.videos = App.videos || {};
         }
         const best = sources && sources[0];
         if (!best || !best.url || best.isLive) return;
-        if (!DIRECT_PLAYABLE_RE.test(best.url)) return;
+        // Sources that require a specific upstream Referer can't be fetched
+        // directly by the browser (it can't forge a cross-origin Referer), so a
+        // probe would always fail -- leave them to the proxy.
+        if (best.refererRequired) return;
         App.videos.probeDirect(best.url);
     };
 
+    // Listing items arrive without formats (meta is null) -- only a page URL --
+    // so there's nothing direct-playable to probe up front. This resolves a
+    // video's real media formats via the backend (yt-dlp), attaches them as
+    // `video.meta` so the player and probe can use them, then probes the best
+    // source. Resolution is per-video and deduped: it runs at most once per
+    // video, triggered lazily by hover/scroll so we don't resolve cards the
+    // user never looks at.
+    const cardVideo = new WeakMap();
+    const metaResolved = new Set();
+    const metaPending = new Map();
+
+    const probeObserver = new IntersectionObserver((entries) => {
+        entries.forEach((entry) => {
+            if (!entry.isIntersecting) return;
+            probeObserver.unobserve(entry.target);
+            const video = cardVideo.get(entry.target);
+            if (video) App.videos.resolveAndProbe(video);
+        });
+    }, { rootMargin: '200px' });
+
+    App.videos.resolveAndProbe = function(video) {
+        if (!video || typeof video !== 'object' || !video.id) return Promise.resolve();
+        // Already have formats (resolved earlier): just (re)probe the best one.
+        if (video.meta && Array.isArray(video.meta.formats) && video.meta.formats.length) {
+            App.videos.probeVideoSources(video);
+            return Promise.resolve();
+        }
+        if (metaResolved.has(video.id)) return Promise.resolve();
+        if (metaPending.has(video.id)) return metaPending.get(video.id);
+        if (!video.url) return Promise.resolve();
+
+        const promise = (async () => {
+            try {
+                const response = await fetch('/api/resolve', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ url: video.url })
+                });
+                if (!response.ok) return;
+                const data = await response.json();
+                if (data && Array.isArray(data.formats) && data.formats.length) {
+                    video.meta = data;
+                    App.videos.probeVideoSources(video);
+                }
+            } catch (err) {
+                // Best-effort: playback still works through the proxy.
+            } finally {
+                metaResolved.add(video.id);
+                metaPending.delete(video.id);
+            }
+        })();
+        metaPending.set(video.id, promise);
+        return promise;
+    };
+
     // Builds a proxied stream URL. Extra params other than `url` are forwarded
     // by the backend as request headers, so use real header names here.
     App.videos.buildStreamUrlFromSource = function(resolved) {