simon/jacuzzi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

advanced probing

Browse Source
This commit is contained in:
Simon
2026-06-23 12:44:13 +00:00
parent 80476a8a42
commit d6865d7c35
2 changed files with 245 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
backend/main.py
View File

@@ -11,6 +11,7 @@ from yt_dlp.networking.impersonate import ImpersonateTarget
from curl_cffi import requests as impersonate_requests
import threading
import io
import time
from urllib.parse import urljoin
# Browser to impersonate at the TLS/HTTP layer. Some origins (e.g. the
@@ -163,6 +164,135 @@ def videos_proxy():
except Exception as e:
return jsonify({"error": str(e)}), 500
# Cache of yt-dlp metadata resolutions so the on-demand probe endpoint doesn't
# re-extract the same video on every hover/scroll. Signed media URLs expire, so
# entries are intentionally short-lived.
RESOLVE_CACHE_TTL = 300
_resolve_cache = {}
_resolve_cache_lock = threading.Lock()
# Per-format fields the frontend needs to rank formats and build stream/probe
# URLs (see resolveStreamSources/rankFormats in videos.js). Everything else in a
# yt-dlp format dict is dropped to keep the payload small.
_RESOLVE_FORMAT_FIELDS = ('url', 'http_headers', 'height', 'width', 'tbr',
'fps', 'vcodec', 'acodec', 'ext', 'video_ext', 'quality')
# Some channels surface pages that yt-dlp can't extract because the video is
# embedded in a third-party JS player iframe (e.g. the xtremestream family used
# by tube.perverzija.com). The player page declares its HLS playlist URL as
# `m3u8_loader_url + video_id`, so we scrape the page for the embed iframe, then
# read those two variables out of the player to reconstruct the stream URL.
_EMBED_IFRAME_RE = re.compile(r'''<iframe[^>]+src=["']([^"']+)''', re.I)
_EMBED_LOADER_RE = re.compile(r'''m3u8_loader_url\s*=\s*[`'"]([^`'"]+)[`'"]''')
_EMBED_VIDEOID_RE = re.compile(r'''video_id\s*=\s*[`'"]([^`'"]+)[`'"]''')
def resolve_unsupported_embed(page_url):
"""Best-effort resolver for iframe-embedded JS players yt-dlp can't handle.
Returns an info-like dict ({'url','formats','http_headers','is_live'}) whose
single format is the embed's HLS playlist, or None if nothing was found."""
try:
sess = get_impersonate_session()
page = sess.get(page_url, headers={'Referer': page_url}, timeout=15)
embed_url = None
for src in _EMBED_IFRAME_RE.findall(page.text):
candidate = urljoin(page_url, src)
if '/player/' in candidate or 'index.php?data=' in candidate:
embed_url = candidate
break
if not embed_url:
return None
player = sess.get(embed_url, headers={'Referer': page_url}, timeout=15)
loader = _EMBED_LOADER_RE.search(player.text)
video_id = _EMBED_VIDEOID_RE.search(player.text)
if not (loader and video_id):
return None
stream_url = loader.group(1) + video_id.group(1)
parsed = urllib.parse.urlparse(embed_url)
referer = f"{parsed.scheme}://{parsed.netloc}/"
headers = {'Referer': referer}
return {
'url': stream_url,
'is_live': False,
'http_headers': headers,
'formats': [{'url': stream_url, 'ext': 'm3u8', 'protocol': 'm3u8', 'http_headers': headers}],
}
except Exception:
return None
@app.route('/api/resolve', methods=['POST', 'GET'])
def resolve_video():
"""Resolve a page URL to its playable formats via yt-dlp and return them as
JSON. The frontend calls this on demand (when a card is hovered or scrolled
into view) to learn the real media URLs so it can background-probe them for
direct, proxy-free playability."""
if request.method == 'POST':
source = request.json or {}
video_url = source.get('url')
else:
source = request.args
video_url = request.args.get('url')
if not video_url:
return jsonify({"error": "No URL provided"}), 400
now = time.time()
with _resolve_cache_lock:
cached = _resolve_cache.get(video_url)
if cached and cached[0] > now:
return jsonify(cached[1])
ydl_opts = {
'quiet': True,
'no_warnings': True,
'skip_download': True,
# Match /api/stream so the resolved formats reflect what playback will
# actually fetch from fingerprinting origins.
'impersonate': ImpersonateTarget.from_str(IMPERSONATE_TARGET),
}
passthrough_headers = collect_passthrough_headers(source)
if passthrough_headers:
ydl_opts['http_headers'] = passthrough_headers
try:
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info = ydl.extract_info(video_url, download=False)
except Exception as e:
# Many channels point at sites yt-dlp can't extract ("Unsupported URL").
# That's not fatal here -- the embed fallback below may still find a
# stream, and otherwise we return empty formats so playback falls back to
# the proxy.
app.logger.info("[resolve] yt-dlp could not extract %s: %s", video_url, e)
info = None
# Fall back to scraping iframe-embedded JS players yt-dlp doesn't support.
if not (info and (info.get('formats') or info.get('url'))):
embed = resolve_unsupported_embed(video_url)
if embed:
info = embed
formats = []
for fmt in ((info.get('formats') if info else None) or []):
if not fmt.get('url'):
continue
formats.append({k: fmt.get(k) for k in _RESOLVE_FORMAT_FIELDS if fmt.get(k) is not None})
result = {
'url': info.get('url') if info else None,
'http_headers': (info.get('http_headers') if info else None) or {},
'isLive': bool(info.get('is_live')) if info else False,
'formats': formats,
}
with _resolve_cache_lock:
# Drop expired entries so the cache doesn't grow without bound.
for key in [k for k, v in _resolve_cache.items() if v[0] <= now]:
_resolve_cache.pop(key, None)
_resolve_cache[video_url] = (now + RESOLVE_CACHE_TTL, result)
return jsonify(result)
@app.route('/api/image', methods=['GET', 'HEAD'])
def image_proxy():
image_url = request.args.get('url')
@@ -664,7 +794,19 @@ def stream_video():
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
# Extract the info
info = ydl.extract_info(video_url, download=False)
try:
info = ydl.extract_info(video_url, download=False)
except Exception as ydl_err:
# yt-dlp can't extract iframe-embedded JS players; scrape the
# embed for its HLS playlist and proxy that directly instead.
embed = resolve_unsupported_embed(video_url)
if not embed:
raise
dbg(f"embed fallback resolved {video_url} -> {embed['url']}")
if request.method == 'HEAD':
return Response("", status=200, content_type='application/vnd.apple.mpegurl')
return proxy_hls_playlist(embed['url'], embed['http_headers'].get('Referer'),
upstream_headers=embed['http_headers'])
dbg(f"yt_dlp extractor={info.get('extractor')} protocol={info.get('protocol')}")
# Try to get the URL from the info dict (works for progressive downloads)

129
frontend/js/videos.js
View File

@@ -290,9 +290,6 @@ App.videos = App.videos || {};
items.forEach(v => {
if (state.renderedVideoIds.has(v.id)) return;
state.loadedVideos.push(v);
// Probe in the background whether this video's best source plays
// directly, so playback can bypass the proxy when proven.
App.videos.probeVideoSources(v);
const card = document.createElement('div');
card.className = 'video-card';
@@ -410,6 +407,11 @@ App.videos = App.videos || {};
App.player.open(v, { originEl: card });
};
grid.appendChild(card);
// Resolve formats + probe direct playability on demand: when the
// card scrolls near the viewport, or the moment it's hovered.
cardVideo.set(card, v);
probeObserver.observe(card);
card.addEventListener('pointerenter', () => App.videos.resolveAndProbe(v), { once: true });
state.renderedVideoIds.add(v.id);
});
@@ -650,7 +652,7 @@ App.videos = App.videos || {};
const isLive = !!(videoOrUrl && typeof videoOrUrl === 'object' &&
(videoOrUrl.isLive || (videoOrUrl.meta && videoOrUrl.meta.isLive)));
if (typeof videoOrUrl === 'string') {
return videoOrUrl ? [{ url: videoOrUrl, referer: deriveReferer(videoOrUrl), userAgent: '', isLive }] : [];
return videoOrUrl ? [{ url: videoOrUrl, referer: deriveReferer(videoOrUrl), userAgent: '', isLive, refererRequired: false }] : [];
}
if (!videoOrUrl || typeof videoOrUrl !== 'object') return [];
@@ -664,9 +666,14 @@ App.videos = App.videos || {};
}
const sources = App.videos.rankFormats(meta.formats, preferredHeight).map((fmt) => {
const referer = headerValue(fmt.http_headers, 'Referer') || metaReferer || deriveReferer(fmt.url);
// An *explicit* Referer (from the extractor) signals the upstream
// enforces it; deriveReferer is only a best-effort fallback. The
// browser can't set a cross-origin Referer, so refererRequired tells
// callers (the probe) that direct playback can't work.
const explicitReferer = headerValue(fmt.http_headers, 'Referer') || metaReferer;
const referer = explicitReferer || deriveReferer(fmt.url);
const userAgent = headerValue(fmt.http_headers, 'User-Agent') || metaUserAgent;
return { url: fmt.url, referer, userAgent, isLive };
return { url: fmt.url, referer, userAgent, isLive, refererRequired: !!explicitReferer };
});
if (!sources.length) {
@@ -676,7 +683,8 @@ App.videos = App.videos || {};
url: fallbackUrl,
referer: metaReferer || deriveReferer(fallbackUrl),
userAgent: metaUserAgent,
isLive
isLive,
refererRequired: !!metaReferer
});
}
}
@@ -685,37 +693,41 @@ App.videos = App.videos || {};
App.videos.resolveStreamSource = function(videoOrUrl, options) {
const sources = App.videos.resolveStreamSources(videoOrUrl, options);
return sources.length ? sources[0] : { url: '', referer: '', userAgent: '', isLive: false };
return sources.length ? sources[0] : { url: '', referer: '', userAgent: '', isLive: false, refererRequired: false };
};
// Background "direct playability" probe. The backend proxy exists to work
// around CORS, hotlink (403) protection, and TLS fingerprinting. When the
// browser can fetch a media URL cross-origin and actually read the response
// (CORS allowed, not blocked/403), playing it directly works and the proxy
// is pure overhead. We probe each loaded video's best source in the
// background and, only when proven, let the player skip the proxy.
// is pure overhead. CORS is an origin-level policy, so the answer is the
// same for every media URL served by a given host: we probe (and cache)
// once per host and let the player skip the proxy for any URL on a host
// that's been proven.
const DIRECT_PROBE_TIMEOUT_MS = 8000;
// Only URLs that the player can hand straight to <video>/hls.js are worth
// probing; anything else (e.g. a live channel page URL) is resolved
// server-side and must keep going through the proxy.
const DIRECT_PLAYABLE_RE = /\.(mp4|m4v|m4s|webm|ts|mov|m3u8)($|\?)/i;
// url -> true (proven directly playable) | false (proven not). Absent means
// unknown/unprobed, in which case the proxy is used.
const directHostOf = (url) => {
try { return new URL(url).host; } catch (err) { return ''; }
};
// host -> true (proven directly playable) | false (proven not). Absent
// means unknown/unprobed, in which case the proxy is used.
App.videos._directStatus = new Map();
const directPending = new Map();
App.videos.isDirectProven = function(url) {
return App.videos._directStatus.get(url) === true;
return App.videos._directStatus.get(directHostOf(url)) === true;
};
App.videos.probeDirect = function(url) {
if (!url) return Promise.resolve(false);
if (App.videos._directStatus.has(url)) {
return Promise.resolve(App.videos._directStatus.get(url));
const host = directHostOf(url);
if (!host) return Promise.resolve(false);
if (App.videos._directStatus.has(host)) {
return Promise.resolve(App.videos._directStatus.get(host));
}
if (directPending.has(url)) {
return directPending.get(url);
if (directPending.has(host)) {
return directPending.get(host);
}
const promise = (async () => {
const controller = new AbortController();
@@ -742,19 +754,24 @@ App.videos = App.videos || {};
} finally {
clearTimeout(timer);
}
App.videos._directStatus.set(url, ok);
directPending.delete(url);
console.log(`[direct-probe] ${ok ? 'DIRECT' : 'PROXY '} (${detail}) ${url}`);
App.videos._directStatus.set(host, ok);
directPending.delete(host);
console.log(`[direct-probe] ${ok ? 'DIRECT' : 'PROXY '} (${detail}) ${host}`);
return ok;
})();
directPending.set(url, promise);
directPending.set(host, promise);
return promise;
};
// Kicks off a background probe of a video's best (first-played) source so a
// later playback can skip the proxy if the direct URL is proven reachable.
// later playback can skip the proxy if its host is proven reachable. Only
// runs once the video has resolved formats (see resolveAndProbe): those are
// real media URLs (or redirects to them), whereas a bare listing item only
// carries a page URL that the player can't use directly.
App.videos.probeVideoSources = function(video) {
if (!video || typeof video !== 'object') return;
const meta = video.meta || video;
if (!meta || !Array.isArray(meta.formats) || !meta.formats.length) return;
let sources;
try {
sources = App.videos.resolveStreamSources(video);
@@ -763,10 +780,68 @@ App.videos = App.videos || {};
}
const best = sources && sources[0];
if (!best || !best.url || best.isLive) return;
if (!DIRECT_PLAYABLE_RE.test(best.url)) return;
// Sources that require a specific upstream Referer can't be fetched
// directly by the browser (it can't forge a cross-origin Referer), so a
// probe would always fail -- leave them to the proxy.
if (best.refererRequired) return;
App.videos.probeDirect(best.url);
};
// Listing items arrive without formats (meta is null) -- only a page URL --
// so there's nothing direct-playable to probe up front. This resolves a
// video's real media formats via the backend (yt-dlp), attaches them as
// `video.meta` so the player and probe can use them, then probes the best
// source. Resolution is per-video and deduped: it runs at most once per
// video, triggered lazily by hover/scroll so we don't resolve cards the
// user never looks at.
const cardVideo = new WeakMap();
const metaResolved = new Set();
const metaPending = new Map();
const probeObserver = new IntersectionObserver((entries) => {
entries.forEach((entry) => {
if (!entry.isIntersecting) return;
probeObserver.unobserve(entry.target);
const video = cardVideo.get(entry.target);
if (video) App.videos.resolveAndProbe(video);
});
}, { rootMargin: '200px' });
App.videos.resolveAndProbe = function(video) {
if (!video || typeof video !== 'object' || !video.id) return Promise.resolve();
// Already have formats (resolved earlier): just (re)probe the best one.
if (video.meta && Array.isArray(video.meta.formats) && video.meta.formats.length) {
App.videos.probeVideoSources(video);
return Promise.resolve();
}
if (metaResolved.has(video.id)) return Promise.resolve();
if (metaPending.has(video.id)) return metaPending.get(video.id);
if (!video.url) return Promise.resolve();
const promise = (async () => {
try {
const response = await fetch('/api/resolve', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url: video.url })
});
if (!response.ok) return;
const data = await response.json();
if (data && Array.isArray(data.formats) && data.formats.length) {
video.meta = data;
App.videos.probeVideoSources(video);
}
} catch (err) {
// Best-effort: playback still works through the proxy.
} finally {
metaResolved.add(video.id);
metaPending.delete(video.id);
}
})();
metaPending.set(video.id, promise);
return promise;
};
// Builds a proxied stream URL. Extra params other than `url` are forwarded
// by the backend as request headers, so use real header names here.
App.videos.buildStreamUrlFromSource = function(resolved) {