diff --git a/backend/main.py b/backend/main.py index 7519abe..15952e7 100644 --- a/backend/main.py +++ b/backend/main.py @@ -450,6 +450,104 @@ def resolve_video(): return jsonify(view_of(info)) +# The picture a page says it has, in the order worth trusting: the card the +# page wants shared, then the one it declares to search engines, then the still +# its own player shows before playing. +_POSTER_META_RE = re.compile( + r''']+(?:property|name|itemprop)\s*=\s*["']?''' + r'''(og:image(?::url)?|twitter:image(?::src)?|thumbnailUrl)["']?[^>]*>''', re.I) +_POSTER_CONTENT_RE = re.compile(r'''content\s*=\s*["']([^"']+)["']''', re.I) +_POSTER_VIDEO_RE = re.compile(r''']+poster\s*=\s*["']([^"']+)["']''', re.I) +# Everything worth having is in the head; a post page's comment section is not. +_POSTER_SCAN_BYTES = 256 * 1024 +POSTER_CACHE_TTL = 600 +_poster_cache = {} +_poster_cache_lock = threading.Lock() + + +def _poster_from_html(html, base_url): + """The first usable image URL declared by `html`, absolute, or None.""" + candidates = [] + for match in _POSTER_META_RE.finditer(html): + content = _POSTER_CONTENT_RE.search(match.group(0)) + if content: + candidates.append((match.group(1).lower(), content.group(1))) + ranked = [] + for key in ('og:image', 'og:image:url', 'twitter:image', 'twitter:image:src', 'thumbnailurl'): + ranked += [url for name, url in candidates if name == key] + video_poster = _POSTER_VIDEO_RE.search(html) + if video_poster: + ranked.append(video_poster.group(1)) + for url in ranked: + absolute = urljoin(base_url, url.strip()) + parsed = urllib.parse.urlparse(absolute) + if parsed.scheme in ('http', 'https') and parsed.netloc: + return absolute + return None + + +@app.route('/api/poster', methods=['GET']) +def page_poster(): + """Ask a video's own page what picture it shows. + + A listing's thumbnail can be dead on arrival. sxyprn hands out a still it + derives from the video (`.../vid//.../full.jpg`) which, for a post + made minutes ago, the CDN has not generated -- while the post page itself + shows one that works (`.../img//.../0.webp`). Nothing on the + client can guess the second from the first: each path carries its own + signature. So when a thumbnail has failed every way we know to ask for it, + the page it came from gets asked what it shows. + + Cheap to be wrong about and expensive to repeat, so answers -- including + "nothing" -- are cached for a few minutes.""" + page_url = request.args.get('url') + if not page_url: + return jsonify({"error": "No URL provided"}), 400 + parsed = urllib.parse.urlparse(page_url) + if parsed.scheme not in ('http', 'https') or not parsed.netloc: + return jsonify({"error": "Invalid target URL"}), 400 + + now = time.time() + with _poster_cache_lock: + for key in [k for k, v in _poster_cache.items() if v[0] <= now]: + _poster_cache.pop(key, None) + hit = _poster_cache.get(page_url) + if hit: + return jsonify({"thumb": hit[1]}) + + thumb = None + sess = _borrow_session() + try: + # Streamed, and only the head of it: a page URL that turns out to + # redirect to the video itself (the Hot Tub proxy does exactly that for + # some channels) must cost one buffer, not a whole download. + resp = sess.get(page_url, headers={'Referer': page_url}, timeout=15, + allow_redirects=True, stream=True) + try: + content_type = (resp.headers.get('Content-Type') or '').lower() + if resp.status_code < 400 and ('html' in content_type or not content_type): + head = b'' + for chunk in resp.iter_content(32 * 1024): + head += chunk + if len(head) >= _POSTER_SCAN_BYTES: + break + thumb = _poster_from_html( + head.decode(resp.encoding or 'utf-8', errors='replace'), + resp.url or page_url) + finally: + resp.close() + except Exception: + _discard_session(sess) + sess = None + finally: + if sess is not None: + _return_session(sess) + + with _poster_cache_lock: + _poster_cache[page_url] = (now + POSTER_CACHE_TTL, thumb) + return jsonify({"thumb": thumb}) + + @app.route('/api/image', methods=['GET', 'HEAD']) def image_proxy(): image_url = request.args.get('url') diff --git a/frontend/js/favorites.js b/frontend/js/favorites.js index da595db..0ce0de6 100644 --- a/frontend/js/favorites.js +++ b/frontend/js/favorites.js @@ -494,7 +494,7 @@ App.favorites = App.favorites || {}; `; const thumb = card.querySelector('img'); if (App.videos && typeof App.videos.attachThumbnail === 'function') { - App.videos.attachThumbnail(thumb, item.thumb); + App.videos.attachThumbnail(thumb, item.thumb, App.videos.sourcePageUrl(item)); } card.onclick = () => { if (card.classList.contains('is-loading')) return; diff --git a/frontend/js/feed.js b/frontend/js/feed.js index dfc74d3..863ee12 100644 --- a/frontend/js/feed.js +++ b/frontend/js/feed.js @@ -631,7 +631,7 @@ App.feed = App.feed || {}; `; const poster = pane.querySelector('.feed-poster'); - App.videos.attachThumbnail(poster, v.thumb); + App.videos.attachThumbnail(poster, v.thumb, App.videos.sourcePageUrl(v)); const slideVideo = pane.querySelector('.feed-video'); bindTimeline(pane, slideVideo); bindSharedControls(pane, slideVideo, v); @@ -1110,7 +1110,7 @@ App.feed = App.feed || {}; const titleText = pane.querySelector('.feed-title-text'); if (titleText) titleText.textContent = v.title || ''; const poster = pane.querySelector('.feed-poster'); - if (poster) App.videos.attachThumbnail(poster, v.thumb); + if (poster) App.videos.attachThumbnail(poster, v.thumb, App.videos.sourcePageUrl(v)); loadSlideSource(pane, v, true); setPipMetadata(v); }; diff --git a/frontend/js/videos.js b/frontend/js/videos.js index 5a91b93..260aff9 100644 --- a/frontend/js/videos.js +++ b/frontend/js/videos.js @@ -254,6 +254,45 @@ App.videos = App.videos || {}; ].filter((step) => !!step.url); }; + // Last resort, once every route to the listed picture has failed: ask the + // page the item came from what picture *it* shows. + // + // A listing's thumbnail URL can be dead on arrival. sxyprn signs its CDN + // paths with an expiry, and the URLs the server hands us have generally + // passed theirs -- they only look alive while Cloudflare still has the + // bytes cached, which for the newest posts it does not. The signature can't + // be recomputed here, but the post page always carries a freshly signed + // one, so the server fetches the page and reads it off (see /api/poster). + // + // One ask per page, answered from the server's cache after that, and the + // replacement is loaded like any other thumbnail -- except that it gets no + // second repair, so a page that keeps handing back a dead picture stops + // being asked. + const posterAsked = new Map(); // page URL -> promise of a replacement, or '' + + const repairThumbnail = function(img, token) { + const page = img.dataset.thumbPage || ''; + if (!page) return; + let pending = posterAsked.get(page); + if (!pending) { + pending = fetch(`/api/poster?url=${encodeURIComponent(page)}`) + .then((response) => (response.ok ? response.json() : null)) + .then((data) => (data && usableThumbUrl(data.thumb)) || '') + .catch(() => ''); + posterAsked.set(page, pending); + } + pending.then((replacement) => { + if (!replacement || img.dataset.thumbToken !== token) return; + // No second repair for this mount: the ladder below must not come + // back here and ask the same page for the same picture forever. + delete img.dataset.thumbPage; + const proxyUrl = App.videos.buildImageProxyUrl(replacement); + const route = imageRoutes.get(imageHostOf(replacement)); + attachRetry(img, retryPlan(replacement, proxyUrl, route), token); + showThumbnail(img, route === IMAGE_PROXY ? proxyUrl : replacement, token); + }); + }; + // Arms `img` with the steps to take if what it is showing fails to load. const attachRetry = function(img, plan, token) { // Checked here too, not just in showThumbnail: this *replaces* whatever @@ -263,15 +302,18 @@ App.videos = App.videos || {}; // behind it if it failed. if (!img || (token !== undefined && img.dataset.thumbToken !== token)) return; detachRetry(img); - if (!plan || !plan.length) return; - const steps = plan.slice(); + // Armed even with nothing left to try: the failure of the *last* step + // is what says the picture itself is gone, and something has to be + // listening to hear it. + const steps = (plan || []).slice(); // Held on the element so detachThumbnail can take it off again. On the // happy path it never fires and `once` never collects it, so a pooled // image would otherwise accumulate one closure per mount it has served. const onError = function() { img._thumbRetry = null; const step = steps.shift(); - if (!step) return; + // Out of routes: the picture itself is gone, not the way to it. + if (!step) { repairThumbnail(img, token); return; } const go = function() { img._thumbRetryTimer = null; if (token !== undefined && img.dataset.thumbToken !== token) return; @@ -455,9 +497,39 @@ App.videos = App.videos || {}; proxyProbe.src = proxyUrl; }; + // The page this item came from, on the site it came from -- where a fresh + // picture can be read off when the listed one is dead (see + // repairThumbnail). Empty when the item doesn't say enough to name it. + // + // A listing item points at the Hot Tub server's own proxy + // (/proxy//post/.html), which answers a request by redirecting + // to the video: there is no page there to read. The site's address is in + // the item all the same -- it's the Referer the server says the media needs + // -- so putting that origin in front of the path the proxy was going to + // fetch names the page a browser would open. + App.videos.sourcePageUrl = function(v) { + if (!v || !v.url) return ''; + const headers = v.http_headers || {}; + const referer = headers.Referer || headers.referer || ''; + if (!referer) return ''; + try { + const page = new URL(v.url); + const site = new URL(referer); + if (site.origin === page.origin) return page.href; + const prefix = `/proxy/${v.channel}/`; + const path = v.channel && page.pathname.startsWith(prefix) ? + page.pathname.slice(prefix.length - 1) : page.pathname; + return site.origin + path + page.search; + } catch (err) { + return ''; + } + }; + // Points `img` at `url` by whichever route is known to work for its host, - // racing the two the first time that host is seen. - App.videos.attachThumbnail = function(img, url) { + // racing the two the first time that host is seen. `page` is optional: the + // item's own page, asked for a replacement if the picture turns out to be + // gone. + App.videos.attachThumbnail = function(img, url, page) { // An address that isn't one is the same thing as no thumbnail: the card // keeps its placeholder rather than chasing it. See usableThumbUrl. const directUrl = usableThumbUrl(url || (img && img.dataset.thumb) || ''); @@ -483,6 +555,8 @@ App.videos = App.videos || {}; // stops being able to touch this element. const token = String(++thumbSeq); img.dataset.thumbToken = token; + if (page) img.dataset.thumbPage = page; + else delete img.dataset.thumbPage; if (route === IMAGE_PROXY) { attachRetry(img, retryPlan(directUrl, proxyUrl, IMAGE_PROXY), token); @@ -515,6 +589,7 @@ App.videos = App.videos || {}; if (!img) return; img.dataset.thumbToken = String(++thumbSeq); delete img.dataset.alt; + delete img.dataset.thumbPage; detachRetry(img); }; @@ -957,7 +1032,7 @@ App.videos = App.videos || {}; // the same frame, so loading a thumbnail for it -- let alone racing one // -- would be pure waste. if (!(options && options.skipThumbnail)) { - App.videos.attachThumbnail(cardRefs(card).img, v.thumb); + App.videos.attachThumbnail(cardRefs(card).img, v.thumb, App.videos.sourcePageUrl(v)); } return card; }; @@ -1643,7 +1718,7 @@ App.videos = App.videos || {}; if (!card || !v) return; unfilled.delete(i); const img = cardRefs(card).img; - App.videos.attachThumbnail(img, v.thumb); + App.videos.attachThumbnail(img, v.thumb, App.videos.sourcePageUrl(v)); // Once the thumbnail loads, drop the 16:9 placeholder so it shows at // its true aspect ratio, clear the shimmer, then correct the height. if (img) { diff --git a/tests/smoke_thumbnails.py b/tests/smoke_thumbnails.py index a779519..d8358fe 100644 --- a/tests/smoke_thumbnails.py +++ b/tests/smoke_thumbnails.py @@ -17,16 +17,32 @@ provider, because what's under test is what the client does with awkward data: * a thumbnail whose first request fails. One blip used to mean an empty box for as long as the card stayed mounted. + + * a thumbnail that is simply gone. sxyprn's CDN paths are signed with an + expiry and the listing hands out URLs that have passed theirs, so the + newest posts arrive pointing at a 404. The item's own page still shows a + working picture, and /api/poster reads it off. """ import base64 import json +import os import sys +import threading +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer from playwright.sync_api import sync_playwright -BASE = "http://127.0.0.1:5000/" +BASE = os.environ.get("JACUZZI_BASE", "http://127.0.0.1:5000/") SERVER = "https://hottubapp.io" CHANNEL = "xvideos" CDN = "https://cdn.example-thumbs.test" +# Where a "provider page" lives, and the picture it says it has -- the shape +# /api/poster reads. The proxy path mirrors the Hot Tub server's own, so the +# client has to rebuild the site address from the item's Referer to find it. +SITE = "https://site.example-thumbs.test" +PROXY_PAGE = "https://proxy.example-thumbs.test/proxy/test/post/gone.html" # 1x1 transparent PNG. PIXEL = base64.b64decode( @@ -49,6 +65,66 @@ STATE = """() => Array.from(document.querySelectorAll('#video-grid .video-card') })""" +PAGE_HTML = """ + + + +""" + + +class PageServer(threading.Thread): + """A page for /api/poster to read, and one that redirects to a video -- + which is what the Hot Tub proxy does, and what must not be downloaded.""" + + def __init__(self): + super().__init__(daemon=True) + outer = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + if self.path.startswith("/post"): + body = PAGE_HTML.encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + if self.path.startswith("/video"): + body = b"\0" * (4 * 1024 * 1024) + self.send_response(200) + self.send_header("Content-Type", "video/mp4") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + self.send_response(404) + self.end_headers() + + self.httpd = HTTPServer(("127.0.0.1", 0), Handler) + outer.port = self.httpd.server_port + + def run(self): + self.httpd.serve_forever() + + def stop(self): + self.httpd.shutdown() + + +GONE_STATE = """() => { + const card = document.querySelector('#video-grid .video-card[data-video-id="test:9"]'); + const img = card && card.querySelector('img'); + return { src: img ? img.getAttribute('src') || '' : '', loaded: !!(img && img.naturalWidth > 0) }; +}""" + + +def gone_state(page): + return page.evaluate(GONE_STATE) + + class Checks: def __init__(self): self.failed = 0 @@ -75,17 +151,51 @@ def listing(): }) items[3]["thumb"] = "https:" # what sxyprn's "latest" actually sends items[7]["thumb"] = f"{CDN}/flaky.png" + # The expired-signature case: a dead thumbnail, and an item that says + # enough about where it came from for the page to be found. + items[9]["thumb"] = f"{CDN}/expired/gone.jpg" + items[9]["url"] = PROXY_PAGE + items[9]["channel"] = "test" + items[9]["http_headers"] = {"Referer": f"{SITE}/"} return {"items": items, "pageInfo": {"hasNextPage": False}} +def poster_of(page_url): + """What /api/poster answers for a page, straight from the backend. + + Returns None if the backend doesn't have the endpoint -- which on a machine + that has been running since before it existed means "restart it", not + "broken".""" + url = BASE.rstrip("/") + "/api/poster?url=" + urllib.parse.quote(page_url, safe="") + try: + with urllib.request.urlopen(url, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as err: + if err.code == 404: + return None + raise + + def main(): c = Checks() image_proxy_calls = [] + poster_asks = [] flaky_hits = {"direct": 0, "proxy": 0} + pages = PageServer() + pages.start() + local = f"http://127.0.0.1:{pages.port}" + with sync_playwright() as p: browser = p.chromium.launch(args=[ - "--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu", + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-gpu", + # Lean flags, matching smoke_grid: this runs alongside the app's own + # server and a default Chromium spikes hard enough at startup to get + # itself killed on a constrained box. + "--renderer-process-limit=1", + "--js-flags=--max-old-space-size=512", ]) page = browser.new_page(viewport={"width": 1400, "height": 1000}) @@ -97,6 +207,9 @@ def main(): # Only the thumbnails matter here; record what the client asked us # to fetch on its behalf, and hand back the picture. image_proxy_calls.append(route.request.url) + if "%2Fexpired%2F" in route.request.url or "/expired/" in route.request.url: + route.fulfill(status=404, content_type="text/html", body="gone") + return if "flaky.png" in route.request.url: flaky_hits["proxy"] += 1 # The flaky picture is refused on *both* routes the first time @@ -106,7 +219,19 @@ def main(): return route.fulfill(status=200, content_type="image/png", body=PIXEL) + def serve_poster(route): + # The endpoint itself is exercised against a real page below; here + # only the client's half is under test, so the answer is canned. + asked = urllib.parse.parse_qs( + urllib.parse.urlparse(route.request.url).query).get("url", [""])[0] + poster_asks.append(asked) + route.fulfill(status=200, content_type="application/json", + body=json.dumps({"thumb": f"{CDN}/repaired.png"})) + def serve_cdn(route): + if "/expired/" in route.request.url: + route.fulfill(status=404, content_type="text/html", body="gone") + return if route.request.url.endswith("/flaky.png"): flaky_hits["direct"] += 1 if flaky_hits["direct"] == 1: @@ -115,6 +240,7 @@ def main(): route.fulfill(status=200, content_type="image/png", body=PIXEL) page.route("**/api/videos", serve_listing) + page.route("**/api/poster*", serve_poster) page.route("**/api/image*", serve_proxy) page.route(f"{CDN}/**", serve_cdn) @@ -140,7 +266,8 @@ def main(): str([u for u in image_proxy_calls if "https%3A" in u][:2])) print("\nthe rest of the page is unaffected by it") - others = [card for card in cards if card["id"] != "test:3"] + # test:9's thumbnail is deliberately dead; it has a section of its own. + others = [card for card in cards if card["id"] not in ("test:3", "test:9")] c.ok("every other card shows its picture", all(card["loaded"] for card in others), str([card["id"] for card in others if not card["loaded"]])) @@ -166,7 +293,38 @@ def main(): flaky_hits["direct"] + flaky_hits["proxy"] <= 4, f"direct={flaky_hits['direct']} proxy={flaky_hits['proxy']}") + print("\na thumbnail that is simply gone") + gone = by_id.get("test:9") + c.ok("the card is mounted", gone is not None) + c.ok("its page was asked what picture it shows", poster_asks, + str(poster_asks)) + # The item points at a proxy path; the page lives on the site named by + # the Referer the item carries. + c.ok("and the page asked for was the item's own, on its own site", + poster_asks and poster_asks[0] == f"{SITE}/post/gone.html", + str(poster_asks[:2])) + c.ok("the card ends up showing the replacement", + gone and gone_state(page)["loaded"], str(gone_state(page))) + c.ok("one ask is enough for that page", len(poster_asks) == 1, str(poster_asks)) + + print("\nwhat /api/poster reads off a page") + answer = poster_of(f"{local}/post.html") + if answer is None: + c.ok("the backend has /api/poster (restart it if this fails)", False) + answer = {} + else: + c.ok("the backend has /api/poster", True) + c.ok("the page's own og:image, made absolute", + answer.get("thumb") == f"http://pictures.example.test/fresh/poster.webp", + str(answer)) + video = poster_of(f"{local}/video.mp4") + c.ok("a URL that turns out to be a video yields nothing", + video.get("thumb") is None, str(video)) + missing = poster_of(f"{local}/nope.html") + c.ok("and so does a page that isn't there", missing.get("thumb") is None, str(missing)) + browser.close() + pages.stop() print(f"\n{'OK' if not c.failed else 'FAILED'}: {c.failed} check(s) failed") return 1 if c.failed else 0