simon/jacuzzi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ask a video's own page for a thumbnail the listing lost

Browse Source 
sxyprn signs its CDN paths with an expiry, and the URLs the listing hands
us have generally passed theirs. They only look alive while Cloudflare
still has the bytes: add a cache-buster to one that returns 200 and it
returns 404, every time, for every one tried. The newest posts are the ones
nobody fetched while the URL was valid, so they are the ones that arrive as
holes in the grid -- which is exactly where this was reported.

Neither route can help, because both ask for the same dead address, and the
right one can't be derived: the token signs the whole path, so swapping
`full.jpg` for `small.jpg` or `vid` for `img` is just another 404. The post
page, though, always carries a freshly signed one in og:image. So when a
thumbnail has failed every way we know to ask for it, /api/poster fetches
that page and reads the picture off it -- streamed, capped, and only if what
comes back is HTML, since a page URL that turns out to redirect to the video
must cost one buffer rather than a download. Answers are cached, including
"nothing", which is the honest answer for a post that has been deleted.

Finding the page is the other half. A listing item points at the Hot Tub
server's proxy, which answers by redirecting to the video, so there is no
page there to read -- but the item also carries the Referer the media needs,
and that names the site. Its origin plus the path the proxy was going to
fetch is the page a browser would open.

Also fixes what this exposed: the retry ladder disarmed itself on its last
step, so the failure that means "the picture is gone, not the route" was
never heard by anything.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MPZFnNdHbPGDTqQUNiE4ZN
This commit is contained in:
Simon
2026-09-21 15:47:47 +00:00
parent a795442634
commit 051dae98cb
5 changed files with 344 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
backend/main.py
View File

@@ -450,6 +450,104 @@ def resolve_video():
return jsonify(view_of(info))
# The picture a page says it has, in the order worth trusting: the card the
# page wants shared, then the one it declares to search engines, then the still
# its own player shows before playing.
_POSTER_META_RE = re.compile(
r'''<meta[^>]+(?:property|name|itemprop)\s*=\s*["']?'''
r'''(og:image(?::url)?|twitter:image(?::src)?|thumbnailUrl)["']?[^>]*>''', re.I)
_POSTER_CONTENT_RE = re.compile(r'''content\s*=\s*["']([^"']+)["']''', re.I)
_POSTER_VIDEO_RE = re.compile(r'''<video[^>]+poster\s*=\s*["']([^"']+)["']''', re.I)
# Everything worth having is in the head; a post page's comment section is not.
_POSTER_SCAN_BYTES = 256 * 1024
POSTER_CACHE_TTL = 600
_poster_cache = {}
_poster_cache_lock = threading.Lock()
def _poster_from_html(html, base_url):
"""The first usable image URL declared by `html`, absolute, or None."""
candidates = []
for match in _POSTER_META_RE.finditer(html):
content = _POSTER_CONTENT_RE.search(match.group(0))
if content:
candidates.append((match.group(1).lower(), content.group(1)))
ranked = []
for key in ('og:image', 'og:image:url', 'twitter:image', 'twitter:image:src', 'thumbnailurl'):
ranked += [url for name, url in candidates if name == key]
video_poster = _POSTER_VIDEO_RE.search(html)
if video_poster:
ranked.append(video_poster.group(1))
for url in ranked:
absolute = urljoin(base_url, url.strip())
parsed = urllib.parse.urlparse(absolute)
if parsed.scheme in ('http', 'https') and parsed.netloc:
return absolute
return None
@app.route('/api/poster', methods=['GET'])
def page_poster():
"""Ask a video's own page what picture it shows.
A listing's thumbnail can be dead on arrival. sxyprn hands out a still it
derives from the video (`.../vid/<token>/.../full.jpg`) which, for a post
made minutes ago, the CDN has not generated -- while the post page itself
shows one that works (`.../img/<other token>/.../0.webp`). Nothing on the
client can guess the second from the first: each path carries its own
signature. So when a thumbnail has failed every way we know to ask for it,
the page it came from gets asked what it shows.
Cheap to be wrong about and expensive to repeat, so answers -- including
"nothing" -- are cached for a few minutes."""
page_url = request.args.get('url')
if not page_url:
return jsonify({"error": "No URL provided"}), 400
parsed = urllib.parse.urlparse(page_url)
if parsed.scheme not in ('http', 'https') or not parsed.netloc:
return jsonify({"error": "Invalid target URL"}), 400
now = time.time()
with _poster_cache_lock:
for key in [k for k, v in _poster_cache.items() if v[0] <= now]:
_poster_cache.pop(key, None)
hit = _poster_cache.get(page_url)
if hit:
return jsonify({"thumb": hit[1]})
thumb = None
sess = _borrow_session()
try:
# Streamed, and only the head of it: a page URL that turns out to
# redirect to the video itself (the Hot Tub proxy does exactly that for
# some channels) must cost one buffer, not a whole download.
resp = sess.get(page_url, headers={'Referer': page_url}, timeout=15,
allow_redirects=True, stream=True)
try:
content_type = (resp.headers.get('Content-Type') or '').lower()
if resp.status_code < 400 and ('html' in content_type or not content_type):
head = b''
for chunk in resp.iter_content(32 * 1024):
head += chunk
if len(head) >= _POSTER_SCAN_BYTES:
break
thumb = _poster_from_html(
head.decode(resp.encoding or 'utf-8', errors='replace'),
resp.url or page_url)
finally:
resp.close()
except Exception:
_discard_session(sess)
sess = None
finally:
if sess is not None:
_return_session(sess)
with _poster_cache_lock:
_poster_cache[page_url] = (now + POSTER_CACHE_TTL, thumb)
return jsonify({"thumb": thumb})
@app.route('/api/image', methods=['GET', 'HEAD'])
def image_proxy():
image_url = request.args.get('url')

2
frontend/js/favorites.js
View File

@@ -494,7 +494,7 @@ App.favorites = App.favorites || {};
`;
const thumb = card.querySelector('img');
if (App.videos && typeof App.videos.attachThumbnail === 'function') {
App.videos.attachThumbnail(thumb, item.thumb);
App.videos.attachThumbnail(thumb, item.thumb, App.videos.sourcePageUrl(item));
}
card.onclick = () => {
if (card.classList.contains('is-loading')) return;

4
frontend/js/feed.js
View File

@@ -631,7 +631,7 @@ App.feed = App.feed || {};
</div>
`;
const poster = pane.querySelector('.feed-poster');
App.videos.attachThumbnail(poster, v.thumb);
App.videos.attachThumbnail(poster, v.thumb, App.videos.sourcePageUrl(v));
const slideVideo = pane.querySelector('.feed-video');
bindTimeline(pane, slideVideo);
bindSharedControls(pane, slideVideo, v);
@@ -1110,7 +1110,7 @@ App.feed = App.feed || {};
const titleText = pane.querySelector('.feed-title-text');
if (titleText) titleText.textContent = v.title || '';
const poster = pane.querySelector('.feed-poster');
if (poster) App.videos.attachThumbnail(poster, v.thumb);
if (poster) App.videos.attachThumbnail(poster, v.thumb, App.videos.sourcePageUrl(v));
loadSlideSource(pane, v, true);
setPipMetadata(v);
};

89
frontend/js/videos.js
View File

@@ -254,6 +254,45 @@ App.videos = App.videos || {};
].filter((step) => !!step.url);
};
// Last resort, once every route to the listed picture has failed: ask the
// page the item came from what picture *it* shows.
//
// A listing's thumbnail URL can be dead on arrival. sxyprn signs its CDN
// paths with an expiry, and the URLs the server hands us have generally
// passed theirs -- they only look alive while Cloudflare still has the
// bytes cached, which for the newest posts it does not. The signature can't
// be recomputed here, but the post page always carries a freshly signed
// one, so the server fetches the page and reads it off (see /api/poster).
//
// One ask per page, answered from the server's cache after that, and the
// replacement is loaded like any other thumbnail -- except that it gets no
// second repair, so a page that keeps handing back a dead picture stops
// being asked.
const posterAsked = new Map(); // page URL -> promise of a replacement, or ''
const repairThumbnail = function(img, token) {
const page = img.dataset.thumbPage || '';
if (!page) return;
let pending = posterAsked.get(page);
if (!pending) {
pending = fetch(`/api/poster?url=${encodeURIComponent(page)}`)
.then((response) => (response.ok ? response.json() : null))
.then((data) => (data && usableThumbUrl(data.thumb)) || '')
.catch(() => '');
posterAsked.set(page, pending);
}
pending.then((replacement) => {
if (!replacement || img.dataset.thumbToken !== token) return;
// No second repair for this mount: the ladder below must not come
// back here and ask the same page for the same picture forever.
delete img.dataset.thumbPage;
const proxyUrl = App.videos.buildImageProxyUrl(replacement);
const route = imageRoutes.get(imageHostOf(replacement));
attachRetry(img, retryPlan(replacement, proxyUrl, route), token);
showThumbnail(img, route === IMAGE_PROXY ? proxyUrl : replacement, token);
});
};
// Arms `img` with the steps to take if what it is showing fails to load.
const attachRetry = function(img, plan, token) {
// Checked here too, not just in showThumbnail: this *replaces* whatever
@@ -263,15 +302,18 @@ App.videos = App.videos || {};
// behind it if it failed.
if (!img || (token !== undefined && img.dataset.thumbToken !== token)) return;
detachRetry(img);
if (!plan || !plan.length) return;
const steps = plan.slice();
// Armed even with nothing left to try: the failure of the *last* step
// is what says the picture itself is gone, and something has to be
// listening to hear it.
const steps = (plan || []).slice();
// Held on the element so detachThumbnail can take it off again. On the
// happy path it never fires and `once` never collects it, so a pooled
// image would otherwise accumulate one closure per mount it has served.
const onError = function() {
img._thumbRetry = null;
const step = steps.shift();
if (!step) return;
// Out of routes: the picture itself is gone, not the way to it.
if (!step) { repairThumbnail(img, token); return; }
const go = function() {
img._thumbRetryTimer = null;
if (token !== undefined && img.dataset.thumbToken !== token) return;
@@ -455,9 +497,39 @@ App.videos = App.videos || {};
proxyProbe.src = proxyUrl;
};
// The page this item came from, on the site it came from -- where a fresh
// picture can be read off when the listed one is dead (see
// repairThumbnail). Empty when the item doesn't say enough to name it.
//
// A listing item points at the Hot Tub server's own proxy
// (/proxy/<channel>/post/<id>.html), which answers a request by redirecting
// to the video: there is no page there to read. The site's address is in
// the item all the same -- it's the Referer the server says the media needs
// -- so putting that origin in front of the path the proxy was going to
// fetch names the page a browser would open.
App.videos.sourcePageUrl = function(v) {
if (!v || !v.url) return '';
const headers = v.http_headers || {};
const referer = headers.Referer || headers.referer || '';
if (!referer) return '';
try {
const page = new URL(v.url);
const site = new URL(referer);
if (site.origin === page.origin) return page.href;
const prefix = `/proxy/${v.channel}/`;
const path = v.channel && page.pathname.startsWith(prefix) ?
page.pathname.slice(prefix.length - 1) : page.pathname;
return site.origin + path + page.search;
} catch (err) {
return '';
}
};
// Points `img` at `url` by whichever route is known to work for its host,
// racing the two the first time that host is seen.
App.videos.attachThumbnail = function(img, url) {
// racing the two the first time that host is seen. `page` is optional: the
// item's own page, asked for a replacement if the picture turns out to be
// gone.
App.videos.attachThumbnail = function(img, url, page) {
// An address that isn't one is the same thing as no thumbnail: the card
// keeps its placeholder rather than chasing it. See usableThumbUrl.
const directUrl = usableThumbUrl(url || (img && img.dataset.thumb) || '');
@@ -483,6 +555,8 @@ App.videos = App.videos || {};
// stops being able to touch this element.
const token = String(++thumbSeq);
img.dataset.thumbToken = token;
if (page) img.dataset.thumbPage = page;
else delete img.dataset.thumbPage;
if (route === IMAGE_PROXY) {
attachRetry(img, retryPlan(directUrl, proxyUrl, IMAGE_PROXY), token);
@@ -515,6 +589,7 @@ App.videos = App.videos || {};
if (!img) return;
img.dataset.thumbToken = String(++thumbSeq);
delete img.dataset.alt;
delete img.dataset.thumbPage;
detachRetry(img);
};
@@ -957,7 +1032,7 @@ App.videos = App.videos || {};
// the same frame, so loading a thumbnail for it -- let alone racing one
// -- would be pure waste.
if (!(options && options.skipThumbnail)) {
App.videos.attachThumbnail(cardRefs(card).img, v.thumb);
App.videos.attachThumbnail(cardRefs(card).img, v.thumb, App.videos.sourcePageUrl(v));
}
return card;
};
@@ -1643,7 +1718,7 @@ App.videos = App.videos || {};
if (!card || !v) return;
unfilled.delete(i);
const img = cardRefs(card).img;
App.videos.attachThumbnail(img, v.thumb);
App.videos.attachThumbnail(img, v.thumb, App.videos.sourcePageUrl(v));
// Once the thumbnail loads, drop the 16:9 placeholder so it shows at
// its true aspect ratio, clear the shimmer, then correct the height.
if (img) {

164
tests/smoke_thumbnails.py
View File

@@ -17,16 +17,32 @@ provider, because what's under test is what the client does with awkward data:
* a thumbnail whose first request fails. One blip used to mean an empty box
for as long as the card stayed mounted.
* a thumbnail that is simply gone. sxyprn's CDN paths are signed with an
expiry and the listing hands out URLs that have passed theirs, so the
newest posts arrive pointing at a 404. The item's own page still shows a
working picture, and /api/poster reads it off.
"""
import base64
import json
import os
import sys
import threading
import urllib.error
import urllib.parse
import urllib.request
from http.server import BaseHTTPRequestHandler, HTTPServer
from playwright.sync_api import sync_playwright
BASE = "http://127.0.0.1:5000/"
BASE = os.environ.get("JACUZZI_BASE", "http://127.0.0.1:5000/")
SERVER = "https://hottubapp.io"
CHANNEL = "xvideos"
CDN = "https://cdn.example-thumbs.test"
# Where a "provider page" lives, and the picture it says it has -- the shape
# /api/poster reads. The proxy path mirrors the Hot Tub server's own, so the
# client has to rebuild the site address from the item's Referer to find it.
SITE = "https://site.example-thumbs.test"
PROXY_PAGE = "https://proxy.example-thumbs.test/proxy/test/post/gone.html"
# 1x1 transparent PNG.
PIXEL = base64.b64decode(
@@ -49,6 +65,66 @@ STATE = """() => Array.from(document.querySelectorAll('#video-grid .video-card')
})"""
PAGE_HTML = """<!doctype html><html><head>
<meta property='og:title' content='A video'/>
<meta property='og:image' content='//pictures.example.test/fresh/poster.webp'/>
<meta itemprop="thumbnailUrl" content="//pictures.example.test/other.webp" />
</head><body><video poster='//pictures.example.test/player.webp'></video></body></html>"""
class PageServer(threading.Thread):
"""A page for /api/poster to read, and one that redirects to a video --
which is what the Hot Tub proxy does, and what must not be downloaded."""
def __init__(self):
super().__init__(daemon=True)
outer = self
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_GET(self):
if self.path.startswith("/post"):
body = PAGE_HTML.encode()
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if self.path.startswith("/video"):
body = b"\0" * (4 * 1024 * 1024)
self.send_response(200)
self.send_header("Content-Type", "video/mp4")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
self.send_response(404)
self.end_headers()
self.httpd = HTTPServer(("127.0.0.1", 0), Handler)
outer.port = self.httpd.server_port
def run(self):
self.httpd.serve_forever()
def stop(self):
self.httpd.shutdown()
GONE_STATE = """() => {
const card = document.querySelector('#video-grid .video-card[data-video-id="test:9"]');
const img = card && card.querySelector('img');
return { src: img ? img.getAttribute('src') || '' : '', loaded: !!(img && img.naturalWidth > 0) };
}"""
def gone_state(page):
return page.evaluate(GONE_STATE)
class Checks:
def __init__(self):
self.failed = 0
@@ -75,17 +151,51 @@ def listing():
})
items[3]["thumb"] = "https:" # what sxyprn's "latest" actually sends
items[7]["thumb"] = f"{CDN}/flaky.png"
# The expired-signature case: a dead thumbnail, and an item that says
# enough about where it came from for the page to be found.
items[9]["thumb"] = f"{CDN}/expired/gone.jpg"
items[9]["url"] = PROXY_PAGE
items[9]["channel"] = "test"
items[9]["http_headers"] = {"Referer": f"{SITE}/"}
return {"items": items, "pageInfo": {"hasNextPage": False}}
def poster_of(page_url):
"""What /api/poster answers for a page, straight from the backend.
Returns None if the backend doesn't have the endpoint -- which on a machine
that has been running since before it existed means "restart it", not
"broken"."""
url = BASE.rstrip("/") + "/api/poster?url=" + urllib.parse.quote(page_url, safe="")
try:
with urllib.request.urlopen(url, timeout=30) as response:
return json.load(response)
except urllib.error.HTTPError as err:
if err.code == 404:
return None
raise
def main():
c = Checks()
image_proxy_calls = []
poster_asks = []
flaky_hits = {"direct": 0, "proxy": 0}
pages = PageServer()
pages.start()
local = f"http://127.0.0.1:{pages.port}"
with sync_playwright() as p:
browser = p.chromium.launch(args=[
"--no-sandbox", "--disable-dev-shm-usage", "--disable-gpu",
"--no-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
# Lean flags, matching smoke_grid: this runs alongside the app's own
# server and a default Chromium spikes hard enough at startup to get
# itself killed on a constrained box.
"--renderer-process-limit=1",
"--js-flags=--max-old-space-size=512",
])
page = browser.new_page(viewport={"width": 1400, "height": 1000})
@@ -97,6 +207,9 @@ def main():
# Only the thumbnails matter here; record what the client asked us
# to fetch on its behalf, and hand back the picture.
image_proxy_calls.append(route.request.url)
if "%2Fexpired%2F" in route.request.url or "/expired/" in route.request.url:
route.fulfill(status=404, content_type="text/html", body="gone")
return
if "flaky.png" in route.request.url:
flaky_hits["proxy"] += 1
# The flaky picture is refused on *both* routes the first time
@@ -106,7 +219,19 @@ def main():
return
route.fulfill(status=200, content_type="image/png", body=PIXEL)
def serve_poster(route):
# The endpoint itself is exercised against a real page below; here
# only the client's half is under test, so the answer is canned.
asked = urllib.parse.parse_qs(
urllib.parse.urlparse(route.request.url).query).get("url", [""])[0]
poster_asks.append(asked)
route.fulfill(status=200, content_type="application/json",
body=json.dumps({"thumb": f"{CDN}/repaired.png"}))
def serve_cdn(route):
if "/expired/" in route.request.url:
route.fulfill(status=404, content_type="text/html", body="gone")
return
if route.request.url.endswith("/flaky.png"):
flaky_hits["direct"] += 1
if flaky_hits["direct"] == 1:
@@ -115,6 +240,7 @@ def main():
route.fulfill(status=200, content_type="image/png", body=PIXEL)
page.route("**/api/videos", serve_listing)
page.route("**/api/poster*", serve_poster)
page.route("**/api/image*", serve_proxy)
page.route(f"{CDN}/**", serve_cdn)
@@ -140,7 +266,8 @@ def main():
str([u for u in image_proxy_calls if "https%3A" in u][:2]))
print("\nthe rest of the page is unaffected by it")
others = [card for card in cards if card["id"] != "test:3"]
# test:9's thumbnail is deliberately dead; it has a section of its own.
others = [card for card in cards if card["id"] not in ("test:3", "test:9")]
c.ok("every other card shows its picture",
all(card["loaded"] for card in others),
str([card["id"] for card in others if not card["loaded"]]))
@@ -166,7 +293,38 @@ def main():
flaky_hits["direct"] + flaky_hits["proxy"] <= 4,
f"direct={flaky_hits['direct']} proxy={flaky_hits['proxy']}")
print("\na thumbnail that is simply gone")
gone = by_id.get("test:9")
c.ok("the card is mounted", gone is not None)
c.ok("its page was asked what picture it shows", poster_asks,
str(poster_asks))
# The item points at a proxy path; the page lives on the site named by
# the Referer the item carries.
c.ok("and the page asked for was the item's own, on its own site",
poster_asks and poster_asks[0] == f"{SITE}/post/gone.html",
str(poster_asks[:2]))
c.ok("the card ends up showing the replacement",
gone and gone_state(page)["loaded"], str(gone_state(page)))
c.ok("one ask is enough for that page", len(poster_asks) == 1, str(poster_asks))
print("\nwhat /api/poster reads off a page")
answer = poster_of(f"{local}/post.html")
if answer is None:
c.ok("the backend has /api/poster (restart it if this fails)", False)
answer = {}
else:
c.ok("the backend has /api/poster", True)
c.ok("the page's own og:image, made absolute",
answer.get("thumb") == f"http://pictures.example.test/fresh/poster.webp",
str(answer))
video = poster_of(f"{local}/video.mp4")
c.ok("a URL that turns out to be a video yields nothing",
video.get("thumb") is None, str(video))
missing = poster_of(f"{local}/nope.html")
c.ok("and so does a page that isn't there", missing.get("thumb") is None, str(missing))
browser.close()
pages.stop()
print(f"\n{'OK' if not c.failed else 'FAILED'}: {c.failed} check(s) failed")
return 1 if c.failed else 0