Load a split step's videos together, and three steps ahead

Preloading was counted in steps and then shared out between the panels,
so with four panels only the very next step was ready and each panel did
its resolving and buffering at the moment it was swiped to -- four
panels, four starts in a row. It is now counted per panel: every panel
keeps its next three videos ready (N * 3 for a step of N), the rendered
window reaches far enough forward to hold them, and the step right
behind the one being watched buffers for real rather than taking its
metadata alone.

Their formats are resolved up front and all at once, rather than one
panel at a time as each is swiped to.

The listing is fetched several pages at a time to keep up: a page number
is claimed before its request goes out, which is what lets pages
overlap, and finished pages wait their turn so the reader still gets
them in order. A page whose request fails puts its number back rather
than leaving a hole. How far ahead to run scales with the split, since a
step of N panels gets through a page N times as fast.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MPZFnNdHbPGDTqQUNiE4ZN
This commit is contained in:
Simon
2026-09-22 09:11:52 +00:00
parent 1b38f9d9e7
commit 8163693150
3 changed files with 536 additions and 84 deletions

View File

@@ -22,7 +22,14 @@ App.feed = App.feed || {};
// Separately we prefetch PREFETCH_PAGES worth of video JSON ahead of the // Separately we prefetch PREFETCH_PAGES worth of video JSON ahead of the
// active slide so the data buffer is always full before we need to render // active slide so the data buffer is always full before we need to render
// a slide from it. // a slide from it.
const PRELOAD_COUNT = 2; // slides ahead kept with a live <video> playing/preloaded // Preloading is counted in steps, so it scales with the split rather than
// being shared out between the panels: every panel keeps its own next three
// videos ready, which is N * 3 videos loading ahead for a step of N panels.
// The step right behind the one being watched buffers for real; the two
// behind that hold their metadata, which is what makes their first frame
// immediate without four more streams downloading in full.
const PRELOAD_STEPS = 3; // steps ahead kept loaded, per panel
const BUFFER_STEPS = 1; // of those, how many buffer rather than just resolve
const RENDER_AHEAD = 5; // slides ahead kept materialized in the DOM const RENDER_AHEAD = 5; // slides ahead kept materialized in the DOM
const HISTORY_COUNT = 5; // slides behind kept materialized in the DOM const HISTORY_COUNT = 5; // slides behind kept materialized in the DOM
const PREFETCH_PAGES = 2; // pages of JSON to keep buffered ahead of the active slide const PREFETCH_PAGES = 2; // pages of JSON to keep buffered ahead of the active slide
@@ -453,7 +460,9 @@ App.feed = App.feed || {};
// `opts.silent` plays without sound whatever the panel's own switch says -- // `opts.silent` plays without sound whatever the panel's own switch says --
// for a panel that is only half on screen (see peek), whose audio would // for a panel that is only half on screen (see peek), whose audio would
// otherwise talk over the one still being watched. // otherwise talk over the one still being watched. `opts.buffer` asks for a
// real buffer rather than metadata alone, for a step close enough that it
// is about to be swiped to.
const loadSlideSource = function(slide, videoData, autoplay, opts) { const loadSlideSource = function(slide, videoData, autoplay, opts) {
const video = slide.querySelector('.feed-video'); const video = slide.querySelector('.feed-video');
if (!video) return; if (!video) return;
@@ -506,11 +515,11 @@ App.feed = App.feed || {};
const isHls = App.videos.classifySource(resolved).isHls; const isHls = App.videos.classifySource(resolved).isHls;
video.muted = silent || slide._muted !== false; video.muted = silent || slide._muted !== false;
// The step being watched buffers properly; the one preloaded behind it // The step being watched and the one right behind it buffer properly,
// only needs enough to start instantly on the swipe. With four panes // so a swipe lands on video that is already there. Steps further ahead
// that is the difference between four extra streams downloading and // take their metadata only: enough to start without delay, without four
// four holding a few seconds each. // more streams downloading in full behind them.
video.preload = autoplay ? 'auto' : 'metadata'; video.preload = (autoplay || (opts && opts.buffer)) ? 'auto' : 'metadata';
video._tearingDown = false; video._tearingDown = false;
applyResume(video, videoData && videoData.id, resolved.isLive); applyResume(video, videoData && videoData.id, resolved.isLive);
@@ -534,11 +543,13 @@ App.feed = App.feed || {};
video._hlsPlayer = hls; video._hlsPlayer = hls;
hls.loadSource(streamUrl); hls.loadSource(streamUrl);
hls.attachMedia(video); hls.attachMedia(video);
if (!autoplay && split) { if (!autoplay && split && !(opts && opts.buffer)) {
// Buffered enough to start instantly, then stopped: four // Buffered enough to start instantly, then stopped: four
// panels' worth of hls.js all fetching and demuxing ahead is // panels' worth of hls.js all fetching and demuxing ahead is
// work for a step the reader has not swiped to yet. loadSlide- // work for a step the reader has not swiped to yet. The step
// Source runs again with autoplay when they do. // right behind the active one is exempt -- it is the one about
// to be swiped to, and maxBufferLength stops it on its own.
// loadSlideSource runs again with autoplay when they do.
hls.on(HlsLib.Events.FRAG_BUFFERED, function once() { hls.on(HlsLib.Events.FRAG_BUFFERED, function once() {
hls.off(HlsLib.Events.FRAG_BUFFERED, once); hls.off(HlsLib.Events.FRAG_BUFFERED, once);
if (video._hlsPlayer === hls) hls.stopLoad(); if (video._hlsPlayer === hls) hls.stopLoad();
@@ -955,10 +966,12 @@ App.feed = App.feed || {};
// that fell outside [active - HISTORY_COUNT, active + RENDER_AHEAD], builds // that fell outside [active - HISTORY_COUNT, active + RENDER_AHEAD], builds
// any missing ones inside it, and sizes the top spacer to stand in for the // any missing ones inside it, and sizes the top spacer to stand in for the
// slides above the window. // slides above the window.
// Which steps are materialised. Steps ahead/behind are counted in steps, // Which steps are materialised. Behind, the window narrows as panes are
// but each one now costs a whole step's worth of <video> elements, so // added -- history is the cheap thing to give up. Ahead it does not: what
// splitting four ways should not put four times as many in the document: // is preloaded has to be in the document to be preloaded at all, so the
// the window narrows as panes are added. // window always reaches at least PRELOAD_STEPS forward however many panels
// there are. A four-way split therefore holds four times as many elements
// as a single panel, which is the price of four panels never waiting.
// //
// The top spacer stands in for the steps below `start`, so anything that // The top spacer stands in for the steps below `start`, so anything that
// sizes the spacer has to agree with this exactly -- disagreeing by a step // sizes the spacer has to agree with this exactly -- disagreeing by a step
@@ -972,7 +985,7 @@ App.feed = App.feed || {};
return { return {
start: Math.max(0, activeIndex - Math.max(1, Math.round(HISTORY_COUNT / per))), start: Math.max(0, activeIndex - Math.max(1, Math.round(HISTORY_COUNT / per))),
end: Math.min(stepCount() - 1, end: Math.min(stepCount() - 1,
activeIndex + Math.max(1, Math.round(RENDER_AHEAD / per))) activeIndex + Math.max(PRELOAD_STEPS, Math.round(RENDER_AHEAD / per)))
}; };
}; };
@@ -1002,13 +1015,38 @@ App.feed = App.feed || {};
if (spacer) spacer.style.height = `${start * slideHeight()}px`; if (spacer) spacer.style.height = `${start * slideHeight()}px`;
}; };
// Once the active slide gets within PREFETCH_PAGES of the end of the loaded // The videos the next few swipes will want, resolved now and all at once.
// JSON, pull the next page so the buffer stays ahead of the rendered window. // Resolving is a round trip per video (yt-dlp, on the backend), and leaving
// it until load time put that round trip between a panel and its first
// frame -- with four panels, four of them, arriving one after another. They
// are deduped per video, so asking early costs nothing but the request, and
// asking for all of them at once is the point: they run in parallel.
const warmFormats = function(activeIndex) {
if (!App.videos || typeof App.videos.ensureFormats !== 'function') return;
const videos = state.loadedVideos || [];
const per = paneCount();
const from = Math.max(0, activeIndex * per);
const to = Math.min(videos.length, from + per * (PRELOAD_STEPS + 1));
for (let i = from; i < to; i++) {
if (videos[i]) App.videos.ensureFormats(videos[i]);
}
};
// Keeps the JSON buffer ahead of what the feed is about to eat. A step of N
// panels consumes N videos per swipe, so the same page of listing goes N
// times as fast -- the buffer it needs, and the number of pages worth having
// on the wire at once, both scale with the split.
const prefetchIfNeeded = function(activeIndex) { const prefetchIfNeeded = function(activeIndex) {
const per = paneCount();
const perPage = state.perPage || 12;
const total = (state.loadedVideos || []).length; const total = (state.loadedVideos || []).length;
const bufferAhead = total - 1 - ((activeIndex + 1) * paneCount() - 1); const bufferAhead = total - 1 - ((activeIndex + 1) * per - 1);
if (bufferAhead < PREFETCH_PAGES * (state.perPage || 12) // Everything the rendered window will reach into, plus room to grow.
&& state.hasNextPage && !state.isLoading) { const needed = Math.max(PREFETCH_PAGES * perPage, per * (PRELOAD_STEPS + 2));
if (App.videos.setPrefetchDepth) {
App.videos.setPrefetchDepth(Math.ceil(needed / perPage));
}
if (bufferAhead < needed && state.hasNextPage && !state.isLoading) {
// The feed is its own reader: the grid's scroll position says // The feed is its own reader: the grid's scroll position says
// nothing about whether it needs the next page. // nothing about whether it needs the next page.
App.videos.loadVideos({ force: true }); App.videos.loadVideos({ force: true });
@@ -1016,7 +1054,7 @@ App.feed = App.feed || {};
}; };
// Promotes the slide at `index` to active: syncs the window, updates the // Promotes the slide at `index` to active: syncs the window, updates the
// active styling, plays it, preloads the next PRELOAD_COUNT, and tears down // active styling, plays it, preloads the next PRELOAD_STEPS, and tears down
// playback for everything else in the window. // playback for everything else in the window.
const setActive = function(index) { const setActive = function(index) {
const clamped = clampIndex(index); const clamped = clampIndex(index);
@@ -1044,14 +1082,14 @@ App.feed = App.feed || {};
slidesByIndex.forEach((slide, i) => { slidesByIndex.forEach((slide, i) => {
if (i === clamped) return; if (i === clamped) return;
// Same floor, same reason: at least the next step is preloaded, so const ahead = i - clamped;
// every panel has its next video ready before the swipe. It is
// preloaded for *all* of that step's panes, which is what makes the
// guarantee hold per panel rather than only for the first.
const preloadAhead = Math.max(1, Math.round(PRELOAD_COUNT / paneCount()));
panesOf(slide).forEach((pane) => { panesOf(slide).forEach((pane) => {
if (i > clamped && i <= clamped + preloadAhead) { if (ahead > 0 && ahead <= PRELOAD_STEPS) {
loadSlideSource(pane, pane._videoData, false); // Every panel of the step, started together: what used to
// make them start one after another is that only the first
// step ahead was loaded, so each panel did its resolving
// and buffering at the moment it was swiped to.
loadSlideSource(pane, pane._videoData, false, { buffer: ahead <= BUFFER_STEPS });
} else if (i >= clamped - KEEP_BEHIND && i < clamped) { } else if (i >= clamped - KEEP_BEHIND && i < clamped) {
// Recently-watched panes stay loaded but paused so scrolling // Recently-watched panes stay loaded but paused so scrolling
// back resumes seamlessly from where it was paused. // back resumes seamlessly from where it was paused.
@@ -1063,6 +1101,7 @@ App.feed = App.feed || {};
}); });
updateAutoPiPTarget(); updateAutoPiPTarget();
warmFormats(clamped);
prefetchIfNeeded(clamped); prefetchIfNeeded(clamped);
}; };
@@ -1729,6 +1768,9 @@ App.feed = App.feed || {};
container.setAttribute('aria-hidden', 'true'); container.setAttribute('aria-hidden', 'true');
document.body.classList.remove('feed-mode-open'); document.body.classList.remove('feed-mode-open');
document.body.style.overflow = 'auto'; document.body.style.overflow = 'auto';
// The grid reveals a page at a time and is back in charge of the
// buffer: the feed's deeper one was for swipes eating N videos each.
if (App.videos.setPrefetchDepth) App.videos.setPrefetchDepth(1);
App.feed.updateToggleButton(); App.feed.updateToggleButton();
}; };

View File

@@ -598,6 +598,31 @@ App.videos = App.videos || {};
// dominating a single interleaved batch. // dominating a single interleaved batch.
const GROUP_CHANNEL_PAGE_SIZE = 4; const GROUP_CHANNEL_PAGE_SIZE = 4;
// A page number is claimed before its request goes out, not after it comes
// back, which is what lets several pages be on the wire at once. A request
// that fails puts its number back in the hat rather than leaving a hole in
// the listing. `cursor` is the group's per-channel cursor, or the session-
// wide one below for a plain channel.
const claimPage = function(cursor) {
if (cursor.retries && cursor.retries.length) return cursor.retries.shift();
return cursor.page++;
};
const returnPage = function(cursor, page) {
cursor.retries = cursor.retries || [];
cursor.retries.push(page);
cursor.retries.sort((a, b) => a - b);
};
// The plain-channel cursor. state.currentPage is the number itself, so
// resetGrid keeps working by setting it back to 1; the retry list is
// cleared alongside it in resetPrefetch.
const channelCursor = {
get page() { return state.currentPage; },
set page(next) { state.currentPage = next; },
retries: []
};
// Fetches one page from every channel in a group and zips the results // Fetches one page from every channel in a group and zips the results
// together round-robin so the feed alternates between sources instead of // together round-robin so the feed alternates between sources instead of
// running through one channel's videos before moving to the next. // running through one channel's videos before moving to the next.
@@ -631,23 +656,28 @@ App.videos = App.videos || {};
} }
const results = await Promise.all(active.map(async (cursor) => { const results = await Promise.all(active.map(async (cursor) => {
const response = await fetch('/api/videos', { const page = claimPage(cursor);
method: 'POST', try {
headers: { 'Content-Type': 'application/json' }, const response = await fetch('/api/videos', {
body: JSON.stringify({ method: 'POST',
channel: cursor.id, headers: { 'Content-Type': 'application/json' },
query: query || "", body: JSON.stringify({
page: cursor.page, channel: cursor.id,
perPage: GROUP_CHANNEL_PAGE_SIZE, query: query || "",
server: session.server page: page,
}), perPage: GROUP_CHANNEL_PAGE_SIZE,
signal: signal server: session.server
}); }),
const data = await response.json(); signal: signal
const items = data && Array.isArray(data.items) ? data.items : []; });
cursor.page++; const data = await response.json();
cursor.hasNextPage = items.length > 0 && (data && data.pageInfo ? data.pageInfo.hasNextPage !== false : true); const items = data && Array.isArray(data.items) ? data.items : [];
return items; cursor.hasNextPage = items.length > 0 && (data && data.pageInfo ? data.pageInfo.hasNextPage !== false : true);
return items;
} catch (err) {
returnPage(cursor, page);
throw err;
}
})); }));
const interleaved = []; const interleaved = [];
@@ -668,10 +698,11 @@ App.videos = App.videos || {};
const searchInput = document.getElementById('search-input'); const searchInput = document.getElementById('search-input');
const query = searchInput ? searchInput.value : ""; const query = searchInput ? searchInput.value : "";
const page = claimPage(channelCursor);
const body = { const body = {
channel: session.channel.id, channel: session.channel.id,
query: query || "", query: query || "",
page: state.currentPage, page: page,
perPage: state.perPage, perPage: state.perPage,
server: session.server server: session.server
}; };
@@ -684,14 +715,19 @@ App.videos = App.videos || {};
} }
}); });
const response = await fetch('/api/videos', { let videos;
method: 'POST', try {
headers: { 'Content-Type': 'application/json' }, const response = await fetch('/api/videos', {
body: JSON.stringify(body), method: 'POST',
signal: signal headers: { 'Content-Type': 'application/json' },
}); body: JSON.stringify(body),
const videos = await response.json(); signal: signal
state.currentPage++; });
videos = await response.json();
} catch (err) {
returnPage(channelCursor, page);
throw err;
}
return { return {
items: videos && Array.isArray(videos.items) ? videos.items : [], items: videos && Array.isArray(videos.items) ? videos.items : [],
hasNextPage: videos && videos.pageInfo ? videos.pageInfo.hasNextPage !== false : true hasNextPage: videos && videos.pageInfo ? videos.pageInfo.hasNextPage !== false : true
@@ -712,11 +748,28 @@ App.videos = App.videos || {};
// reader reaches the second-to-last row (see the virtualizer's update()). // reader reaches the second-to-last row (see the virtualizer's update()).
// So the cards that come into view are complete on arrival instead of // So the cards that come into view are complete on arrival instead of
// shimmering their way in while the request is still on the wire. // shimmering their way in while the request is still on the wire.
//
// How far ahead is `prefetch.depth`: one page for the grid, more when the
// reels feed is running a split and gets through a page in two swipes.
// Pages are fetched at once and given to the reader in page order.
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
const prefetch = { const prefetch = {
batch: null, // fetched + warmed, waiting for the reader ready: [], // fetched pages waiting for the reader, in page order
inFlight: null, // promise of the fetch currently running inFlight: [], // pages on the wire, in the order their numbers were claimed
controller: null depth: 1, // how many pages to keep fetched or in flight
exhausted: false // a page came back saying it was the last one
};
// How deep to run ahead. One page is right for the grid, which reveals a
// page at a time. The reels feed asks for more: a step of N panels eats N
// videos per swipe, so a page is a couple of swipes and fetching them one
// after another leaves the feed waiting on a round trip it could have made
// earlier. See prefetchIfNeeded in feed.js.
App.videos.setPrefetchDepth = function(depth) {
const next = Math.max(1, Math.round(depth) || 1);
if (next === prefetch.depth) return;
prefetch.depth = next;
if (next > 1) App.videos.prefetchNextBatch();
}; };
// One slow thumbnail must not hold a whole page back. // One slow thumbnail must not hold a whole page back.
@@ -768,29 +821,54 @@ App.videos = App.videos || {};
App.videos.prefetchNextBatch(); // stay one page ahead again App.videos.prefetchNextBatch(); // stay one page ahead again
}; };
App.videos.prefetchNextBatch = function() { // Pages finish out of order -- page 3 can beat page 2 back -- so a finished
if (prefetch.batch || prefetch.inFlight) return Promise.resolve(); // page waits its turn here. Only a run of finished pages from the front
if (!state.hasNextPage) return Promise.resolve(); // moves across, which keeps what the reader is given in page order.
const session = App.storage.getSession(); const drainFinished = function() {
if (!session || !session.channel) return Promise.resolve(); while (prefetch.inFlight.length && prefetch.inFlight[0].done) {
const entry = prefetch.inFlight.shift();
if (entry.batch) prefetch.ready.push(entry.batch);
}
};
prefetch.controller = new AbortController(); const startPrefetch = function(session) {
const signal = prefetch.controller.signal; const controller = new AbortController();
prefetch.inFlight = (async () => { const entry = { controller: controller, batch: null, done: false };
prefetch.inFlight.push(entry);
entry.promise = (async () => {
try { try {
const batch = await fetchNextBatch(session, signal); const batch = await fetchNextBatch(session, controller.signal);
await warmThumbnails(batch.items); if (!controller.signal.aborted) entry.batch = batch;
if (!signal.aborted) prefetch.batch = batch; // Running several pages ahead means learning where the end is
// from a page the reader hasn't been given yet; without this,
// depth more requests would be claimed past it.
if (batch && batch.hasNextPage === false) prefetch.exhausted = true;
// Warming decodes the thumbnails so cards are complete when
// they come into view. It runs alongside the pages behind it
// rather than in front of them: a page held back for a slow
// image is a page the feed is waiting on.
warmThumbnails(batch.items);
} catch (err) { } catch (err) {
if (err.name !== 'AbortError') { if (err.name !== 'AbortError') {
console.error("Failed to prefetch videos:", err); console.error("Failed to prefetch videos:", err);
} }
} finally { } finally {
prefetch.inFlight = null; entry.done = true;
prefetch.controller = null; drainFinished();
} }
})(); })();
return prefetch.inFlight; return entry.promise;
};
App.videos.prefetchNextBatch = function() {
if (!state.hasNextPage || prefetch.exhausted) return Promise.resolve();
const session = App.storage.getSession();
if (!session || !session.channel) return Promise.resolve();
while (prefetch.ready.length + prefetch.inFlight.length < prefetch.depth) {
startPrefetch(session);
}
return prefetch.inFlight.length ? prefetch.inFlight[0].promise : Promise.resolve();
}; };
// Throws away a video's resolved formats and asks the server again. Media // Throws away a video's resolved formats and asks the server again. Media
@@ -808,20 +886,20 @@ App.videos = App.videos || {};
// Called by the virtualizer once the reader is within two rows of the end. // Called by the virtualizer once the reader is within two rows of the end.
App.videos.releasePrefetched = function() { App.videos.releasePrefetched = function() {
if (!prefetch.batch || state.isLoading) return false; if (!prefetch.ready.length || state.isLoading) return false;
const batch = prefetch.batch; commitBatch(prefetch.ready.shift());
prefetch.batch = null;
commitBatch(batch);
return true; return true;
}; };
// Drops anything in flight or held, for when the result set changes out // Drops anything in flight or held, for when the result set changes out
// from under it (new search, channel, or filters). // from under it (new search, channel, or filters).
App.videos.resetPrefetch = function() { App.videos.resetPrefetch = function() {
if (prefetch.controller) prefetch.controller.abort(); prefetch.inFlight.forEach((entry) => entry.controller.abort());
prefetch.batch = null; prefetch.inFlight = [];
prefetch.inFlight = null; prefetch.ready = [];
prefetch.controller = null; prefetch.exhausted = false;
// Page numbers claimed against the old result set mean nothing now.
channelCursor.retries = [];
}; };
// Is the reader actually out of content? True when nothing is rendered yet, // Is the reader actually out of content? True when nothing is rendered yet,
@@ -863,16 +941,16 @@ App.videos = App.videos || {};
loadRunning = true; loadRunning = true;
try { try {
if (prefetch.batch || prefetch.inFlight) { if (prefetch.ready.length || prefetch.inFlight.length) {
// A page is already here or on its way; revealing it before the // A page is already here or on its way; revealing it before the
// reader gets near the end would defeat the point of holding it. // reader gets near the end would defeat the point of holding it.
if (!force && !needsContentNow()) return; if (!force && !needsContentNow()) return;
if (App.videos.releasePrefetched()) return; if (App.videos.releasePrefetched()) return;
if (prefetch.inFlight) { if (prefetch.inFlight.length) {
// Wait for it rather than asking for the same page twice -- // Wait for the oldest rather than asking for the same page
// the fetch advances the page cursor, so a second request // twice -- its number is already claimed, so a request from
// here would skip a page entirely. // here would skip a page entirely.
await prefetch.inFlight; await prefetch.inFlight[0].promise;
if (App.videos.releasePrefetched()) return; if (App.videos.releasePrefetched()) return;
} }
} }

332
tests/smoke_parallel.py Normal file
View File

@@ -0,0 +1,332 @@
#!/usr/bin/env python3
"""A split step loads its videos together and ahead of time, not one by one.
Run against a locally running backend:
backend/main.py &
.venv/bin/python tests/smoke_parallel.py
The listing and the media are fixtures, so what is measured is the feed and not
somebody's CDN: every clip takes the same MEDIA_DELAY_MS to arrive, which is
far longer than a step that was preloaded needs to start.
Four panels, which is the case this is for. With one step preloaded between
them, a panel did its loading at the moment it was swiped to -- on a real CDN
that is four starts in a row, and here it is four late ones. The second swipe
is where it shows: the first lands on the step that was preloaded either way,
and it comes quickly enough that the step after it cannot have loaded since.
"""
import base64
import http.server
import json
import socketserver
import sys
import threading
import time
from playwright.sync_api import sync_playwright
BASE = "http://127.0.0.1:5000/"
SERVER = "https://hottubapp.io"
CHANNEL = "xvideos"
PER_PAGE = 12
PAGES = 8
MEDIA_DELAY_MS = 2000 # what a clip costs if it isn't already loading
READY_MS = 400 # a start this quick means the clip was already there
# One second of black, 64x64, h264 -- a real file, because the panels have to
# decode a frame from it. Same blob as smoke_open.py.
MP4_B64 = (
"AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAOxbW9vdgAAAGxtdmhkAAAAAAAAAAAA"
"AAAAAAAD6AAAA+gAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA"
"AABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAtx0cmFrAAAAXHRraGQAAAADAAAA"
"AAAAAAAAAAABAAAAAAAAA+gAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAA"
"AAAAAAAAAABAAAAAAEAAAABAAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAPoAAAIAAABAAAA"
"AAJUbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAAAoAAAAKABVxAAAAAAALWhkbHIAAAAAAAAAAHZp"
"ZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAAB/21pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAA"
"ACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAAb9zdGJsAAAAv3N0c2QAAAAAAAAA"
"AQAAAK9hdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAEAAQABIAAAASAAAAAAAAAABFUxhdmM2"
"MS4xOS4xMDEgbGlieDI2NAAAAAAAAAAAAAAAGP//AAAANWF2Y0MBZAAK/+EAGGdkAAqs2UQmwEQA"
"AAMABAAAAwBQPEiWWAEABmjr48siwP34+AAAAAAQcGFzcAAAAAEAAAABAAAAFGJ0cnQAAAAAAAAa"
"uAAAAAAAAAAYc3R0cwAAAAAAAAABAAAACgAABAAAAAAUc3RzcwAAAAAAAAABAAAAAQAAAGBjdHRz"
"AAAAAAAAAAoAAAABAAAIAAAAAAEAABQAAAAAAQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAA"
"AAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAIAAAAABxzdHNjAAAAAAAAAAEAAAABAAAACgAA"
"AAEAAAA8c3RzegAAAAAAAAAAAAAACgAAAtcAAAAOAAAADAAAAAwAAAAMAAAAFAAAAA4AAAAMAAAA"
"DAAAABQAAAAUc3RjbwAAAAAAAAABAAAD4QAAAGF1ZHRhAAAAWW1ldGEAAAAAAAAAIWhkbHIAAAAA"
"AAAAAG1kaXJhcHBsAAAAAAAAAAAAAAAALGlsc3QAAAAkqXRvbwAAABxkYXRhAAAAAQAAAABMYXZm"
"NjEuNy4xMDAAAAAIZnJlZQAAA19tZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBj"
"b3JlIDE2NCByMzEwOCAzMWUxOWY5IC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0"
"IDIwMDMtMjAyMyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6"
"IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3Vi"
"bWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9t"
"YV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lw"
"PTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTIgbG9va2FoZWFkX3RocmVhZHM9MSBzbGlj"
"ZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0w"
"IGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2Jp"
"YXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBr"
"ZXlpbnRfbWluPTEwIHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAg"
"cmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0"
"ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAAAIWWIhAAR//73iB8yy2+catdyEeesVP1G"
"Ixltc+dmuhineQAAAApBmiRsQQ/+qlfeAAAACEGeQniHfwW9AAAACAGeYXRDfwd8AAAACAGeY2pD"
"fwd9AAAAEEGaaEmoQWiZTAh3//6pnTUAAAAKQZ6GRREsO/8FvQAAAAgBnqV0Q38HfQAAAAgBnqdq"
"Q38HfAAAABBBmqlJqEFsmUwIb//+p4+I"
)
def make_server():
"""Serves the fixture clip, after a delay, however many at a time."""
class Handler(http.server.BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args):
pass
def do_GET(self):
time.sleep(MEDIA_DELAY_MS / 1000)
body = base64.b64decode(MP4_B64)
self.send_response(200)
self.send_header("Content-Type", "video/mp4")
self.send_header("Content-Length", str(len(body)))
self.send_header("Accept-Ranges", "none")
self.send_header("Access-Control-Allow-Origin", "*")
self.end_headers()
self.wfile.write(body)
def do_HEAD(self):
self.send_response(200)
self.send_header("Content-Type", "video/mp4")
self.send_header("Content-Length", "0")
self.end_headers()
httpd = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
httpd.daemon_threads = True
threading.Thread(target=httpd.serve_forever, daemon=True).start()
return httpd
SEED = """([server, channel]) => {
localStorage.setItem('config', JSON.stringify({ servers: [{ [server]: {} }] }));
localStorage.setItem('preferences', JSON.stringify({ [server]: { channelId: channel } }));
localStorage.removeItem('session');
localStorage.setItem('favorites', JSON.stringify([]));
}"""
# Which videos were asked to resolve their formats. Prefilled formats make the
# answer immediate, but the asking is what says when the feed goes looking.
SPY = """() => {
window.__resolved = [];
const real = App.videos.ensureFormats;
App.videos.ensureFormats = function(video) {
if (video && video.id) window.__resolved.push(String(video.id));
return real.apply(this, arguments);
};
}"""
PANES = """() => Array.from(document.querySelectorAll('.feed-slide')).map((slide) => ({
step: Number(slide.dataset.step),
loaded: Array.from(slide.querySelectorAll('.feed-pane'))
.map((pane) => pane.classList.contains('is-loaded')),
ids: Array.from(slide.querySelectorAll('.feed-pane')).map((pane) => pane.dataset.videoId),
}))"""
WATCH = """() => {
window.__t0 = performance.now();
window.__started = {};
document.addEventListener('playing', (e) => {
const pane = e.target.closest && e.target.closest('.feed-pane');
if (!pane) return;
const id = pane.dataset.videoId;
if (!(id in window.__started)) window.__started[id] = Math.round(performance.now() - window.__t0);
}, true);
}"""
SWIPE = """() => { const s = document.getElementById('feed-scroll'); s.scrollTop += s.clientHeight; }"""
CLICK = """(sel) => {
const el = document.querySelector(sel);
if (!el) return 'missing';
el.click(); // dispatched, so the pointer never wakes the HUD
return 'ok';
}"""
class Checks:
def __init__(self):
self.failed = 0
def ok(self, label, condition, detail=""):
mark = "PASS" if condition else "FAIL"
if not condition:
self.failed += 1
print(f" [{mark}] {label}" + (f" -- {detail}" if detail and not condition else ""))
def make_items(page_no, media):
"""One page of the listing, formats already resolved so nothing but the
media itself stands between a panel and its first frame."""
start = (page_no - 1) * PER_PAGE
items = []
for i in range(PER_PAGE):
n = start + i
url = f"{media}/v{n}.mp4"
items.append({
"id": f"fix-{n}",
"title": f"Fixture {n}",
"url": "https://example.test/watch/%d" % n,
"channel": "test",
"duration": 1,
"thumb": "",
"tags": [],
"meta": {
"url": url,
"http_headers": {},
"isLive": False,
"formats": [{"url": url, "ext": "mp4", "height": 240, "protocol": "https",
"vcodec": "avc1", "acodec": "none"}],
},
})
return items
def main():
c = Checks()
httpd = make_server()
media = f"http://127.0.0.1:{httpd.server_address[1]}"
# Listing requests are parked as they arrive and answered a beat later, so
# how many the feed has out at once is something the test can see.
parked = []
busiest = {"max": 0}
with sync_playwright() as p:
browser = p.chromium.launch(args=[
"--no-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
"--autoplay-policy=no-user-gesture-required",
"--renderer-process-limit=1",
"--js-flags=--max-old-space-size=768",
])
page = browser.new_page(viewport={"width": 800, "height": 900})
crashes = []
page.on("pageerror", lambda e: crashes.append(str(e)))
def park(route):
parked.append(route)
busiest["max"] = max(busiest["max"], len(parked))
def answer():
while parked:
route = parked.pop(0)
try:
body = json.loads(route.request.post_data or "{}")
except Exception:
body = {}
page_no = int(body.get("page", 1) or 1)
route.fulfill(status=200, content_type="application/json", body=json.dumps({
"items": make_items(page_no, media) if page_no <= PAGES else [],
"pageInfo": {"hasNextPage": page_no < PAGES},
}))
def settle(ms):
"""Let the page run, answering listing requests as they pile up."""
for _ in range(max(1, ms // 100)):
page.wait_for_timeout(100)
answer()
page.route("**/api/videos", park)
page.goto(BASE, wait_until="domcontentloaded")
page.evaluate(SEED, [SERVER, CHANNEL])
page.goto(BASE, wait_until="load")
settle(1500)
page.wait_for_selector(".video-card", timeout=30000)
page.evaluate(SPY)
page.evaluate("() => App.feed.open()")
page.wait_for_selector(".feed-slide", timeout=20000)
settle(3000)
print("\nsplitting the step four ways")
page.evaluate(CLICK, ".feed-slide.is-active .feed-pane:nth-of-type(1) .feed-pane-split-right")
settle(1500)
for nth in (1, 2):
page.evaluate(CLICK, f".feed-slide.is-active .feed-pane:nth-of-type({nth}) .feed-pane-split-down")
settle(1500)
settle(3000)
per = page.evaluate("() => App.feed.paneCount()")
c.ok("four panels", per == 4, str(per))
panes = page.evaluate(PANES)
active = page.evaluate("() => App.state.feedActiveIndex")
ahead = [s for s in panes if s["step"] > active]
loaded_ahead = sum(1 for s in ahead for is_loaded in s["loaded"] if is_loaded)
want = per * 3
print(f" ({loaded_ahead} videos loading ahead, over {len(ahead)} steps)")
c.ok(f"every panel keeps its next three videos ready ({want} of them)",
loaded_ahead >= want, f"loaded {loaded_ahead} of {sum(len(s['loaded']) for s in ahead)}")
resolved = set(page.evaluate("() => window.__resolved"))
missing = [i for s in ahead for i in s["ids"] if i not in resolved]
c.ok("and their formats were asked for up front, not at load time",
not missing, f"never resolved: {missing[:4]}")
def swipe_and_time(label, watch_ms):
page.evaluate(WATCH)
page.evaluate(SWIPE)
settle(watch_ms)
started = page.evaluate("() => window.__started")
ids = page.evaluate("""() => Array.from(
document.querySelectorAll('.feed-slide.is-active .feed-pane')).map(p => p.dataset.videoId)""")
times = [started[i] for i in ids if i in started]
print(f" ({label}: started at {sorted(times)} ms after the swipe)")
return ids, times
print("\nswiping to the next step")
# Measured briefly, and on purpose: the step after this one must not get
# a quiet moment to load in, or the swipe below tests nothing.
ids, times = swipe_and_time("first swipe", 800)
c.ok("it landed on the next step",
page.evaluate("() => App.state.feedActiveIndex") == active + 1)
c.ok("all four panels started, together",
len(times) == len(ids) and (max(times) - min(times)) < READY_MS,
f"{len(times)} of {len(ids)} started; at {sorted(times)}")
print("\nand straight on to the one after it")
ids, times = swipe_and_time("second swipe", 3000)
c.ok("it landed there too",
page.evaluate("() => App.state.feedActiveIndex") == active + 2)
c.ok("every panel of it is playing",
page.evaluate("""() => Array.from(document.querySelectorAll('.feed-slide.is-active .feed-video'))
.every(v => !v.paused)"""))
c.ok("all four started, together",
len(times) == len(ids) and (max(times) - min(times)) < READY_MS,
f"{len(times)} of {len(ids)} started; at {sorted(times)}")
c.ok(f"and at once, because the clips were already there (a clip takes {MEDIA_DELAY_MS}ms)",
times and max(times) < READY_MS,
f"slowest {max(times) if times else 'n/a'}ms")
print("\nfeeding the split")
c.ok("more than one page of listing is fetched at a time",
busiest["max"] > 1, f"most at once: {busiest['max']}")
now_active = page.evaluate("() => App.state.feedActiveIndex")
c.ok("and the buffer stays ahead of what a swipe eats",
page.evaluate("() => App.state.loadedVideos.length") > (now_active + 1) * per + per * 3,
str(page.evaluate("() => App.state.loadedVideos.length")))
print("\nclosing a panel")
page.evaluate(CLICK, ".feed-slide.is-active .feed-pane:nth-of-type(2) .feed-pane-close")
settle(2000)
c.ok("what is left plays on", page.evaluate("""() => {
const v = document.querySelector('.feed-slide.is-active .feed-video');
return !!v && !v.paused;
}"""))
c.ok("nothing threw along the way", not crashes, str(crashes[:2]))
browser.close()
httpd.shutdown()
print(f"\n{'OK' if not c.failed else 'FAILED'}: {c.failed} check(s) failed")
return 1 if c.failed else 0
sys.exit(main())