// Someone at the pages, filmed: a browser logged in as them, Chrome's own // screencast saving each frame it paints, and a cursor and keyboard moving // at a person's pace. A take is a folder of frames under public/footage // with a take.json of when each was painted, the marks the scene set, and // where each click landed, which the video plays back (video/Footage.tsx). import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { chromium } from 'playwright-core'; const root = join(dirname(fileURLToPath(import.meta.url)), '..'); export const SIZES = { wide: { viewport: { width: 1280, height: 800 }, deviceScaleFactor: 1.5 }, phone: { viewport: { width: 390, height: 844 }, deviceScaleFactor: 3, isMobile: true, hasTouch: true } }; /// A browser for the world: its gateway's name resolved here, its /// certificate taken, drawn dark as the videos are. export async function launch(size) { const browser = await chromium.launch({ headless: true, // Without it a headless screencast sends frames at 1x, whatever the // context's deviceScaleFactor. args: [ `--force-device-scale-factor=${SIZES[size].deviceScaleFactor}`, '--host-resolver-rules=MAP gateway.knotless.internal 127.0.0.1' ] }); const context = await browser.newContext({ ...SIZES[size], ignoreHTTPSErrors: true, colorScheme: 'dark', locale: 'en-US' }); return { browser, context }; } /// Until the page's script has taken it over, as the root says once mounted. export const ready = (page) => page.locator('html[data-ready]').waitFor({ state: 'attached' }); /// Logs `who` in from the header's dialog, through their PDS's page: done /// before filming, so the take starts logged in. export async function logIn(page, world, who) { await page.getByRole('button', { name: 'Log in' }).click(); const dialog = page.getByRole('dialog', { name: 'Log in with your Atmosphere account' }); await dialog.getByLabel('Handle').fill(who.handle); const back = page.url(); await dialog.getByRole('button', { name: 'Log in' }).click(); await page.waitForURL((u) => u.origin === new URL(world.pds).origin); await page.getByRole('textbox', { name: 'Password' }).fill(who.password); await page.getByRole('button', { name: 'Sign in' }).click(); await page.getByRole('button', { name: /^(accept|authorize|allow)$/i }).click(); await page.waitForURL(back); await ready(page); await page.getByLabel('Your account').waitFor(); } /// Film `page` as `take`, at `size`: frames into public/footage/. export async function film(take, page, size) { const dir = join(root, 'public/footage', take); rmSync(dir, { recursive: true, force: true }); mkdirSync(dir, { recursive: true }); await page.addInitScript(CURSOR); await page.addInitScript(`(${LIVE})()`); // A phone is touched, not pointed at: no cursor, just the taps the video draws. if (size === 'phone') await page.addInitScript(HIDE_CURSOR); const { width, height } = SIZES[size].viewport; const scale = SIZES[size].deviceScaleFactor; const cdp = await page.context().newCDPSession(page); const frames = []; const marks = {}; const clicks = []; let t0 = 0; let n = 0; // Written as they come: a sync write here held up the scene's own moves. const writing = []; cdp.on('Page.screencastFrame', (e) => { void cdp.send('Page.screencastFrameAck', { sessionId: e.sessionId }).catch(() => {}); const t = e.metadata.timestamp ?? Date.now() / 1000; if (!t0) t0 = t; const f = `${String(n++).padStart(5, '0')}.jpg`; frames.push({ t: +(t - t0).toFixed(3), f }); writing.push(writeFile(join(dir, f), Buffer.from(e.data, 'base64'))); }); await cdp.send('Page.startScreencast', { format: 'jpeg', quality: 92, maxWidth: width * scale, maxHeight: height * scale, everyNthFrame: 1 }); const now = () => (t0 ? Date.now() / 1000 - t0 : 0); const at = { x: width / 2, y: height / 2 }; const cursor = (x, y, hand) => page.evaluate(([cx, cy, h]) => window.__cursor?.(cx, cy, h), [x, y, hand]).catch(() => {}); /// To `target`, the cursor a hand as it nears it when it is to be clicked. async function moveTo(target, hand = false) { await target.scrollIntoViewIfNeeded().catch(() => {}); const box = await target.boundingBox(); if (!box) throw new Error(`nothing to point at: ${target}`); const x = box.x + Math.min(box.width / 2, 40 + Math.random() * 10); const y = box.y + box.height / 2; const steps = 24; for (let i = 1; i <= steps; i++) { const k = i / steps; const e = k < 0.5 ? 2 * k * k : 1 - (-2 * k + 2) ** 2 / 2; const px = at.x + (x - at.x) * e; const py = at.y + (y - at.y) * e; await page.mouse.move(px, py); await cursor(px, py, hand && k > 0.6); await page.waitForTimeout(16); } at.x = x; at.y = y; } const press = async () => { clicks.push({ t: +now().toFixed(3), x: Math.round(at.x), y: Math.round(at.y) }); await page.evaluate(() => window.__press?.()).catch(() => {}); }; return { page, /// Go to `url`, and wait for the page to be ready. open: async (url) => { await page.goto(url); await ready(page); await cursor(at.x, at.y, false); }, /// Note where the scene is, for the edit to cut on. mark: (name) => { marks[name] = +now().toFixed(3); }, /// Move the cursor to something, without clicking. point: (target) => moveTo(target), click: async (target, { pause = 180 } = {}) => { await moveTo(target, true); await page.waitForTimeout(pause); await press(); await target.click(); await cursor(at.x, at.y, false); await page.waitForTimeout(250); }, /// Click into a field and type, a key at a time. type: async (target, text, { delay = 55 } = {}) => { await moveTo(target, true); await press(); await target.click(); // Off what is about to be typed, as anyone moves the mouse aside. const box = await target.boundingBox(); if (box) { at.x = Math.min(box.x + box.width - 24, width - 30); at.y = box.y + box.height - 10; await cursor(at.x, at.y, false); } await page.waitForTimeout(200); await target.pressSequentially(text, { delay }); }, hold: (ms) => page.waitForTimeout(ms), /// Scroll the window, smoothly, by `dy`. scroll: async (dy) => { const steps = Math.max(10, Math.round(Math.abs(dy) / 12)); for (let i = 0; i < steps; i++) { await page.mouse.wheel(0, dy / steps); await page.waitForTimeout(16); } }, cut: async () => { await cdp.send('Page.stopScreencast').catch(() => {}); await Promise.all(writing); writeFileSync( join(dir, 'take.json'), JSON.stringify({ size, width, height, scale, frames, marks, clicks }, null, 1) ); console.log( ` ${take}: ${frames.length} frames, ${now().toFixed(1)} s, marks ${JSON.stringify(marks)}` ); } }; } // The world's own address, as knotless.dev's would read, in the text a page // shows (never its scripts). const LIVE = `() => { if (window.__live) return; window.__live = true; const at = /gateway\\.knotless\\.internal(:\\d+)?/g; const live = (n) => { if (n.nodeType === 3) { const p = n.parentElement?.tagName; if (p !== 'SCRIPT' && p !== 'STYLE' && at.test(n.data)) n.data = n.data.replace(at, 'knotless.dev'); } else n.childNodes.forEach(live); }; const start = () => { live(document); new MutationObserver((ms) => ms.forEach((m) => (m.type === 'characterData' ? live(m.target) : m.addedNodes.forEach(live)))).observe(document, { childList: true, subtree: true, characterData: true }); }; if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', start); else start(); }`; const HIDE_CURSOR = `document.addEventListener('DOMContentLoaded', () => { const s = document.createElement('style'); s.textContent = '#film-cursor svg{display:none!important}'; document.head.append(s); });`; // A cursor drawn into the page, in its top layer over any dialog, since a // screencast does not show the real one: moved by the camera (moveTo). An // arrow, or a hand on its way to a click, which presses in as it clicks; the // video draws the click itself. A pixel blinking in a corner keeps the // screencast painting while a page is still. const CURSOR = `(() => { if (window !== window.top) return; const draw = () => { if (document.getElementById('film-cursor')) return; const c = document.createElement('div'); c.id = 'film-cursor'; c.setAttribute('popover', 'manual'); c.innerHTML = ''; c.style.cssText = 'position:fixed;left:0;top:0;z-index:2147483647;pointer-events:none;transform:translate(640px,400px);will-change:transform'; const s = document.createElement('style'); s.textContent = '#film-cursor{inset:auto;margin:0;border:0;padding:0;background:none;overflow:visible;width:0;height:0}#film-cursor svg{position:absolute;max-width:none;max-height:none;filter:drop-shadow(0 1px 2px rgba(0,0,0,.5));transform-origin:30% 10%}#film-cursor .arrow{left:-5px;top:-3px}#film-cursor .hand{left:-10px;top:-3px;display:none}#film-cursor.hand .arrow{display:none}#film-cursor.hand .hand{display:block}#film-cursor.press svg{animation:film-press .3s ease-out}@keyframes film-press{40%{transform:scale(.82)}100%{transform:scale(1)}}#film-tick{position:fixed;right:0;bottom:0;width:1px;height:1px;z-index:2147483647;pointer-events:none;animation:film-tick 1s steps(2) infinite}@keyframes film-tick{0%{background:rgba(128,128,128,.01)}100%{background:rgba(128,128,128,.02)}}'; const tick = document.createElement('div'); tick.id = 'film-tick'; document.documentElement.append(s, c, tick); const last = sessionStorage.getItem('film-cursor'); if (last) c.style.transform = last; }; window.__cursor = (x, y, hand) => { draw(); const c = document.getElementById('film-cursor'); c.classList.toggle('hand', !!hand); try { if (c.matches(':popover-open')) c.hidePopover(); c.showPopover(); } catch {} c.style.transform = 'translate(' + x + 'px,' + y + 'px)'; sessionStorage.setItem('film-cursor', c.style.transform); }; window.__press = () => { const c = document.getElementById('film-cursor'); if (!c) return; c.classList.remove('press'); void c.offsetWidth; c.classList.add('press'); }; if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', draw); else draw(); })();`;