From 989e647fc2cb130cd10b2c2c98af83ea6155046d Mon Sep 17 00:00:00 2001 From: Anton Balabin Date: Mon, 7 Sep 2026 13:16:30 +0300 Subject: [PATCH] fix(web): keep preview recordings valid across viewport resizing --- .../scripts/browser-recording-media.test.mjs | 173 ++++++++++++++++++ apps/web/src/browser/browserMediaRecorder.ts | 23 +++ apps/web/src/browser/browserRecording.test.ts | 28 ++- apps/web/src/browser/browserRecording.ts | 29 +-- 4 files changed, 224 insertions(+), 29 deletions(-) create mode 100644 apps/desktop/scripts/browser-recording-media.test.mjs create mode 100644 apps/web/src/browser/browserMediaRecorder.ts diff --git a/apps/desktop/scripts/browser-recording-media.test.mjs b/apps/desktop/scripts/browser-recording-media.test.mjs new file mode 100644 index 000000000000..83a1c44e1d68 --- /dev/null +++ b/apps/desktop/scripts/browser-recording-media.test.mjs @@ -0,0 +1,173 @@ +import * as NodeAssert from "node:assert/strict"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeModule from "node:module"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +if (process.versions.electron) { + const { app, BrowserWindow, ipcMain } = await import("electron"); + const directory = process.env.T3CODE_RECORDING_TEST_DIR; + NodeAssert.ok(directory); + app.setPath("userData", NodePath.join(directory, "profile")); + ipcMain.handle("recording-result", async (_event, bytes) => { + await NodeFSP.writeFile(NodePath.join(directory, "recording.mp4"), Buffer.from(bytes)); + app.quit(); + }); + ipcMain.on("recording-error", (_event, message) => { + console.error(message); + app.exit(1); + }); + app.whenReady().then(async () => { + const window = new BrowserWindow({ + width: 900, + height: 700, + webPreferences: { + nodeIntegration: true, + contextIsolation: false, + backgroundThrottling: false, + }, + }); + window.webContents.on("console-message", (event) => console.error(event.message)); + window.setFullScreen(true); + await window.loadFile(NodePath.join(directory, "recording.html")); + }); +} else { + const { test } = await import("node:test"); + const require = NodeModule.createRequire(import.meta.url); + + const run = (command, args, options = {}) => + new Promise((resolve, reject) => { + const child = NodeChildProcess.spawn(command, args, { + ...options, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.once("error", (cause) => reject(new Error(`${command} failed\n${stderr}`, { cause }))); + child.once("close", (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(`${command} exited with ${code}\n${stderr}`)); + }); + }); + + test("exported recording decodes through desktop, mobile, and desktop frames", async () => { + const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-recording-media-")); + try { + const source = await NodeFSP.readFile( + new URL("../../web/src/browser/browserMediaRecorder.ts", import.meta.url), + "utf8", + ); + await NodeFSP.writeFile( + NodePath.join(directory, "recorder.mjs"), + NodeModule.stripTypeScriptTypes(source), + ); + await NodeFSP.writeFile( + NodePath.join(directory, "recording.html"), + `Recording regression +`, + ); + const env = { ...process.env, T3CODE_RECORDING_TEST_DIR: directory }; + delete env.ELECTRON_RUN_AS_NODE; + delete env.NODE_OPTIONS; + await run(require("electron"), [NodeURL.fileURLToPath(import.meta.url)], { + env, + signal: AbortSignal.timeout(30_000), + }); + const recording = NodePath.join(directory, "recording.mp4"); + await run("ffmpeg", [ + "-v", + "error", + "-xerror", + "-i", + recording, + "-fps_mode", + "passthrough", + "-enc_time_base", + "1:1000000", + "-f", + "null", + "-", + ]); + const { frames, format } = JSON.parse( + await run("ffprobe", [ + "-v", + "error", + "-show_frames", + "-show_entries", + "frame=width,height,best_effort_timestamp_time:format=duration", + "-of", + "json", + recording, + ]), + ); + const phases = frames.filter( + (frame, index) => + index === 0 || + frame.width !== frames[index - 1].width || + frame.height !== frames[index - 1].height, + ); + NodeAssert.deepEqual( + phases.map(({ width, height }) => [width, height]), + [ + [1600, 1000], + [488, 1054], + [1600, 1000], + ], + ); + NodeAssert.ok(Number(format.duration) >= 3); + NodeAssert.ok(Number(frames.at(-1).best_effort_timestamp_time) >= 3); + for (let index = 1; index < frames.length; index += 1) { + NodeAssert.ok( + Number(frames[index].best_effort_timestamp_time) > + Number(frames[index - 1].best_effort_timestamp_time), + ); + } + } finally { + await NodeFSP.rm(directory, { recursive: true, force: true }); + } + }); +} diff --git a/apps/web/src/browser/browserMediaRecorder.ts b/apps/web/src/browser/browserMediaRecorder.ts new file mode 100644 index 000000000000..441ba5772230 --- /dev/null +++ b/apps/web/src/browser/browserMediaRecorder.ts @@ -0,0 +1,23 @@ +const preferredMimeTypes = [ + "video/mp4;codecs=avc3", + "video/mp4;codecs=avc3.640028", + "video/mp4;codecs=avc3.42e01e", + "video/webm;codecs=vp9", + "video/webm;codecs=vp8", + "video/webm", +] as const; + +export function createBrowserMediaRecorder(stream: MediaStream): MediaRecorder { + const mimeType = preferredMimeTypes.find((candidate) => MediaRecorder.isTypeSupported(candidate)); + const settings = stream.getVideoTracks()[0]?.getSettings(); + const videoBitsPerSecond = Math.round( + Math.min( + 50_000_000, + Math.max( + 2_500_000, + (settings?.width ?? 1920) * (settings?.height ?? 1080) * (settings?.frameRate ?? 30) * 0.05, + ), + ), + ); + return new MediaRecorder(stream, { ...(mimeType ? { mimeType } : {}), videoBitsPerSecond }); +} diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 218684c54fbc..66014571e61b 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -300,20 +300,44 @@ describe("browser recording", () => { expect(stopTrack).toHaveBeenCalledOnce(); }); - it("uses the best supported encoder and saves the recorder's actual format", async () => { + it.each([ + ["video/mp4;codecs=avc3", "video/mp4;codecs=avc3"], + ["video/mp4;codecs=avc3.640028", "video/mp4;codecs=avc3.640028"], + ["video/mp4;codecs=avc3.42e01e", "video/mp4;codecs=avc3.42e01e"], + ["video/mp4;codecs=avc1", "video/webm;codecs=vp9"], + ])( + "selects a resize-capable recording format when %s is available", + async (supported, expected) => { + FakeMediaRecorder.supportedTypes = new Set([ + "video/mp4;codecs=avc1", + "video/mp4;codecs=avc1.640028", + "video/mp4;codecs=avc1.42e01e", + "video/webm;codecs=vp9", + supported, + ]); + + await startBrowserRecording("recording-tab"); + await stopBrowserRecording("recording-tab"); + + expect(FakeMediaRecorder.instances[0]?.options?.mimeType).toBe(expected); + }, + ); + + it("saves the recorder's actual format", async () => { FakeMediaRecorder.supportedTypes = new Set([ "video/mp4;codecs=avc1", "video/mp4;codecs=avc1.42e01e", "video/webm;codecs=vp9", "video/webm;codecs=av1", ]); + FakeMediaRecorder.supportedTypes.add("video/mp4;codecs=avc3"); FakeMediaRecorder.outputMimeType = "video/webm;codecs=av01"; await startBrowserRecording("recording-tab"); await stopBrowserRecording("recording-tab"); expect(FakeMediaRecorder.instances[0]?.options).toEqual({ - mimeType: "video/mp4;codecs=avc1", + mimeType: "video/mp4;codecs=avc3", videoBitsPerSecond: 3_110_400, }); expect(save).toHaveBeenCalledWith( diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 92050394a1c9..153824d54c9e 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -9,6 +9,7 @@ import { ensureClientSettingsHydrated, getClientSettings } from "~/hooks/useSett import { appAtomRegistry } from "~/rpc/atomRegistry"; import { acquireBrowserSurfaceActivity } from "./browserSurfaceStore"; +import { createBrowserMediaRecorder } from "./browserMediaRecorder"; export class BrowserRecordingUnavailableError extends Schema.TaggedErrorClass()( "BrowserRecordingUnavailableError", @@ -237,32 +238,6 @@ export function findActiveBrowserRecordingRuntimeTabId( ); } -const preferredMimeTypes = [ - "video/mp4;codecs=avc1", - "video/mp4;codecs=avc1.640028", - "video/mp4;codecs=avc1.42e01e", - "video/webm;codecs=vp9", - "video/webm;codecs=vp8", - "video/webm", -] as const; - -const createMediaRecorder = (stream: MediaStream): MediaRecorder => { - const mimeType = preferredMimeTypes.find((candidate) => MediaRecorder.isTypeSupported(candidate)); - const settings = stream.getVideoTracks()[0]?.getSettings(); - // Browser defaults under-budget native-resolution text and motion. Scale with captured pixels - // and frames, while bounding storage and encoder load for very large displays. - const videoBitsPerSecond = Math.round( - Math.min( - 50_000_000, - Math.max( - 2_500_000, - (settings?.width ?? 1920) * (settings?.height ?? 1080) * (settings?.frameRate ?? 30) * 0.05, - ), - ), - ); - return new MediaRecorder(stream, { ...(mimeType ? { mimeType } : {}), videoBitsPerSecond }); -}; - const captureTabMediaStream = (frameRate: number): Promise => // The desktop main process routes this request to the tab that `startScreencast` armed, so the // stream already arrives at that tab's native size and needs no source or dimension constraints. @@ -612,7 +587,7 @@ export async function startBrowserRecording( let recorder: MediaRecorder; try { - recorder = createMediaRecorder(stream); + recorder = createBrowserMediaRecorder(stream); recording.recorder = recorder; recorder.addEventListener("dataavailable", (event) => { if (event.data.size > 0) chunks.push(event.data);