diff --git a/electron/appPaths.ts b/electron/appPaths.ts index 75f7b51bf..68d6cfc39 100644 --- a/electron/appPaths.ts +++ b/electron/appPaths.ts @@ -8,4 +8,4 @@ if (process.env["VITE_DEV_SERVER_URL"]) { } export const USER_DATA_PATH = app.getPath("userData"); -export const RECORDINGS_DIR = path.join(USER_DATA_PATH, "recordings"); \ No newline at end of file +export const RECORDINGS_DIR = path.join(USER_DATA_PATH, "recordings"); diff --git a/electron/cursorHider.ts b/electron/cursorHider.ts index 228aa7dc5..266d111a9 100644 --- a/electron/cursorHider.ts +++ b/electron/cursorHider.ts @@ -1,4 +1,4 @@ -import { spawnSync } from 'node:child_process' +import { spawnSync } from "node:child_process"; const PY_HIDE_WIN = ` import ctypes, sys @@ -25,7 +25,7 @@ for _ in range(32): user32.ShowCursor(False) sys.exit(0) -`.trim() +`.trim(); const PY_SHOW_WIN = ` import ctypes, sys @@ -52,90 +52,93 @@ for _ in range(32): user32.ShowCursor(True) sys.exit(0) -`.trim() +`.trim(); function getPowerShellCommand(show: boolean) { - const desiredFlag = show ? 1 : 0 - const showLiteral = show ? '$true' : '$false' - - return [ - '$signature = @"', - 'using System;', - 'using System.Runtime.InteropServices;', - 'public struct POINT { public int X; public int Y; }', - 'public struct CURSORINFO { public int cbSize; public int flags; public IntPtr hCursor; public POINT ptScreenPos; }', - 'public static class CursorNative {', - ' [DllImport("user32.dll")] public static extern int ShowCursor(bool show);', - ' [DllImport("user32.dll")] public static extern bool GetCursorInfo(ref CURSORINFO info);', - '}', - '"@;', - 'Add-Type -TypeDefinition $signature -Language CSharp -ErrorAction SilentlyContinue | Out-Null;', - '$info = New-Object CURSORINFO;', - '$info.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type]CURSORINFO);', - 'for ($i = 0; $i -lt 32; $i++) {', - ' if ([CursorNative]::GetCursorInfo([ref]$info) -and (($info.flags -band 1) -eq ' + desiredFlag + ')) { exit 0 }', - ' [CursorNative]::ShowCursor(' + showLiteral + ') | Out-Null;', - '}', - 'exit 0', - ].join(' ') + const desiredFlag = show ? 1 : 0; + const showLiteral = show ? "$true" : "$false"; + + return [ + '$signature = @"', + "using System;", + "using System.Runtime.InteropServices;", + "public struct POINT { public int X; public int Y; }", + "public struct CURSORINFO { public int cbSize; public int flags; public IntPtr hCursor; public POINT ptScreenPos; }", + "public static class CursorNative {", + ' [DllImport("user32.dll")] public static extern int ShowCursor(bool show);', + ' [DllImport("user32.dll")] public static extern bool GetCursorInfo(ref CURSORINFO info);', + "}", + '"@;', + "Add-Type -TypeDefinition $signature -Language CSharp -ErrorAction SilentlyContinue | Out-Null;", + "$info = New-Object CURSORINFO;", + "$info.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type]CURSORINFO);", + "for ($i = 0; $i -lt 32; $i++) {", + " if ([CursorNative]::GetCursorInfo([ref]$info) -and (($info.flags -band 1) -eq " + + desiredFlag + + ")) { exit 0 }", + " [CursorNative]::ShowCursor(" + showLiteral + ") | Out-Null;", + "}", + "exit 0", + ].join(" "); } function runPythonSnippet(code: string) { - for (const executable of ['python', 'python3', 'py']) { - const result = spawnSync(executable, ['-c', code], { timeout: 5000 }) - if (!result.error && result.status === 0) { - return true - } - } - - return false + for (const executable of ["python", "python3", "py"]) { + const result = spawnSync(executable, ["-c", code], { timeout: 5000 }); + if (!result.error && result.status === 0) { + return true; + } + } + + return false; } function runPowerShellSnippet(command: string) { - const result = spawnSync( - 'powershell.exe', - ['-NoProfile', '-NonInteractive', '-WindowStyle', 'Hidden', '-Command', command], - { timeout: 8000 }, - ) + const result = spawnSync( + "powershell.exe", + ["-NoProfile", "-NonInteractive", "-WindowStyle", "Hidden", "-Command", command], + { timeout: 8000 }, + ); - return !result.error && result.status === 0 + return !result.error && result.status === 0; } -let cursorHidden = false +let cursorHidden = false; export function hideCursor() { - if (process.platform !== 'win32' || cursorHidden) { - return false - } - - try { - const didHide = runPythonSnippet(PY_HIDE_WIN) - || runPowerShellSnippet(getPowerShellCommand(false)) - - if (didHide) { - cursorHidden = true - } - - return didHide - } catch (error) { - console.error('[cursorHider] Failed to hide Windows cursor:', error) - return false - } + if (process.platform !== "win32" || cursorHidden) { + return false; + } + + try { + const didHide = + runPythonSnippet(PY_HIDE_WIN) || runPowerShellSnippet(getPowerShellCommand(false)); + + if (didHide) { + cursorHidden = true; + } + + return didHide; + } catch (error) { + console.error("[cursorHider] Failed to hide Windows cursor:", error); + return false; + } } export function showCursor() { - if (process.platform !== 'win32' || !cursorHidden) { - return false - } - - try { - const didShow = runPythonSnippet(PY_SHOW_WIN) - || runPowerShellSnippet(getPowerShellCommand(true)) - return didShow - } catch (error) { - console.error('[cursorHider] Failed to show Windows cursor:', error) - return false - } finally { - cursorHidden = false - } -} \ No newline at end of file + if (process.platform !== "win32" || !cursorHidden) { + return false; + } + + try { + const didShow = + runPythonSnippet(PY_SHOW_WIN) || runPowerShellSnippet(getPowerShellCommand(true)); + if (didShow) { + cursorHidden = false; + } + return didShow; + } catch (error) { + console.error("[cursorHider] Failed to show Windows cursor:", error); + return false; + } +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index c6929171f..c6abd25be 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -1,5 +1,6 @@ /// +// biome-ignore lint/style/noNamespace: NodeJS.ProcessEnv augmentation requires a namespace declaration. declare namespace NodeJS { interface ProcessEnv { /** @@ -60,6 +61,14 @@ interface UpdateStatusSummary { detail?: string; } +type RendererExtensionInfo = import("./extensions/extensionTypes").ExtensionInfo; +type RendererExtensionReview = import("./extensions/extensionTypes").ExtensionReview; +type RendererMarketplaceExtension = import("./extensions/extensionTypes").MarketplaceExtension; +type RendererMarketplaceReviewStatus = + import("./extensions/extensionTypes").MarketplaceReviewStatus; +type RendererMarketplaceSearchResult = + import("./extensions/extensionTypes").MarketplaceSearchResult; + interface Window { electronAPI: { hudOverlaySetIgnoreMouse: (ignore: boolean) => void; @@ -73,15 +82,18 @@ interface Window { setHudOverlayCaptureProtection: ( enabled: boolean, ) => Promise<{ success: boolean; enabled: boolean }>; + getAssetBasePath: () => Promise; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; openSourceSelector: () => Promise; - selectSource: (source: any) => Promise; - showSourceHighlight: (source: any) => Promise<{ success: boolean }>; - getSelectedSource: () => Promise; - onSelectedSourceChanged: (callback: (source: any) => void) => () => void; + selectSource: (source: ProcessedDesktopSource) => Promise; + showSourceHighlight: (source: ProcessedDesktopSource) => Promise<{ success: boolean }>; + getSelectedSource: () => Promise; + onSelectedSourceChanged: ( + callback: (source: ProcessedDesktopSource | null) => void, + ) => () => void; startNativeScreenRecording: ( - source: any, + source: ProcessedDesktopSource, options?: { capturesSystemAudio?: boolean; capturesMicrophone?: boolean; @@ -123,7 +135,7 @@ interface Window { error?: string; }>; startFfmpegRecording: ( - source: any, + source: ProcessedDesktopSource, ) => Promise<{ success: boolean; path?: string; message?: string; error?: string }>; stopFfmpegRecording: () => Promise<{ success: boolean; @@ -145,7 +157,6 @@ interface Window { files?: string[]; error?: string; }>; - getAssetBasePath: () => Promise; readLocalFile: ( filePath: string, ) => Promise<{ success: boolean; data?: Uint8Array; error?: string }>; @@ -158,6 +169,7 @@ interface Window { frameRate: number; bitrate: number; encodingMode: "fast" | "balanced" | "quality"; + inputMode?: "rawvideo" | "h264-stream"; }) => Promise<{ success: boolean; sessionId?: string; @@ -459,14 +471,14 @@ interface Window { cancelCountdown: () => Promise<{ success: boolean }>; getActiveCountdown: () => Promise<{ success: boolean; seconds: number | null }>; onCountdownTick: (callback: (seconds: number) => void) => () => void; - extensionsDiscover: () => Promise; - extensionsList: () => Promise; - extensionsGet: (id: string) => Promise; + extensionsDiscover: () => Promise; + extensionsList: () => Promise; + extensionsGet: (id: string) => Promise; extensionsEnable: (id: string) => Promise<{ success: boolean; error?: string }>; extensionsDisable: (id: string) => Promise<{ success: boolean; error?: string }>; extensionsInstallFromFolder: () => Promise<{ success: boolean; - extension?: any; + extension?: RendererExtensionInfo; message?: string; error?: string; canceled?: boolean; @@ -480,8 +492,8 @@ interface Window { sort?: string; page?: number; pageSize?: number; - }) => Promise; - extensionsMarketplaceGet: (id: string) => Promise; + }) => Promise; + extensionsMarketplaceGet: (id: string) => Promise; extensionsMarketplaceInstall: ( extensionId: string, downloadUrl: string, @@ -490,13 +502,13 @@ interface Window { extensionId: string, ) => Promise<{ success: boolean; reviewId?: string; error?: string }>; extensionsReviewsList: (params: { - status?: string; + status?: RendererMarketplaceReviewStatus; page?: number; pageSize?: number; - }) => Promise<{ reviews: any[]; total: number }>; + }) => Promise<{ reviews: RendererExtensionReview[]; total: number; error?: string }>; extensionsReviewUpdate: ( reviewId: string, - status: string, + status: RendererMarketplaceReviewStatus, notes?: string, ) => Promise<{ success: boolean; error?: string }>; }; @@ -518,6 +530,7 @@ interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + pressure?: number; interactionType?: | "move" | "click" diff --git a/electron/extensions/errorUtils.ts b/electron/extensions/errorUtils.ts new file mode 100644 index 000000000..ae5ccfdac --- /dev/null +++ b/electron/extensions/errorUtils.ts @@ -0,0 +1,3 @@ +export function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} \ No newline at end of file diff --git a/electron/extensions/extensionIpc.ts b/electron/extensions/extensionIpc.ts index 7d6a6b454..d1a32b673 100644 --- a/electron/extensions/extensionIpc.ts +++ b/electron/extensions/extensionIpc.ts @@ -23,6 +23,7 @@ import { submitExtensionForReview, updateReviewStatus, } from "./extensionMarketplace"; +import { getErrorMessage } from "./errorUtils"; import type { ExtensionInfo, MarketplaceReviewStatus } from "./extensionTypes"; /** @@ -129,13 +130,13 @@ export function registerExtensionIpcHandlers(): void { ) => { try { return await searchMarketplace(params); - } catch (err: unknown) { + } catch (error: unknown) { return { extensions: [], total: 0, page: 1, pageSize: 20, - error: err instanceof Error ? err.message : String(err), + error: getErrorMessage(error), }; } }, @@ -174,12 +175,8 @@ export function registerExtensionIpcHandlers(): void { ) => { try { return await fetchPendingReviews(params); - } catch (err: unknown) { - return { - reviews: [], - total: 0, - error: err instanceof Error ? err.message : String(err), - }; + } catch (error: unknown) { + return { reviews: [], total: 0, error: getErrorMessage(error) }; } }, ); diff --git a/electron/extensions/extensionMarketplace.ts b/electron/extensions/extensionMarketplace.ts index 9a951bc1d..e8bc66ddf 100644 --- a/electron/extensions/extensionMarketplace.ts +++ b/electron/extensions/extensionMarketplace.ts @@ -5,25 +5,27 @@ * Recordly marketplace API. Also provides admin review endpoints. */ -import path from 'node:path'; -import fs from 'node:fs/promises'; -import { createWriteStream, existsSync } from 'node:fs'; -import { app } from 'electron'; -import { pipeline } from 'node:stream/promises'; -import { Readable } from 'node:stream'; -import { installExtensionFromPath, getRegisteredExtensions } from './extensionLoader'; +import { createWriteStream, existsSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream as NodeReadableStream } from "node:stream/web"; +import { app } from "electron"; +import { getErrorMessage } from "./errorUtils"; +import { getRegisteredExtensions, installExtensionFromPath } from "./extensionLoader"; import type { - MarketplaceExtension, - MarketplaceSearchResult, - MarketplaceReviewStatus, - ExtensionReview, -} from './extensionTypes'; + ExtensionReview, + MarketplaceExtension, + MarketplaceReviewStatus, + MarketplaceSearchResult, +} from "./extensionTypes"; // --------------------------------------------------------------------------- // Configuration // --------------------------------------------------------------------------- -const MARKETPLACE_API_BASE = 'https://marketplace.recordly.dev/extensions/api/v1'; +const MARKETPLACE_API_BASE = "https://marketplace.recordly.dev/extensions/api/v1"; const REQUEST_TIMEOUT_MS = 15_000; // --------------------------------------------------------------------------- @@ -33,29 +35,31 @@ const REQUEST_TIMEOUT_MS = 15_000; // --------------------------------------------------------------------------- async function assertNoEscapedFiles(dir: string, root: string): Promise { - const entries = await fs.readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const entryPath = path.join(dir, entry.name); - const real = await fs.realpath(entryPath); - if (!real.startsWith(root + path.sep) && real !== root) { - // Nuke the escaped file/symlink and throw - await fs.rm(entryPath, { recursive: true, force: true }).catch(() => {}); - throw new Error(`Zip-slip detected: ${entry.name} resolves outside extraction directory`); - } - if (entry.isDirectory()) { - await assertNoEscapedFiles(entryPath, root); - } - } + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const entryPath = path.join(dir, entry.name); + const real = await fs.realpath(entryPath); + if (!real.startsWith(root + path.sep) && real !== root) { + // Nuke the escaped file/symlink and throw + await fs.rm(entryPath, { recursive: true, force: true }).catch(() => undefined); + throw new Error( + `Zip-slip detected: ${entry.name} resolves outside extraction directory`, + ); + } + if (entry.isDirectory()) { + await assertNoEscapedFiles(entryPath, root); + } + } } function getMarketplaceUrl(): string { - // Allow explicit override for local marketplace development. - if (process.env.RECORDLY_MARKETPLACE_URL) return process.env.RECORDLY_MARKETPLACE_URL; - return MARKETPLACE_API_BASE; + // Allow explicit override for local marketplace development. + if (process.env.RECORDLY_MARKETPLACE_URL) return process.env.RECORDLY_MARKETPLACE_URL; + return MARKETPLACE_API_BASE; } function getAdminKey(): string | undefined { - return process.env.RECORDLY_ADMIN_KEY; + return process.env.RECORDLY_ADMIN_KEY; } // --------------------------------------------------------------------------- @@ -63,43 +67,43 @@ function getAdminKey(): string | undefined { // --------------------------------------------------------------------------- async function marketplaceFetch( - endpoint: string, - options: { method?: string; body?: unknown; timeout?: number; admin?: boolean } = {}, + endpoint: string, + options: { method?: string; body?: unknown; timeout?: number; admin?: boolean } = {}, ): Promise { - const url = `${getMarketplaceUrl()}${endpoint}`; - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), options.timeout ?? REQUEST_TIMEOUT_MS); - - try { - const headers: Record = { - 'Content-Type': 'application/json', - 'X-Recordly-Version': app.getVersion(), - 'X-Recordly-Platform': process.platform, - }; - - // Attach admin key for privileged endpoints - if (options.admin) { - const key = getAdminKey(); - if (!key) throw new Error('Admin key not configured (set RECORDLY_ADMIN_KEY env var)'); - headers['X-Admin-Key'] = key; - } - - const response = await fetch(url, { - method: options.method ?? 'GET', - headers, - body: options.body ? JSON.stringify(options.body) : undefined, - signal: controller.signal, - }); - - if (!response.ok) { - const text = await response.text().catch(() => ''); - throw new Error(`Marketplace API error ${response.status}: ${text}`); - } - - return await response.json() as T; - } finally { - clearTimeout(timeoutId); - } + const url = `${getMarketplaceUrl()}${endpoint}`; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), options.timeout ?? REQUEST_TIMEOUT_MS); + + try { + const headers: Record = { + "Content-Type": "application/json", + "X-Recordly-Version": app.getVersion(), + "X-Recordly-Platform": process.platform, + }; + + // Attach admin key for privileged endpoints + if (options.admin) { + const key = getAdminKey(); + if (!key) throw new Error("Admin key not configured (set RECORDLY_ADMIN_KEY env var)"); + headers["X-Admin-Key"] = key; + } + + const response = await fetch(url, { + method: options.method ?? "GET", + headers, + body: options.body ? JSON.stringify(options.body) : undefined, + signal: controller.signal, + }); + + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`Marketplace API error ${response.status}: ${text}`); + } + + return (await response.json()) as T; + } finally { + clearTimeout(timeoutId); + } } // --------------------------------------------------------------------------- @@ -110,46 +114,48 @@ async function marketplaceFetch( * Search/browse marketplace extensions. */ export async function searchMarketplace(params: { - query?: string; - tags?: string[]; - sort?: 'popular' | 'recent' | 'rating'; - page?: number; - pageSize?: number; + query?: string; + tags?: string[]; + sort?: "popular" | "recent" | "rating"; + page?: number; + pageSize?: number; }): Promise { - const searchParams = new URLSearchParams(); - if (params.query) searchParams.set('query', params.query); - if (params.tags?.length) searchParams.set('tags', params.tags.join(',')); - if (params.sort) searchParams.set('sort', params.sort); - if (params.page) searchParams.set('page', String(params.page)); - if (params.pageSize) searchParams.set('pageSize', String(params.pageSize)); - - const qs = searchParams.toString(); - const result = await marketplaceFetch( - `/extensions${qs ? `?${qs}` : ''}`, - ); - - // Mark installed extensions - const installed = getRegisteredExtensions(); - const installedIds = new Set(installed.map(e => e.manifest.id)); - for (const ext of result.extensions) { - ext.installed = installedIds.has(ext.id); - } - - return result; + const searchParams = new URLSearchParams(); + if (params.query) searchParams.set("query", params.query); + if (params.tags?.length) searchParams.set("tags", params.tags.join(",")); + if (params.sort) searchParams.set("sort", params.sort); + if (params.page) searchParams.set("page", String(params.page)); + if (params.pageSize) searchParams.set("pageSize", String(params.pageSize)); + + const qs = searchParams.toString(); + const result = await marketplaceFetch( + `/extensions${qs ? `?${qs}` : ""}`, + ); + + // Mark installed extensions + const installed = getRegisteredExtensions(); + const installedIds = new Set(installed.map((e) => e.manifest.id)); + for (const ext of result.extensions) { + ext.installed = installedIds.has(ext.id); + } + + return result; } /** * Get a single marketplace extension by ID. */ export async function getMarketplaceExtension(id: string): Promise { - try { - const ext = await marketplaceFetch(`/extensions/${encodeURIComponent(id)}`); - const installed = getRegisteredExtensions(); - ext.installed = installed.some(e => e.manifest.id === ext.id); - return ext; - } catch { - return null; - } + try { + const ext = await marketplaceFetch( + `/extensions/${encodeURIComponent(id)}`, + ); + const installed = getRegisteredExtensions(); + ext.installed = installed.some((e) => e.manifest.id === ext.id); + return ext; + } catch { + return null; + } } /** @@ -157,130 +163,138 @@ export async function getMarketplaceExtension(id: string): Promise { - // Validate download URL against allowed marketplace origins - const allowedOrigins = [ - 'https://marketplace.recordly.dev', - 'https://recordly.dev', - ...(app.isPackaged ? [] : ['http://localhost:3001']), - ]; - try { - const url = new URL(downloadUrl); - if (!allowedOrigins.some(o => url.origin === o)) { - return { success: false, error: `Untrusted download origin: ${url.origin}` }; - } - } catch { - return { success: false, error: 'Invalid download URL' }; - } - - const tempDir = path.join(app.getPath('temp'), `recordly-ext-${extensionId}-${Date.now()}`); - const zipPath = path.join(tempDir, 'extension.zip'); - - try { - // Create temp directory - await fs.mkdir(tempDir, { recursive: true }); - - // Download the archive - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 60_000); - - let response: Response; - try { - response = await fetch(downloadUrl, { - signal: controller.signal, - headers: { - 'X-Recordly-Version': app.getVersion(), - }, - }); - } finally { - clearTimeout(timeoutId); - } - - if (!response.ok) { - throw new Error(`Download failed with status ${response.status}`); - } - - if (!response.body) { - throw new Error('Download response has no body'); - } - - // Write to disk - const fileStream = createWriteStream(zipPath); - await pipeline(Readable.fromWeb(response.body as any), fileStream); - - // Extract the zip — use the built-in decompress or shell unzip - const extractDir = path.join(tempDir, 'extracted'); - await fs.mkdir(extractDir, { recursive: true }); - - // Use Node's built-in unzip capability via child_process (execFile — no shell) - const { execFile } = await import('node:child_process'); - await new Promise((resolve, reject) => { - if (process.platform === 'win32') { - // Use -LiteralPath to avoid PowerShell injection via single-quote in paths - execFile('powershell', [ - '-NoProfile', - '-NonInteractive', - '-command', - 'Expand-Archive', - '-LiteralPath', zipPath, - '-DestinationPath', extractDir, - '-Force', - ], (error) => { - if (error) reject(error); - else resolve(); - }); - } else { - execFile('unzip', ['-o', zipPath, '-d', extractDir], (error) => { - if (error) reject(error); - else resolve(); - }); - } - }); - - // Security: verify no extracted file escaped the extraction directory - // (protects against zip-slip / path traversal entries in malicious archives) - // Use fs.realpath so the root matches what fs.realpath returns for children - // (on macOS /var is a symlink to /private/var — path.resolve does not - // resolve symlinks, so root and children would mismatch). - const resolvedExtractDir = await fs.realpath(extractDir); - await assertNoEscapedFiles(resolvedExtractDir, resolvedExtractDir); - - // Find the manifest — it might be in a subfolder - const entries = await fs.readdir(extractDir, { withFileTypes: true }); - let manifestDir = extractDir; - - // If there's a single directory, look inside it - const dirs = entries.filter(e => e.isDirectory()); - if (dirs.length === 1 && !existsSync(path.join(extractDir, 'recordly-extension.json'))) { - manifestDir = path.join(extractDir, dirs[0].name); - } - - // Verify manifest exists - if (!existsSync(path.join(manifestDir, 'recordly-extension.json'))) { - throw new Error('Downloaded extension does not contain a recordly-extension.json manifest'); - } - - // Install from the extracted directory - const info = await installExtensionFromPath(manifestDir); - if (!info) { - throw new Error('Extension validation failed after download'); - } - - // Track download count (fire-and-forget — CDN may cache the GET, so POST separately) - fetch(`${MARKETPLACE_API_BASE}/extensions/${encodeURIComponent(extensionId)}/download`, { - method: 'POST', - headers: { 'X-Recordly-Version': app.getVersion() }, - }).catch(() => {}); - - return { success: true }; - } catch (err: any) { - return { success: false, error: err.message ?? String(err) }; - } finally { - // Clean up temp directory - await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {}); - } + // Validate download URL against allowed marketplace origins + const allowedOrigins = [ + "https://marketplace.recordly.dev", + "https://recordly.dev", + ...(app.isPackaged ? [] : ["http://localhost:3001"]), + ]; + try { + const url = new URL(downloadUrl); + if (!allowedOrigins.some((o) => url.origin === o)) { + return { success: false, error: `Untrusted download origin: ${url.origin}` }; + } + } catch { + return { success: false, error: "Invalid download URL" }; + } + + const tempDir = path.join(app.getPath("temp"), `recordly-ext-${extensionId}-${Date.now()}`); + const zipPath = path.join(tempDir, "extension.zip"); + + try { + // Create temp directory + await fs.mkdir(tempDir, { recursive: true }); + + // Download the archive + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 60_000); + + let response: Response; + try { + response = await fetch(downloadUrl, { + signal: controller.signal, + headers: { + "X-Recordly-Version": app.getVersion(), + }, + }); + } finally { + clearTimeout(timeoutId); + } + + if (!response.ok) { + throw new Error(`Download failed with status ${response.status}`); + } + + if (!response.body) { + throw new Error("Download response has no body"); + } + + // Write to disk + const fileStream = createWriteStream(zipPath); + await pipeline(Readable.fromWeb(response.body as NodeReadableStream), fileStream); + + // Extract the zip — use the built-in decompress or shell unzip + const extractDir = path.join(tempDir, "extracted"); + await fs.mkdir(extractDir, { recursive: true }); + + // Use Node's built-in unzip capability via child_process (execFile — no shell) + const { execFile } = await import("node:child_process"); + await new Promise((resolve, reject) => { + if (process.platform === "win32") { + // Use -LiteralPath to avoid PowerShell injection via single-quote in paths + execFile( + "powershell", + [ + "-NoProfile", + "-NonInteractive", + "-command", + "Expand-Archive", + "-LiteralPath", + zipPath, + "-DestinationPath", + extractDir, + "-Force", + ], + (error) => { + if (error) reject(error); + else resolve(); + }, + ); + } else { + execFile("unzip", ["-o", zipPath, "-d", extractDir], (error) => { + if (error) reject(error); + else resolve(); + }); + } + }); + + // Security: verify no extracted file escaped the extraction directory + // (protects against zip-slip / path traversal entries in malicious archives) + // Use fs.realpath so the root matches what fs.realpath returns for children + // (on macOS /var is a symlink to /private/var — path.resolve does not + // resolve symlinks, so root and children would mismatch). + const resolvedExtractDir = await fs.realpath(extractDir); + await assertNoEscapedFiles(resolvedExtractDir, resolvedExtractDir); + + // Find the manifest — it might be in a subfolder + const entries = await fs.readdir(extractDir, { withFileTypes: true }); + let manifestDir = extractDir; + + // If there's a single directory, look inside it for the manifest. + const dirs = entries.filter((e) => e.isDirectory()); + if (dirs.length === 1 && !existsSync(path.join(extractDir, "recordly-extension.json"))) { + manifestDir = path.join(extractDir, dirs[0].name); + } + + // Verify manifest exists + if (!existsSync(path.join(manifestDir, "recordly-extension.json"))) { + throw new Error( + "Downloaded extension does not contain a recordly-extension.json manifest", + ); + } + + // Install from the extracted directory + const info = await installExtensionFromPath(manifestDir); + if (!info) { + throw new Error("Extension validation failed after download"); + } + + // Track download count (fire-and-forget — CDN may cache the GET, so POST separately) + fetch(`${getMarketplaceUrl()}/extensions/${encodeURIComponent(extensionId)}/download`, { + method: "POST", + headers: { "X-Recordly-Version": app.getVersion() }, + }).catch(() => undefined); + + return { success: true }; + } catch (error: unknown) { + return { success: false, error: getErrorMessage(error) }; + } finally { + // Clean up temp directory + await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined); + } } // --------------------------------------------------------------------------- @@ -291,47 +305,52 @@ export async function downloadAndInstallExtension( * Fetch extensions pending review (admin only). */ export async function fetchPendingReviews(params: { - status?: MarketplaceReviewStatus; - page?: number; - pageSize?: number; + status?: MarketplaceReviewStatus; + page?: number; + pageSize?: number; }): Promise<{ reviews: ExtensionReview[]; total: number }> { - const searchParams = new URLSearchParams(); - if (params.status) searchParams.set('status', params.status); - if (params.page) searchParams.set('page', String(params.page)); - if (params.pageSize) searchParams.set('pageSize', String(params.pageSize)); - - const qs = searchParams.toString(); - return marketplaceFetch<{ reviews: ExtensionReview[]; total: number }>( - `/admin/reviews${qs ? `?${qs}` : ''}`, - { admin: true }, - ); + const searchParams = new URLSearchParams(); + if (params.status) searchParams.set("status", params.status); + if (params.page) searchParams.set("page", String(params.page)); + if (params.pageSize) searchParams.set("pageSize", String(params.pageSize)); + + const qs = searchParams.toString(); + return marketplaceFetch<{ reviews: ExtensionReview[]; total: number }>( + `/admin/reviews${qs ? `?${qs}` : ""}`, + { admin: true }, + ); } /** * Update the review status of a submitted extension (admin only). */ export async function updateReviewStatus( - reviewId: string, - status: MarketplaceReviewStatus, - notes?: string, + reviewId: string, + status: MarketplaceReviewStatus, + notes?: string, ): Promise<{ success: boolean }> { - return marketplaceFetch<{ success: boolean }>(`/admin/reviews/${encodeURIComponent(reviewId)}`, { - method: 'PATCH', - body: { status, notes }, - admin: true, - }); + return marketplaceFetch<{ success: boolean }>( + `/admin/reviews/${encodeURIComponent(reviewId)}`, + { + method: "PATCH", + body: { status, notes }, + admin: true, + }, + ); } /** * Submit an extension for marketplace review. */ -export async function submitExtensionForReview(extensionId: string): Promise<{ success: boolean; reviewId?: string; error?: string }> { - try { - return await marketplaceFetch<{ success: boolean; reviewId?: string }>( - `/extensions/${encodeURIComponent(extensionId)}/submit`, - { method: 'POST' }, - ); - } catch (err: any) { - return { success: false, error: err.message ?? String(err) }; - } +export async function submitExtensionForReview( + extensionId: string, +): Promise<{ success: boolean; reviewId?: string; error?: string }> { + try { + return await marketplaceFetch<{ success: boolean; reviewId?: string }>( + `/extensions/${encodeURIComponent(extensionId)}/submit`, + { method: "POST" }, + ); + } catch (error: unknown) { + return { success: false, error: getErrorMessage(error) }; + } } diff --git a/electron/extensions/extensionTypes.ts b/electron/extensions/extensionTypes.ts index 8f21218ef..c89e3a780 100644 --- a/electron/extensions/extensionTypes.ts +++ b/electron/extensions/extensionTypes.ts @@ -1,99 +1,16 @@ /** * Re-export of extension types for use in the main process. * The canonical types live in src/lib/extensions/types.ts. - * This file mirrors the subset needed by the electron layer. */ -export type ExtensionPermission = - | "render" - | "cursor" - | "audio" - | "timeline" - | "ui" - | "assets" - | "export"; - -export interface ExtensionManifest { - id: string; - name: string; - version: string; - description: string; - author?: string; - homepage?: string; - license?: string; - engine?: string; - icon?: string; - main: string; - permissions: ExtensionPermission[]; - contributes?: ExtensionContributions; -} - -export interface ExtensionContributions { - /** Informational manifest metadata only; runtime registration still happens from activate(). */ - cursorStyles?: { - id: string; - label: string; - defaultImage: string; - clickImage?: string; - hotspot?: { x: number; y: number }; - }[]; - sounds?: { id: string; label: string; category: string; file: string; durationMs?: number }[]; - wallpapers?: { id: string; label: string; file: string; thumbnail?: string; isVideo?: boolean }[]; - webcamFrames?: { id: string; label: string; file: string; thumbnail?: string }[]; -} - -export type ExtensionStatus = "installed" | "active" | "disabled" | "error"; - -export interface ExtensionInfo { - manifest: ExtensionManifest; - status: ExtensionStatus; - path: string; - error?: string; - builtin?: boolean; -} - -// --------------------------------------------------------------------------- -// Marketplace Types -// --------------------------------------------------------------------------- - -export type MarketplaceReviewStatus = "pending" | "approved" | "rejected" | "flagged"; - -export interface MarketplaceExtension { - id: string; - name: string; - version: string; - description: string; - author: string; - downloadUrl: string; - iconUrl?: string; - screenshots?: string[]; - downloads: number; - rating: number; - ratingCount: number; - tags: string[]; - permissions: ExtensionPermission[]; - reviewStatus: MarketplaceReviewStatus; - publishedAt: string; - updatedAt: string; - installed?: boolean; -} - -export interface MarketplaceSearchResult { - extensions: MarketplaceExtension[]; - total: number; - page: number; - pageSize: number; -} - -export interface ExtensionReview { - id: string; - extensionId: string; - extensionName: string; - version: string; - author: string; - submittedAt: string; - status: MarketplaceReviewStatus; - reviewNotes?: string; - manifest: ExtensionManifest; - downloadUrl: string; -} +export type { + ExtensionContributions, + ExtensionInfo, + ExtensionManifest, + ExtensionPermission, + ExtensionReview, + ExtensionStatus, + MarketplaceExtension, + MarketplaceReviewStatus, + MarketplaceSearchResult, +} from "../../src/lib/extensions/types"; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index fc138e34e..9e7d56be7 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -22,6 +22,7 @@ import { RECORDINGS_DIR, USER_DATA_PATH } from "../appPaths"; import { hideCursor, showCursor } from "../cursorHider"; import { closeCountdownWindow, createCountdownWindow, getCountdownWindow } from "../windows"; import { + buildNativeH264StreamExportArgs, buildNativeVideoExportArgs, buildTrimmedSourceAudioFilter, getEditedAudioExtension, @@ -167,6 +168,17 @@ let nativeScreenRecordingActive = false; let currentVideoPath: string | null = null; let currentRecordingSession: RecordingSessionData | null = null; const approvedLocalReadPaths = new Set(); +function approveUserPath(filePath: string | null | undefined) { + if (!filePath) { + return; + } + + try { + approvedLocalReadPaths.add(path.resolve(filePath)); + } catch { + // Ignore invalid paths; later reads will surface the underlying error. + } +} let nativeCaptureProcess: ChildProcessWithoutNullStreams | null = null; let nativeCaptureOutputBuffer = ""; let nativeCaptureTargetPath: string | null = null; @@ -631,6 +643,21 @@ async function loadProjectFromPath(projectPath: string) { currentProjectPath = normalizedPath; currentVideoPath = mediaSources.videoPath; + const projectObj = project as Record; + const editorObj = projectObj?.editor as Record | undefined; + const audioTracks = editorObj?.audioTracks as { sourcePath?: unknown }[] | undefined; + const approvedProjectPaths: Array = [ + mediaSources.videoPath, + mediaSources.webcamPath, + ]; + if (Array.isArray(audioTracks)) { + for (const track of audioTracks) { + if (typeof track?.sourcePath === "string") { + approvedProjectPaths.push(track.sourcePath); + } + } + } + await replaceApprovedSessionLocalReadPaths(approvedProjectPaths); currentRecordingSession = { videoPath: mediaSources.videoPath, webcamPath: mediaSources.webcamPath, @@ -675,12 +702,7 @@ function isPathInsideDirectory(candidatePath: string, directoryPath: string) { } function isAllowedLocalReadPath(candidatePath: string) { - const allowedPrefixes = [ - RECORDINGS_DIR, - USER_DATA_PATH, - getAssetRootPath(), - app.getPath("temp"), - ]; + const allowedPrefixes = [RECORDINGS_DIR, USER_DATA_PATH, getAssetRootPath(), app.getPath("temp")]; return ( existsSync(candidatePath) || @@ -1083,11 +1105,7 @@ let nativeHelperMigrationPromise: Promise | null = null; async function migrateLegacyNativeHelperBinaries() { const legacyToCurrentPaths: Array<[string, string]> = [ [ - path.join( - app.getPath("userData"), - "native-tools", - "openscreen-screencapturekit-helper", - ), + path.join(app.getPath("userData"), "native-tools", "openscreen-screencapturekit-helper"), getNativeCaptureHelperBinaryPath(), ], [ @@ -1319,26 +1337,9 @@ function loadFfmpegStatic() { return null; } -function resolveSystemFfmpegBinaryPath() { - const locator = process.platform === "win32" ? "where" : "which"; - const result = spawnSync(locator, ["ffmpeg"], { - encoding: "utf-8", - windowsHide: true, - }); - - if (result.status !== 0) { - return null; - } - - const candidate = result.stdout - .split(/\r?\n/) - .map((line) => line.trim()) - .find((line) => line.length > 0); - - return candidate || null; -} +type HookEventName = "mousedown" | "mouseup" | "mousemove"; -type HookCursorEvent = { +type HookMouseEvent = { button?: number; mouseButton?: number; x?: number; @@ -1355,61 +1356,107 @@ type HookCursorEvent = { }; }; -type HookEventName = "mousedown" | "mouseup" | "mousemove"; - -type HookCursorHandler = (event: HookCursorEvent) => void; +type HookEventListener = (event: HookMouseEvent) => void; type UiohookLike = { - on?: (eventName: HookEventName, listener: HookCursorHandler) => void; - off?: (eventName: HookEventName, listener: HookCursorHandler) => void; - removeListener?: (eventName: HookEventName, listener: HookCursorHandler) => void; - start?: () => void; + on: (eventName: HookEventName, listener: HookEventListener) => void; + off?: (eventName: HookEventName, listener: HookEventListener) => void; + removeListener?: (eventName: HookEventName, listener: HookEventListener) => void; + start: () => void; stop?: () => void; +}; + +type UiohookModuleNamespace = { uIOhook?: UiohookLike; uiohook?: UiohookLike; Uiohook?: UiohookLike; - default?: UiohookLike; + default?: UiohookLike | UiohookModuleNamespace; }; -function loadUiohookModule() { - const moduleExports = nodeRequire("uiohook-napi") as UiohookLike; - return ( - moduleExports.uIOhook ?? - moduleExports.uiohook ?? - moduleExports.Uiohook ?? - moduleExports.default?.uIOhook ?? - moduleExports.default?.uiohook ?? - moduleExports.default ?? - moduleExports - ); +function isUiohookLike(value: unknown): value is UiohookLike { + const candidate = value as Partial | null; + return typeof candidate?.on === "function" && typeof candidate?.start === "function"; } -function getFfmpegBinaryPath() { - const ffmpegStatic = loadFfmpegStatic(); - if (ffmpegStatic && typeof ffmpegStatic === "string") { - const bundledPath = app.isPackaged - ? ffmpegStatic.replace(/\.asar([/\\])/, ".asar.unpacked$1") - : ffmpegStatic; - - if (existsSync(bundledPath)) { - return bundledPath; - } +function resolveSystemFfmpegBinaryPath() { + const locator = process.platform === 'win32' ? 'where' : 'which' + const result = spawnSync(locator, ['ffmpeg'], { + encoding: 'utf-8', + windowsHide: true, + }) + + if (result.status !== 0) { + return null + } + + const candidate = result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line.length > 0) + + return candidate || null +} + +function loadUiohookModule() { + const moduleExports = nodeRequire("uiohook-napi") as UiohookModuleNamespace; + const defaultExport = moduleExports.default; + + if (moduleExports.uIOhook) { + return moduleExports.uIOhook; } - const systemFfmpeg = resolveSystemFfmpegBinaryPath(); - if (systemFfmpeg) { - return systemFfmpeg; + if (moduleExports.uiohook) { + return moduleExports.uiohook; } - throw new Error( - "FFmpeg binary is unavailable. Install ffmpeg-static for this platform or make ffmpeg available on PATH.", - ); + if (moduleExports.Uiohook) { + return moduleExports.Uiohook; + } + + if (isUiohookLike(defaultExport)) { + return defaultExport; + } + + if (defaultExport?.uIOhook) { + return defaultExport.uIOhook; + } + + if (defaultExport?.uiohook) { + return defaultExport.uiohook; + } + + if (defaultExport?.Uiohook) { + return defaultExport.Uiohook; + } + + return null; +} + +function getFfmpegBinaryPath() { + const ffmpegStatic = loadFfmpegStatic() + if (ffmpegStatic && typeof ffmpegStatic === 'string') { + const bundledPath = app.isPackaged + ? ffmpegStatic.replace(/\.asar([\/\\])/, '.asar.unpacked$1') + : ffmpegStatic + + if (existsSync(bundledPath)) { + return bundledPath + } + } + + const systemFfmpeg = resolveSystemFfmpegBinaryPath() + if (systemFfmpeg) { + return systemFfmpeg + } + + throw new Error('FFmpeg binary is unavailable. Install ffmpeg-static for this platform or make ffmpeg available on PATH.') } type NativeVideoExportSession = { ffmpegProcess: ChildProcessByStdio; outputPath: string; inputByteSize: number; + inputMode: 'rawvideo' | 'h264-stream'; maxQueuedWriteBytes: number; stderrOutput: string; encoderName: string; @@ -1445,6 +1492,7 @@ export function cleanupNativeVideoExportSessions() { } function getNativeVideoExportMaxQueuedWriteBytes(inputByteSize: number) { + if (inputByteSize === 0) return 8 * 1024 * 1024; // H264 stream: variable-size chunks return Math.min(64 * 1024 * 1024, Math.max(16 * 1024 * 1024, inputByteSize * 4)); } @@ -1591,7 +1639,7 @@ async function writeNativeVideoExportFrame( session: NativeVideoExportSession, frameData: Uint8Array | ArrayBuffer, ) { - if (getNativeVideoExportFrameLength(frameData) !== session.inputByteSize) { + if (session.inputMode !== 'h264-stream' && getNativeVideoExportFrameLength(frameData) !== session.inputByteSize) { throw new Error( `Native video export expected ${session.inputByteSize} bytes per frame but received ${getNativeVideoExportFrameLength(frameData)}`, ); @@ -1827,31 +1875,31 @@ async function muxNativeVideoExportAudio( } async function muxExportedVideoAudioBuffer( - videoData: ArrayBuffer, - options: NativeVideoExportFinishOptions, + videoData: ArrayBuffer, + options: NativeVideoExportFinishOptions, ) { - const tempVideoPath = path.join( - app.getPath("temp"), - `recordly-export-video-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.mp4`, - ); - - try { - await fs.writeFile(tempVideoPath, Buffer.from(videoData)); - const finalizedPath = await muxNativeVideoExportAudio(tempVideoPath, options); - const muxedData = await fs.readFile(finalizedPath); - return new Uint8Array(muxedData); - } finally { - await Promise.allSettled([ - removeTemporaryExportFile(tempVideoPath), - removeTemporaryExportFile(`${tempVideoPath}.muxed.mp4`), - removeTemporaryExportFile( - path.join( - path.dirname(tempVideoPath), - `${path.basename(tempVideoPath, path.extname(tempVideoPath))}-final.mp4`, - ), - ), - ]); - } + const tempVideoPath = path.join( + app.getPath('temp'), + `recordly-export-video-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.mp4`, + ) + + try { + await fs.writeFile(tempVideoPath, Buffer.from(videoData)) + const finalizedPath = await muxNativeVideoExportAudio(tempVideoPath, options) + const muxedData = await fs.readFile(finalizedPath) + return new Uint8Array(muxedData) + } finally { + await Promise.allSettled([ + removeTemporaryExportFile(tempVideoPath), + removeTemporaryExportFile(`${tempVideoPath}.muxed.mp4`), + removeTemporaryExportFile( + path.join( + path.dirname(tempVideoPath), + `${path.basename(tempVideoPath, path.extname(tempVideoPath))}-final.mp4`, + ), + ), + ]) + } } /** Probe the duration of a media file (in seconds) using the container header. */ @@ -3818,7 +3866,7 @@ function normalizeHookMouseButton(rawButton: unknown): 1 | 2 | 3 { return 1; } -function getHookMouseButton(event: HookCursorEvent | null | undefined): 1 | 2 | 3 { +function getHookMouseButton(event: HookMouseEvent | null | undefined): 1 | 2 | 3 { return normalizeHookMouseButton( event?.button ?? event?.mouseButton ?? event?.data?.button ?? event?.data?.mouseButton, ); @@ -3983,7 +4031,7 @@ function getNormalizedCursorPoint() { } function getHookCursorScreenPoint( - event: HookCursorEvent | null | undefined, + event: HookMouseEvent | null | undefined, ): { x: number; y: number } | null { const rawX = event?.x ?? event?.data?.x ?? event?.screenX ?? event?.data?.screenX; const rawY = event?.y ?? event?.data?.y ?? event?.screenY ?? event?.data?.screenY; @@ -4203,7 +4251,7 @@ async function startInteractionCapture() { return; } - const onMouseDown = (event: HookCursorEvent) => { + const onMouseDown = (event: HookMouseEvent) => { if (!isCursorCaptureActive) { return; } @@ -4241,7 +4289,7 @@ async function startInteractionCapture() { pushCursorSample(point.cx, point.cy, timeMs, interactionType); }; - const onMouseUp = (_event: HookCursorEvent) => { + const onMouseUp = () => { if (!isCursorCaptureActive) { return; } @@ -4255,7 +4303,7 @@ async function startInteractionCapture() { pushCursorSample(point.cx, point.cy, timeMs, "mouseup"); }; - const onMouseMove = (event: HookCursorEvent) => { + const onMouseMove = (event: HookMouseEvent) => { if (process.platform !== "linux" || !isCursorCaptureActive) { return; } @@ -4670,2488 +4718,2266 @@ body{background:transparent;overflow:hidden;width:100vw;height:100vh}
-`; - - await highlightWin.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(html)}`); - - setTimeout(() => { - if (!highlightWin.isDestroyed()) highlightWin.close(); - }, 1700); - - return { success: true }; - } catch (error) { - console.error("Failed to show source highlight:", error); - return { success: false }; - } - }); - - ipcMain.handle("get-selected-source", () => { - return selectedSource; - }); - - ipcMain.handle("open-source-selector", () => { - const sourceSelectorWin = getSourceSelectorWindow(); - if (sourceSelectorWin) { - sourceSelectorWin.focus(); - return; - } - createSourceSelectorWindow(); - }); - ipcMain.handle("switch-to-editor", () => { - console.log("[switch-to-editor] Opening editor window"); - const sourceSelectorWin = getSourceSelectorWindow(); - if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) { - sourceSelectorWin.close(); - } - createEditorWindow(); - }); - - ipcMain.handle( - "start-native-screen-recording", - async (_, source: SelectedSource, options?: NativeMacRecordingOptions) => { - // Windows native capture path - if (process.platform === "win32") { - const windowsCaptureAvailable = await isNativeWindowsCaptureAvailable(); - if (!windowsCaptureAvailable) { - return { - success: false, - message: "Native Windows capture is not available on this system.", - }; - } - - if (windowsCaptureProcess && !windowsNativeCaptureActive) { - try { - windowsCaptureProcess.kill(); - } catch { - /* ignore */ - } - windowsCaptureProcess = null; - windowsCaptureTargetPath = null; - windowsCaptureStopRequested = false; - } - - if (windowsCaptureProcess) { - return { - success: false, - message: "A native Windows screen recording is already active.", - }; - } - - try { - const exePath = getWindowsCaptureExePath(); - const recordingsDir = await getRecordingsDir(); - const timestamp = Date.now(); - const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); - const displayBounds = source?.id?.startsWith("window:") - ? null - : getDisplayBoundsForSource(source); - - const config: Record = { - outputPath, - fps: 60, - }; - - if (options?.capturesSystemAudio) { - const audioPath = path.join( - recordingsDir, - `recording-${timestamp}.system.wav`, - ); - config.captureSystemAudio = true; - config.audioOutputPath = audioPath; - windowsSystemAudioPath = audioPath; - } - - if (options?.capturesMicrophone) { - const micPath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`); - config.captureMic = true; - config.micOutputPath = micPath; - if (options.microphoneLabel) { - config.micDeviceName = options.microphoneLabel; - } - windowsMicAudioPath = micPath; - } - - const windowId = parseWindowId(source?.id); - if (windowId && source?.id?.startsWith("window:")) { - config.windowHandle = windowId; - } else { - const resolvedDisplay = resolveWindowsCaptureDisplay( - source, - getScreen().getAllDisplays(), - getScreen().getPrimaryDisplay(), - ); - config.displayId = resolvedDisplay.displayId; - - // Monitor handle IDs can drift across Electron/Windows capture boundaries, - // so also provide display bounds for a coordinate-based native fallback. - config.displayX = Math.round(resolvedDisplay.bounds.x); - config.displayY = Math.round(resolvedDisplay.bounds.y); - config.displayW = Math.round(resolvedDisplay.bounds.width); - config.displayH = Math.round(resolvedDisplay.bounds.height); - } - - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - displayId: typeof config.displayId === "number" ? config.displayId : null, - displayBounds, - windowHandle: - typeof config.windowHandle === "number" ? config.windowHandle : null, - helperPath: exePath, - outputPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - }); - - windowsCaptureOutputBuffer = ""; - windowsCaptureTargetPath = outputPath; - windowsCaptureStopRequested = false; - windowsCapturePaused = false; - windowsCaptureProcess = spawn(exePath, [JSON.stringify(config)], { - cwd: recordingsDir, - stdio: ["pipe", "pipe", "pipe"], - }); - attachWindowsCaptureLifecycle(windowsCaptureProcess); - - windowsCaptureProcess.stdout.on("data", (chunk: Buffer) => { - windowsCaptureOutputBuffer += chunk.toString(); - }); - windowsCaptureProcess.stderr.on("data", (chunk: Buffer) => { - windowsCaptureOutputBuffer += chunk.toString(); - }); - - await waitForWindowsCaptureStart(windowsCaptureProcess); - windowsNativeCaptureActive = true; - nativeScreenRecordingActive = true; - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - displayId: typeof config.displayId === "number" ? config.displayId : null, - displayBounds, - windowHandle: - typeof config.windowHandle === "number" ? config.windowHandle : null, - helperPath: exePath, - outputPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - }); - return { success: true }; - } catch (error) { - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - helperPath: windowsCaptureTargetPath ? getWindowsCaptureExePath() : null, - outputPath: windowsCaptureTargetPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - }); - console.error("Failed to start native Windows capture:", error); - try { - windowsCaptureProcess?.kill(); - } catch { - /* ignore */ - } - windowsNativeCaptureActive = false; - nativeScreenRecordingActive = false; - windowsCaptureProcess = null; - windowsCaptureTargetPath = null; - windowsCaptureStopRequested = false; - windowsCapturePaused = false; - return { - success: false, - message: "Failed to start native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (nativeCaptureProcess && !nativeScreenRecordingActive) { - try { - nativeCaptureProcess.kill(); - } catch { - // ignore stale helper cleanup failures - } - nativeCaptureProcess = null; - nativeCaptureTargetPath = null; - nativeCaptureStopRequested = false; - } - - if (nativeCaptureProcess) { - return { success: false, message: "A native screen recording is already active." }; - } - - try { - const recordingsDir = await getRecordingsDir(); - - // Warm up TCC: trigger an Electron-level screen capture API call so macOS - // activates the screen-recording grant for this process tree before the - // native helper binary spawns and calls SCStream.startCapture(). - try { - await desktopCapturer.getSources({ - types: ["screen"], - thumbnailSize: { width: 1, height: 1 }, - }); - } catch { - // non-fatal – the helper will report its own TCC status - } - - // Ensure microphone TCC is granted for this process tree when mic capture - // is requested, so the child helper inherits the grant. - if (options?.capturesMicrophone) { - const micStatus = systemPreferences.getMediaAccessStatus("microphone"); - if (micStatus !== "granted") { - await systemPreferences.askForMediaAccess("microphone"); - } - } - - const appName = normalizeDesktopSourceName(String(source?.appName ?? "")); - const ownAppName = normalizeDesktopSourceName(app.getName()); - if ( - !ALLOW_RECORDLY_WINDOW_CAPTURE && - source?.id?.startsWith("window:") && - appName && - (appName === ownAppName || appName === "recordly") - ) { - return { - success: false, - message: - "Cannot record Recordly windows. Please select another app window.", - }; - } - - const helperPath = await ensureNativeCaptureHelperBinary(); - const timestamp = Date.now(); - const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); - const capturesSystemAudio = Boolean(options?.capturesSystemAudio); - const capturesMicrophone = Boolean(options?.capturesMicrophone); - const systemAudioOutputPath = capturesSystemAudio - ? path.join(recordingsDir, `recording-${timestamp}.system.m4a`) - : null; - const microphoneOutputPath = capturesMicrophone - ? path.join(recordingsDir, `recording-${timestamp}.mic.m4a`) - : null; - const config: Record = { - fps: 60, - outputPath, - capturesSystemAudio, - capturesMicrophone, - }; - - if (options?.microphoneDeviceId) { - config.microphoneDeviceId = options.microphoneDeviceId; - } - - if (options?.microphoneLabel) { - config.microphoneLabel = options.microphoneLabel; - } - - if (systemAudioOutputPath) { - config.systemAudioOutputPath = systemAudioOutputPath; - } - - if (microphoneOutputPath) { - config.microphoneOutputPath = microphoneOutputPath; - } - - const windowId = parseWindowId(source?.id); - const screenId = Number(source?.display_id); - - if (Number.isFinite(windowId) && windowId && source?.id?.startsWith("window:")) { - config.windowId = windowId; - } else if (Number.isFinite(screenId) && screenId > 0) { - config.displayId = screenId; - } else { - config.displayId = Number(getScreen().getPrimaryDisplay().id); - } - - nativeCaptureOutputBuffer = ""; - nativeCaptureTargetPath = outputPath; - nativeCaptureSystemAudioPath = systemAudioOutputPath; - nativeCaptureMicrophonePath = microphoneOutputPath; - nativeCaptureStopRequested = false; - nativeCapturePaused = false; - nativeCaptureProcess = spawn(helperPath, [JSON.stringify(config)], { - cwd: recordingsDir, - stdio: ["pipe", "pipe", "pipe"], - }); - attachNativeCaptureLifecycle(nativeCaptureProcess); - - nativeCaptureProcess.stdout.on("data", (chunk: Buffer) => { - nativeCaptureOutputBuffer += chunk.toString(); - }); - nativeCaptureProcess.stderr.on("data", (chunk: Buffer) => { - nativeCaptureOutputBuffer += chunk.toString(); - }); - - await waitForNativeCaptureStart(nativeCaptureProcess); - nativeScreenRecordingActive = true; - - // If the native helper reported MICROPHONE_CAPTURE_UNAVAILABLE, it started - // capture without microphone. Clear the mic path so the renderer can fall - // back to a browser-side sidecar recording for the microphone track. - const micUnavailableNatively = nativeCaptureOutputBuffer.includes( - "MICROPHONE_CAPTURE_UNAVAILABLE", - ); - if (micUnavailableNatively) { - nativeCaptureMicrophonePath = null; - } - - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - displayId: typeof config.displayId === "number" ? config.displayId : null, - helperPath, - outputPath, - systemAudioPath: systemAudioOutputPath, - microphonePath: nativeCaptureMicrophonePath, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - }); - return { success: true, microphoneFallbackRequired: micUnavailableNatively }; - } catch (error) { - console.error("Failed to start native ScreenCaptureKit recording:", error); - const errorStr = String(error); - - // Detect TCC (screen recording permission) errors and show a helpful dialog - if ( - errorStr.includes("declined TCC") || - errorStr.includes("declined TCCs") || - errorStr.includes("SCREEN_RECORDING_PERMISSION_DENIED") - ) { - const { response } = await dialog.showMessageBox({ - type: "warning", - title: "Screen Recording Permission Required", - message: - "Recordly needs screen recording permission to capture your screen.", - detail: "Please open System Settings > Privacy & Security > Screen Recording, make sure Recordly is toggled ON, then try recording again.", - buttons: ["Open System Settings", "Cancel"], - defaultId: 0, - cancelId: 1, - }); - if (response === 0) { - await shell.openExternal(getMacPrivacySettingsUrl("screen")); - } - try { - nativeCaptureProcess?.kill(); - } catch { - /* ignore */ - } - nativeScreenRecordingActive = false; - nativeCaptureProcess = null; - nativeCaptureTargetPath = null; - nativeCaptureSystemAudioPath = null; - nativeCaptureMicrophonePath = null; - nativeCaptureStopRequested = false; - nativeCapturePaused = false; - return { - success: false, - message: - "Screen recording permission not granted. Please allow access in System Settings and restart the app.", - userNotified: true, - }; - } - - if (errorStr.includes("MICROPHONE_PERMISSION_DENIED")) { - const { response } = await dialog.showMessageBox({ - type: "warning", - title: "Microphone Permission Required", - message: "Recordly needs microphone permission to record audio.", - detail: "Please open System Settings > Privacy & Security > Microphone, make sure Recordly is toggled ON, then try recording again.", - buttons: ["Open System Settings", "Cancel"], - defaultId: 0, - cancelId: 1, - }); - if (response === 0) { - await shell.openExternal(getMacPrivacySettingsUrl("microphone")); - } - try { - nativeCaptureProcess?.kill(); - } catch { - /* ignore */ - } - nativeScreenRecordingActive = false; - nativeCaptureProcess = null; - nativeCaptureTargetPath = null; - nativeCaptureSystemAudioPath = null; - nativeCaptureMicrophonePath = null; - nativeCaptureStopRequested = false; - nativeCapturePaused = false; - return { - success: false, - message: - "Microphone permission not granted. Please allow access in System Settings.", - userNotified: true, - }; - } - - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "start", - sourceId: source?.id ?? null, - sourceType: source?.sourceType ?? "unknown", - helperPath: getNativeCaptureHelperBinaryPath(), - outputPath: nativeCaptureTargetPath, - systemAudioPath: nativeCaptureSystemAudioPath, - microphonePath: nativeCaptureMicrophonePath, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(nativeCaptureTargetPath), - error: String(error), - }); - try { - nativeCaptureProcess?.kill(); - } catch { - // ignore cleanup failures - } - nativeScreenRecordingActive = false; - nativeCaptureProcess = null; - nativeCaptureTargetPath = null; - nativeCaptureSystemAudioPath = null; - nativeCaptureMicrophonePath = null; - nativeCaptureStopRequested = false; - nativeCapturePaused = false; - return { - success: false, - message: "Failed to start native ScreenCaptureKit recording", - error: String(error), - }; - } - }, - ); - - ipcMain.handle("stop-native-screen-recording", async () => { - // Windows native capture stop path - if (process.platform === "win32" && windowsNativeCaptureActive) { - try { - if (!windowsCaptureProcess) { - throw new Error("Native Windows capture process is not running"); - } - - const proc = windowsCaptureProcess; - const preferredVideoPath = windowsCaptureTargetPath; - windowsCaptureStopRequested = true; - proc.stdin.write("stop\n"); - const tempVideoPath = await waitForWindowsCaptureStop(proc); - windowsCaptureProcess = null; - windowsNativeCaptureActive = false; - nativeScreenRecordingActive = false; - windowsCaptureTargetPath = null; - windowsCaptureStopRequested = false; - windowsCapturePaused = false; - - const finalVideoPath = preferredVideoPath ?? tempVideoPath; - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } - - windowsPendingVideoPath = finalVideoPath; - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(finalVideoPath), - }); - return { success: true, path: finalVideoPath }; - } catch (error) { - console.error("Failed to stop native Windows capture:", error); - const fallbackPath = windowsCaptureTargetPath; - windowsNativeCaptureActive = false; - nativeScreenRecordingActive = false; - windowsCaptureProcess = null; - windowsCaptureTargetPath = null; - windowsCaptureStopRequested = false; - windowsCapturePaused = false; - windowsSystemAudioPath = null; - windowsMicAudioPath = null; - windowsPendingVideoPath = null; - - if (fallbackPath) { - try { - await fs.access(fallbackPath); - windowsPendingVideoPath = fallbackPath; - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - error: String(error), - }); - return { success: true, path: fallbackPath }; - } catch { - // File doesn't exist - } - } - - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - }); - - return { - success: false, - message: "Failed to stop native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive) { - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - - return { success: false, message: "No native screen recording is active." }; - } - - try { - if (!nativeCaptureProcess) { - throw new Error("Native capture helper process is not running"); - } - - const process = nativeCaptureProcess; - const preferredVideoPath = nativeCaptureTargetPath; - const preferredSystemAudioPath = nativeCaptureSystemAudioPath; - const preferredMicrophonePath = nativeCaptureMicrophonePath; - console.log( - "[stop-native] Audio paths — system:", - preferredSystemAudioPath, - "mic:", - preferredMicrophonePath, - ); - nativeCaptureStopRequested = true; - process.stdin.write("stop\n"); - const tempVideoPath = await waitForNativeCaptureStop(process); - console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); - nativeCaptureProcess = null; - nativeScreenRecordingActive = false; - nativeCaptureTargetPath = null; - nativeCaptureSystemAudioPath = null; - nativeCaptureMicrophonePath = null; - nativeCaptureStopRequested = false; - nativeCapturePaused = false; - - const finalVideoPath = preferredVideoPath ?? tempVideoPath; - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } - - if (preferredSystemAudioPath || preferredMicrophonePath) { - console.log( - "[stop-native] Attempting audio mux (merging separate tracks) into:", - finalVideoPath, - ); - try { - await muxNativeMacRecordingWithAudio( - finalVideoPath, - preferredSystemAudioPath, - preferredMicrophonePath, - ); - console.log("[stop-native] Audio mux completed successfully"); - } catch (error) { - console.warn( - "[stop-native] Audio mux failed (video still has inline audio):", - error, - ); - } - } else { - console.log("[stop-native] No separate audio tracks to mux"); - } - - return await finalizeStoredVideo(finalVideoPath); - } catch (error) { - console.error("Failed to stop native ScreenCaptureKit recording:", error); - const fallbackPath = nativeCaptureTargetPath; - const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; - const fallbackMicrophonePath = nativeCaptureMicrophonePath; - const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); - nativeScreenRecordingActive = false; - nativeCaptureProcess = null; - nativeCaptureTargetPath = null; - nativeCaptureSystemAudioPath = null; - nativeCaptureMicrophonePath = null; - nativeCaptureStopRequested = false; - nativeCapturePaused = false; - - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "stop", - sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, - sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", - displayId: lastNativeCaptureDiagnostics?.displayId ?? null, - displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, - windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, - helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, - outputPath: fallbackPath, - systemAudioPath: fallbackSystemAudioPath, - microphonePath: fallbackMicrophonePath, - osRelease: lastNativeCaptureDiagnostics?.osRelease, - supported: lastNativeCaptureDiagnostics?.supported, - helperExists: lastNativeCaptureDiagnostics?.helperExists, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: fallbackFileSizeBytes, - error: String(error), - }); - - // Try to recover: if the target file exists on disk, finalize with it - if (fallbackPath) { - try { - await fs.access(fallbackPath); - console.log( - "[stop-native-screen-recording] Recovering with fallback path:", - fallbackPath, - ); - if (fallbackSystemAudioPath || fallbackMicrophonePath) { - try { - await muxNativeMacRecordingWithAudio( - fallbackPath, - fallbackSystemAudioPath, - fallbackMicrophonePath, - ); - } catch (muxError) { - console.warn( - "Failed to mux recovered native macOS audio into capture:", - muxError, - ); - } - } - return await finalizeStoredVideo(fallbackPath); - } catch { - // File doesn't exist or isn't accessible - } - } - - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - - return { - success: false, - message: "Failed to stop native ScreenCaptureKit recording", - error: String(error), - }; - } - }); - - ipcMain.handle("recover-native-screen-recording", async () => { - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording recovery is only available on macOS.", - }; - } - - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } - - return { - success: false, - message: "No recoverable native macOS recording output was found.", - }; - }); - - ipcMain.handle("pause-native-screen-recording", async () => { - if (process.platform === "win32") { - if (!windowsNativeCaptureActive || !windowsCaptureProcess) { - return { success: false, message: "No native Windows screen recording is active." }; - } - - if (windowsCapturePaused) { - return { success: true }; - } - - try { - windowsCaptureProcess.stdin.write("pause\n"); - windowsCapturePaused = true; - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to pause native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive || !nativeCaptureProcess) { - return { success: false, message: "No native screen recording is active." }; - } - - if (nativeCapturePaused) { - return { success: true }; - } - - try { - nativeCaptureProcess.stdin.write("pause\n"); - nativeCapturePaused = true; - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to pause native screen recording", - error: String(error), - }; - } - }); - - ipcMain.handle("resume-native-screen-recording", async () => { - if (process.platform === "win32") { - if (!windowsNativeCaptureActive || !windowsCaptureProcess) { - return { success: false, message: "No native Windows screen recording is active." }; - } - - if (!windowsCapturePaused) { - return { success: true }; - } - - try { - windowsCaptureProcess.stdin.write("resume\n"); - windowsCapturePaused = false; - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to resume native Windows capture", - error: String(error), - }; - } - } - - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } - - if (!nativeScreenRecordingActive || !nativeCaptureProcess) { - return { success: false, message: "No native screen recording is active." }; - } - - if (!nativeCapturePaused) { - return { success: true }; - } - - try { - nativeCaptureProcess.stdin.write("resume\n"); - nativeCapturePaused = false; - return { success: true }; - } catch (error) { - return { - success: false, - message: "Failed to resume native screen recording", - error: String(error), - }; - } - }); - - ipcMain.handle("get-system-cursor-assets", async () => { - try { - return { success: true, cursors: await getSystemCursorAssets() }; - } catch (error) { - console.error("Failed to load system cursor assets:", error); - return { success: false, cursors: {}, error: String(error) }; - } - }); - - ipcMain.handle("is-native-windows-capture-available", async () => { - return { available: await isNativeWindowsCaptureAvailable() }; - }); - - ipcMain.handle("get-last-native-capture-diagnostics", async () => { - return { success: true, diagnostics: lastNativeCaptureDiagnostics }; - }); - - ipcMain.handle("get-video-audio-fallback-paths", async (_event, videoPath: string) => { - if (!videoPath) { - return { success: true, paths: [] }; - } - - try { - const paths = await getCompanionAudioFallbackPaths(videoPath); - await Promise.all([ - rememberApprovedLocalReadPath(videoPath), - ...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)), - ]); - return { success: true, paths }; - } catch (error) { - console.error("Failed to resolve companion audio fallback paths:", error); - return { success: false, paths: [], error: String(error) }; - } - }); - - ipcMain.handle( - "mux-native-windows-recording", - async (_event, pauseSegments?: PauseSegment[]) => { - const videoPath = windowsPendingVideoPath; - windowsPendingVideoPath = null; - - if (!videoPath) { - return { success: false, message: "No native Windows video pending for mux" }; - } - - try { - if (windowsSystemAudioPath || windowsMicAudioPath) { - await muxNativeWindowsVideoWithAudio( - videoPath, - windowsSystemAudioPath, - windowsMicAudioPath, - pauseSegments ?? [], - ); - windowsSystemAudioPath = null; - windowsMicAudioPath = null; - } - - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "mux", - outputPath: videoPath, - fileSizeBytes: await getFileSizeIfPresent(videoPath), - }); - return await finalizeStoredVideo(videoPath); - } catch (error) { - console.error("Failed to mux native Windows recording:", error); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "mux", - outputPath: videoPath, - systemAudioPath: windowsSystemAudioPath, - microphonePath: windowsMicAudioPath, - fileSizeBytes: await getFileSizeIfPresent(videoPath), - error: String(error), - }); - windowsSystemAudioPath = null; - windowsMicAudioPath = null; - try { - return await finalizeStoredVideo(videoPath); - } catch { - return { - success: false, - message: "Failed to mux native Windows recording", - error: String(error), - }; - } - } - }, - ); - - ipcMain.handle("start-ffmpeg-recording", async (_, source: SelectedSource) => { - if (ffmpegCaptureProcess) { - return { success: false, message: "An FFmpeg recording is already active." }; - } - - try { - const recordingsDir = await getRecordingsDir(); - const ffmpegPath = getFfmpegBinaryPath(); - const outputPath = path.join(recordingsDir, `recording-${Date.now()}.mp4`); - const args = await buildFfmpegCaptureArgs(source, outputPath); - - ffmpegCaptureOutputBuffer = ""; - ffmpegCaptureTargetPath = outputPath; - ffmpegCaptureProcess = spawn(ffmpegPath, args, { - cwd: recordingsDir, - stdio: ["pipe", "pipe", "pipe"], - }); - - ffmpegCaptureProcess.stdout.on("data", (chunk: Buffer) => { - ffmpegCaptureOutputBuffer += chunk.toString(); - }); - ffmpegCaptureProcess.stderr.on("data", (chunk: Buffer) => { - ffmpegCaptureOutputBuffer += chunk.toString(); - }); - - await waitForFfmpegCaptureStart(ffmpegCaptureProcess); - ffmpegScreenRecordingActive = true; - return { success: true }; - } catch (error) { - console.error("Failed to start FFmpeg recording:", error); - ffmpegScreenRecordingActive = false; - ffmpegCaptureProcess = null; - ffmpegCaptureTargetPath = null; - return { - success: false, - message: "Failed to start FFmpeg recording", - error: String(error), - }; - } - }); - - ipcMain.handle("stop-ffmpeg-recording", async () => { - if (!ffmpegScreenRecordingActive) { - return { success: false, message: "No FFmpeg recording is active." }; - } - - try { - if (!ffmpegCaptureProcess || !ffmpegCaptureTargetPath) { - throw new Error("FFmpeg process is not running"); - } - - const process = ffmpegCaptureProcess; - const outputPath = ffmpegCaptureTargetPath; - process.stdin.write("q\n"); - const finalVideoPath = await waitForFfmpegCaptureStop(process, outputPath); - - ffmpegCaptureProcess = null; - ffmpegCaptureTargetPath = null; - ffmpegScreenRecordingActive = false; - - return await finalizeStoredVideo(finalVideoPath); - } catch (error) { - console.error("Failed to stop FFmpeg recording:", error); - ffmpegCaptureProcess = null; - ffmpegCaptureTargetPath = null; - ffmpegScreenRecordingActive = false; - return { - success: false, - message: "Failed to stop FFmpeg recording", - error: String(error), - }; - } - }); - - ipcMain.handle( - "store-microphone-sidecar", - async (_, audioData: ArrayBuffer, videoPath: string) => { - try { - const baseName = videoPath.replace(/\.[^.]+$/, ""); - const sidecarPath = `${baseName}.mic.webm`; - await fs.writeFile(sidecarPath, Buffer.from(audioData)); - return { success: true, path: sidecarPath }; - } catch (error) { - console.error("Failed to store microphone sidecar:", error); - return { success: false, error: String(error) }; - } - }, - ); - - ipcMain.handle("store-recorded-video", async (_, videoData: ArrayBuffer, fileName: string) => { - try { - const recordingsDir = await getRecordingsDir(); - const videoPath = path.join(recordingsDir, fileName); - await fs.writeFile(videoPath, Buffer.from(videoData)); - return await finalizeStoredVideo(videoPath); - } catch (error) { - console.error("Failed to store video:", error); - return { - success: false, - message: "Failed to store video", - error: String(error), - }; - } - }); - - ipcMain.handle("get-recorded-video-path", async () => { - try { - const recordingsDir = await getRecordingsDir(); - const files = await fs.readdir(recordingsDir); - const videoFiles = files.filter((file) => /\.(webm|mov|mp4)$/i.test(file)); - - if (videoFiles.length === 0) { - return { success: false, message: "No recorded video found" }; - } - - const latestVideo = videoFiles.sort().reverse()[0]; - const videoPath = path.join(recordingsDir, latestVideo); - - return { success: true, path: videoPath }; - } catch (error) { - console.error("Failed to get video path:", error); - return { success: false, message: "Failed to get video path", error: String(error) }; - } - }); - - ipcMain.handle("set-recording-state", (_, recording: boolean) => { - if (recording) { - stopCursorCapture(); - stopInteractionCapture(); - startWindowBoundsCapture(); - void startNativeCursorMonitor(); - isCursorCaptureActive = true; - activeCursorSamples = []; - pendingCursorSamples = []; - cursorCaptureStartTimeMs = Date.now(); - linuxCursorScreenPoint = null; - lastLeftClick = null; - sampleCursorPoint(); - cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS); - void startInteractionCapture(); - } else { - isCursorCaptureActive = false; - stopCursorCapture(); - stopInteractionCapture(); - stopWindowBoundsCapture(); - stopNativeCursorMonitor(); - showCursor(); - linuxCursorScreenPoint = null; - snapshotCursorTelemetryForPersistence(); - activeCursorSamples = []; - } - - const source = selectedSource || { name: "Screen" }; - BrowserWindow.getAllWindows().forEach((window) => { - if (!window.isDestroyed()) { - window.webContents.send("recording-state-changed", { - recording, - sourceName: source.name, - }); - } - }); - - if (onRecordingStateChange) { - onRecordingStateChange(recording, source.name); - } - }); - - ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { - const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); - if (!targetVideoPath) { - return { success: true, samples: [] }; - } - - const telemetryPath = getTelemetryPathForVideo(targetVideoPath); - try { - const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = JSON.parse(content); - const rawSamples = Array.isArray(parsed) - ? parsed - : Array.isArray(parsed?.samples) - ? parsed.samples - : []; - - const samples: CursorTelemetryPoint[] = rawSamples - .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) - .map((sample: unknown) => { - const point = sample as Partial; - return { - timeMs: - typeof point.timeMs === "number" && Number.isFinite(point.timeMs) - ? Math.max(0, point.timeMs) - : 0, - cx: - typeof point.cx === "number" && Number.isFinite(point.cx) - ? clamp(point.cx, 0, 1) - : 0.5, - cy: - typeof point.cy === "number" && Number.isFinite(point.cy) - ? clamp(point.cy, 0, 1) - : 0.5, - interactionType: - point.interactionType === "click" || - point.interactionType === "double-click" || - point.interactionType === "right-click" || - point.interactionType === "middle-click" || - point.interactionType === "move" || - point.interactionType === "mouseup" - ? point.interactionType - : undefined, - cursorType: - point.cursorType === "arrow" || - point.cursorType === "text" || - point.cursorType === "pointer" || - point.cursorType === "crosshair" || - point.cursorType === "open-hand" || - point.cursorType === "closed-hand" || - point.cursorType === "resize-ew" || - point.cursorType === "resize-ns" || - point.cursorType === "not-allowed" - ? point.cursorType - : undefined, - }; - }) - .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs); - - return { success: true, samples }; - } catch (error) { - const nodeError = error as NodeJS.ErrnoException; - if (nodeError.code === "ENOENT") { - return { success: true, samples: [] }; - } - console.error("Failed to load cursor telemetry:", error); - return { - success: false, - message: "Failed to load cursor telemetry", - error: String(error), - samples: [], - }; - } - }); - - ipcMain.handle("open-external-url", async (_, url: string) => { - try { - // Security: only allow http/https URLs to prevent file:// or custom protocol abuse - const parsed = new URL(url); - if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { - return { success: false, error: `Blocked non-HTTP URL: ${parsed.protocol}` }; - } - await shell.openExternal(url); - return { success: true }; - } catch (error) { - console.error("Failed to open URL:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("get-accessibility-permission-status", () => { - if (process.platform !== "darwin") { - return { success: true, trusted: true, prompted: false }; - } - - return { - success: true, - trusted: systemPreferences.isTrustedAccessibilityClient(false), - prompted: false, - }; - }); - - ipcMain.handle("request-accessibility-permission", () => { - if (process.platform !== "darwin") { - return { success: true, trusted: true, prompted: false }; - } - - return { - success: true, - trusted: systemPreferences.isTrustedAccessibilityClient(true), - prompted: true, - }; - }); - - ipcMain.handle("get-screen-recording-permission-status", () => { - if (process.platform !== "darwin") { - return { success: true, status: "granted" }; - } - - try { - return { - success: true, - status: systemPreferences.getMediaAccessStatus("screen"), - }; - } catch (error) { - console.error("Failed to get screen recording permission status:", error); - return { success: false, status: "unknown", error: String(error) }; - } - }); - - ipcMain.handle("open-screen-recording-preferences", async () => { - if (process.platform !== "darwin") { - return { success: true }; - } - - try { - await shell.openExternal(getMacPrivacySettingsUrl("screen")); - return { success: true }; - } catch (error) { - console.error("Failed to open Screen Recording preferences:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("open-accessibility-preferences", async () => { - if (process.platform !== "darwin") { - return { success: true }; - } - - try { - await shell.openExternal(getMacPrivacySettingsUrl("accessibility")); - return { success: true }; - } catch (error) { - console.error("Failed to open Accessibility preferences:", error); - return { success: false, error: String(error) }; - } - }); - - // Generate a tiny thumbnail for a wallpaper image and cache it in userData. - // Returns the cached thumbnail as raw JPEG bytes for fast grid rendering. - // Serialized to prevent concurrent nativeImage operations from eating memory. - const THUMB_SIZE = 96; - const thumbCacheDir = path.join(USER_DATA_PATH, "wallpaper-thumbs"); - let thumbGenerationQueue: Promise = Promise.resolve(); - - ipcMain.handle("generate-wallpaper-thumbnail", async (_, filePath: string) => { - try { - const resolved = normalizePath(filePath); - const realResolved = await fs.realpath(resolved).catch(() => resolved); - - if (!isAllowedLocalReadPath(resolved) && !isAllowedLocalReadPath(realResolved)) { - return { success: false, error: "Access denied" }; - } - - // Deterministic cache key from file path + mtime - const stat = await fs.stat(resolved); - const cacheKey = Buffer.from(`${resolved}:${stat.mtimeMs}`).toString("base64url"); - const thumbPath = path.join(thumbCacheDir, `${cacheKey}.jpg`); - - // Return cached thumbnail if it exists (no queue needed) - if (existsSync(thumbPath)) { - const data = await fs.readFile(thumbPath); - return { success: true, data }; - } - - // Serialize nativeImage operations to avoid OOM from concurrent full-res decodes - let jpegData: Buffer; - const generation = thumbGenerationQueue.then(async () => { - const { nativeImage } = await import("electron"); - const img = nativeImage.createFromPath(resolved); - if (img.isEmpty()) { - throw new Error("Failed to load image"); - } - const { width, height } = img.getSize(); - const scale = THUMB_SIZE / Math.min(width, height); - const resized = img.resize({ - width: Math.round(width * scale), - height: Math.round(height * scale), - quality: "good", - }); - jpegData = resized.toJPEG(70); - - // Cache to disk - await fs.mkdir(thumbCacheDir, { recursive: true }); - await fs.writeFile(thumbPath, jpegData); - }); - // Keep the queue moving even if one fails - thumbGenerationQueue = generation.catch(() => undefined); - await generation; - - return { success: true, data: jpegData! }; - } catch (error) { - return { success: false, error: String(error) }; - } - }); - - // Return base path for assets so renderer can resolve file:// paths in production - ipcMain.handle("get-asset-base-path", () => { - try { - const assetPath = getAssetRootPath(); - return pathToFileURL(`${assetPath}${path.sep}`).toString(); - } catch (err) { - console.error("Failed to resolve asset base path:", err); - return null; - } - }); - - ipcMain.handle("list-asset-directory", async (_, relativeDir: string) => { - try { - const normalizedRelativeDir = String(relativeDir ?? "") - .replace(/\\/g, "/") - .replace(/^\/+/, ""); - - const assetRootPath = path.resolve(getAssetRootPath()); - const targetDirPath = path.resolve(assetRootPath, normalizedRelativeDir); - if ( - targetDirPath !== assetRootPath && - !targetDirPath.startsWith(`${assetRootPath}${path.sep}`) - ) { - return { success: false, error: "Invalid asset directory" }; - } - - const entries = await fs.readdir(targetDirPath, { withFileTypes: true }); - const files = entries - .filter((entry) => entry.isFile()) - .map((entry) => entry.name) - .sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare); - - return { success: true, files }; - } catch (error) { - console.error("Failed to list asset directory:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("read-local-file", async (_, filePath: string) => { - try { - const resolved = normalizePath(filePath); - const realResolved = await fs.realpath(resolved).catch(() => resolved); - if (!isAllowedLocalReadPath(resolved) && !isAllowedLocalReadPath(realResolved)) { - console.warn( - `[read-local-file] Blocked read outside allowed directories: ${resolved}`, - ); - return { success: false, error: "Access denied: path outside allowed directories" }; - } - - const data = await fs.readFile(resolved); - return { success: true, data }; - } catch (error) { - console.error("Failed to read local file:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle( - "native-video-export-start", - async ( - event, - options: { - width: number; - height: number; - frameRate: number; - bitrate: number; - encodingMode: NativeExportEncodingMode; - }, - ) => { - try { - if (options.width % 2 !== 0 || options.height % 2 !== 0) { - throw new Error("Native export requires even output dimensions"); - } - - const ffmpegPath = getFfmpegBinaryPath(); - const encoderName = await resolveNativeVideoEncoder( - ffmpegPath, - options.encodingMode, - ); - const sessionId = `recordly-export-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - const outputPath = path.join(app.getPath("temp"), `${sessionId}.mp4`); - const ffmpegArgs = buildNativeVideoExportArgs(encoderName, options, outputPath); - const ffmpegProcess = spawn(ffmpegPath, ffmpegArgs, { - stdio: ["pipe", "ignore", "pipe"], - }) as ChildProcessByStdio; - const inputByteSize = getNativeVideoInputByteSize(options.width, options.height); - - const session: NativeVideoExportSession = { - ffmpegProcess, - outputPath, - inputByteSize, - maxQueuedWriteBytes: getNativeVideoExportMaxQueuedWriteBytes(inputByteSize), - stderrOutput: "", - encoderName, - processError: null, - stdinError: null, - terminating: false, - writeSequence: Promise.resolve(), - sender: event.sender, - pendingWriteRequestIds: new Set(), - completionPromise: new Promise((resolve, reject) => { - ffmpegProcess.once("error", (error) => { - const processError = - error instanceof Error ? error : new Error(String(error)); - if (session.terminating) { - resolve(); - return; - } - - session.processError = processError; - reject(processError); - }); - ffmpegProcess.stdin.once("error", (error) => { - const stdinError = - error instanceof Error ? error : new Error(String(error)); - if ( - session.terminating && - isIgnorableNativeVideoExportStreamError(stdinError) - ) { - return; - } - - session.stdinError = stdinError; - }); - ffmpegProcess.once("close", (code, signal) => { - if (session.terminating) { - resolve(); - return; - } - - if (code === 0) { - resolve(); - return; - } - - reject( - new Error( - getNativeVideoExportSessionError( - session, - `FFmpeg exited with code ${code ?? "unknown"}${signal ? ` (signal ${signal})` : ""}`, - ), - ), - ); - }); - }), - }; - void session.completionPromise.catch(() => undefined); - - ffmpegProcess.stderr.on("data", (chunk: Buffer) => { - session.stderrOutput += chunk.toString(); - }); - - nativeVideoExportSessions.set(sessionId, session); - - console.log( - `[native-export] Started ${isHardwareAcceleratedVideoEncoder(encoderName) ? "hardware" : "software"} session ${sessionId} with ${encoderName}`, - ); - - return { - success: true, - sessionId, - encoderName, - }; - } catch (error) { - console.error( - "[native-export] Failed to start native video export session:", - error, - ); - return { - success: false, - error: String(error), - }; - } - }, - ); - - ipcMain.on( - "native-video-export-write-frame-async", - ( - event, - payload: { - sessionId: string; - requestId: number; - frameData: Uint8Array; - }, - ) => { - const sessionId = payload?.sessionId; - const requestId = payload?.requestId; - const frameData = payload?.frameData; - - if (typeof sessionId !== "string" || typeof requestId !== "number" || !frameData) { - return; - } - - const session = nativeVideoExportSessions.get(sessionId); - if (!session) { - sendNativeVideoExportWriteFrameResult(event.sender, sessionId, requestId, { - success: false, - error: "Invalid native export session", - }); - return; - } - - session.sender = event.sender; - session.pendingWriteRequestIds.add(requestId); - - if (session.terminating) { - settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { - success: false, - error: "Native video export session was cancelled", - }); - return; - } - - if (frameData.byteLength !== session.inputByteSize) { - settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { - success: false, - error: `Native video export expected ${session.inputByteSize} bytes per frame but received ${frameData.byteLength}`, - }); - return; - } - - void enqueueNativeVideoExportFrameWrite(session, frameData) - .then(() => { - settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { - success: true, - }); - }) - .catch((error) => { - session.stdinError = error instanceof Error ? error : new Error(String(error)); - settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { - success: false, - error: getNativeVideoExportSessionError( - session, - session.stdinError.message, - ), - }); - }); - }, - ); - - ipcMain.handle( - "native-video-export-finish", - async (_, sessionId: string, options?: NativeVideoExportFinishOptions) => { - const session = nativeVideoExportSessions.get(sessionId); - if (!session) { - return { success: false, error: "Invalid native export session" }; - } - - try { - await session.writeSequence; - if ( - !session.ffmpegProcess.stdin.destroyed && - !session.ffmpegProcess.stdin.writableEnded - ) { - session.ffmpegProcess.stdin.end(); - } - await session.completionPromise; - - const finalizedPath = await muxNativeVideoExportAudio( - session.outputPath, - options ?? {}, - ); - const data = await fs.readFile(finalizedPath); - nativeVideoExportSessions.delete(sessionId); - await removeTemporaryExportFile(finalizedPath); - - return { - success: true, - data: new Uint8Array(data), - encoderName: session.encoderName, - }; - } catch (error) { - flushNativeVideoExportPendingWriteRequests(sessionId, session, String(error)); - nativeVideoExportSessions.delete(sessionId); - await removeTemporaryExportFile(session.outputPath); - const finalizedSuffix = session.outputPath.replace(/\.mp4$/, "-final.mp4"); - await removeTemporaryExportFile(finalizedSuffix); - return { - success: false, - error: String(error), - }; - } - }, - ); - - ipcMain.handle( - "mux-exported-video-audio", - async (_, videoData: ArrayBuffer, options?: NativeVideoExportFinishOptions) => { - try { - const data = await muxExportedVideoAudioBuffer(videoData, options ?? {}); - return { - success: true, - data, - }; - } catch (error) { - return { - success: false, - error: String(error), - }; - } - }, - ); - - ipcMain.handle("native-video-export-cancel", async (_, sessionId: string) => { - const session = nativeVideoExportSessions.get(sessionId); - if (!session) { - return { success: true }; - } - - session.terminating = true; - nativeVideoExportSessions.delete(sessionId); - flushNativeVideoExportPendingWriteRequests( - sessionId, - session, - "Native video export session was cancelled", - ); - - try { - if ( - !session.ffmpegProcess.stdin.destroyed && - !session.ffmpegProcess.stdin.writableEnded - ) { - session.ffmpegProcess.stdin.destroy(); - } - } catch { - // Stream may already be closed. - } - - try { - session.ffmpegProcess.kill("SIGKILL"); - } catch { - // Process may already be closed. - } - - await session.completionPromise.catch(() => undefined); - await removeTemporaryExportFile(session.outputPath); - return { success: true }; - }); - - ipcMain.handle( - "save-exported-video", - async (event, videoData: ArrayBuffer, fileName: string) => { - try { - // Determine file type from extension - const isGif = fileName.toLowerCase().endsWith(".gif"); - const filters = isGif - ? [{ name: "GIF Image", extensions: ["gif"] }] - : [{ name: "MP4 Video", extensions: ["mp4"] }]; - const parentWindow = BrowserWindow.fromWebContents(event.sender); - const saveDialogOptions: SaveDialogOptions = { - title: isGif ? "Save Exported GIF" : "Save Exported Video", - defaultPath: path.join(app.getPath("downloads"), fileName), - filters, - properties: ["createDirectory", "showOverwriteConfirmation"], - }; - - const result = parentWindow - ? await dialog.showSaveDialog(parentWindow, saveDialogOptions) - : await dialog.showSaveDialog(saveDialogOptions); - - if (result.canceled || !result.filePath) { - return { - success: false, - canceled: true, - message: "Export canceled", - }; - } - - await fs.writeFile(result.filePath, Buffer.from(videoData)); - - return { - success: true, - path: result.filePath, - message: "Video exported successfully", - }; - } catch (error) { - console.error("Failed to save exported video:", error); - return { - success: false, - message: "Failed to save exported video", - error: String(error), - }; - } - }, - ); - - ipcMain.handle( - "write-exported-video-to-path", - async (_event, videoData: ArrayBuffer, outputPath: string) => { - try { - const resolvedPath = path.resolve(outputPath); - await fs.mkdir(path.dirname(resolvedPath), { recursive: true }); - await fs.writeFile(resolvedPath, Buffer.from(videoData)); - - return { - success: true, - path: outputPath, - message: "Video exported successfully", - canceled: false, - }; - } catch (error) { - console.error("Failed to write exported video to path:", error); - return { - success: false, - message: "Failed to write exported video", - canceled: false, - error: String(error), - }; - } - }, - ); - - ipcMain.handle("open-video-file-picker", async () => { - try { - const recordingsDir = await getRecordingsDir(); - const result = await dialog.showOpenDialog({ - title: "Select Video File", - defaultPath: recordingsDir, - filters: [ - { name: "Video Files", extensions: ["webm", "mp4", "mov", "avi", "mkv"] }, - { name: "All Files", extensions: ["*"] }, - ], - properties: ["openFile"], - }); - - if (result.canceled || result.filePaths.length === 0) { - return { success: false, canceled: true }; - } - - currentProjectPath = null; - return { - success: true, - path: result.filePaths[0], - }; - } catch (error) { - console.error("Failed to open file picker:", error); - return { - success: false, - message: "Failed to open file picker", - error: String(error), - }; - } - }); - - ipcMain.handle("open-audio-file-picker", async () => { - try { - const result = await dialog.showOpenDialog({ - title: "Select Audio File", - filters: [ - { - name: "Audio Files", - extensions: ["mp3", "wav", "aac", "m4a", "flac", "ogg"], - }, - { name: "All Files", extensions: ["*"] }, - ], - properties: ["openFile"], - }); - - if (result.canceled || result.filePaths.length === 0) { - return { success: false, canceled: true }; - } - - return { - success: true, - path: result.filePaths[0], - }; - } catch (error) { - console.error("Failed to open audio file picker:", error); - return { - success: false, - message: "Failed to open audio file picker", - error: String(error), - }; - } - }); - - ipcMain.handle("open-whisper-executable-picker", async () => { - try { - const result = await dialog.showOpenDialog({ - title: "Select Whisper Executable", - filters: [ - { - name: "Executables", - extensions: process.platform === "win32" ? ["exe", "cmd", "bat"] : ["*"], - }, - { name: "All Files", extensions: ["*"] }, - ], - properties: ["openFile"], - }); - - if (result.canceled || result.filePaths.length === 0) { - return { success: false, canceled: true }; - } - - return { success: true, path: result.filePaths[0] }; - } catch (error) { - console.error("Failed to open Whisper executable picker:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("open-whisper-model-picker", async () => { - try { - const result = await dialog.showOpenDialog({ - title: "Select Whisper Model", - filters: [ - { name: "Whisper Models", extensions: ["bin"] }, - { name: "All Files", extensions: ["*"] }, - ], - properties: ["openFile"], - }); - - if (result.canceled || result.filePaths.length === 0) { - return { success: false, canceled: true }; - } - - return { success: true, path: result.filePaths[0] }; - } catch (error) { - console.error("Failed to open Whisper model picker:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("get-whisper-small-model-status", async () => { - try { - return await getWhisperSmallModelStatus(); - } catch (error) { - return { success: false, exists: false, path: null, error: String(error) }; - } - }); - - ipcMain.handle("download-whisper-small-model", async (event) => { - try { - const existing = await getWhisperSmallModelStatus(); - if (existing.exists) { - sendWhisperModelDownloadProgress(event.sender, { - status: "downloaded", - progress: 100, - path: existing.path, - }); - return { success: true, path: existing.path, alreadyDownloaded: true }; - } - - const modelPath = await downloadWhisperSmallModel(event.sender); - return { success: true, path: modelPath }; - } catch (error) { - console.error("Failed to download Whisper small model:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("delete-whisper-small-model", async (event) => { - try { - await deleteWhisperSmallModel(); - sendWhisperModelDownloadProgress(event.sender, { - status: "idle", - progress: 0, - path: null, - }); - return { success: true }; - } catch (error) { - console.error("Failed to delete Whisper small model:", error); - // Verify whether the file was actually removed despite the error - const status = await getWhisperSmallModelStatus(); - if (!status.exists) { - // File is gone — treat as success - sendWhisperModelDownloadProgress(event.sender, { - status: "idle", - progress: 0, - path: null, - }); - return { success: true }; - } - sendWhisperModelDownloadProgress(event.sender, { - status: "error", - progress: 0, - path: null, - error: String(error), - }); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle( - "generate-auto-captions", - async ( - _, - options: { - videoPath: string; - whisperExecutablePath: string; - whisperModelPath: string; - language?: string; - }, - ) => { - try { - const result = await generateAutoCaptionsFromVideo(options); - return { - success: true, - cues: result.cues, - message: - result.audioSourceLabel === "recording" - ? `Generated ${result.cues.length} caption cues.` - : `Generated ${result.cues.length} caption cues from the ${result.audioSourceLabel}.`, - }; - } catch (error) { - console.error("Failed to generate auto captions:", error); - return { - success: false, - error: String(error), - message: "Failed to generate auto captions", - }; - } - }, - ); - - ipcMain.handle("reveal-in-folder", async (_, filePath: string) => { - try { - // shell.showItemInFolder doesn't return a value, it throws on error - shell.showItemInFolder(filePath); - return { success: true }; - } catch (error) { - console.error(`Error revealing item in folder: ${filePath}`, error); - // Fallback to open the directory if revealing the item fails - // This might happen if the file was moved or deleted after export, - // or if the path is somehow invalid for showItemInFolder - try { - const openPathResult = await shell.openPath(path.dirname(filePath)); - if (openPathResult) { - // openPath returned an error message - return { success: false, error: openPathResult }; - } - return { success: true, message: "Could not reveal item, but opened directory." }; - } catch (openError) { - console.error(`Error opening directory: ${path.dirname(filePath)}`, openError); - return { success: false, error: String(error) }; - } - } - }); - - ipcMain.handle("open-recordings-folder", async () => { - try { - const recordingsDir = await getRecordingsDir(); - const openPathResult = await shell.openPath(recordingsDir); - if (openPathResult) { - return { - success: false, - error: openPathResult, - message: "Failed to open recordings folder.", - }; - } - - return { success: true }; - } catch (error) { - console.error("Failed to open recordings folder:", error); - return { - success: false, - error: String(error), - message: "Failed to open recordings folder.", - }; - } - }); - - ipcMain.handle("get-recordings-directory", async () => { - try { - const recordingsDir = await getRecordingsDir(); - return { - success: true, - path: recordingsDir, - isDefault: recordingsDir === RECORDINGS_DIR, - }; - } catch (error) { - return { - success: false, - path: RECORDINGS_DIR, - isDefault: true, - error: String(error), - }; - } - }); - - ipcMain.handle("choose-recordings-directory", async () => { - try { - const current = await getRecordingsDir(); - const result = await dialog.showOpenDialog({ - title: "Choose recordings folder", - defaultPath: current, - properties: ["openDirectory", "createDirectory", "promptToCreate"], - }); - - if (result.canceled || result.filePaths.length === 0) { - return { success: false, canceled: true, path: current }; - } - - const selectedPath = path.resolve(result.filePaths[0]); - await fs.mkdir(selectedPath, { recursive: true }); - await fs.access(selectedPath, fsConstants.W_OK); - await persistRecordingsDirectorySetting(selectedPath); - - return { - success: true, - path: selectedPath, - isDefault: selectedPath === RECORDINGS_DIR, - }; - } catch (error) { - return { - success: false, - error: String(error), - message: "Failed to set recordings folder", - }; - } - }); - - ipcMain.handle( - "save-project-file", - async ( - _, - projectData: unknown, - suggestedName?: string, - existingProjectPath?: string, - thumbnailDataUrl?: string | null, - ) => { - try { - const projectsDir = await getProjectsDir(); - const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) - ? existingProjectPath - : null; - - if (trustedExistingProjectPath) { - await fs.writeFile( - trustedExistingProjectPath, - JSON.stringify(projectData, null, 2), - "utf-8", - ); - currentProjectPath = trustedExistingProjectPath; - await saveProjectThumbnail(trustedExistingProjectPath, thumbnailDataUrl); - await rememberRecentProject(trustedExistingProjectPath); - return { - success: true, - path: trustedExistingProjectPath, - message: "Project saved successfully", - }; - } - - const safeName = (suggestedName || `project-${Date.now()}`).replace( - /[^a-zA-Z0-9-_]/g, - "_", - ); - const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) - ? safeName - : `${safeName}.${PROJECT_FILE_EXTENSION}`; - - const result = await dialog.showSaveDialog({ - title: "Save Recordly Project", - defaultPath: path.join(projectsDir, defaultName), - filters: [ - { name: "Recordly Project", extensions: [PROJECT_FILE_EXTENSION] }, - { name: "JSON", extensions: ["json"] }, - ], - properties: ["createDirectory", "showOverwriteConfirmation"], - }); - - if (result.canceled || !result.filePath) { - return { - success: false, - canceled: true, - message: "Save project canceled", - }; - } - - await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); - currentProjectPath = result.filePath; - await saveProjectThumbnail(result.filePath, thumbnailDataUrl); - await rememberRecentProject(result.filePath); - - return { - success: true, - path: result.filePath, - message: "Project saved successfully", - }; - } catch (error) { - console.error("Failed to save project file:", error); - return { - success: false, - message: "Failed to save project file", - error: String(error), - }; - } - }, - ); - - ipcMain.handle("load-project-file", async () => { - try { - const projectsDir = await getProjectsDir(); - const result = await dialog.showOpenDialog({ - title: "Open Recordly Project", - defaultPath: projectsDir, - filters: [ - { - name: "Recordly Project", - extensions: [PROJECT_FILE_EXTENSION, ...LEGACY_PROJECT_FILE_EXTENSIONS], - }, - { name: "JSON", extensions: ["json"] }, - { name: "All Files", extensions: ["*"] }, - ], - properties: ["openFile"], - }); - - if (result.canceled || result.filePaths.length === 0) { - return { success: false, canceled: true, message: "Open project canceled" }; - } - - return await loadProjectFromPath(result.filePaths[0]); - } catch (error) { - console.error("Failed to load project file:", error); - return { - success: false, - message: "Failed to load project file", - error: String(error), - }; - } - }); - - ipcMain.handle("load-current-project-file", async () => { - try { - if (!currentProjectPath) { - return { success: false, message: "No active project" }; - } - - return await loadProjectFromPath(currentProjectPath); - } catch (error) { - console.error("Failed to load current project file:", error); - return { - success: false, - message: "Failed to load current project file", - error: String(error), - }; - } - }); - - ipcMain.handle("get-projects-directory", async () => { - try { - return { - success: true, - path: await getProjectsDir(), - }; - } catch (error) { - return { - success: false, - error: String(error), - }; - } - }); - - ipcMain.handle("list-project-files", async () => { - try { - const library = await listProjectLibraryEntries(); - return { - success: true, - projectsDir: library.projectsDir, - entries: library.entries, - }; - } catch (error) { - return { - success: false, - projectsDir: null, - entries: [], - error: String(error), - }; - } - }); - - ipcMain.handle("open-project-file-at-path", async (_, filePath: string) => { - try { - return await loadProjectFromPath(filePath); - } catch (error) { - console.error("Failed to open project file at path:", error); - return { - success: false, - message: "Failed to open project file", - error: String(error), - }; - } - }); - - ipcMain.handle("open-projects-directory", async () => { - try { - const projectsDir = await getProjectsDir(); - const openPathResult = await shell.openPath(projectsDir); - if (openPathResult) { - return { - success: false, - error: openPathResult, - message: "Failed to open projects folder.", - }; - } - - return { success: true, path: projectsDir }; - } catch (error) { - console.error("Failed to open projects folder:", error); - return { - success: false, - error: String(error), - message: "Failed to open projects folder.", - }; - } - }); - ipcMain.handle("set-current-video-path", async (_, path: string) => { - currentVideoPath = normalizeVideoSourcePath(path) ?? path; - const resolvedSession = (await resolveRecordingSession(currentVideoPath)) ?? { - videoPath: currentVideoPath, - webcamPath: null, - timeOffsetMs: 0, - }; - - currentRecordingSession = resolvedSession; - await replaceApprovedSessionLocalReadPaths([ - resolvedSession.videoPath, - resolvedSession.webcamPath, - ]); - - if (resolvedSession.webcamPath) { - await persistRecordingSessionManifest(resolvedSession); - } - - currentProjectPath = null; - return { success: true, webcamPath: resolvedSession.webcamPath ?? null }; - }); - - ipcMain.handle( - "set-current-recording-session", - async ( - _, - session: { videoPath: string; webcamPath?: string | null; timeOffsetMs?: number }, - ) => { - const normalizedVideoPath = - normalizeVideoSourcePath(session.videoPath) ?? session.videoPath; - currentVideoPath = normalizedVideoPath; - currentRecordingSession = { - videoPath: normalizedVideoPath, - webcamPath: normalizeVideoSourcePath(session.webcamPath ?? null), - timeOffsetMs: normalizeRecordingTimeOffsetMs(session.timeOffsetMs), - }; - await replaceApprovedSessionLocalReadPaths([ - currentRecordingSession.videoPath, - currentRecordingSession.webcamPath, - ]); - currentProjectPath = null; - await persistRecordingSessionManifest(currentRecordingSession); - return { success: true }; - }, - ); - - ipcMain.handle("get-current-recording-session", () => { - if (!currentRecordingSession) { - return { success: false }; - } - - return { - success: true, - session: currentRecordingSession, - }; - }); - - ipcMain.handle("get-current-video-path", () => { - return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; - }); - - ipcMain.handle("clear-current-video-path", () => { - currentVideoPath = null; - currentRecordingSession = null; - return { success: true }; - }); - - ipcMain.handle("delete-recording-file", async (_, filePath: string) => { - try { - if (!filePath || !isAutoRecordingPath(filePath)) { - return { success: false, error: "Only auto-generated recordings can be deleted" }; - } - await fs.unlink(filePath); - // Also delete the cursor telemetry sidecar if it exists - const telemetryPath = getTelemetryPathForVideo(filePath); - await fs.unlink(telemetryPath).catch(() => undefined); - if (currentVideoPath === filePath) { - currentVideoPath = null; - currentRecordingSession = null; - } - return { success: true }; - } catch (error) { - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("app:getVersion", () => { - return app.getVersion(); - }); - - ipcMain.handle("get-platform", () => { - return process.platform; - }); - - // --------------------------------------------------------------------------- - // Cursor hiding for the browser-capture fallback. - // The IPC promise resolves only after the cursor hide attempt completes. - // --------------------------------------------------------------------------- - ipcMain.handle("hide-cursor", () => { - if (process.platform !== "win32") { - return { success: true }; - } - - return { success: hideCursor() }; - }); - - ipcMain.handle("get-shortcuts", async () => { - try { - const data = await fs.readFile(SHORTCUTS_FILE, "utf-8"); - return JSON.parse(data); - } catch { - return null; - } - }); - - ipcMain.handle("save-shortcuts", async (_, shortcuts: unknown) => { - try { - await fs.writeFile(SHORTCUTS_FILE, JSON.stringify(shortcuts, null, 2), "utf-8"); - return { success: true }; - } catch (error) { - console.error("Failed to save shortcuts:", error); - return { success: false, error: String(error) }; - } - }); - - // --------------------------------------------------------------------------- - // Countdown timer before recording - // --------------------------------------------------------------------------- - ipcMain.handle("get-recording-preferences", async () => { - try { - const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, "utf-8"); - const parsed = JSON.parse(content) as Record; - return { - success: true, - microphoneEnabled: parsed.microphoneEnabled === true, - microphoneDeviceId: - typeof parsed.microphoneDeviceId === "string" - ? parsed.microphoneDeviceId - : undefined, - systemAudioEnabled: parsed.systemAudioEnabled !== false, - }; - } catch { - return { - success: true, - microphoneEnabled: false, - microphoneDeviceId: undefined, - systemAudioEnabled: true, - }; - } - }); - - ipcMain.handle( - "set-recording-preferences", - async ( - _, - prefs: { - microphoneEnabled?: boolean; - microphoneDeviceId?: string; - systemAudioEnabled?: boolean; - }, - ) => { - try { - let existing: Record = {}; - try { - const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, "utf-8"); - existing = JSON.parse(content) as Record; - } catch { - // file doesn't exist yet - } - const merged = { ...existing, ...prefs }; - await fs.writeFile( - RECORDINGS_SETTINGS_FILE, - JSON.stringify(merged, null, 2), - "utf-8", - ); - return { success: true }; - } catch (error) { - console.error("Failed to save recording preferences:", error); - return { success: false, error: String(error) }; - } - }, - ); - - ipcMain.handle("get-countdown-delay", async () => { - try { - const content = await fs.readFile(COUNTDOWN_SETTINGS_FILE, "utf-8"); - const parsed = JSON.parse(content) as { delay?: number }; - return { success: true, delay: parsed.delay ?? 3 }; - } catch { - return { success: true, delay: 3 }; - } - }); - - ipcMain.handle("set-countdown-delay", async (_, delay: number) => { - try { - await fs.writeFile( - COUNTDOWN_SETTINGS_FILE, - JSON.stringify({ delay }, null, 2), - "utf-8", - ); - return { success: true }; - } catch (error) { - console.error("Failed to save countdown delay:", error); - return { success: false, error: String(error) }; - } - }); - - ipcMain.handle("start-countdown", async (_, seconds: number) => { - if (countdownInProgress) { - return { success: false, error: "Countdown already in progress" }; - } - - countdownInProgress = true; - countdownCancelled = false; - countdownRemaining = seconds; - - const countdownWin = createCountdownWindow(); - - if (countdownWin.webContents.isLoadingMainFrame()) { - await new Promise((resolve) => { - countdownWin.webContents.once("did-finish-load", () => { - resolve(); - }); - }); - } - - return new Promise<{ success: boolean; cancelled?: boolean }>((resolve) => { - let remaining = seconds; - countdownRemaining = remaining; - - countdownWin.webContents.send("countdown-tick", remaining); - - countdownTimer = setInterval(() => { - if (countdownCancelled) { - if (countdownTimer) { - clearInterval(countdownTimer); - countdownTimer = null; - } - closeCountdownWindow(); - countdownInProgress = false; - countdownRemaining = null; - resolve({ success: false, cancelled: true }); - return; - } - - remaining--; - countdownRemaining = remaining; - - if (remaining <= 0) { - if (countdownTimer) { - clearInterval(countdownTimer); - countdownTimer = null; - } - closeCountdownWindow(); - countdownInProgress = false; - countdownRemaining = null; - resolve({ success: true }); - } else { - const win = getCountdownWindow(); - if (win && !win.isDestroyed()) { - win.webContents.send("countdown-tick", remaining); - } - } - }, 1000); - }); - }); - - ipcMain.handle("cancel-countdown", () => { - countdownCancelled = true; - countdownInProgress = false; - countdownRemaining = null; - if (countdownTimer) { - clearInterval(countdownTimer); - countdownTimer = null; - } - closeCountdownWindow(); - return { success: true }; - }); - - ipcMain.handle("get-active-countdown", () => { - return { - success: true, - seconds: countdownInProgress ? countdownRemaining : null, - }; - }); +` + + await highlightWin.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(html)}`) + + setTimeout(() => { + if (!highlightWin.isDestroyed()) highlightWin.close() + }, 1700) + + return { success: true } + } catch (error) { + console.error('Failed to show source highlight:', error) + return { success: false } + } + }) + + ipcMain.handle('get-selected-source', () => { + return selectedSource + }) + + ipcMain.handle('open-source-selector', () => { + const sourceSelectorWin = getSourceSelectorWindow() + if (sourceSelectorWin) { + sourceSelectorWin.focus() + return + } + createSourceSelectorWindow() + }) + ipcMain.handle('switch-to-editor', () => { + console.log('[switch-to-editor] Opening editor window') + const sourceSelectorWin = getSourceSelectorWindow() + if (sourceSelectorWin && !sourceSelectorWin.isDestroyed()) { + sourceSelectorWin.close() + } + createEditorWindow() + }) + + ipcMain.handle('start-native-screen-recording', async (_, source: SelectedSource, options?: NativeMacRecordingOptions) => { + // Windows native capture path + if (process.platform === 'win32') { + const windowsCaptureAvailable = await isNativeWindowsCaptureAvailable() + if (!windowsCaptureAvailable) { + return { success: false, message: 'Native Windows capture is not available on this system.' } + } + + if (windowsCaptureProcess && !windowsNativeCaptureActive) { + try { windowsCaptureProcess.kill() } catch { /* ignore */ } + windowsCaptureProcess = null + windowsCaptureTargetPath = null + windowsCaptureStopRequested = false + } + + if (windowsCaptureProcess) { + return { success: false, message: 'A native Windows screen recording is already active.' } + } + + try { + const exePath = getWindowsCaptureExePath() + const recordingsDir = await getRecordingsDir() + const timestamp = Date.now() + const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`) + const displayBounds = source?.id?.startsWith('window:') ? null : getDisplayBoundsForSource(source) + + const config: Record = { + outputPath, + fps: 60, + } + + if (options?.capturesSystemAudio) { + const audioPath = path.join(recordingsDir, `recording-${timestamp}.system.wav`) + config.captureSystemAudio = true + config.audioOutputPath = audioPath + windowsSystemAudioPath = audioPath + } + + if (options?.capturesMicrophone) { + const micPath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`) + config.captureMic = true + config.micOutputPath = micPath + if (options.microphoneLabel) { + config.micDeviceName = options.microphoneLabel + } + windowsMicAudioPath = micPath + } + + const windowId = parseWindowId(source?.id) + if (windowId && source?.id?.startsWith('window:')) { + config.windowHandle = windowId + } else { + const resolvedDisplay = resolveWindowsCaptureDisplay( + source, + getScreen().getAllDisplays(), + getScreen().getPrimaryDisplay(), + ) + config.displayId = resolvedDisplay.displayId + + // Monitor handle IDs can drift across Electron/Windows capture boundaries, + // so also provide display bounds for a coordinate-based native fallback. + config.displayX = Math.round(resolvedDisplay.bounds.x) + config.displayY = Math.round(resolvedDisplay.bounds.y) + config.displayW = Math.round(resolvedDisplay.bounds.width) + config.displayH = Math.round(resolvedDisplay.bounds.height) + } + + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'start', + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? 'unknown', + displayId: typeof config.displayId === 'number' ? config.displayId : null, + displayBounds, + windowHandle: typeof config.windowHandle === 'number' ? config.windowHandle : null, + helperPath: exePath, + outputPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + }) + + windowsCaptureOutputBuffer = '' + windowsCaptureTargetPath = outputPath + windowsCaptureStopRequested = false + windowsCapturePaused = false + windowsCaptureProcess = spawn(exePath, [JSON.stringify(config)], { + cwd: recordingsDir, + stdio: ['pipe', 'pipe', 'pipe'], + }) + attachWindowsCaptureLifecycle(windowsCaptureProcess) + + windowsCaptureProcess.stdout.on('data', (chunk: Buffer) => { + windowsCaptureOutputBuffer += chunk.toString() + }) + windowsCaptureProcess.stderr.on('data', (chunk: Buffer) => { + windowsCaptureOutputBuffer += chunk.toString() + }) + + await waitForWindowsCaptureStart(windowsCaptureProcess) + windowsNativeCaptureActive = true + nativeScreenRecordingActive = true + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'start', + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? 'unknown', + displayId: typeof config.displayId === 'number' ? config.displayId : null, + displayBounds, + windowHandle: typeof config.windowHandle === 'number' ? config.windowHandle : null, + helperPath: exePath, + outputPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + }) + return { success: true } + } catch (error) { + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'start', + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? 'unknown', + helperPath: windowsCaptureTargetPath ? getWindowsCaptureExePath() : null, + outputPath: windowsCaptureTargetPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + }) + console.error('Failed to start native Windows capture:', error) + try { windowsCaptureProcess?.kill() } catch { /* ignore */ } + windowsNativeCaptureActive = false + nativeScreenRecordingActive = false + windowsCaptureProcess = null + windowsCaptureTargetPath = null + windowsCaptureStopRequested = false + windowsCapturePaused = false + return { + success: false, + message: 'Failed to start native Windows capture', + error: String(error), + } + } + } + + if (process.platform !== 'darwin') { + return { success: false, message: 'Native screen recording is only available on macOS.' } + } + + if (nativeCaptureProcess && !nativeScreenRecordingActive) { + try { + nativeCaptureProcess.kill() + } catch { + // ignore stale helper cleanup failures + } + nativeCaptureProcess = null + nativeCaptureTargetPath = null + nativeCaptureStopRequested = false + } + + if (nativeCaptureProcess) { + return { success: false, message: 'A native screen recording is already active.' } + } + + try { + const recordingsDir = await getRecordingsDir() + + // Warm up TCC: trigger an Electron-level screen capture API call so macOS + // activates the screen-recording grant for this process tree before the + // native helper binary spawns and calls SCStream.startCapture(). + try { + await desktopCapturer.getSources({ types: ['screen'], thumbnailSize: { width: 1, height: 1 } }) + } catch { + // non-fatal – the helper will report its own TCC status + } + + // Ensure microphone TCC is granted for this process tree when mic capture + // is requested, so the child helper inherits the grant. + if (options?.capturesMicrophone) { + const micStatus = systemPreferences.getMediaAccessStatus('microphone') + if (micStatus !== 'granted') { + await systemPreferences.askForMediaAccess('microphone') + } + } + + const appName = normalizeDesktopSourceName(String(source?.appName ?? '')) + const ownAppName = normalizeDesktopSourceName(app.getName()) + if ( + !ALLOW_RECORDLY_WINDOW_CAPTURE + && + source?.id?.startsWith('window:') + && appName + && (appName === ownAppName || appName === 'recordly') + ) { + return { success: false, message: 'Cannot record Recordly windows. Please select another app window.' } + } + + const helperPath = await ensureNativeCaptureHelperBinary() + const timestamp = Date.now() + const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`) + const capturesSystemAudio = Boolean(options?.capturesSystemAudio) + const capturesMicrophone = Boolean(options?.capturesMicrophone) + const systemAudioOutputPath = capturesSystemAudio + ? path.join(recordingsDir, `recording-${timestamp}.system.m4a`) + : null + const microphoneOutputPath = capturesMicrophone + ? path.join(recordingsDir, `recording-${timestamp}.mic.m4a`) + : null + const config: Record = { + fps: 60, + outputPath, + capturesSystemAudio, + capturesMicrophone, + } + + if (options?.microphoneDeviceId) { + config.microphoneDeviceId = options.microphoneDeviceId + } + + if (options?.microphoneLabel) { + config.microphoneLabel = options.microphoneLabel + } + + if (systemAudioOutputPath) { + config.systemAudioOutputPath = systemAudioOutputPath + } + + if (microphoneOutputPath) { + config.microphoneOutputPath = microphoneOutputPath + } + + const windowId = parseWindowId(source?.id) + const screenId = Number(source?.display_id) + + if (Number.isFinite(windowId) && windowId && source?.id?.startsWith('window:')) { + config.windowId = windowId + } else if (Number.isFinite(screenId) && screenId > 0) { + config.displayId = screenId + } else { + config.displayId = Number(getScreen().getPrimaryDisplay().id) + } + + nativeCaptureOutputBuffer = '' + nativeCaptureTargetPath = outputPath + nativeCaptureSystemAudioPath = systemAudioOutputPath + nativeCaptureMicrophonePath = microphoneOutputPath + nativeCaptureStopRequested = false + nativeCapturePaused = false + nativeCaptureProcess = spawn(helperPath, [JSON.stringify(config)], { + cwd: recordingsDir, + stdio: ['pipe', 'pipe', 'pipe'], + }) + attachNativeCaptureLifecycle(nativeCaptureProcess) + + nativeCaptureProcess.stdout.on('data', (chunk: Buffer) => { + nativeCaptureOutputBuffer += chunk.toString() + }) + nativeCaptureProcess.stderr.on('data', (chunk: Buffer) => { + nativeCaptureOutputBuffer += chunk.toString() + }) + + await waitForNativeCaptureStart(nativeCaptureProcess) + nativeScreenRecordingActive = true + + // If the native helper reported MICROPHONE_CAPTURE_UNAVAILABLE, it started + // capture without microphone. Clear the mic path so the renderer can fall + // back to a browser-side sidecar recording for the microphone track. + const micUnavailableNatively = nativeCaptureOutputBuffer.includes('MICROPHONE_CAPTURE_UNAVAILABLE') + if (micUnavailableNatively) { + nativeCaptureMicrophonePath = null + } + + recordNativeCaptureDiagnostics({ + backend: 'mac-screencapturekit', + phase: 'start', + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? 'unknown', + displayId: typeof config.displayId === 'number' ? config.displayId : null, + helperPath, + outputPath, + systemAudioPath: systemAudioOutputPath, + microphonePath: nativeCaptureMicrophonePath, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + }) + return { success: true, microphoneFallbackRequired: micUnavailableNatively } + } catch (error) { + console.error('Failed to start native ScreenCaptureKit recording:', error) + const errorStr = String(error) + + // Detect TCC (screen recording permission) errors and show a helpful dialog + if (errorStr.includes('declined TCC') || errorStr.includes('declined TCCs') || errorStr.includes('SCREEN_RECORDING_PERMISSION_DENIED')) { + const { response } = await dialog.showMessageBox({ + type: 'warning', + title: 'Screen Recording Permission Required', + message: 'Recordly needs screen recording permission to capture your screen.', + detail: 'Please open System Settings > Privacy & Security > Screen Recording, make sure Recordly is toggled ON, then try recording again.', + buttons: ['Open System Settings', 'Cancel'], + defaultId: 0, + cancelId: 1, + }) + if (response === 0) { + await shell.openExternal(getMacPrivacySettingsUrl('screen')) + } + try { nativeCaptureProcess?.kill() } catch { /* ignore */ } + nativeScreenRecordingActive = false + nativeCaptureProcess = null + nativeCaptureTargetPath = null + nativeCaptureSystemAudioPath = null + nativeCaptureMicrophonePath = null + nativeCaptureStopRequested = false + nativeCapturePaused = false + return { + success: false, + message: 'Screen recording permission not granted. Please allow access in System Settings and restart the app.', + userNotified: true, + } + } + + if (errorStr.includes('MICROPHONE_PERMISSION_DENIED')) { + const { response } = await dialog.showMessageBox({ + type: 'warning', + title: 'Microphone Permission Required', + message: 'Recordly needs microphone permission to record audio.', + detail: 'Please open System Settings > Privacy & Security > Microphone, make sure Recordly is toggled ON, then try recording again.', + buttons: ['Open System Settings', 'Cancel'], + defaultId: 0, + cancelId: 1, + }) + if (response === 0) { + await shell.openExternal(getMacPrivacySettingsUrl('microphone')) + } + try { nativeCaptureProcess?.kill() } catch { /* ignore */ } + nativeScreenRecordingActive = false + nativeCaptureProcess = null + nativeCaptureTargetPath = null + nativeCaptureSystemAudioPath = null + nativeCaptureMicrophonePath = null + nativeCaptureStopRequested = false + nativeCapturePaused = false + return { + success: false, + message: 'Microphone permission not granted. Please allow access in System Settings.', + userNotified: true, + } + } + + recordNativeCaptureDiagnostics({ + backend: 'mac-screencapturekit', + phase: 'start', + sourceId: source?.id ?? null, + sourceType: source?.sourceType ?? 'unknown', + helperPath: getNativeCaptureHelperBinaryPath(), + outputPath: nativeCaptureTargetPath, + systemAudioPath: nativeCaptureSystemAudioPath, + microphonePath: nativeCaptureMicrophonePath, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: await getFileSizeIfPresent(nativeCaptureTargetPath), + error: String(error), + }) + try { + nativeCaptureProcess?.kill() + } catch { + // ignore cleanup failures + } + nativeScreenRecordingActive = false + nativeCaptureProcess = null + nativeCaptureTargetPath = null + nativeCaptureSystemAudioPath = null + nativeCaptureMicrophonePath = null + nativeCaptureStopRequested = false + nativeCapturePaused = false + return { + success: false, + message: 'Failed to start native ScreenCaptureKit recording', + error: String(error), + } + } + }) + + ipcMain.handle('stop-native-screen-recording', async () => { + // Windows native capture stop path + if (process.platform === 'win32' && windowsNativeCaptureActive) { + try { + if (!windowsCaptureProcess) { + throw new Error('Native Windows capture process is not running') + } + + const proc = windowsCaptureProcess + const preferredVideoPath = windowsCaptureTargetPath + windowsCaptureStopRequested = true + proc.stdin.write('stop\n') + const tempVideoPath = await waitForWindowsCaptureStop(proc) + windowsCaptureProcess = null + windowsNativeCaptureActive = false + nativeScreenRecordingActive = false + windowsCaptureTargetPath = null + windowsCaptureStopRequested = false + windowsCapturePaused = false + + const finalVideoPath = preferredVideoPath ?? tempVideoPath + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath) + } + + windowsPendingVideoPath = finalVideoPath + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'stop', + outputPath: finalVideoPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: await getFileSizeIfPresent(finalVideoPath), + }) + return { success: true, path: finalVideoPath } + } catch (error) { + console.error('Failed to stop native Windows capture:', error) + const fallbackPath = windowsCaptureTargetPath + windowsNativeCaptureActive = false + nativeScreenRecordingActive = false + windowsCaptureProcess = null + windowsCaptureTargetPath = null + windowsCaptureStopRequested = false + windowsCapturePaused = false + windowsSystemAudioPath = null + windowsMicAudioPath = null + windowsPendingVideoPath = null + + if (fallbackPath) { + try { + await fs.access(fallbackPath) + windowsPendingVideoPath = fallbackPath + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'stop', + outputPath: fallbackPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: await getFileSizeIfPresent(fallbackPath), + error: String(error), + }) + return { success: true, path: fallbackPath } + } catch { + // File doesn't exist + } + } + + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'stop', + outputPath: fallbackPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + }) + + return { + success: false, + message: 'Failed to stop native Windows capture', + error: String(error), + } + } + } + + if (process.platform !== 'darwin') { + return { success: false, message: 'Native screen recording is only available on macOS.' } + } + + if (!nativeScreenRecordingActive) { + const recovered = await recoverNativeMacCaptureOutput() + if (recovered) { + return recovered + } + + return { success: false, message: 'No native screen recording is active.' } + } + + try { + if (!nativeCaptureProcess) { + throw new Error('Native capture helper process is not running') + } + + const process = nativeCaptureProcess + const preferredVideoPath = nativeCaptureTargetPath + const preferredSystemAudioPath = nativeCaptureSystemAudioPath + const preferredMicrophonePath = nativeCaptureMicrophonePath + console.log('[stop-native] Audio paths — system:', preferredSystemAudioPath, 'mic:', preferredMicrophonePath) + nativeCaptureStopRequested = true + process.stdin.write('stop\n') + const tempVideoPath = await waitForNativeCaptureStop(process) + console.log('[stop-native] Helper stopped, tempVideoPath:', tempVideoPath) + nativeCaptureProcess = null + nativeScreenRecordingActive = false + nativeCaptureTargetPath = null + nativeCaptureSystemAudioPath = null + nativeCaptureMicrophonePath = null + nativeCaptureStopRequested = false + nativeCapturePaused = false + + const finalVideoPath = preferredVideoPath ?? tempVideoPath + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath) + } + + if (preferredSystemAudioPath || preferredMicrophonePath) { + console.log('[stop-native] Attempting audio mux (merging separate tracks) into:', finalVideoPath) + try { + await muxNativeMacRecordingWithAudio(finalVideoPath, preferredSystemAudioPath, preferredMicrophonePath) + console.log('[stop-native] Audio mux completed successfully') + } catch (error) { + console.warn('[stop-native] Audio mux failed (video still has inline audio):', error) + } + } else { + console.log('[stop-native] No separate audio tracks to mux') + } + + return await finalizeStoredVideo(finalVideoPath) + } catch (error) { + console.error('Failed to stop native ScreenCaptureKit recording:', error) + const fallbackPath = nativeCaptureTargetPath + const fallbackSystemAudioPath = nativeCaptureSystemAudioPath + const fallbackMicrophonePath = nativeCaptureMicrophonePath + const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath) + nativeScreenRecordingActive = false + nativeCaptureProcess = null + nativeCaptureTargetPath = null + nativeCaptureSystemAudioPath = null + nativeCaptureMicrophonePath = null + nativeCaptureStopRequested = false + nativeCapturePaused = false + + recordNativeCaptureDiagnostics({ + backend: 'mac-screencapturekit', + phase: 'stop', + sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, + sourceType: lastNativeCaptureDiagnostics?.sourceType ?? 'unknown', + displayId: lastNativeCaptureDiagnostics?.displayId ?? null, + displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, + windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, + helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, + outputPath: fallbackPath, + systemAudioPath: fallbackSystemAudioPath, + microphonePath: fallbackMicrophonePath, + osRelease: lastNativeCaptureDiagnostics?.osRelease, + supported: lastNativeCaptureDiagnostics?.supported, + helperExists: lastNativeCaptureDiagnostics?.helperExists, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: fallbackFileSizeBytes, + error: String(error), + }) + + // Try to recover: if the target file exists on disk, finalize with it + if (fallbackPath) { + try { + await fs.access(fallbackPath) + console.log('[stop-native-screen-recording] Recovering with fallback path:', fallbackPath) + if (fallbackSystemAudioPath || fallbackMicrophonePath) { + try { + await muxNativeMacRecordingWithAudio( + fallbackPath, + fallbackSystemAudioPath, + fallbackMicrophonePath, + ) + } catch (muxError) { + console.warn('Failed to mux recovered native macOS audio into capture:', muxError) + } + } + return await finalizeStoredVideo(fallbackPath) + } catch { + // File doesn't exist or isn't accessible + } + } + + const recovered = await recoverNativeMacCaptureOutput() + if (recovered) { + return recovered + } + + return { + success: false, + message: 'Failed to stop native ScreenCaptureKit recording', + error: String(error), + } + } + }) + + ipcMain.handle('recover-native-screen-recording', async () => { + if (process.platform !== 'darwin') { + return { success: false, message: 'Native screen recording recovery is only available on macOS.' } + } + + const recovered = await recoverNativeMacCaptureOutput() + if (recovered) { + return recovered + } + + return { + success: false, + message: 'No recoverable native macOS recording output was found.', + } + }) + + ipcMain.handle('pause-native-screen-recording', async () => { + if (process.platform === 'win32') { + if (!windowsNativeCaptureActive || !windowsCaptureProcess) { + return { success: false, message: 'No native Windows screen recording is active.' } + } + + if (windowsCapturePaused) { + return { success: true } + } + + try { + windowsCaptureProcess.stdin.write('pause\n') + windowsCapturePaused = true + return { success: true } + } catch (error) { + return { success: false, message: 'Failed to pause native Windows capture', error: String(error) } + } + } + + if (process.platform !== 'darwin') { + return { success: false, message: 'Native screen recording is only available on macOS.' } + } + + if (!nativeScreenRecordingActive || !nativeCaptureProcess) { + return { success: false, message: 'No native screen recording is active.' } + } + + if (nativeCapturePaused) { + return { success: true } + } + + try { + nativeCaptureProcess.stdin.write('pause\n') + nativeCapturePaused = true + return { success: true } + } catch (error) { + return { success: false, message: 'Failed to pause native screen recording', error: String(error) } + } + }) + + ipcMain.handle('resume-native-screen-recording', async () => { + if (process.platform === 'win32') { + if (!windowsNativeCaptureActive || !windowsCaptureProcess) { + return { success: false, message: 'No native Windows screen recording is active.' } + } + + if (!windowsCapturePaused) { + return { success: true } + } + + try { + windowsCaptureProcess.stdin.write('resume\n') + windowsCapturePaused = false + return { success: true } + } catch (error) { + return { success: false, message: 'Failed to resume native Windows capture', error: String(error) } + } + } + + if (process.platform !== 'darwin') { + return { success: false, message: 'Native screen recording is only available on macOS.' } + } + + if (!nativeScreenRecordingActive || !nativeCaptureProcess) { + return { success: false, message: 'No native screen recording is active.' } + } + + if (!nativeCapturePaused) { + return { success: true } + } + + try { + nativeCaptureProcess.stdin.write('resume\n') + nativeCapturePaused = false + return { success: true } + } catch (error) { + return { success: false, message: 'Failed to resume native screen recording', error: String(error) } + } + }) + + ipcMain.handle('get-system-cursor-assets', async () => { + try { + return { success: true, cursors: await getSystemCursorAssets() } + } catch (error) { + console.error('Failed to load system cursor assets:', error) + return { success: false, cursors: {}, error: String(error) } + } + }) + + ipcMain.handle('is-native-windows-capture-available', async () => { + return { available: await isNativeWindowsCaptureAvailable() } + }) + + ipcMain.handle('get-last-native-capture-diagnostics', async () => { + return { success: true, diagnostics: lastNativeCaptureDiagnostics } + }) + + ipcMain.handle('get-video-audio-fallback-paths', async (_event, videoPath: string) => { + if (!videoPath) { + return { success: true, paths: [] } + } + + try { + const paths = await getCompanionAudioFallbackPaths(videoPath) + await Promise.all([ + rememberApprovedLocalReadPath(videoPath), + ...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)), + ]) + return { success: true, paths } + } catch (error) { + console.error('Failed to resolve companion audio fallback paths:', error) + return { success: false, paths: [], error: String(error) } + } + }) + + ipcMain.handle('mux-native-windows-recording', async (_event, pauseSegments?: PauseSegment[]) => { + const videoPath = windowsPendingVideoPath + windowsPendingVideoPath = null + + if (!videoPath) { + return { success: false, message: 'No native Windows video pending for mux' } + } + + try { + if (windowsSystemAudioPath || windowsMicAudioPath) { + await muxNativeWindowsVideoWithAudio(videoPath, windowsSystemAudioPath, windowsMicAudioPath, pauseSegments ?? []) + windowsSystemAudioPath = null + windowsMicAudioPath = null + } + + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'mux', + outputPath: videoPath, + fileSizeBytes: await getFileSizeIfPresent(videoPath), + }) + return await finalizeStoredVideo(videoPath) + } catch (error) { + console.error('Failed to mux native Windows recording:', error) + recordNativeCaptureDiagnostics({ + backend: 'windows-wgc', + phase: 'mux', + outputPath: videoPath, + systemAudioPath: windowsSystemAudioPath, + microphonePath: windowsMicAudioPath, + fileSizeBytes: await getFileSizeIfPresent(videoPath), + error: String(error), + }) + windowsSystemAudioPath = null + windowsMicAudioPath = null + try { + return await finalizeStoredVideo(videoPath) + } catch { + return { success: false, message: 'Failed to mux native Windows recording', error: String(error) } + } + } + }) + + ipcMain.handle('start-ffmpeg-recording', async (_, source: SelectedSource) => { + if (ffmpegCaptureProcess) { + return { success: false, message: 'An FFmpeg recording is already active.' } + } + + try { + const recordingsDir = await getRecordingsDir() + const ffmpegPath = getFfmpegBinaryPath() + const outputPath = path.join(recordingsDir, `recording-${Date.now()}.mp4`) + const args = await buildFfmpegCaptureArgs(source, outputPath) + + ffmpegCaptureOutputBuffer = '' + ffmpegCaptureTargetPath = outputPath + ffmpegCaptureProcess = spawn(ffmpegPath, args, { + cwd: recordingsDir, + stdio: ['pipe', 'pipe', 'pipe'], + }) + + ffmpegCaptureProcess.stdout.on('data', (chunk: Buffer) => { + ffmpegCaptureOutputBuffer += chunk.toString() + }) + ffmpegCaptureProcess.stderr.on('data', (chunk: Buffer) => { + ffmpegCaptureOutputBuffer += chunk.toString() + }) + + await waitForFfmpegCaptureStart(ffmpegCaptureProcess) + ffmpegScreenRecordingActive = true + return { success: true } + } catch (error) { + console.error('Failed to start FFmpeg recording:', error) + ffmpegScreenRecordingActive = false + ffmpegCaptureProcess = null + ffmpegCaptureTargetPath = null + return { + success: false, + message: 'Failed to start FFmpeg recording', + error: String(error), + } + } + }) + + ipcMain.handle('stop-ffmpeg-recording', async () => { + if (!ffmpegScreenRecordingActive) { + return { success: false, message: 'No FFmpeg recording is active.' } + } + + try { + if (!ffmpegCaptureProcess || !ffmpegCaptureTargetPath) { + throw new Error('FFmpeg process is not running') + } + + const process = ffmpegCaptureProcess + const outputPath = ffmpegCaptureTargetPath + process.stdin.write('q\n') + const finalVideoPath = await waitForFfmpegCaptureStop(process, outputPath) + + ffmpegCaptureProcess = null + ffmpegCaptureTargetPath = null + ffmpegScreenRecordingActive = false + + return await finalizeStoredVideo(finalVideoPath) + } catch (error) { + console.error('Failed to stop FFmpeg recording:', error) + ffmpegCaptureProcess = null + ffmpegCaptureTargetPath = null + ffmpegScreenRecordingActive = false + return { + success: false, + message: 'Failed to stop FFmpeg recording', + error: String(error), + } + } + }) + + + + ipcMain.handle('store-microphone-sidecar', async (_, audioData: ArrayBuffer, videoPath: string) => { + try { + const baseName = videoPath.replace(/\.[^.]+$/, '') + const sidecarPath = `${baseName}.mic.webm` + await fs.writeFile(sidecarPath, Buffer.from(audioData)) + return { success: true, path: sidecarPath } + } catch (error) { + console.error('Failed to store microphone sidecar:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('store-recorded-video', async (_, videoData: ArrayBuffer, fileName: string) => { + try { + const recordingsDir = await getRecordingsDir() + const videoPath = path.join(recordingsDir, fileName) + await fs.writeFile(videoPath, Buffer.from(videoData)) + return await finalizeStoredVideo(videoPath) + } catch (error) { + console.error('Failed to store video:', error) + return { + success: false, + message: 'Failed to store video', + error: String(error) + } + } + }) + + + + ipcMain.handle('get-recorded-video-path', async () => { + try { + const recordingsDir = await getRecordingsDir() + const files = await fs.readdir(recordingsDir) + const videoFiles = files.filter(file => /\.(webm|mov|mp4)$/i.test(file)) + + if (videoFiles.length === 0) { + return { success: false, message: 'No recorded video found' } + } + + const latestVideo = videoFiles.sort().reverse()[0] + const videoPath = path.join(recordingsDir, latestVideo) + + return { success: true, path: videoPath } + } catch (error) { + console.error('Failed to get video path:', error) + return { success: false, message: 'Failed to get video path', error: String(error) } + } + }) + + ipcMain.handle('set-recording-state', (_, recording: boolean) => { + if (recording) { + stopCursorCapture() + stopInteractionCapture() + startWindowBoundsCapture() + void startNativeCursorMonitor() + isCursorCaptureActive = true + activeCursorSamples = [] + pendingCursorSamples = [] + cursorCaptureStartTimeMs = Date.now() + linuxCursorScreenPoint = null + lastLeftClick = null + sampleCursorPoint() + cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS) + void startInteractionCapture() + } else { + isCursorCaptureActive = false + stopCursorCapture() + stopInteractionCapture() + stopWindowBoundsCapture() + stopNativeCursorMonitor() + showCursor() + linuxCursorScreenPoint = null + snapshotCursorTelemetryForPersistence() + activeCursorSamples = [] + } + + const source = selectedSource || { name: 'Screen' } + BrowserWindow.getAllWindows().forEach((window) => { + if (!window.isDestroyed()) { + window.webContents.send('recording-state-changed', { + recording, + sourceName: source.name, + }) + } + }) + + if (onRecordingStateChange) { + onRecordingStateChange(recording, source.name) + } + }) + + ipcMain.handle('get-cursor-telemetry', async (_, videoPath?: string) => { + const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath) + if (!targetVideoPath) { + return { success: true, samples: [] } + } + + const telemetryPath = getTelemetryPathForVideo(targetVideoPath) + try { + const content = await fs.readFile(telemetryPath, 'utf-8') + const parsed = JSON.parse(content) + const rawSamples = Array.isArray(parsed) + ? parsed + : (Array.isArray(parsed?.samples) ? parsed.samples : []) + + const samples: CursorTelemetryPoint[] = rawSamples + .filter((sample: unknown) => Boolean(sample && typeof sample === 'object')) + .map((sample: unknown) => { + const point = sample as Partial + return { + timeMs: typeof point.timeMs === 'number' && Number.isFinite(point.timeMs) ? Math.max(0, point.timeMs) : 0, + cx: typeof point.cx === 'number' && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, + cy: typeof point.cy === 'number' && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, + interactionType: point.interactionType === 'click' + || point.interactionType === 'double-click' + || point.interactionType === 'right-click' + || point.interactionType === 'middle-click' + || point.interactionType === 'move' + || point.interactionType === 'mouseup' + ? point.interactionType + : undefined, + cursorType: point.cursorType === 'arrow' + || point.cursorType === 'text' + || point.cursorType === 'pointer' + || point.cursorType === 'crosshair' + || point.cursorType === 'open-hand' + || point.cursorType === 'closed-hand' + || point.cursorType === 'resize-ew' + || point.cursorType === 'resize-ns' + || point.cursorType === 'not-allowed' + ? point.cursorType + : undefined, + } + }) + .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs) + + return { success: true, samples } + } catch (error) { + const nodeError = error as NodeJS.ErrnoException + if (nodeError.code === 'ENOENT') { + return { success: true, samples: [] } + } + console.error('Failed to load cursor telemetry:', error) + return { success: false, message: 'Failed to load cursor telemetry', error: String(error), samples: [] } + } + }) + + + ipcMain.handle('open-external-url', async (_, url: string) => { + try { + // Security: only allow http/https URLs to prevent file:// or custom protocol abuse + const parsed = new URL(url) + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + return { success: false, error: `Blocked non-HTTP URL: ${parsed.protocol}` } + } + await shell.openExternal(url) + return { success: true } + } catch (error) { + console.error('Failed to open URL:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('get-accessibility-permission-status', () => { + if (process.platform !== 'darwin') { + return { success: true, trusted: true, prompted: false } + } + + return { + success: true, + trusted: systemPreferences.isTrustedAccessibilityClient(false), + prompted: false, + } + }) + + ipcMain.handle('request-accessibility-permission', () => { + if (process.platform !== 'darwin') { + return { success: true, trusted: true, prompted: false } + } + + return { + success: true, + trusted: systemPreferences.isTrustedAccessibilityClient(true), + prompted: true, + } + }) + + ipcMain.handle('get-screen-recording-permission-status', () => { + if (process.platform !== 'darwin') { + return { success: true, status: 'granted' } + } + + try { + return { + success: true, + status: systemPreferences.getMediaAccessStatus('screen'), + } + } catch (error) { + console.error('Failed to get screen recording permission status:', error) + return { success: false, status: 'unknown', error: String(error) } + } + }) + + ipcMain.handle('open-screen-recording-preferences', async () => { + if (process.platform !== 'darwin') { + return { success: true } + } + + try { + await shell.openExternal(getMacPrivacySettingsUrl('screen')) + return { success: true } + } catch (error) { + console.error('Failed to open Screen Recording preferences:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('open-accessibility-preferences', async () => { + if (process.platform !== 'darwin') { + return { success: true } + } + + try { + await shell.openExternal(getMacPrivacySettingsUrl('accessibility')) + return { success: true } + } catch (error) { + console.error('Failed to open Accessibility preferences:', error) + return { success: false, error: String(error) } + } + }) + + // Generate a tiny thumbnail for a wallpaper image and cache it in userData. + // Returns the cached thumbnail as raw JPEG bytes for fast grid rendering. + // Serialized to prevent concurrent nativeImage operations from eating memory. + const THUMB_SIZE = 96 + const thumbCacheDir = path.join(USER_DATA_PATH, 'wallpaper-thumbs') + let thumbGenerationQueue: Promise = Promise.resolve() + + ipcMain.handle('generate-wallpaper-thumbnail', async (_, filePath: string) => { + try { + const resolved = normalizePath(filePath) + const realResolved = await fs.realpath(resolved).catch(() => resolved) + + if (!isAllowedLocalReadPath(resolved) && !isAllowedLocalReadPath(realResolved)) { + return { success: false, error: 'Access denied' } + } + + // Deterministic cache key from file path + mtime + const stat = await fs.stat(resolved) + const cacheKey = Buffer.from(`${resolved}:${stat.mtimeMs}`).toString('base64url') + const thumbPath = path.join(thumbCacheDir, `${cacheKey}.jpg`) + + // Return cached thumbnail if it exists (no queue needed) + if (existsSync(thumbPath)) { + const data = await fs.readFile(thumbPath) + return { success: true, data } + } + + // Serialize nativeImage operations to avoid OOM from concurrent full-res decodes + let jpegData: Buffer + const generation = thumbGenerationQueue.then(async () => { + const { nativeImage } = await import('electron') + const img = nativeImage.createFromPath(resolved) + if (img.isEmpty()) { + throw new Error('Failed to load image') + } + const { width, height } = img.getSize() + const scale = THUMB_SIZE / Math.min(width, height) + const resized = img.resize({ + width: Math.round(width * scale), + height: Math.round(height * scale), + quality: 'good', + }) + jpegData = resized.toJPEG(70) + + // Cache to disk + await fs.mkdir(thumbCacheDir, { recursive: true }) + await fs.writeFile(thumbPath, jpegData) + }) + // Keep the queue moving even if one fails + thumbGenerationQueue = generation.catch(() => {}) + await generation + + return { success: true, data: jpegData! } + } catch (error) { + return { success: false, error: String(error) } + } + }) + + // Return base path for assets so renderer can resolve file:// paths in production + ipcMain.handle('get-asset-base-path', () => { + try { + const assetPath = getAssetRootPath() + return pathToFileURL(`${assetPath}${path.sep}`).toString() + } catch (err) { + console.error('Failed to resolve asset base path:', err) + return null + } + }) + + ipcMain.handle('list-asset-directory', async (_, relativeDir: string) => { + try { + const normalizedRelativeDir = String(relativeDir ?? '') + .replace(/\\/g, '/') + .replace(/^\/+/, '') + + const assetRootPath = path.resolve(getAssetRootPath()) + const targetDirPath = path.resolve(assetRootPath, normalizedRelativeDir) + if (targetDirPath !== assetRootPath && !targetDirPath.startsWith(`${assetRootPath}${path.sep}`)) { + return { success: false, error: 'Invalid asset directory' } + } + + const entries = await fs.readdir(targetDirPath, { withFileTypes: true }) + const files = entries + .filter((entry) => entry.isFile()) + .map((entry) => entry.name) + .sort(new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }).compare) + + return { success: true, files } + } catch (error) { + console.error('Failed to list asset directory:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('read-local-file', async (_, filePath: string) => { + try { + const resolved = normalizePath(filePath) + const realResolved = await fs.realpath(resolved).catch(() => resolved) + if (!isAllowedLocalReadPath(resolved) && !isAllowedLocalReadPath(realResolved)) { + console.warn(`[read-local-file] Blocked read outside allowed directories: ${resolved}`) + return { success: false, error: 'Access denied: path outside allowed directories' } + } + + const data = await fs.readFile(resolved) + return { success: true, data } + } catch (error) { + console.error('Failed to read local file:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle( + 'native-video-export-start', + async ( + event, + options: { + width: number + height: number + frameRate: number + bitrate: number + encodingMode: NativeExportEncodingMode + inputMode?: 'rawvideo' | 'h264-stream' + }, + ) => { + try { + if (options.width % 2 !== 0 || options.height % 2 !== 0) { + throw new Error('Native export requires even output dimensions') + } + + const ffmpegPath = getFfmpegBinaryPath() + const inputMode = options.inputMode ?? 'rawvideo' + const sessionId = `recordly-export-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + const outputPath = path.join(app.getPath('temp'), `${sessionId}.mp4`) + + let encoderName: string + let ffmpegArgs: string[] + + if (inputMode === 'h264-stream') { + // Pre-encoded H.264 Annex B from browser VideoEncoder — just stream-copy into MP4 + encoderName = 'h264-stream-copy' + ffmpegArgs = buildNativeH264StreamExportArgs({ frameRate: options.frameRate, outputPath }) + } else { + encoderName = await resolveNativeVideoEncoder(ffmpegPath, options.encodingMode) + ffmpegArgs = buildNativeVideoExportArgs(encoderName, options, outputPath) + } + + const ffmpegProcess = spawn(ffmpegPath, ffmpegArgs, { + stdio: ['pipe', 'ignore', 'pipe'], + }) as ChildProcessByStdio + // For rawvideo, frames are a fixed RGBA size. For h264-stream, chunks are variable. + const inputByteSize = inputMode === 'rawvideo' ? getNativeVideoInputByteSize(options.width, options.height) : 0 + + const session: NativeVideoExportSession = { + ffmpegProcess, + outputPath, + inputByteSize, + inputMode, + maxQueuedWriteBytes: inputMode === 'h264-stream' ? 8 * 1024 * 1024 : getNativeVideoExportMaxQueuedWriteBytes(inputByteSize), + stderrOutput: '', + encoderName, + processError: null, + stdinError: null, + terminating: false, + writeSequence: Promise.resolve(), + sender: event.sender, + pendingWriteRequestIds: new Set(), + completionPromise: new Promise((resolve, reject) => { + ffmpegProcess.once('error', (error) => { + const processError = error instanceof Error ? error : new Error(String(error)) + if (session.terminating) { + resolve() + return + } + + session.processError = processError + reject(processError) + }) + ffmpegProcess.stdin.once('error', (error) => { + const stdinError = error instanceof Error ? error : new Error(String(error)) + if (session.terminating && isIgnorableNativeVideoExportStreamError(stdinError)) { + return + } + + session.stdinError = stdinError + }) + ffmpegProcess.once('close', (code, signal) => { + if (session.terminating) { + resolve() + return + } + + if (code === 0) { + resolve() + return + } + + reject( + new Error( + getNativeVideoExportSessionError( + session, + `FFmpeg exited with code ${code ?? 'unknown'}${signal ? ` (signal ${signal})` : ''}`, + ), + ), + ) + }) + }), + } + void session.completionPromise.catch(() => undefined) + + ffmpegProcess.stderr.on('data', (chunk: Buffer) => { + session.stderrOutput += chunk.toString() + }) + + nativeVideoExportSessions.set(sessionId, session) + + console.log( + `[native-export] Started ${isHardwareAcceleratedVideoEncoder(encoderName) ? 'hardware' : 'software'} session ${sessionId} with ${encoderName}`, + ) + + return { + success: true, + sessionId, + encoderName, + } + } catch (error) { + console.error('[native-export] Failed to start native video export session:', error) + return { + success: false, + error: String(error), + } + } + }, + ) + + ipcMain.on( + 'native-video-export-write-frame-async', + ( + event, + payload: { + sessionId: string + requestId: number + frameData: Uint8Array + }, + ) => { + const sessionId = payload?.sessionId + const requestId = payload?.requestId + const frameData = payload?.frameData + + if (typeof sessionId !== 'string' || typeof requestId !== 'number' || !frameData) { + return + } + + const session = nativeVideoExportSessions.get(sessionId) + if (!session) { + sendNativeVideoExportWriteFrameResult(event.sender, sessionId, requestId, { + success: false, + error: 'Invalid native export session', + }) + return + } + + session.sender = event.sender + session.pendingWriteRequestIds.add(requestId) + + if (session.terminating) { + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: false, + error: 'Native video export session was cancelled', + }) + return + } + + if (session.inputMode !== 'h264-stream' && frameData.byteLength !== session.inputByteSize) { + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: false, + error: `Native video export expected ${session.inputByteSize} bytes per frame but received ${frameData.byteLength}`, + }) + return + } + + void enqueueNativeVideoExportFrameWrite(session, frameData) + .then(() => { + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: true, + }) + }) + .catch((error) => { + session.stdinError = error instanceof Error ? error : new Error(String(error)) + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: false, + error: getNativeVideoExportSessionError( + session, + session.stdinError.message, + ), + }) + }) + }, + ) + + ipcMain.handle( + 'native-video-export-finish', + async (_, sessionId: string, options?: NativeVideoExportFinishOptions) => { + const session = nativeVideoExportSessions.get(sessionId) + if (!session) { + return { success: false, error: 'Invalid native export session' } + } + + try { + await session.writeSequence + if (!session.ffmpegProcess.stdin.destroyed && !session.ffmpegProcess.stdin.writableEnded) { + session.ffmpegProcess.stdin.end() + } + await session.completionPromise + + const finalizedPath = await muxNativeVideoExportAudio(session.outputPath, options ?? {}) + const data = await fs.readFile(finalizedPath) + nativeVideoExportSessions.delete(sessionId) + await removeTemporaryExportFile(finalizedPath) + + return { + success: true, + data: new Uint8Array(data), + encoderName: session.encoderName, + } + } catch (error) { + flushNativeVideoExportPendingWriteRequests( + sessionId, + session, + String(error), + ) + nativeVideoExportSessions.delete(sessionId) + await removeTemporaryExportFile(session.outputPath) + const finalizedSuffix = session.outputPath.replace(/\.mp4$/, '-final.mp4') + await removeTemporaryExportFile(finalizedSuffix) + return { + success: false, + error: String(error), + } + } + }, + ) + + ipcMain.handle( + 'mux-exported-video-audio', + async (_, videoData: ArrayBuffer, options?: NativeVideoExportFinishOptions) => { + try { + const data = await muxExportedVideoAudioBuffer(videoData, options ?? {}) + return { + success: true, + data, + } + } catch (error) { + return { + success: false, + error: String(error), + } + } + }, + ) + + ipcMain.handle('native-video-export-cancel', async (_, sessionId: string) => { + const session = nativeVideoExportSessions.get(sessionId) + if (!session) { + return { success: true } + } + + session.terminating = true + nativeVideoExportSessions.delete(sessionId) + flushNativeVideoExportPendingWriteRequests( + sessionId, + session, + 'Native video export session was cancelled', + ) + + try { + if (!session.ffmpegProcess.stdin.destroyed && !session.ffmpegProcess.stdin.writableEnded) { + session.ffmpegProcess.stdin.destroy() + } + } catch { + // Stream may already be closed. + } + + try { + session.ffmpegProcess.kill('SIGKILL') + } catch { + // Process may already be closed. + } + + await session.completionPromise.catch(() => undefined) + await removeTemporaryExportFile(session.outputPath) + return { success: true } + }) + + ipcMain.handle('save-exported-video', async (event, videoData: ArrayBuffer, fileName: string) => { + try { + // Determine file type from extension + const isGif = fileName.toLowerCase().endsWith('.gif'); + const filters = isGif + ? [{ name: 'GIF Image', extensions: ['gif'] }] + : [{ name: 'MP4 Video', extensions: ['mp4'] }]; + const parentWindow = BrowserWindow.fromWebContents(event.sender) + const saveDialogOptions: SaveDialogOptions = { + title: isGif ? 'Save Exported GIF' : 'Save Exported Video', + defaultPath: path.join(app.getPath('downloads'), fileName), + filters, + properties: ['createDirectory', 'showOverwriteConfirmation'], + } + + const result = parentWindow + ? await dialog.showSaveDialog(parentWindow, saveDialogOptions) + : await dialog.showSaveDialog(saveDialogOptions) + + if (result.canceled || !result.filePath) { + return { + success: false, + canceled: true, + message: 'Export canceled' + }; + } + + await fs.writeFile(result.filePath, Buffer.from(videoData)); + + return { + success: true, + path: result.filePath, + message: 'Video exported successfully' + }; + } catch (error) { + console.error('Failed to save exported video:', error) + return { + success: false, + message: 'Failed to save exported video', + error: String(error) + } + } + }) + + ipcMain.handle('write-exported-video-to-path', async (_event, videoData: ArrayBuffer, outputPath: string) => { + try { + const resolvedPath = path.resolve(outputPath) + await fs.mkdir(path.dirname(resolvedPath), { recursive: true }); + await fs.writeFile(resolvedPath, Buffer.from(videoData)); + + return { + success: true, + path: outputPath, + message: 'Video exported successfully', + canceled: false, + }; + } catch (error) { + console.error('Failed to write exported video to path:', error) + return { + success: false, + message: 'Failed to write exported video', + canceled: false, + error: String(error) + } + } + }) + + ipcMain.handle('open-video-file-picker', async () => { + try { + const recordingsDir = await getRecordingsDir() + const result = await dialog.showOpenDialog({ + title: 'Select Video File', + defaultPath: recordingsDir, + filters: [ + { name: 'Video Files', extensions: ['webm', 'mp4', 'mov', 'avi', 'mkv'] }, + { name: 'All Files', extensions: ['*'] } + ], + properties: ['openFile'] + }); + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + + approveUserPath(result.filePaths[0]) + currentProjectPath = null + return { + success: true, + path: result.filePaths[0] + }; + } catch (error) { + console.error('Failed to open file picker:', error); + return { + success: false, + message: 'Failed to open file picker', + error: String(error) + }; + } + }); + + ipcMain.handle('open-audio-file-picker', async () => { + try { + const result = await dialog.showOpenDialog({ + title: 'Select Audio File', + filters: [ + { name: 'Audio Files', extensions: ['mp3', 'wav', 'aac', 'm4a', 'flac', 'ogg'] }, + { name: 'All Files', extensions: ['*'] } + ], + properties: ['openFile'] + }); + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + + approveUserPath(result.filePaths[0]) + return { + success: true, + path: result.filePaths[0] + }; + } catch (error) { + console.error('Failed to open audio file picker:', error); + return { + success: false, + message: 'Failed to open audio file picker', + error: String(error) + }; + } + }); + + ipcMain.handle('open-whisper-executable-picker', async () => { + try { + const result = await dialog.showOpenDialog({ + title: 'Select Whisper Executable', + filters: [ + { name: 'Executables', extensions: process.platform === 'win32' ? ['exe', 'cmd', 'bat'] : ['*'] }, + { name: 'All Files', extensions: ['*'] }, + ], + properties: ['openFile'], + }) + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true } + } + + approveUserPath(result.filePaths[0]) + return { success: true, path: result.filePaths[0] } + } catch (error) { + console.error('Failed to open Whisper executable picker:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('open-whisper-model-picker', async () => { + try { + const result = await dialog.showOpenDialog({ + title: 'Select Whisper Model', + filters: [ + { name: 'Whisper Models', extensions: ['bin'] }, + { name: 'All Files', extensions: ['*'] }, + ], + properties: ['openFile'], + }) + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true } + } + + approveUserPath(result.filePaths[0]) + return { success: true, path: result.filePaths[0] } + } catch (error) { + console.error('Failed to open Whisper model picker:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('get-whisper-small-model-status', async () => { + try { + return await getWhisperSmallModelStatus() + } catch (error) { + return { success: false, exists: false, path: null, error: String(error) } + } + }) + + ipcMain.handle('download-whisper-small-model', async (event) => { + try { + const existing = await getWhisperSmallModelStatus() + if (existing.exists) { + sendWhisperModelDownloadProgress(event.sender, { + status: 'downloaded', + progress: 100, + path: existing.path, + }) + return { success: true, path: existing.path, alreadyDownloaded: true } + } + + const modelPath = await downloadWhisperSmallModel(event.sender) + return { success: true, path: modelPath } + } catch (error) { + console.error('Failed to download Whisper small model:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('delete-whisper-small-model', async (event) => { + try { + await deleteWhisperSmallModel() + sendWhisperModelDownloadProgress(event.sender, { + status: 'idle', + progress: 0, + path: null, + }) + return { success: true } + } catch (error) { + console.error('Failed to delete Whisper small model:', error) + // Verify whether the file was actually removed despite the error + const status = await getWhisperSmallModelStatus() + if (!status.exists) { + // File is gone — treat as success + sendWhisperModelDownloadProgress(event.sender, { + status: 'idle', + progress: 0, + path: null, + }) + return { success: true } + } + sendWhisperModelDownloadProgress(event.sender, { + status: 'error', + progress: 0, + path: null, + error: String(error), + }) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('generate-auto-captions', async (_, options: { + videoPath: string + whisperExecutablePath: string + whisperModelPath: string + language?: string + }) => { + try { + const result = await generateAutoCaptionsFromVideo(options) + return { + success: true, + cues: result.cues, + message: result.audioSourceLabel === 'recording' + ? `Generated ${result.cues.length} caption cues.` + : `Generated ${result.cues.length} caption cues from the ${result.audioSourceLabel}.`, + } + } catch (error) { + console.error('Failed to generate auto captions:', error) + return { + success: false, + error: String(error), + message: 'Failed to generate auto captions', + } + } + }) + + ipcMain.handle('reveal-in-folder', async (_, filePath: string) => { + try { + // shell.showItemInFolder doesn't return a value, it throws on error + shell.showItemInFolder(filePath); + return { success: true }; + } catch (error) { + console.error(`Error revealing item in folder: ${filePath}`, error); + // Fallback to open the directory if revealing the item fails + // This might happen if the file was moved or deleted after export, + // or if the path is somehow invalid for showItemInFolder + try { + const openPathResult = await shell.openPath(path.dirname(filePath)); + if (openPathResult) { + // openPath returned an error message + return { success: false, error: openPathResult }; + } + return { success: true, message: 'Could not reveal item, but opened directory.' }; + } catch (openError) { + console.error(`Error opening directory: ${path.dirname(filePath)}`, openError); + return { success: false, error: String(error) }; + } + } + }); + + ipcMain.handle('open-recordings-folder', async () => { + try { + const recordingsDir = await getRecordingsDir(); + const openPathResult = await shell.openPath(recordingsDir); + if (openPathResult) { + return { success: false, error: openPathResult, message: 'Failed to open recordings folder.' }; + } + + return { success: true }; + } catch (error) { + console.error('Failed to open recordings folder:', error); + return { success: false, error: String(error), message: 'Failed to open recordings folder.' }; + } + }); + + ipcMain.handle('get-recordings-directory', async () => { + try { + const recordingsDir = await getRecordingsDir() + return { + success: true, + path: recordingsDir, + isDefault: recordingsDir === RECORDINGS_DIR, + } + } catch (error) { + return { + success: false, + path: RECORDINGS_DIR, + isDefault: true, + error: String(error), + } + } + }) + + ipcMain.handle('choose-recordings-directory', async () => { + try { + const current = await getRecordingsDir() + const result = await dialog.showOpenDialog({ + title: 'Choose recordings folder', + defaultPath: current, + properties: ['openDirectory', 'createDirectory', 'promptToCreate'], + }) + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true, path: current } + } + + const selectedPath = path.resolve(result.filePaths[0]) + await fs.mkdir(selectedPath, { recursive: true }) + await fs.access(selectedPath, fsConstants.W_OK) + await persistRecordingsDirectorySetting(selectedPath) + + return { success: true, path: selectedPath, isDefault: selectedPath === RECORDINGS_DIR } + } catch (error) { + return { success: false, error: String(error), message: 'Failed to set recordings folder' } + } + }) + + ipcMain.handle('save-project-file', async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string, thumbnailDataUrl?: string | null) => { + try { + const projectsDir = await getProjectsDir() + const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) + ? existingProjectPath + : null + + if (trustedExistingProjectPath) { + await fs.writeFile(trustedExistingProjectPath, JSON.stringify(projectData, null, 2), 'utf-8') + currentProjectPath = trustedExistingProjectPath + await saveProjectThumbnail(trustedExistingProjectPath, thumbnailDataUrl) + await rememberRecentProject(trustedExistingProjectPath) + return { + success: true, + path: trustedExistingProjectPath, + message: 'Project saved successfully' + } + } + + const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, '_') + const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) + ? safeName + : `${safeName}.${PROJECT_FILE_EXTENSION}` + + const result = await dialog.showSaveDialog({ + title: 'Save Recordly Project', + defaultPath: path.join(projectsDir, defaultName), + filters: [ + { name: 'Recordly Project', extensions: [PROJECT_FILE_EXTENSION] }, + { name: 'JSON', extensions: ['json'] } + ], + properties: ['createDirectory', 'showOverwriteConfirmation'] + }) + + if (result.canceled || !result.filePath) { + return { + success: false, + canceled: true, + message: 'Save project canceled' + } + } + + await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), 'utf-8') + currentProjectPath = result.filePath + await saveProjectThumbnail(result.filePath, thumbnailDataUrl) + await rememberRecentProject(result.filePath) + + return { + success: true, + path: result.filePath, + message: 'Project saved successfully' + } + } catch (error) { + console.error('Failed to save project file:', error) + return { + success: false, + message: 'Failed to save project file', + error: String(error) + } + } + }) + + ipcMain.handle('load-project-file', async () => { + try { + const projectsDir = await getProjectsDir() + const result = await dialog.showOpenDialog({ + title: 'Open Recordly Project', + defaultPath: projectsDir, + filters: [ + { name: 'Recordly Project', extensions: [PROJECT_FILE_EXTENSION, ...LEGACY_PROJECT_FILE_EXTENSIONS] }, + { name: 'JSON', extensions: ['json'] }, + { name: 'All Files', extensions: ['*'] } + ], + properties: ['openFile'] + }) + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true, message: 'Open project canceled' } + } + + return await loadProjectFromPath(result.filePaths[0]) + } catch (error) { + console.error('Failed to load project file:', error) + return { + success: false, + message: 'Failed to load project file', + error: String(error) + } + } + }) + + ipcMain.handle('load-current-project-file', async () => { + try { + if (!currentProjectPath) { + return { success: false, message: 'No active project' } + } + + return await loadProjectFromPath(currentProjectPath) + } catch (error) { + console.error('Failed to load current project file:', error) + return { + success: false, + message: 'Failed to load current project file', + error: String(error), + } + } + }) + + ipcMain.handle('get-projects-directory', async () => { + try { + return { + success: true, + path: await getProjectsDir(), + } + } catch (error) { + return { + success: false, + error: String(error), + } + } + }) + + ipcMain.handle('list-project-files', async () => { + try { + const library = await listProjectLibraryEntries() + return { + success: true, + projectsDir: library.projectsDir, + entries: library.entries, + } + } catch (error) { + return { + success: false, + projectsDir: null, + entries: [], + error: String(error), + } + } + }) + + ipcMain.handle('open-project-file-at-path', async (_, filePath: string) => { + try { + return await loadProjectFromPath(filePath) + } catch (error) { + console.error('Failed to open project file at path:', error) + return { + success: false, + message: 'Failed to open project file', + error: String(error), + } + } + }) + + ipcMain.handle('open-projects-directory', async () => { + try { + const projectsDir = await getProjectsDir() + const openPathResult = await shell.openPath(projectsDir) + if (openPathResult) { + return { success: false, error: openPathResult, message: 'Failed to open projects folder.' } + } + + return { success: true, path: projectsDir } + } catch (error) { + console.error('Failed to open projects folder:', error) + return { success: false, error: String(error), message: 'Failed to open projects folder.' } + } + }) + ipcMain.handle('set-current-video-path', async (_, path: string) => { + currentVideoPath = normalizeVideoSourcePath(path) ?? path + approveUserPath(currentVideoPath) + const resolvedSession = await resolveRecordingSession(currentVideoPath) + ?? { + videoPath: currentVideoPath, + webcamPath: null, + timeOffsetMs: 0, + } + + currentRecordingSession = resolvedSession + await replaceApprovedSessionLocalReadPaths([ + resolvedSession.videoPath, + resolvedSession.webcamPath, + ]) + + if (resolvedSession.webcamPath) { + await persistRecordingSessionManifest(resolvedSession) + } + + currentProjectPath = null + return { success: true, webcamPath: resolvedSession.webcamPath ?? null } + }) + + ipcMain.handle('set-current-recording-session', async (_, session: { videoPath: string; webcamPath?: string | null; timeOffsetMs?: number }) => { + const normalizedVideoPath = normalizeVideoSourcePath(session.videoPath) ?? session.videoPath + currentVideoPath = normalizedVideoPath + currentRecordingSession = { + videoPath: normalizedVideoPath, + webcamPath: normalizeVideoSourcePath(session.webcamPath ?? null), + timeOffsetMs: normalizeRecordingTimeOffsetMs(session.timeOffsetMs), + } + await replaceApprovedSessionLocalReadPaths([ + currentRecordingSession.videoPath, + currentRecordingSession.webcamPath, + ]) + currentProjectPath = null + await persistRecordingSessionManifest(currentRecordingSession) + return { success: true } + }) + + ipcMain.handle('get-current-recording-session', () => { + if (!currentRecordingSession) { + return { success: false } + } + + return { + success: true, + session: currentRecordingSession, + } + }) + + ipcMain.handle('get-current-video-path', () => { + return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; + }); + + ipcMain.handle('clear-current-video-path', () => { + currentVideoPath = null; + currentRecordingSession = null; + return { success: true }; + }); + + ipcMain.handle('delete-recording-file', async (_, filePath: string) => { + try { + if (!filePath || !isAutoRecordingPath(filePath)) { + return { success: false, error: 'Only auto-generated recordings can be deleted' }; + } + await fs.unlink(filePath); + // Also delete the cursor telemetry sidecar if it exists + const telemetryPath = getTelemetryPathForVideo(filePath); + await fs.unlink(telemetryPath).catch(() => {}); + if (currentVideoPath === filePath) { + currentVideoPath = null; + currentRecordingSession = null; + } + return { success: true }; + } catch (error) { + return { success: false, error: String(error) }; + } + }); + + ipcMain.handle('app:getVersion', () => { + return app.getVersion() + }) + + ipcMain.handle('get-platform', () => { + return process.platform; + }); + + // --------------------------------------------------------------------------- + // Cursor hiding for the browser-capture fallback. + // The IPC promise resolves only after the cursor hide attempt completes. + // --------------------------------------------------------------------------- + ipcMain.handle('hide-cursor', () => { + if (process.platform !== 'win32') { + return { success: true } + } + + return { success: hideCursor() } + }) + + ipcMain.handle('get-shortcuts', async () => { + try { + const data = await fs.readFile(SHORTCUTS_FILE, 'utf-8'); + return JSON.parse(data); + } catch { + return null; + } + }); + + ipcMain.handle('save-shortcuts', async (_, shortcuts: unknown) => { + try { + await fs.writeFile(SHORTCUTS_FILE, JSON.stringify(shortcuts, null, 2), 'utf-8'); + return { success: true }; + } catch (error) { + console.error('Failed to save shortcuts:', error); + return { success: false, error: String(error) }; + } + }); + + // --------------------------------------------------------------------------- + // Countdown timer before recording + // --------------------------------------------------------------------------- + ipcMain.handle('get-recording-preferences', async () => { + try { + const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, 'utf-8') + const parsed = JSON.parse(content) as Record + return { + success: true, + microphoneEnabled: parsed.microphoneEnabled === true, + microphoneDeviceId: typeof parsed.microphoneDeviceId === 'string' ? parsed.microphoneDeviceId : undefined, + systemAudioEnabled: parsed.systemAudioEnabled !== false, + } + } catch { + return { success: true, microphoneEnabled: false, microphoneDeviceId: undefined, systemAudioEnabled: true } + } + }) + + ipcMain.handle('set-recording-preferences', async (_, prefs: { microphoneEnabled?: boolean; microphoneDeviceId?: string; systemAudioEnabled?: boolean }) => { + try { + let existing: Record = {} + try { + const content = await fs.readFile(RECORDINGS_SETTINGS_FILE, 'utf-8') + existing = JSON.parse(content) as Record + } catch { + // file doesn't exist yet + } + const merged = { ...existing, ...prefs } + await fs.writeFile(RECORDINGS_SETTINGS_FILE, JSON.stringify(merged, null, 2), 'utf-8') + return { success: true } + } catch (error) { + console.error('Failed to save recording preferences:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('get-countdown-delay', async () => { + try { + const content = await fs.readFile(COUNTDOWN_SETTINGS_FILE, 'utf-8') + const parsed = JSON.parse(content) as { delay?: number } + return { success: true, delay: parsed.delay ?? 3 } + } catch { + return { success: true, delay: 3 } + } + }) + + ipcMain.handle('set-countdown-delay', async (_, delay: number) => { + try { + await fs.writeFile(COUNTDOWN_SETTINGS_FILE, JSON.stringify({ delay }, null, 2), 'utf-8') + return { success: true } + } catch (error) { + console.error('Failed to save countdown delay:', error) + return { success: false, error: String(error) } + } + }) + + ipcMain.handle('start-countdown', async (_, seconds: number) => { + if (countdownInProgress) { + return { success: false, error: 'Countdown already in progress' } + } + + countdownInProgress = true + countdownCancelled = false + countdownRemaining = seconds + + const countdownWin = createCountdownWindow() + + if (countdownWin.webContents.isLoadingMainFrame()) { + await new Promise((resolve) => { + countdownWin.webContents.once('did-finish-load', () => { + resolve() + }) + }) + } + + return new Promise<{ success: boolean; cancelled?: boolean }>((resolve) => { + let remaining = seconds + countdownRemaining = remaining + + countdownWin.webContents.send('countdown-tick', remaining) + + countdownTimer = setInterval(() => { + if (countdownCancelled) { + if (countdownTimer) { + clearInterval(countdownTimer) + countdownTimer = null + } + closeCountdownWindow() + countdownInProgress = false + countdownRemaining = null + resolve({ success: false, cancelled: true }) + return + } + + remaining-- + countdownRemaining = remaining + + if (remaining <= 0) { + if (countdownTimer) { + clearInterval(countdownTimer) + countdownTimer = null + } + closeCountdownWindow() + countdownInProgress = false + countdownRemaining = null + resolve({ success: true }) + } else { + const win = getCountdownWindow() + if (win && !win.isDestroyed()) { + win.webContents.send('countdown-tick', remaining) + } + } + }, 1000) + }) + }) + + ipcMain.handle('cancel-countdown', () => { + countdownCancelled = true + countdownInProgress = false + countdownRemaining = null + if (countdownTimer) { + clearInterval(countdownTimer) + countdownTimer = null + } + closeCountdownWindow() + return { success: true } + }) + + ipcMain.handle('get-active-countdown', () => { + return { + success: true, + seconds: countdownInProgress ? countdownRemaining : null, + } + }) } diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index abbdabbb6..1861bce2d 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -1,85 +1,86 @@ -const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4 +const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4; -export type NativeExportEncodingMode = 'fast' | 'balanced' | 'quality' +export type NativeExportEncodingMode = "fast" | "balanced" | "quality"; -export type NativeVideoExportAudioMode = 'none' | 'copy-source' | 'trim-source' | 'edited-track' +export type NativeVideoExportAudioMode = "none" | "copy-source" | "trim-source" | "edited-track"; export interface NativeVideoExportStartOptions { - width: number - height: number - frameRate: number - bitrate: number - encodingMode: NativeExportEncodingMode + width: number; + height: number; + frameRate: number; + bitrate: number; + encodingMode: NativeExportEncodingMode; + inputMode?: "rawvideo" | "h264-stream"; } export interface NativeVideoExportAudioSegment { - startMs: number - endMs: number + startMs: number; + endMs: number; } export interface NativeVideoExportFinishOptions { - audioMode?: NativeVideoExportAudioMode - audioSourcePath?: string | null - trimSegments?: NativeVideoExportAudioSegment[] - editedAudioData?: ArrayBuffer - editedAudioMimeType?: string | null + audioMode?: NativeVideoExportAudioMode; + audioSourcePath?: string | null; + trimSegments?: NativeVideoExportAudioSegment[]; + editedAudioData?: ArrayBuffer; + editedAudioMimeType?: string | null; } export function getNativeVideoInputByteSize(width: number, height: number): number { - return width * height * NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL + return width * height * NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL; } export function parseAvailableFfmpegEncoders(stdout: string): Set { - const encoders = new Set() + const encoders = new Set(); for (const line of stdout.split(/\r?\n/)) { - const match = line.match(/^\s*[A-Z.]{6}\s+([a-z0-9_]+)/i) + const match = line.match(/^\s*[A-Z.]{6}\s+([a-z0-9_]+)/i); if (match?.[1]) { - encoders.add(match[1]) + encoders.add(match[1]); } } - return encoders + return encoders; } export function getPreferredNativeVideoEncoders(platform: NodeJS.Platform): string[] { switch (platform) { - case 'darwin': - return ['h264_videotoolbox', 'libx264'] - case 'win32': - return ['h264_nvenc', 'h264_qsv', 'h264_amf', 'h264_mf', 'libx264'] - case 'linux': - return ['h264_nvenc', 'h264_qsv', 'libx264'] + case "darwin": + return ["h264_videotoolbox", "libx264"]; + case "win32": + return ["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf", "libx264"]; + case "linux": + return ["h264_nvenc", "h264_qsv", "libx264"]; default: - return ['libx264'] + return ["libx264"]; } } function getLibx264ModeArgs(encodingMode: NativeExportEncodingMode): string[] { switch (encodingMode) { - case 'fast': - return ['-preset', 'ultrafast', '-tune', 'zerolatency'] - case 'quality': - return ['-preset', 'slow'] - case 'balanced': + case "fast": + return ["-preset", "ultrafast", "-tune", "zerolatency"]; + case "quality": + return ["-preset", "slow"]; + case "balanced": default: - return ['-preset', 'medium'] + return ["-preset", "medium"]; } } function getBitrateArgs(bitrate: number): string[] { - const effectiveBitrate = Math.max(1_500_000, Math.round(bitrate)) - const maxRate = Math.max(effectiveBitrate, Math.round(effectiveBitrate * 1.2)) - const bufferSize = Math.max(maxRate * 2, effectiveBitrate * 2) + const effectiveBitrate = Math.max(1_500_000, Math.round(bitrate)); + const maxRate = Math.max(effectiveBitrate, Math.round(effectiveBitrate * 1.2)); + const bufferSize = Math.max(maxRate * 2, effectiveBitrate * 2); return [ - '-b:v', + "-b:v", String(effectiveBitrate), - '-maxrate', + "-maxrate", String(maxRate), - '-bufsize', + "-bufsize", String(bufferSize), - ] + ]; } export function buildNativeVideoExportArgs( @@ -88,85 +89,111 @@ export function buildNativeVideoExportArgs( outputPath: string, ): string[] { const args = [ - '-y', - '-hide_banner', - '-loglevel', - 'error', - '-f', - 'rawvideo', - '-pix_fmt', - 'rgba', - '-s:v', + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "rawvideo", + "-pix_fmt", + "rgba", + "-s:v", `${options.width}x${options.height}`, - '-framerate', + "-framerate", String(options.frameRate), - '-i', - 'pipe:0', - '-vf', - 'vflip', - '-an', - '-c:v', + "-i", + "pipe:0", + "-vf", + "vflip", + "-an", + "-c:v", encoder, - '-g', + "-g", String(Math.max(1, Math.round(options.frameRate * 5))), ...getBitrateArgs(options.bitrate), - ] + ]; - if (encoder === 'libx264') { - args.push(...getLibx264ModeArgs(options.encodingMode)) + if (encoder === "libx264") { + args.push(...getLibx264ModeArgs(options.encodingMode)); } - args.push('-pix_fmt', 'yuv420p', '-movflags', '+faststart', outputPath) - return args + args.push("-pix_fmt", "yuv420p", "-movflags", "+faststart", outputPath); + return args; } function formatFfmpegSeconds(milliseconds: number): string { - return (milliseconds / 1000).toFixed(3) + return (milliseconds / 1000).toFixed(3); } export function buildTrimmedSourceAudioFilter( segments: NativeVideoExportAudioSegment[], ): string | null { if (segments.length === 0) { - return null + return null; } - const filterParts: string[] = [] - const segmentLabels: string[] = [] + const filterParts: string[] = []; + const segmentLabels: string[] = []; segments.forEach((segment, index) => { - const label = `trimmed_audio_${index}` + const label = `trimmed_audio_${index}`; filterParts.push( `[1:a]atrim=start=${formatFfmpegSeconds(segment.startMs)}:end=${formatFfmpegSeconds(segment.endMs)},asetpts=PTS-STARTPTS[${label}]`, - ) - segmentLabels.push(`[${label}]`) - }) + ); + segmentLabels.push(`[${label}]`); + }); if (segmentLabels.length === 1) { - filterParts.push(`${segmentLabels[0]}anull[aout]`) + filterParts.push(`${segmentLabels[0]}anull[aout]`); } else { - filterParts.push(`${segmentLabels.join('')}concat=n=${segmentLabels.length}:v=0:a=1[aout]`) + filterParts.push(`${segmentLabels.join("")}concat=n=${segmentLabels.length}:v=0:a=1[aout]`); } - return filterParts.join(';') + return filterParts.join(";"); +} + +/** + * Builds FFmpeg arguments for a zero-copy H.264 stream export. + * FFmpeg receives a pre-encoded Annex B H.264 stream on stdin (produced by the + * browser's hardware VideoEncoder) and copies it straight into an MP4 container + * — no re-encoding step, no raw pixel IPC traffic. + */ +export function buildNativeH264StreamExportArgs(config: { + frameRate: number + outputPath: string +}): string[] { + return [ + '-y', + '-hide_banner', + '-loglevel', + 'error', + // Input 0: pre-encoded H.264 Annex B stream from browser VideoEncoder via stdin + '-f', 'h264', + '-r', String(config.frameRate), + '-i', 'pipe:0', + '-an', // audio handled separately by muxNativeVideoExportAudio + '-c:v', 'copy', + '-movflags', '+faststart', + config.outputPath, + ] } export function getEditedAudioExtension(mimeType?: string | null): string { if (!mimeType) { - return '.webm' + return ".webm"; } - if (mimeType.includes('wav')) { - return '.wav' + if (mimeType.includes("wav")) { + return ".wav"; } - if (mimeType.includes('mp4') || mimeType.includes('m4a')) { - return '.m4a' + if (mimeType.includes("mp4") || mimeType.includes("m4a")) { + return ".m4a"; } - if (mimeType.includes('ogg')) { - return '.ogg' + if (mimeType.includes("ogg")) { + return ".ogg"; } - return '.webm' -} \ No newline at end of file + return ".webm"; +} diff --git a/electron/ipc/windowsCaptureSelection.test.ts b/electron/ipc/windowsCaptureSelection.test.ts index 870ebbd2e..de6a36cfd 100644 --- a/electron/ipc/windowsCaptureSelection.test.ts +++ b/electron/ipc/windowsCaptureSelection.test.ts @@ -1,60 +1,64 @@ -import { describe, expect, it } from 'vitest' - -import { resolveWindowsCaptureDisplay } from './windowsCaptureSelection' - -describe('resolveWindowsCaptureDisplay', () => { - const primaryDisplay = { - id: 101, - bounds: { - x: 0, - y: 0, - width: 1920, - height: 1080, - }, - } - - const secondaryDisplay = { - id: 202, - bounds: { - x: 1920, - y: -40, - width: 2560, - height: 1440, - }, - } - - it('uses the requested secondary display bounds for WGC fallback metadata', () => { - const resolved = resolveWindowsCaptureDisplay( - { display_id: String(secondaryDisplay.id) }, - [primaryDisplay, secondaryDisplay], - primaryDisplay, - ) - - expect(resolved).toEqual({ - displayId: secondaryDisplay.id, - bounds: secondaryDisplay.bounds, - }) - }) - - it('falls back to the primary display when the source has no display id', () => { - const resolved = resolveWindowsCaptureDisplay(undefined, [primaryDisplay, secondaryDisplay], primaryDisplay) - - expect(resolved).toEqual({ - displayId: primaryDisplay.id, - bounds: primaryDisplay.bounds, - }) - }) - - it('keeps the requested display id even if Electron cannot rematch it, while using primary bounds', () => { - const resolved = resolveWindowsCaptureDisplay( - { display_id: '303' }, - [primaryDisplay, secondaryDisplay], - primaryDisplay, - ) - - expect(resolved).toEqual({ - displayId: 303, - bounds: primaryDisplay.bounds, - }) - }) -}) \ No newline at end of file +import { describe, expect, it } from "vitest"; + +import { resolveWindowsCaptureDisplay } from "./windowsCaptureSelection"; + +describe("resolveWindowsCaptureDisplay", () => { + const primaryDisplay = { + id: 101, + bounds: { + x: 0, + y: 0, + width: 1920, + height: 1080, + }, + }; + + const secondaryDisplay = { + id: 202, + bounds: { + x: 1920, + y: -40, + width: 2560, + height: 1440, + }, + }; + + it("uses the requested secondary display bounds for WGC fallback metadata", () => { + const resolved = resolveWindowsCaptureDisplay( + { display_id: String(secondaryDisplay.id) }, + [primaryDisplay, secondaryDisplay], + primaryDisplay, + ); + + expect(resolved).toEqual({ + displayId: secondaryDisplay.id, + bounds: secondaryDisplay.bounds, + }); + }); + + it("falls back to the primary display when the source has no display id", () => { + const resolved = resolveWindowsCaptureDisplay( + undefined, + [primaryDisplay, secondaryDisplay], + primaryDisplay, + ); + + expect(resolved).toEqual({ + displayId: primaryDisplay.id, + bounds: primaryDisplay.bounds, + }); + }); + + it("keeps the requested display id even if Electron cannot rematch it, while using primary bounds", () => { + const resolved = resolveWindowsCaptureDisplay( + { display_id: "303" }, + [primaryDisplay, secondaryDisplay], + primaryDisplay, + ); + + expect(resolved).toEqual({ + displayId: 303, + bounds: primaryDisplay.bounds, + }); + }); +}); diff --git a/electron/ipc/windowsCaptureSelection.ts b/electron/ipc/windowsCaptureSelection.ts index ee7ad1c7b..5d1157d0f 100644 --- a/electron/ipc/windowsCaptureSelection.ts +++ b/electron/ipc/windowsCaptureSelection.ts @@ -1,40 +1,42 @@ export type WindowsCaptureSourceLike = { - display_id?: string -} + display_id?: string; +}; export type WindowsCaptureDisplayBounds = { - x: number - y: number - width: number - height: number -} + x: number; + y: number; + width: number; + height: number; +}; export type WindowsCaptureDisplayLike = { - id: number - bounds: WindowsCaptureDisplayBounds -} + id: number; + bounds: WindowsCaptureDisplayBounds; +}; export type ResolvedWindowsCaptureDisplay = { - displayId: number - bounds: WindowsCaptureDisplayBounds -} + displayId: number; + bounds: WindowsCaptureDisplayBounds; +}; export function resolveWindowsCaptureDisplay( - source: WindowsCaptureSourceLike | null | undefined, - allDisplays: WindowsCaptureDisplayLike[], - primaryDisplay: WindowsCaptureDisplayLike, + source: WindowsCaptureSourceLike | null | undefined, + allDisplays: WindowsCaptureDisplayLike[], + primaryDisplay: WindowsCaptureDisplayLike, ): ResolvedWindowsCaptureDisplay { - const requestedDisplayId = Number(source?.display_id) - const primaryDisplayId = Number(primaryDisplay.id) - const displayId = Number.isFinite(requestedDisplayId) && requestedDisplayId > 0 - ? requestedDisplayId - : primaryDisplayId + const requestedDisplayId = Number(source?.display_id); + const primaryDisplayId = Number(primaryDisplay.id); + const requestedOrPrimaryDisplayId = + Number.isFinite(requestedDisplayId) && requestedDisplayId > 0 + ? requestedDisplayId + : primaryDisplayId; - const matchedDisplay = allDisplays.find((display) => String(display.id) === String(displayId)) - ?? primaryDisplay + const matchedDisplay = + allDisplays.find((display) => String(display.id) === String(requestedOrPrimaryDisplayId)) ?? + primaryDisplay; - return { - displayId, - bounds: matchedDisplay.bounds, - } -} \ No newline at end of file + return { + displayId: Number(matchedDisplay.id), + bounds: matchedDisplay.bounds, + }; +} diff --git a/electron/main.ts b/electron/main.ts index 6cd19bbd0..04c4efa65 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -16,13 +16,13 @@ import { } from "electron"; import { RECORDINGS_DIR } from "./appPaths"; import { showCursor } from "./cursorHider"; +import { registerExtensionIpcHandlers } from "./extensions/extensionIpc"; import { cleanupNativeVideoExportSessions, getSelectedSourceId, killWindowsCaptureProcess, registerIpcHandlers, } from "./ipc/handlers"; -import { registerExtensionIpcHandlers } from "./extensions/extensionIpc"; import { ensurePackagedRendererServer } from "./rendererServer"; import type { UpdateToastPayload } from "./updater"; import { @@ -226,7 +226,11 @@ function focusOrCreateMainWindow() { // work because they receive an XDG activation token via StatusNotifierItem.ProvideXdgActivationToken; // Electron's tray doesn't handle that yet. Workaround: destroy and recreate the HUD so the new // window gets focus (creation path works). Only for HUD, not editor. - if (process.platform === "linux" && !mainWindow.isFocused() && !isEditorWindow(mainWindow)) { + if ( + process.platform === "linux" && + !mainWindow.isFocused() && + !isEditorWindow(mainWindow) + ) { const win = mainWindow; mainWindow = null; win.once("closed", () => createWindow()); @@ -509,9 +513,11 @@ function sendUpdateToastToWindows(channel: "update-toast-state", payload: unknow return false; } - const notificationKey = [updatePayload.phase, updatePayload.version, updatePayload.detail].join( - ":", - ); + const notificationKey = [ + updatePayload.phase, + updatePayload.version, + updatePayload.detail, + ].join(":"); if (activeUpdateNotificationKey === notificationKey) { return true; } @@ -874,7 +880,9 @@ app.whenReady().then(async () => { try { const sources = await desktopCapturer.getSources({ types: ["screen", "window"] }); const sourceId = getSelectedSourceId(); - const source = sourceId ? (sources.find((s) => s.id === sourceId) ?? sources[0]) : sources[0]; + const source = sourceId + ? (sources.find((s) => s.id === sourceId) ?? sources[0]) + : sources[0]; if (source) { callback({ video: { id: source.id, name: source.name }, diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index 437bfdaa1..c5cd8fabf 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -20,7 +20,7 @@ let targetCaptureFPS = 60 let maxInlineAudioTailExtension = CMTime(seconds: 2.0, preferredTimescale: 600) final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { - private let queue = DispatchQueue(label: "openscreen.screencapturekit.video") + private let queue = DispatchQueue(label: "recordly.screencapturekit.video") private var assetWriter: AVAssetWriter? private var videoInput: AVAssetWriterInput? private var systemAudioWriter: AVAssetWriter? @@ -596,7 +596,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { final class RecorderService { private let recorder = ScreenCaptureRecorder() - private let queue = DispatchQueue(label: "openscreen.screencapturekit.commands") + private let queue = DispatchQueue(label: "recordly.screencapturekit.commands") private let completionGroup = DispatchGroup() func start(configJSON: String) { diff --git a/electron/native/bin/darwin-arm64/openscreen-native-cursor-monitor b/electron/native/bin/darwin-arm64/openscreen-native-cursor-monitor deleted file mode 100755 index 8ce9f414f..000000000 Binary files a/electron/native/bin/darwin-arm64/openscreen-native-cursor-monitor and /dev/null differ diff --git a/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper b/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper deleted file mode 100755 index ddc32b25d..000000000 Binary files a/electron/native/bin/darwin-arm64/openscreen-screencapturekit-helper and /dev/null differ diff --git a/electron/native/bin/darwin-arm64/openscreen-system-cursors b/electron/native/bin/darwin-arm64/openscreen-system-cursors deleted file mode 100755 index 416418a81..000000000 Binary files a/electron/native/bin/darwin-arm64/openscreen-system-cursors and /dev/null differ diff --git a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor new file mode 100755 index 000000000..fdd0bd08f Binary files /dev/null and b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper b/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper new file mode 100755 index 000000000..ff4b8aa5a Binary files /dev/null and b/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper differ diff --git a/electron/native/bin/darwin-arm64/recordly-system-cursors b/electron/native/bin/darwin-arm64/recordly-system-cursors new file mode 100755 index 000000000..d6564426f Binary files /dev/null and b/electron/native/bin/darwin-arm64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-arm64/recordly-window-list b/electron/native/bin/darwin-arm64/recordly-window-list new file mode 100755 index 000000000..aa268ffd9 Binary files /dev/null and b/electron/native/bin/darwin-arm64/recordly-window-list differ diff --git a/electron/native/bin/darwin-arm64/whisper-runtime.json b/electron/native/bin/darwin-arm64/whisper-runtime.json index 47e7440ed..380d1adf6 100644 --- a/electron/native/bin/darwin-arm64/whisper-runtime.json +++ b/electron/native/bin/darwin-arm64/whisper-runtime.json @@ -1,6 +1,6 @@ { - "version": "v1.8.4", - "platform": "darwin", - "arch": "arm64", - "binary": "whisper-cli" -} \ No newline at end of file + "version": "v1.8.4", + "platform": "darwin", + "arch": "arm64", + "binary": "whisper-cli" +} diff --git a/electron/native/bin/darwin-x64/openscreen-native-cursor-monitor b/electron/native/bin/darwin-x64/openscreen-native-cursor-monitor deleted file mode 100755 index 46e4b5a89..000000000 Binary files a/electron/native/bin/darwin-x64/openscreen-native-cursor-monitor and /dev/null differ diff --git a/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper b/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper deleted file mode 100755 index c9010785b..000000000 Binary files a/electron/native/bin/darwin-x64/openscreen-screencapturekit-helper and /dev/null differ diff --git a/electron/native/bin/darwin-x64/openscreen-system-cursors b/electron/native/bin/darwin-x64/openscreen-system-cursors deleted file mode 100755 index b6d8151c4..000000000 Binary files a/electron/native/bin/darwin-x64/openscreen-system-cursors and /dev/null differ diff --git a/electron/native/bin/darwin-x64/openscreen-window-list b/electron/native/bin/darwin-x64/openscreen-window-list deleted file mode 100755 index 67b5182c7..000000000 Binary files a/electron/native/bin/darwin-x64/openscreen-window-list and /dev/null differ diff --git a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor new file mode 100755 index 000000000..58bdf488f Binary files /dev/null and b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-x64/recordly-screencapturekit-helper b/electron/native/bin/darwin-x64/recordly-screencapturekit-helper new file mode 100755 index 000000000..831b86905 Binary files /dev/null and b/electron/native/bin/darwin-x64/recordly-screencapturekit-helper differ diff --git a/electron/native/bin/darwin-x64/recordly-system-cursors b/electron/native/bin/darwin-x64/recordly-system-cursors new file mode 100755 index 000000000..2c53fd683 Binary files /dev/null and b/electron/native/bin/darwin-x64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-x64/recordly-window-list b/electron/native/bin/darwin-x64/recordly-window-list new file mode 100755 index 000000000..949a66bdb Binary files /dev/null and b/electron/native/bin/darwin-x64/recordly-window-list differ diff --git a/electron/native/bin/win32-x64/helpers-manifest.json b/electron/native/bin/win32-x64/helpers-manifest.json index 62c77822e..dbae2baeb 100644 --- a/electron/native/bin/win32-x64/helpers-manifest.json +++ b/electron/native/bin/win32-x64/helpers-manifest.json @@ -1,21 +1,21 @@ { - "version": 1, - "platform": "win32", - "arch": "x64", - "helpers": { - "wgc-capture": { - "binaryName": "wgc-capture.exe", - "binarySha256": "013164c0a1391d334e5aa2fe0ff2e47b507407e36c7e44e7333a70a84b944643", - "sourceDir": "electron/native/wgc-capture", - "sourceFingerprint": "9e9bce082266ca5968cf5f0b535b469d47c4ec3a775a93726171c0dfdbcdaa44", - "updatedAt": "2026-03-29T02:15:34.516Z" - }, - "cursor-monitor": { - "binaryName": "cursor-monitor.exe", - "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0", - "sourceDir": "electron/native/cursor-monitor", - "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e", - "updatedAt": "2026-03-29T02:15:38.286Z" - } - } + "version": 1, + "platform": "win32", + "arch": "x64", + "helpers": { + "wgc-capture": { + "binaryName": "wgc-capture.exe", + "binarySha256": "013164c0a1391d334e5aa2fe0ff2e47b507407e36c7e44e7333a70a84b944643", + "sourceDir": "electron/native/wgc-capture", + "sourceFingerprint": "9e9bce082266ca5968cf5f0b535b469d47c4ec3a775a93726171c0dfdbcdaa44", + "updatedAt": "2026-03-29T02:15:34.516Z" + }, + "cursor-monitor": { + "binaryName": "cursor-monitor.exe", + "binarySha256": "b0732abc06998a40c3e95078465ad750a6169901944571c39cfd7996effe39c0", + "sourceDir": "electron/native/cursor-monitor", + "sourceFingerprint": "6ad1b8b50bb336f2a48937b06f5ec56d90b6ab4a3e56a4bca278cf67a5d3e52e", + "updatedAt": "2026-03-29T02:15:38.286Z" + } + } } diff --git a/electron/preload.ts b/electron/preload.ts index aa17c976e..1c5eb6fa8 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -108,6 +108,7 @@ contextBridge.exposeInMainWorld("electronAPI", { frameRate: number; bitrate: number; encodingMode: "fast" | "balanced" | "quality"; + inputMode?: "rawvideo" | "h264-stream"; }) => { return ipcRenderer.invoke("native-video-export-start", options); }, @@ -138,20 +139,24 @@ contextBridge.exposeInMainWorld("electronAPI", { editedAudioMimeType?: string | null; }, ) => { - return ipcRenderer.invoke("native-video-export-finish", sessionId, options).then((result) => { - settleNativeVideoExportPendingRequests(sessionId, result?.success - ? { success: true } - : { - success: false, - error: - typeof result?.error === "string" - ? result.error - : "Native video export session finished before all frame writes settled.", - }, - ); + return ipcRenderer + .invoke("native-video-export-finish", sessionId, options) + .then((result) => { + settleNativeVideoExportPendingRequests( + sessionId, + result?.success + ? { success: true } + : { + success: false, + error: + typeof result?.error === "string" + ? result.error + : "Native video export session finished before all frame writes settled.", + }, + ); - return result; - }); + return result; + }); }, nativeVideoExportCancel: (sessionId: string) => { return ipcRenderer.invoke("native-video-export-cancel", sessionId).finally(() => { @@ -185,22 +190,25 @@ contextBridge.exposeInMainWorld("electronAPI", { openSourceSelector: () => { return ipcRenderer.invoke("open-source-selector"); }, - selectSource: (source: any) => { + selectSource: (source: ProcessedDesktopSource) => { return ipcRenderer.invoke("select-source", source); }, - showSourceHighlight: (source: any) => { + showSourceHighlight: (source: ProcessedDesktopSource) => { return ipcRenderer.invoke("show-source-highlight", source); }, getSelectedSource: () => { return ipcRenderer.invoke("get-selected-source"); }, - onSelectedSourceChanged: (callback: (source: any) => void) => { - const listener = (_event: Electron.IpcRendererEvent, payload: any) => callback(payload); + onSelectedSourceChanged: (callback: (source: ProcessedDesktopSource | null) => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: ProcessedDesktopSource | null, + ) => callback(payload); ipcRenderer.on("selected-source-changed", listener); return () => ipcRenderer.removeListener("selected-source-changed", listener); }, startNativeScreenRecording: ( - source: any, + source: ProcessedDesktopSource, options?: { capturesSystemAudio?: boolean; capturesMicrophone?: boolean; @@ -225,7 +233,7 @@ contextBridge.exposeInMainWorld("electronAPI", { resumeNativeScreenRecording: () => { return ipcRenderer.invoke("resume-native-screen-recording"); }, - startFfmpegRecording: (source: any) => { + startFfmpegRecording: (source: ProcessedDesktopSource) => { return ipcRenderer.invoke("start-ffmpeg-recording", source); }, stopFfmpegRecording: () => { @@ -331,11 +339,21 @@ contextBridge.exposeInMainWorld("electronAPI", { return ipcRenderer.invoke("delete-whisper-small-model"); }, onWhisperSmallModelDownloadProgress: ( - callback: (state: { status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; error?: string }) => void, + callback: (state: { + status: "idle" | "downloading" | "downloaded" | "error"; + progress: number; + path?: string | null; + error?: string; + }) => void, ) => { const listener = ( _event: Electron.IpcRendererEvent, - payload: { status: "idle" | "downloading" | "downloaded" | "error"; progress: number; path?: string | null; error?: string }, + payload: { + status: "idle" | "downloading" | "downloaded" | "error"; + progress: number; + path?: string | null; + error?: string; + }, ) => callback(payload); ipcRenderer.on("whisper-small-model-download-progress", listener); return () => ipcRenderer.removeListener("whisper-small-model-download-progress", listener); @@ -351,7 +369,11 @@ contextBridge.exposeInMainWorld("electronAPI", { setCurrentVideoPath: (path: string) => { return ipcRenderer.invoke("set-current-video-path", path); }, - setCurrentRecordingSession: (session: { videoPath: string; webcamPath?: string | null; timeOffsetMs?: number }) => { + setCurrentRecordingSession: (session: { + videoPath: string; + webcamPath?: string | null; + timeOffsetMs?: number; + }) => { return ipcRenderer.invoke("set-current-recording-session", session); }, getCurrentRecordingSession: () => { @@ -426,29 +448,29 @@ contextBridge.exposeInMainWorld("electronAPI", { return ipcRenderer.invoke("check-for-app-updates"); }, onUpdateToastStateChanged: ( - callback: (payload: { - version: string; - detail: string; - phase: "available" | "downloading" | "ready" | "error"; - delayMs: number; - isPreview?: boolean; - progressPercent?: number; - primaryAction?: "download-update" | "install-update" | "retry-check"; - } | null) => void, + callback: ( + payload: { + version: string; + detail: string; + phase: "available" | "downloading" | "ready" | "error"; + delayMs: number; + isPreview?: boolean; + progressPercent?: number; + primaryAction?: "download-update" | "install-update" | "retry-check"; + } | null, + ) => void, ) => { const listener = ( _event: Electron.IpcRendererEvent, - payload: - | { - version: string; - detail: string; - phase: "available" | "downloading" | "ready" | "error"; - delayMs: number; - isPreview?: boolean; - progressPercent?: number; - primaryAction?: "download-update" | "install-update" | "retry-check"; - } - | null, + payload: { + version: string; + detail: string; + phase: "available" | "downloading" | "ready" | "error"; + delayMs: number; + isPreview?: boolean; + progressPercent?: number; + primaryAction?: "download-update" | "install-update" | "retry-check"; + } | null, ) => callback(payload); ipcRenderer.on("update-toast-state", listener); return () => ipcRenderer.removeListener("update-toast-state", listener); @@ -520,12 +542,18 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.on("request-save-before-close", listener); return () => ipcRenderer.removeListener("request-save-before-close", listener); }, - isNativeWindowsCaptureAvailable: () => ipcRenderer.invoke("is-native-windows-capture-available"), - muxNativeWindowsRecording: (pauseSegments?: Array<{ startMs: number; endMs: number }>) => ipcRenderer.invoke("mux-native-windows-recording", pauseSegments), + isNativeWindowsCaptureAvailable: () => + ipcRenderer.invoke("is-native-windows-capture-available"), + muxNativeWindowsRecording: (pauseSegments?: Array<{ startMs: number; endMs: number }>) => + ipcRenderer.invoke("mux-native-windows-recording", pauseSegments), hideOsCursor: () => ipcRenderer.invoke("hide-cursor"), getAppVersion: () => ipcRenderer.invoke("app:getVersion"), getRecordingPreferences: () => ipcRenderer.invoke("get-recording-preferences"), - setRecordingPreferences: (prefs: { microphoneEnabled?: boolean; microphoneDeviceId?: string; systemAudioEnabled?: boolean }) => ipcRenderer.invoke("set-recording-preferences", prefs), + setRecordingPreferences: (prefs: { + microphoneEnabled?: boolean; + microphoneDeviceId?: string; + systemAudioEnabled?: boolean; + }) => ipcRenderer.invoke("set-recording-preferences", prefs), getCountdownDelay: () => ipcRenderer.invoke("get-countdown-delay"), setCountdownDelay: (delay: number) => ipcRenderer.invoke("set-countdown-delay", delay), startCountdown: (seconds: number) => ipcRenderer.invoke("start-countdown", seconds), diff --git a/electron/rendererServer.ts b/electron/rendererServer.ts index a8491b733..8f0d56627 100644 --- a/electron/rendererServer.ts +++ b/electron/rendererServer.ts @@ -1,5 +1,5 @@ -import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import fs from "node:fs/promises"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import path from "node:path"; const MIME_TYPES: Record = { @@ -134,4 +134,4 @@ export async function ensurePackagedRendererServer(rootDir: string): Promise { progressPercent = Math.min(100, progressPercent + DEV_UPDATE_PREVIEW_PROGRESS_INCREMENT); @@ -316,19 +312,15 @@ function simulateDevPreviewDownload(sendToRenderer?: UpdateToastSender) { return; } - emitUpdateToastState( - sendToRenderer, - { - ...createDownloadingUpdateToastPayload(DEV_UPDATE_PREVIEW_VERSION, progressPercent), - isPreview: true, - }, - ); + emitUpdateToastState(sendToRenderer, { + ...createDownloadingUpdateToastPayload(DEV_UPDATE_PREVIEW_VERSION, progressPercent), + isPreview: true, + }); }, DEV_UPDATE_PREVIEW_PROGRESS_STEP_MS); return { success: true }; } - export function dismissUpdateToast( getMainWindow: () => BrowserWindow | null, sendToRenderer?: UpdateToastSender, @@ -524,7 +516,7 @@ async function showAvailableUpdateDialog( async function showDownloadedUpdateDialog( getMainWindow: () => BrowserWindow | null, version: string, - options?: { isPreview?: boolean }, + options?: { isPreview?: boolean }, ) { const isPreview = Boolean(options?.isPreview); const result = await showMessageBox(getMainWindow, { @@ -658,7 +650,11 @@ export function setupAutoUpdates( writeUpdaterLog(`Updater initialized. logPath=${UPDATER_LOG_PATH}`); autoUpdater.on("checking-for-update", () => { - setUpdateStatusSummary({ status: "checking", availableVersion: null, detail: "Checking for updates..." }); + setUpdateStatusSummary({ + status: "checking", + availableVersion: null, + detail: "Checking for updates...", + }); writeUpdaterLog("electron-updater emitted checking-for-update."); }); @@ -782,7 +778,9 @@ export function setupAutoUpdates( }); clearDeferredReminderTimer(); - if (emitUpdateToastState(sendToRenderer, createDownloadedUpdateToastPayload(info.version))) { + if ( + emitUpdateToastState(sendToRenderer, createDownloadedUpdateToastPayload(info.version)) + ) { return; } @@ -802,4 +800,4 @@ export function setupAutoUpdates( periodicCheckTimer = null; } }); -} \ No newline at end of file +} diff --git a/electron/windows.ts b/electron/windows.ts index b4028d87d..f84669156 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -307,7 +307,11 @@ ipcMain.on("hud-overlay-drag", (_event, phase: string, screenX: number, screenY: hudDragLastCursor = { x: screenX, y: screenY }; hudDragFixedSize = { width: bounds.width, height: bounds.height }; } else if (phase === "move" && hudDragOffset) { - if (hudDragLastCursor && hudDragLastCursor.x === screenX && hudDragLastCursor.y === screenY) { + if ( + hudDragLastCursor && + hudDragLastCursor.x === screenX && + hudDragLastCursor.y === screenY + ) { return; } diff --git a/src/lib/exporter/annotationRenderer.ts b/src/lib/exporter/annotationRenderer.ts index 4407e1469..a8822a81b 100644 --- a/src/lib/exporter/annotationRenderer.ts +++ b/src/lib/exporter/annotationRenderer.ts @@ -1,472 +1,474 @@ -import { BLUR_ANNOTATION_STRENGTH, type AnnotationRegion, type ArrowDirection } from "@/components/video-editor/types"; - +import { + type AnnotationRegion, + type ArrowDirection, + BLUR_ANNOTATION_STRENGTH, +} from "@/components/video-editor/types"; export interface AnnotationRenderAssets { - imageCache: Map; + imageCache: Map; } const annotationImagePromiseCache = new Map>(); let blurBufferCanvas: HTMLCanvasElement | null = null; -function getBlurBufferCanvas(): HTMLCanvasElement { - if (typeof document === "undefined") return {} as HTMLCanvasElement; - if (!blurBufferCanvas) { - blurBufferCanvas = document.createElement("canvas"); - } - return blurBufferCanvas; +function getBlurBufferCanvas(): HTMLCanvasElement | null { + if (typeof document === "undefined") return null; + if (!blurBufferCanvas) { + blurBufferCanvas = document.createElement("canvas"); + } + return blurBufferCanvas; } - function getAnnotationImageContent(annotation: AnnotationRegion): string | null { - const source = annotation.imageContent || annotation.content; - if (!source || !source.startsWith("data:image")) { - return null; - } + const source = annotation.imageContent || annotation.content; + if (!source || !source.startsWith("data:image")) { + return null; + } - return source; + return source; } function loadAnnotationImage(source: string): Promise { - const cachedPromise = annotationImagePromiseCache.get(source); - if (cachedPromise) { - return cachedPromise; - } - - const loadPromise = new Promise((resolve) => { - const img = new Image(); - img.onload = () => resolve(img); - img.onerror = () => { - console.error("[AnnotationRenderer] Failed to load image annotation"); - resolve(null); - }; - img.src = source; - }); - - annotationImagePromiseCache.set(source, loadPromise); - return loadPromise; + const cachedPromise = annotationImagePromiseCache.get(source); + if (cachedPromise) { + return cachedPromise; + } + + const loadPromise = new Promise((resolve) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = () => { + console.error("[AnnotationRenderer] Failed to load image annotation"); + resolve(null); + }; + img.src = source; + }); + + annotationImagePromiseCache.set(source, loadPromise); + return loadPromise; } export async function preloadAnnotationAssets( - annotations: AnnotationRegion[] = [], + annotations: AnnotationRegion[] = [], ): Promise { - const uniqueSources = [ - ...new Set( - annotations - .filter((annotation) => annotation.type === "image") - .map((annotation) => getAnnotationImageContent(annotation)) - .filter((source): source is string => !!source), - ), - ]; - - if (uniqueSources.length === 0) { - return { imageCache: new Map() }; - } - - const loadedSources = await Promise.all( - uniqueSources.map(async (source) => { - const image = await loadAnnotationImage(source); - return image ? ([source, image] as const) : null; - }), - ); - - return { - imageCache: new Map( - loadedSources.filter((entry): entry is readonly [string, HTMLImageElement] => !!entry), - ), - }; + const uniqueSources = [ + ...new Set( + annotations + .filter((annotation) => annotation.type === "image") + .map((annotation) => getAnnotationImageContent(annotation)) + .filter((source): source is string => !!source), + ), + ]; + + if (uniqueSources.length === 0) { + return { imageCache: new Map() }; + } + + const loadedSources = await Promise.all( + uniqueSources.map(async (source) => { + const image = await loadAnnotationImage(source); + return image ? ([source, image] as const) : null; + }), + ); + + return { + imageCache: new Map( + loadedSources.filter((entry): entry is readonly [string, HTMLImageElement] => !!entry), + ), + }; } const ARROW_PATHS: Record = { - up: ["M 50 20 L 50 80", "M 50 20 L 35 35", "M 50 20 L 65 35"], - down: ["M 50 20 L 50 80", "M 50 80 L 35 65", "M 50 80 L 65 65"], - left: ["M 80 50 L 20 50", "M 20 50 L 35 35", "M 20 50 L 35 65"], - right: ["M 20 50 L 80 50", "M 80 50 L 65 35", "M 80 50 L 65 65"], - "up-right": ["M 25 75 L 75 25", "M 75 25 L 60 30", "M 75 25 L 70 40"], - "up-left": ["M 75 75 L 25 25", "M 25 25 L 40 30", "M 25 25 L 30 40"], - "down-right": ["M 25 25 L 75 75", "M 75 75 L 70 60", "M 75 75 L 60 70"], - "down-left": ["M 75 25 L 25 75", "M 25 75 L 30 60", "M 25 75 L 40 70"], + up: ["M 50 20 L 50 80", "M 50 20 L 35 35", "M 50 20 L 65 35"], + down: ["M 50 20 L 50 80", "M 50 80 L 35 65", "M 50 80 L 65 65"], + left: ["M 80 50 L 20 50", "M 20 50 L 35 35", "M 20 50 L 35 65"], + right: ["M 20 50 L 80 50", "M 80 50 L 65 35", "M 80 50 L 65 65"], + "up-right": ["M 25 75 L 75 25", "M 75 25 L 60 30", "M 75 25 L 70 40"], + "up-left": ["M 75 75 L 25 25", "M 25 25 L 40 30", "M 25 25 L 30 40"], + "down-right": ["M 25 25 L 75 75", "M 75 75 L 70 60", "M 75 75 L 60 70"], + "down-left": ["M 75 25 L 25 75", "M 25 75 L 30 60", "M 25 75 L 40 70"], }; function parseSvgPath( - pathString: string, - scaleX: number, - scaleY: number, + pathString: string, + scaleX: number, + scaleY: number, ): Array<{ cmd: string; args: number[] }> { - const commands: Array<{ cmd: string; args: number[] }> = []; - const parts = pathString.trim().split(/\s+/); - - let i = 0; - while (i < parts.length) { - const cmd = parts[i]; - if (cmd === "M" || cmd === "L") { - const x = parseFloat(parts[i + 1]) * scaleX; - const y = parseFloat(parts[i + 2]) * scaleY; - commands.push({ cmd, args: [x, y] }); - i += 3; - } else { - i++; - } - } - - return commands; + const commands: Array<{ cmd: string; args: number[] }> = []; + const parts = pathString.trim().split(/\s+/); + + let i = 0; + while (i < parts.length) { + const cmd = parts[i]; + if (cmd === "M" || cmd === "L") { + const x = parseFloat(parts[i + 1]) * scaleX; + const y = parseFloat(parts[i + 2]) * scaleY; + commands.push({ cmd, args: [x, y] }); + i += 3; + } else { + i++; + } + } + + return commands; } function renderArrow( - ctx: CanvasRenderingContext2D, - direction: ArrowDirection, - color: string, - strokeWidth: number, - x: number, - y: number, - width: number, - height: number, - _scaleFactor: number, + ctx: CanvasRenderingContext2D, + direction: ArrowDirection, + color: string, + strokeWidth: number, + x: number, + y: number, + width: number, + height: number, + _scaleFactor: number, ) { - const paths = ARROW_PATHS[direction]; - if (!paths) return; + const paths = ARROW_PATHS[direction]; + if (!paths) return; - ctx.save(); - ctx.translate(x, y); + ctx.save(); + ctx.translate(x, y); - const padding = 8 * _scaleFactor; - const availableWidth = Math.max(0, width - padding * 2); - const availableHeight = Math.max(0, height - padding * 2); + const padding = 8 * _scaleFactor; + const availableWidth = Math.max(0, width - padding * 2); + const availableHeight = Math.max(0, height - padding * 2); - const scale = Math.min(availableWidth / 100, availableHeight / 100); + const scale = Math.min(availableWidth / 100, availableHeight / 100); - const offsetX = padding + (availableWidth - 100 * scale) / 2; - const offsetY = padding + (availableHeight - 100 * scale) / 2; + const offsetX = padding + (availableWidth - 100 * scale) / 2; + const offsetY = padding + (availableHeight - 100 * scale) / 2; - ctx.translate(offsetX, offsetY); + ctx.translate(offsetX, offsetY); - ctx.shadowColor = "rgba(0, 0, 0, 0.3)"; - ctx.shadowBlur = 8 * scale; - ctx.shadowOffsetX = 0; - ctx.shadowOffsetY = 4 * scale; + ctx.shadowColor = "rgba(0, 0, 0, 0.3)"; + ctx.shadowBlur = 8 * scale; + ctx.shadowOffsetX = 0; + ctx.shadowOffsetY = 4 * scale; - ctx.strokeStyle = color; - ctx.lineWidth = strokeWidth * scale; - ctx.lineCap = "round"; - ctx.lineJoin = "round"; + ctx.strokeStyle = color; + ctx.lineWidth = strokeWidth * scale; + ctx.lineCap = "round"; + ctx.lineJoin = "round"; - ctx.beginPath(); + ctx.beginPath(); - for (const pathString of paths) { - const commands = parseSvgPath(pathString, scale, scale); + for (const pathString of paths) { + const commands = parseSvgPath(pathString, scale, scale); - for (const { cmd, args } of commands) { - if (cmd === "M") { - ctx.moveTo(args[0], args[1]); - } else if (cmd === "L") { - ctx.lineTo(args[0], args[1]); - } - } - } + for (const { cmd, args } of commands) { + if (cmd === "M") { + ctx.moveTo(args[0], args[1]); + } else if (cmd === "L") { + ctx.lineTo(args[0], args[1]); + } + } + } - ctx.stroke(); + ctx.stroke(); - ctx.restore(); + ctx.restore(); } function renderText( - ctx: CanvasRenderingContext2D, - annotation: AnnotationRegion, - x: number, - y: number, - width: number, - height: number, - scaleFactor: number, + ctx: CanvasRenderingContext2D, + annotation: AnnotationRegion, + x: number, + y: number, + width: number, + height: number, + scaleFactor: number, ) { - const style = annotation.style; - - ctx.save(); - - ctx.beginPath(); - ctx.rect(x, y, width, height); - ctx.clip(); - - const fontWeight = style.fontWeight === "bold" ? "bold" : "normal"; - const fontStyle = style.fontStyle === "italic" ? "italic" : "normal"; - const scaledFontSize = style.fontSize * scaleFactor; - ctx.font = `${fontStyle} ${fontWeight} ${scaledFontSize}px ${style.fontFamily}`; - ctx.textBaseline = "middle"; - - const containerPadding = 8 * scaleFactor; - - let textX = x; - const textY = y + height / 2; - - if (style.textAlign === "center") { - textX = x + width / 2; - ctx.textAlign = "center"; - } else if (style.textAlign === "right") { - textX = x + width - containerPadding; - ctx.textAlign = "right"; - } else { - textX = x + containerPadding; - ctx.textAlign = "left"; - } - - const availableWidth = width - containerPadding * 2; - const rawLines = annotation.content.split("\n"); - const lines: string[] = []; - for (const rawLine of rawLines) { - if (!rawLine) { - lines.push(""); - continue; - } - const words = rawLine.split(/(\s+)/); - let current = ""; - for (const word of words) { - const test = current + word; - if (current && ctx.measureText(test).width > availableWidth) { - lines.push(current); - current = word.trimStart(); - } else { - current = test; - } - } - if (current) lines.push(current); - } - const lineHeight = scaledFontSize * 1.4; - - const startY = textY - ((lines.length - 1) * lineHeight) / 2; - - lines.forEach((line, index) => { - const currentY = startY + index * lineHeight; - - if (style.backgroundColor && style.backgroundColor !== "transparent") { - const metrics = ctx.measureText(line); - const verticalPadding = scaledFontSize * 0.1; - const horizontalPadding = scaledFontSize * 0.2; - const borderRadius = 4 * scaleFactor; - - let bgX = textX - horizontalPadding; - const bgWidth = metrics.width + horizontalPadding * 2; - - const contentHeight = scaledFontSize * 1.4; - const bgHeight = contentHeight + verticalPadding * 2; - const bgY = currentY - bgHeight / 2; - - if (style.textAlign === "center") { - bgX = textX - bgWidth / 2; - } else if (style.textAlign === "right") { - bgX = textX - bgWidth; - } - - ctx.fillStyle = style.backgroundColor; - ctx.beginPath(); - ctx.roundRect(bgX, bgY, bgWidth, bgHeight, borderRadius); - ctx.fill(); - } - - ctx.fillStyle = style.color; - ctx.fillText(line, textX, currentY); - - if (style.textDecoration === "underline") { - const metrics = ctx.measureText(line); - let underlineX = textX; - const underlineY = currentY + scaledFontSize * 0.15; - - if (style.textAlign === "center") { - underlineX = textX - metrics.width / 2; - } else if (style.textAlign === "right") { - underlineX = textX - metrics.width; - } - - ctx.strokeStyle = style.color; - ctx.lineWidth = Math.max(1, scaledFontSize / 16); - ctx.beginPath(); - ctx.moveTo(underlineX, underlineY); - ctx.lineTo(underlineX + metrics.width, underlineY); - ctx.stroke(); - } - }); - - ctx.restore(); + const style = annotation.style; + + ctx.save(); + + ctx.beginPath(); + ctx.rect(x, y, width, height); + ctx.clip(); + + const fontWeight = style.fontWeight === "bold" ? "bold" : "normal"; + const fontStyle = style.fontStyle === "italic" ? "italic" : "normal"; + const scaledFontSize = style.fontSize * scaleFactor; + ctx.font = `${fontStyle} ${fontWeight} ${scaledFontSize}px ${style.fontFamily}`; + ctx.textBaseline = "middle"; + + const containerPadding = 8 * scaleFactor; + + let textX = x; + const textY = y + height / 2; + + if (style.textAlign === "center") { + textX = x + width / 2; + ctx.textAlign = "center"; + } else if (style.textAlign === "right") { + textX = x + width - containerPadding; + ctx.textAlign = "right"; + } else { + textX = x + containerPadding; + ctx.textAlign = "left"; + } + + const availableWidth = width - containerPadding * 2; + const rawLines = annotation.content.split("\n"); + const lines: string[] = []; + for (const rawLine of rawLines) { + if (!rawLine) { + lines.push(""); + continue; + } + const words = rawLine.split(/(\s+)/); + let current = ""; + for (const word of words) { + const test = current + word; + if (current && ctx.measureText(test).width > availableWidth) { + lines.push(current); + current = word.trimStart(); + } else { + current = test; + } + } + if (current) lines.push(current); + } + const lineHeight = scaledFontSize * 1.4; + + const startY = textY - ((lines.length - 1) * lineHeight) / 2; + + lines.forEach((line, index) => { + const currentY = startY + index * lineHeight; + + if (style.backgroundColor && style.backgroundColor !== "transparent") { + const metrics = ctx.measureText(line); + const verticalPadding = scaledFontSize * 0.1; + const horizontalPadding = scaledFontSize * 0.2; + const borderRadius = 4 * scaleFactor; + + let bgX = textX - horizontalPadding; + const bgWidth = metrics.width + horizontalPadding * 2; + + const contentHeight = scaledFontSize * 1.4; + const bgHeight = contentHeight + verticalPadding * 2; + const bgY = currentY - bgHeight / 2; + + if (style.textAlign === "center") { + bgX = textX - bgWidth / 2; + } else if (style.textAlign === "right") { + bgX = textX - bgWidth; + } + + ctx.fillStyle = style.backgroundColor; + ctx.beginPath(); + ctx.roundRect(bgX, bgY, bgWidth, bgHeight, borderRadius); + ctx.fill(); + } + + ctx.fillStyle = style.color; + ctx.fillText(line, textX, currentY); + + if (style.textDecoration === "underline") { + const metrics = ctx.measureText(line); + let underlineX = textX; + const underlineY = currentY + scaledFontSize * 0.15; + + if (style.textAlign === "center") { + underlineX = textX - metrics.width / 2; + } else if (style.textAlign === "right") { + underlineX = textX - metrics.width; + } + + ctx.strokeStyle = style.color; + ctx.lineWidth = Math.max(1, scaledFontSize / 16); + ctx.beginPath(); + ctx.moveTo(underlineX, underlineY); + ctx.lineTo(underlineX + metrics.width, underlineY); + ctx.stroke(); + } + }); + + ctx.restore(); } async function renderImage( - ctx: CanvasRenderingContext2D, - annotation: AnnotationRegion, - x: number, - y: number, - width: number, - height: number, - assets?: AnnotationRenderAssets, + ctx: CanvasRenderingContext2D, + annotation: AnnotationRegion, + x: number, + y: number, + width: number, + height: number, + assets?: AnnotationRenderAssets, ): Promise { - const source = getAnnotationImageContent(annotation); - if (!source) { - return; - } - - const img = assets?.imageCache.get(source) ?? (await loadAnnotationImage(source)); - if (!img) { - return; - } - - const imgAspect = img.width / img.height; - const boxAspect = width / height; - - let drawWidth = width; - let drawHeight = height; - let drawX = x; - let drawY = y; - - if (imgAspect > boxAspect) { - drawHeight = width / imgAspect; - drawY = y + (height - drawHeight) / 2; - } else { - drawWidth = height * imgAspect; - drawX = x + (width - drawWidth) / 2; - } - - ctx.drawImage(img, drawX, drawY, drawWidth, drawHeight); + const source = getAnnotationImageContent(annotation); + if (!source) { + return; + } + + const img = assets?.imageCache.get(source) ?? (await loadAnnotationImage(source)); + if (!img) { + return; + } + + const imgAspect = img.width / img.height; + const boxAspect = width / height; + + let drawWidth = width; + let drawHeight = height; + let drawX = x; + let drawY = y; + + if (imgAspect > boxAspect) { + drawHeight = width / imgAspect; + drawY = y + (height - drawHeight) / 2; + } else { + drawWidth = height * imgAspect; + drawX = x + (width - drawWidth) / 2; + } + + ctx.drawImage(img, drawX, drawY, drawWidth, drawHeight); } export async function renderAnnotations( - ctx: CanvasRenderingContext2D, - annotations: AnnotationRegion[], - canvasWidth: number, - canvasHeight: number, - currentTimeMs: number, - scaleFactor: number = 1.0, - assets?: AnnotationRenderAssets, + ctx: CanvasRenderingContext2D, + annotations: AnnotationRegion[], + canvasWidth: number, + canvasHeight: number, + currentTimeMs: number, + scaleFactor: number = 1.0, + assets?: AnnotationRenderAssets, ): Promise { - const activeAnnotations = annotations.filter( - (ann) => currentTimeMs >= ann.startMs && currentTimeMs <= ann.endMs, - ); - - const sortedAnnotations = [...activeAnnotations].sort((a, b) => a.zIndex - b.zIndex); - - for (const annotation of sortedAnnotations) { - const x = (annotation.position.x / 100) * canvasWidth; - const y = (annotation.position.y / 100) * canvasHeight; - const width = (annotation.size.width / 100) * canvasWidth; - const height = (annotation.size.height / 100) * canvasHeight; - - switch (annotation.type) { - case "text": - renderText(ctx, annotation, x, y, width, height, scaleFactor); - break; - - case "image": - await renderImage(ctx, annotation, x, y, width, height, assets); - break; - - case "figure": - if (annotation.figureData) { - renderArrow( - ctx, - annotation.figureData.arrowDirection, - annotation.figureData.color, - annotation.figureData.strokeWidth, - x, - y, - width, - height, - scaleFactor, - ); - } - break; - - case "blur": { - const blurStrength = (annotation.blurIntensity ?? BLUR_ANNOTATION_STRENGTH) * scaleFactor; - const padding = Math.ceil(blurStrength * 2); - - ctx.save(); - - ctx.beginPath(); - const borderRadius = (annotation.style.borderRadius ?? 0) * scaleFactor; - ctx.roundRect(x, y, width, height, borderRadius); - ctx.clip(); - - const sx = Math.max(0, x - padding); - const sy = Math.max(0, y - padding); - const sw = Math.min(canvasWidth - sx, width + padding * 2); - const sh = Math.min(canvasHeight - sy, height + padding * 2); - - if (sw > 0 && sh > 0) { - const buffer = getBlurBufferCanvas(); - buffer.width = sw; - buffer.height = sh; - const bCtx = buffer.getContext("2d"); - if (bCtx) { - bCtx.drawImage(ctx.canvas, sx, sy, sw, sh, 0, 0, sw, sh); - - ctx.filter = `blur(${blurStrength}px)`; - ctx.drawImage(buffer, sx, sy); - - if (annotation.blurColor && annotation.blurColor !== "transparent") { - ctx.filter = "none"; - ctx.fillStyle = annotation.blurColor; - ctx.fillRect(x, y, width, height); - } - } - } - - ctx.restore(); - break; - } - - - } - } + const activeAnnotations = annotations.filter( + (ann) => currentTimeMs >= ann.startMs && currentTimeMs <= ann.endMs, + ); + + const sortedAnnotations = [...activeAnnotations].sort((a, b) => a.zIndex - b.zIndex); + + for (const annotation of sortedAnnotations) { + const x = (annotation.position.x / 100) * canvasWidth; + const y = (annotation.position.y / 100) * canvasHeight; + const width = (annotation.size.width / 100) * canvasWidth; + const height = (annotation.size.height / 100) * canvasHeight; + + switch (annotation.type) { + case "text": + renderText(ctx, annotation, x, y, width, height, scaleFactor); + break; + + case "image": + await renderImage(ctx, annotation, x, y, width, height, assets); + break; + + case "figure": + if (annotation.figureData) { + renderArrow( + ctx, + annotation.figureData.arrowDirection, + annotation.figureData.color, + annotation.figureData.strokeWidth, + x, + y, + width, + height, + scaleFactor, + ); + } + break; + + case "blur": { + const blurStrength = + (annotation.blurIntensity ?? BLUR_ANNOTATION_STRENGTH) * scaleFactor; + const padding = Math.ceil(blurStrength * 2); + + ctx.save(); + + ctx.beginPath(); + const borderRadius = (annotation.style.borderRadius ?? 0) * scaleFactor; + ctx.roundRect(x, y, width, height, borderRadius); + ctx.clip(); + + const sx = Math.max(0, x - padding); + const sy = Math.max(0, y - padding); + const sw = Math.min(canvasWidth - sx, width + padding * 2); + const sh = Math.min(canvasHeight - sy, height + padding * 2); + + if (sw > 0 && sh > 0) { + const buffer = getBlurBufferCanvas(); + if (buffer) { + buffer.width = sw; + buffer.height = sh; + const bCtx = buffer.getContext("2d"); + if (bCtx) { + bCtx.drawImage(ctx.canvas, sx, sy, sw, sh, 0, 0, sw, sh); + + ctx.filter = `blur(${blurStrength}px)`; + ctx.drawImage(buffer, sx, sy); + + if (annotation.blurColor && annotation.blurColor !== "transparent") { + ctx.filter = "none"; + ctx.fillStyle = annotation.blurColor; + ctx.fillRect(x, y, width, height); + } + } + } + } + + ctx.restore(); + break; + } + } + } } export async function renderAnnotationToCanvas( - annotation: AnnotationRegion, - width: number, - height: number, - scaleFactor: number = 1.0, - assets?: AnnotationRenderAssets, + annotation: AnnotationRegion, + width: number, + height: number, + scaleFactor: number = 1.0, + assets?: AnnotationRenderAssets, ): Promise { - const canvasWidth = Math.max(1, Math.ceil(width)); - const canvasHeight = Math.max(1, Math.ceil(height)); - const canvas = document.createElement("canvas"); - canvas.width = canvasWidth; - canvas.height = canvasHeight; - - const ctx = canvas.getContext("2d"); - if (!ctx) { - return null; - } - - ctx.imageSmoothingEnabled = true; - ctx.imageSmoothingQuality = "high"; - - switch (annotation.type) { - case "text": - renderText(ctx, annotation, 0, 0, canvasWidth, canvasHeight, scaleFactor); - break; - - case "image": - await renderImage(ctx, annotation, 0, 0, canvasWidth, canvasHeight, assets); - break; - - case "figure": - if (!annotation.figureData) { - return null; - } - - renderArrow( - ctx, - annotation.figureData.arrowDirection, - annotation.figureData.color, - annotation.figureData.strokeWidth, - 0, - 0, - canvasWidth, - canvasHeight, - scaleFactor, - ); - break; - case "blur": - // Blur annotations must sample already-rendered scene pixels, - // so they cannot be rasterized as standalone sprites. - return null; - } - - return canvas; + const canvasWidth = Math.max(1, Math.ceil(width)); + const canvasHeight = Math.max(1, Math.ceil(height)); + const canvas = document.createElement("canvas"); + canvas.width = canvasWidth; + canvas.height = canvasHeight; + + const ctx = canvas.getContext("2d"); + if (!ctx) { + return null; + } + + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = "high"; + + switch (annotation.type) { + case "text": + renderText(ctx, annotation, 0, 0, canvasWidth, canvasHeight, scaleFactor); + break; + + case "image": + await renderImage(ctx, annotation, 0, 0, canvasWidth, canvasHeight, assets); + break; + + case "figure": + if (!annotation.figureData) { + return null; + } + + renderArrow( + ctx, + annotation.figureData.arrowDirection, + annotation.figureData.color, + annotation.figureData.strokeWidth, + 0, + 0, + canvasWidth, + canvasHeight, + scaleFactor, + ); + break; + case "blur": + // Blur annotations must sample already-rendered scene pixels, + // so they cannot be rasterized as standalone sprites. + return null; + } + + return canvas; } - diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 2367b3e50..16d919f33 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,1023 +1,1087 @@ -import { WebDemuxer } from 'web-demuxer' -import type { SpeedRegion, TrimRegion, ClipRegion, AudioRegion } from '@/components/video-editor/types' +import { WebDemuxer } from "web-demuxer"; +import type { + AudioRegion, + ClipRegion, + SpeedRegion, + TrimRegion, +} from "@/components/video-editor/types"; import { - clampMediaTimeToDuration, - estimateCompanionAudioStartDelaySeconds, - getMediaSyncPlaybackRate, -} from '@/lib/mediaTiming' -import { resolveMediaElementSource } from './localMediaSource' -import type { VideoMuxer } from './muxer' - -const AUDIO_BITRATE = 128_000 -const DECODE_BACKPRESSURE_LIMIT = 20 -const ENCODE_BACKPRESSURE_LIMIT = 20 -const MIN_SPEED_REGION_DELTA_MS = 0.0001 -const MP4_AUDIO_CODEC = 'mp4a.40.2' + clampMediaTimeToDuration, + estimateCompanionAudioStartDelaySeconds, + getMediaSyncPlaybackRate, +} from "@/lib/mediaTiming"; +import { resolveMediaElementSource } from "./localMediaSource"; +import type { VideoMuxer } from "./muxer"; + +const AUDIO_BITRATE = 128_000; +const DECODE_BACKPRESSURE_LIMIT = 20; +const ENCODE_BACKPRESSURE_LIMIT = 20; +const MIN_SPEED_REGION_DELTA_MS = 0.0001; +const MP4_AUDIO_CODEC = "mp4a.40.2"; export async function isAacAudioEncodingSupported( - sampleRate = 48_000, - numberOfChannels = 2, + sampleRate = 48_000, + numberOfChannels = 2, ): Promise { - try { - const support = await AudioEncoder.isConfigSupported({ - codec: MP4_AUDIO_CODEC, - sampleRate, - numberOfChannels, - bitrate: AUDIO_BITRATE, - }) - return support.supported === true - } catch { - return false - } + try { + const support = await AudioEncoder.isConfigSupported({ + codec: MP4_AUDIO_CODEC, + sampleRate, + numberOfChannels, + bitrate: AUDIO_BITRATE, + }); + return support.supported === true; + } catch { + return false; + } } -type TrimLikeRegion = TrimRegion | ClipRegion +type TrimLikeRegion = TrimRegion | ClipRegion; export class AudioProcessor { - private cancelled = false - private onProgress?: (progress: number) => void - - private isPassthroughAudioCodec(codec: string | undefined): boolean { - if (!codec) { - return false - } - - const normalizedCodec = codec.toLowerCase() - return ( - normalizedCodec === MP4_AUDIO_CODEC - || normalizedCodec === 'aac' - || normalizedCodec.startsWith('mp4a.40.2') - ) - } - - private async passthroughAudioStream( - audioStream: ReadableStream, - audioConfig: AudioDecoderConfig, - muxer: VideoMuxer, - ): Promise { - if (!this.isPassthroughAudioCodec(audioConfig.codec)) { - return false - } - - const reader = audioStream.getReader() - let wroteAudio = false - let passthroughTimestampOffsetUs: number | null = null - - try { - while (!this.cancelled) { - const { done, value: chunk } = await reader.read() - if (done || !chunk) break - - if (passthroughTimestampOffsetUs === null) { - passthroughTimestampOffsetUs = chunk.timestamp - } - - const normalizedTimestamp = Math.max( - 0, - chunk.timestamp - passthroughTimestampOffsetUs, - ) - const outputChunk = passthroughTimestampOffsetUs === 0 - ? chunk - : this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp) - - await muxer.addAudioChunk( - outputChunk, - wroteAudio - ? undefined - : { - decoderConfig: audioConfig, - }, - ) - wroteAudio = true - } - } finally { - try { - await reader.cancel() - } catch { - // reader already closed - } - } - - return wroteAudio - } - - /** - * Audio export has two modes: - * 1) no speed regions -> fast WebCodecs trim-only pipeline - * 2) speed regions present -> pitch-preserving rendered timeline pipeline - */ - setOnProgress(callback: (progress: number) => void) { - this.onProgress = callback - } - - async process( - demuxer: WebDemuxer | null, - muxer: VideoMuxer, - videoUrl: string, - trimRegions?: TrimLikeRegion[], - speedRegions?: SpeedRegion[], - readEndSec?: number, - audioRegions?: AudioRegion[], - sourceAudioFallbackPaths?: string[], - ): Promise { - const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : [] - const sortedSpeedRegions = speedRegions - ? [...speedRegions] - .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) - .sort((a, b) => a.startMs - b.startMs) - : [] - const sortedAudioRegions = audioRegions - ? [...audioRegions].sort((a, b) => a.startMs - b.startMs) - : [] - const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths - ? sourceAudioFallbackPaths.filter((audioPath) => typeof audioPath === 'string' && audioPath.trim().length > 0) - : [] - - // When speed edits, audio regions, or multiple audio sources need mixing, use AudioContext mixing path. - // Note: real-time rendering is required here; it plays audio at 1x speed via HTMLMediaElement. - if ( - sortedSpeedRegions.length > 0 - || sortedAudioRegions.length > 0 - || sortedSourceAudioFallbackPaths.length > 1 - ) { - const renderedAudioBlob = await this.renderMixedTimelineAudio( - videoUrl, - sortedTrims, - sortedSpeedRegions, - sortedAudioRegions, - sortedSourceAudioFallbackPaths, - ) - if (!this.cancelled) { - await this.muxRenderedAudioBlob(renderedAudioBlob, muxer) - } - return - } - - // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). - if (sortedSourceAudioFallbackPaths.length === 1) { - const sidecarDemuxer = await this.loadAudioFileDemuxer(sortedSourceAudioFallbackPaths[0]) - if (sidecarDemuxer) { - try { - await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims) - } finally { - try { sidecarDemuxer.destroy() } catch { /* cleanup */ } - } - return - } - // Fallback to real-time rendering if demuxer creation failed - console.warn('[AudioProcessor] Fast sidecar demux failed, falling back to real-time rendering') - const renderedAudioBlob = await this.renderMixedTimelineAudio( - videoUrl, sortedTrims, sortedSpeedRegions, sortedAudioRegions, sortedSourceAudioFallbackPaths, - ) - if (!this.cancelled) { - await this.muxRenderedAudioBlob(renderedAudioBlob, muxer) - } - return - } - - // No speed edits or audio regions: keep the original demux/decode/encode path with trim timestamp remap. - if (!demuxer) { - console.warn('[AudioProcessor] No demuxer available, skipping audio') - return - } - - if (sortedTrims.length === 0) { - let audioConfig: AudioDecoderConfig - try { - audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig - } catch { - console.warn('[AudioProcessor] No audio track found, skipping') - return - } - - const audioStream = typeof readEndSec === 'number' - ? demuxer.read('audio', 0, readEndSec) - : demuxer.read('audio') - - const copiedSourceAudio = await this.passthroughAudioStream( - audioStream as ReadableStream, - audioConfig, - muxer, - ) - - if (copiedSourceAudio) { - return - } - } - - await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec) - } - - async renderEditedAudioTrack( - videoUrl: string, - trimRegions?: TrimLikeRegion[], - speedRegions?: SpeedRegion[], - audioRegions?: AudioRegion[], - sourceAudioFallbackPaths?: string[], - ): Promise { - const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : [] - const sortedSpeedRegions = speedRegions - ? [...speedRegions] - .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) - .sort((a, b) => a.startMs - b.startMs) - : [] - const sortedAudioRegions = audioRegions - ? [...audioRegions].sort((a, b) => a.startMs - b.startMs) - : [] - const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths - ? sourceAudioFallbackPaths.filter( - (audioPath) => typeof audioPath === 'string' && audioPath.trim().length > 0, - ) - : [] - - return this.renderMixedTimelineAudio( - videoUrl, - sortedTrims, - sortedSpeedRegions, - sortedAudioRegions, - sortedSourceAudioFallbackPaths, - ) - } - - // Legacy trim-only path used when no speed regions are configured. - private async processTrimOnlyAudio( - demuxer: WebDemuxer, - muxer: VideoMuxer, - sortedTrims: TrimLikeRegion[], - readEndSec?: number, - ): Promise { - let audioConfig: AudioDecoderConfig - try { - audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig - } catch { - console.warn('[AudioProcessor] No audio track found, skipping') - return - } - - const codecCheck = await AudioDecoder.isConfigSupported(audioConfig) - if (!codecCheck.supported) { - console.warn('[AudioProcessor] Audio codec not supported:', audioConfig.codec) - return - } - - const audioStream = typeof readEndSec === 'number' - ? demuxer.read('audio', 0, readEndSec) - : demuxer.read('audio') - - let sourceTimestampOffsetUs: number | null = null - - await this.transcodeAudioStream( - audioStream as ReadableStream, - audioConfig, - muxer, - { - observeChunkTimestampUs: (timestampUs) => { - if (sourceTimestampOffsetUs === null) { - sourceTimestampOffsetUs = timestampUs - } - }, - shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims), - transformAudioData: (data) => { - const timestampMs = data.timestamp / 1000 - const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims) - const adjustedTimestampUs = - data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000 - return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)) - }, - }, - ) - } - - private async transcodeAudioStream( - audioStream: ReadableStream, - audioConfig: AudioDecoderConfig, - muxer: VideoMuxer, - options: { - observeChunkTimestampUs?: (timestampUs: number) => void - shouldSkipChunk?: (timestampMs: number) => boolean - transformAudioData?: (data: AudioData) => AudioData | null - } = {}, - ): Promise { - const pendingFrames: AudioData[] = [] - let decodeError: Error | null = null - let encodeError: Error | null = null - let muxError: Error | null = null - let pendingMuxing = Promise.resolve() - - const failIfNeeded = () => { - if (decodeError) throw decodeError - if (encodeError) throw encodeError - if (muxError) throw muxError - } - - const pumpEncodedFrames = () => { - while (!this.cancelled && pendingFrames.length > 0) { - if (encodeError || muxError) { - break - } - if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) { - break - } - - const frame = pendingFrames.shift() - if (!frame) { - break - } - - encoder.encode(frame) - frame.close() - } - } - - const cleanupPendingFrames = () => { - for (const frame of pendingFrames) { - frame.close() - } - pendingFrames.length = 0 - } - - const sampleRate = audioConfig.sampleRate || 48_000 - const channels = audioConfig.numberOfChannels || 2 - const encodeConfig: AudioEncoderConfig = { - codec: MP4_AUDIO_CODEC, - sampleRate, - numberOfChannels: channels, - bitrate: AUDIO_BITRATE, - } - - const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig) - if (!encodeSupport.supported) { - console.warn('[AudioProcessor] AAC encoding not supported, skipping audio') - return - } - - const encoder = new AudioEncoder({ - output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { - pendingMuxing = pendingMuxing - .then(async () => { - if (this.cancelled) { - return - } - await muxer.addAudioChunk(chunk, meta) - }) - .catch((error) => { - muxError = error instanceof Error ? error : new Error(String(error)) - }) - }, - error: (error: DOMException) => { - encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`) - }, - }) - - encoder.configure(encodeConfig) - - const decoder = new AudioDecoder({ - output: (data: AudioData) => { - if (this.cancelled || encodeError || muxError) { - data.close() - return - } - - const transformed = options.transformAudioData ? options.transformAudioData(data) : data - - if (transformed !== data) { - data.close() - } - - if (!transformed) { - return - } - - pendingFrames.push(transformed) - }, - error: (error: DOMException) => { - decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`) - }, - }) - decoder.configure(audioConfig) - - const reader = audioStream.getReader() - - try { - while (!this.cancelled) { - failIfNeeded() - - const { done, value: chunk } = await reader.read() - if (done || !chunk) break - - options.observeChunkTimestampUs?.(chunk.timestamp) - const timestampMs = chunk.timestamp / 1000 - if (options.shouldSkipChunk?.(timestampMs)) continue - - decoder.decode(chunk) - pumpEncodedFrames() - - while ( - !this.cancelled && - (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT - || pendingFrames.length > DECODE_BACKPRESSURE_LIMIT - || encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) - ) { - failIfNeeded() - pumpEncodedFrames() - await new Promise((resolve) => setTimeout(resolve, 1)) - } - } - - if (decoder.state === 'configured') { - await decoder.flush() - } - - while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) { - failIfNeeded() - pumpEncodedFrames() - if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) { - await new Promise((resolve) => setTimeout(resolve, 1)) - } - } - - failIfNeeded() - - if (encoder.state === 'configured') { - await encoder.flush() - } - - await pendingMuxing - failIfNeeded() - } finally { - try { - await reader.cancel() - } catch { - // reader already closed - } - - cleanupPendingFrames() - - if (decoder.state === 'configured') { - decoder.close() - } - - if (encoder.state === 'configured') { - encoder.close() - } - } - - if (this.cancelled) { - return - } - } - - // Renders mixed audio: original video audio (with speed/trim) + external audio regions. - // Uses AudioContext to mix all sources into a single recorded stream. - private async renderMixedTimelineAudio( - videoUrl: string, - trimRegions: TrimLikeRegion[], - speedRegions: SpeedRegion[], - audioRegions: AudioRegion[], - sourceAudioFallbackPaths: string[] = [], - ): Promise { - const timelineMediaSource = await resolveMediaElementSource(videoUrl) - const timelineMedia = document.createElement('video') - timelineMedia.src = timelineMediaSource.src - timelineMedia.preload = 'auto' - timelineMedia.playsInline = true - - const pitchMedia = timelineMedia as HTMLMediaElement & { - preservesPitch?: boolean - mozPreservesPitch?: boolean - webkitPreservesPitch?: boolean - } - pitchMedia.preservesPitch = true - pitchMedia.mozPreservesPitch = true - pitchMedia.webkitPreservesPitch = true - - await this.waitForLoadedMetadata(timelineMedia) - if (this.cancelled) { - throw new Error('Export cancelled') - } - - const audioContext = new AudioContext() - const destinationNode = audioContext.createMediaStreamDestination() - - let timelineAudioSourceNode: MediaElementAudioSourceNode | null = null - if (sourceAudioFallbackPaths.length === 0) { - timelineAudioSourceNode = audioContext.createMediaElementSource(timelineMedia) - timelineAudioSourceNode.connect(destinationNode) - } - - const sourceAudioElements: { - media: HTMLAudioElement - sourceNode: MediaElementAudioSourceNode - startDelaySeconds: number - cleanup: () => void - }[] = [] - - for (const sourceAudioPath of sourceAudioFallbackPaths) { - const sourceFileSource = await resolveMediaElementSource(sourceAudioPath) - const audioEl = document.createElement('audio') - audioEl.src = sourceFileSource.src - audioEl.preload = 'auto' - try { - await this.waitForLoadedMetadata(audioEl) - } catch { - sourceFileSource.revoke() - console.warn('[AudioProcessor] Failed to load source audio fallback:', sourceAudioPath) - continue - } - if (this.cancelled) throw new Error('Export cancelled') - - const sourceNode = audioContext.createMediaElementSource(audioEl) - sourceNode.connect(destinationNode) - - sourceAudioElements.push({ - media: audioEl, - sourceNode, - startDelaySeconds: estimateCompanionAudioStartDelaySeconds( - timelineMedia.duration, - audioEl.duration, - ), - cleanup: sourceFileSource.revoke, - }) - } - - // Prepare external audio region elements - const audioRegionElements: { - media: HTMLAudioElement - sourceNode: MediaElementAudioSourceNode - gainNode: GainNode - region: AudioRegion - cleanup: () => void - }[] = [] - - for (const region of audioRegions) { - const regionFileSource = await resolveMediaElementSource(region.audioPath) - const audioEl = document.createElement('audio') - audioEl.src = regionFileSource.src - audioEl.preload = 'auto' - try { - await this.waitForLoadedMetadata(audioEl) - } catch { - regionFileSource.revoke() - console.warn('[AudioProcessor] Failed to load audio region:', region.audioPath) - continue - } - if (this.cancelled) throw new Error('Export cancelled') - - const regionSourceNode = audioContext.createMediaElementSource(audioEl) - const gainNode = audioContext.createGain() - gainNode.gain.value = Math.max(0, Math.min(1, region.volume)) - regionSourceNode.connect(gainNode) - gainNode.connect(destinationNode) - - audioRegionElements.push({ - media: audioEl, - sourceNode: regionSourceNode, - gainNode, - region, - cleanup: regionFileSource.revoke, - }) - } - - const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream) - let tickTimerId: ReturnType | null = null - - try { - if (audioContext.state === 'suspended') { - await audioContext.resume() - } - - await this.seekTo(timelineMedia, 0) - await timelineMedia.play() - - const totalDurationMs = (timelineMedia.duration || 0) * 1000 - let lastProgressReport = 0 - - await new Promise((resolve, reject) => { - const cleanup = () => { - if (tickTimerId !== null) { - clearTimeout(tickTimerId) - tickTimerId = null - } - timelineMedia.removeEventListener('error', onError) - timelineMedia.removeEventListener('ended', onEnded) - } - - const onError = () => { - cleanup() - reject(new Error('Failed while rendering mixed audio timeline')) - } - - const onEnded = () => { - cleanup() - resolve() - } - - const tick = () => { - if (this.cancelled) { - cleanup() - resolve() - return - } - - // Report audio rendering progress - if (this.onProgress && totalDurationMs > 0) { - const now = performance.now() - if (now - lastProgressReport > 250) { - lastProgressReport = now - const progress = Math.min((timelineMedia.currentTime * 1000) / totalDurationMs, 1) - this.onProgress(progress) - } - } - - let currentTimeMs = timelineMedia.currentTime * 1000 - const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions) - - if (activeTrimRegion && !timelineMedia.paused && !timelineMedia.ended) { - const skipToTime = activeTrimRegion.endMs / 1000 - if (skipToTime >= timelineMedia.duration) { - timelineMedia.pause() - cleanup() - resolve() - return - } - timelineMedia.currentTime = skipToTime - currentTimeMs = skipToTime * 1000 - } - - const activeSpeedRegion = this.findActiveSpeedRegion(currentTimeMs, speedRegions) - const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1 - if (Math.abs(timelineMedia.playbackRate - playbackRate) > 0.0001) { - timelineMedia.playbackRate = playbackRate - } - - for (const entry of sourceAudioElements) { - const audioEl = entry.media - const audioDuration = Number.isFinite(audioEl.duration) ? audioEl.duration : null - const beforeAudioStart = currentTimeMs + 1 < entry.startDelaySeconds * 1000 - const targetTimeSec = clampMediaTimeToDuration( - currentTimeMs / 1000 - entry.startDelaySeconds, - audioDuration, - ) - - const atEnd = audioDuration !== null && targetTimeSec >= audioDuration - if (beforeAudioStart || atEnd) { - if (!audioEl.paused) { - audioEl.pause() - } - continue - } - - if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) { - audioEl.currentTime = targetTimeSec - } - - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: playbackRate, - currentTime: audioEl.currentTime, - targetTime: targetTimeSec, - }) - if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { - audioEl.playbackRate = syncedPlaybackRate - } - - if (audioEl.paused) { - audioEl.currentTime = targetTimeSec - audioEl.play().catch(() => undefined) - } - } - - // Sync external audio regions with the video timeline position - for (const entry of audioRegionElements) { - const { media: audioEl, region } = entry - const isInRegion = currentTimeMs >= region.startMs && currentTimeMs < region.endMs - - if (isInRegion) { - const audioOffset = (currentTimeMs - region.startMs) / 1000 - if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) { - audioEl.currentTime = audioOffset - } - - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: playbackRate, - currentTime: audioEl.currentTime, - targetTime: audioOffset, - }) - if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { - audioEl.playbackRate = syncedPlaybackRate - } - - if (audioEl.paused) { - audioEl.currentTime = audioOffset - audioEl.play().catch(() => undefined) - } - } else { - if (!audioEl.paused) { - audioEl.pause() - } - } - } - - if (!timelineMedia.paused && !timelineMedia.ended) { - tickTimerId = setTimeout(tick, 16) - } else { - cleanup() - resolve() - } - } - - timelineMedia.addEventListener('error', onError, { once: true }) - timelineMedia.addEventListener('ended', onEnded, { once: true }) - tickTimerId = setTimeout(tick, 16) - }) - } finally { - if (tickTimerId !== null) { - clearTimeout(tickTimerId) - } - timelineMedia.pause() - timelineAudioSourceNode?.disconnect() - timelineMedia.src = '' - timelineMedia.load() - timelineMediaSource.revoke() - for (const entry of sourceAudioElements) { - entry.media.pause() - entry.sourceNode.disconnect() - entry.media.src = '' - entry.media.load() - entry.cleanup() - } - for (const entry of audioRegionElements) { - entry.media.pause() - entry.sourceNode.disconnect() - entry.gainNode.disconnect() - entry.media.src = '' - entry.media.load() - entry.cleanup() - } - if (recorder.state !== 'inactive') { - recorder.stop() - } - destinationNode.stream.getTracks().forEach((track) => track.stop()) - timelineAudioSourceNode?.disconnect() - destinationNode.disconnect() - await audioContext.close() - timelineMedia.src = '' - timelineMedia.load() - timelineMediaSource.revoke() - } - - const recordedBlob = await recordedBlobPromise - if (this.cancelled) { - throw new Error('Export cancelled') - } - return recordedBlob - } - - // Demuxes the rendered speed-adjusted blob, decodes it, and re-encodes it to AAC for MP4 output. - private async muxRenderedAudioBlob(blob: Blob, muxer: VideoMuxer): Promise { - if (this.cancelled) return - - const file = new File([blob], 'speed-audio.webm', { type: blob.type || 'audio/webm' }) - const wasmUrl = new URL('./wasm/web-demuxer.wasm', window.location.href).href - const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }) - - try { - await demuxer.load(file) - const audioConfig = (await demuxer.getDecoderConfig('audio')) as AudioDecoderConfig - const codecCheck = await AudioDecoder.isConfigSupported(audioConfig) - if (!codecCheck.supported) { - console.warn('[AudioProcessor] Rendered audio codec not supported:', audioConfig.codec) - return - } - - await this.transcodeAudioStream( - demuxer.read('audio') as ReadableStream, - audioConfig, - muxer, - ) - } finally { - try { - demuxer.destroy() - } catch { - // ignore - } - } - } - - // Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering). - private async loadAudioFileDemuxer(audioPath: string): Promise { - try { - const source = await resolveMediaElementSource(audioPath) - try { - const response = await fetch(source.src) - const blob = await response.blob() - const filename = audioPath.split('/').pop() || 'sidecar-audio' - const file = new File([blob], filename, { type: blob.type || 'audio/webm' }) - const wasmUrl = new URL('./wasm/web-demuxer.wasm', window.location.href).href - const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }) - await demuxer.load(file) - return demuxer - } finally { - source.revoke() - } - } catch (error) { - console.warn('[AudioProcessor] Failed to create demuxer for sidecar audio:', error) - return null - } - } - - private startAudioRecording(stream: MediaStream): { - recorder: MediaRecorder - recordedBlobPromise: Promise - } { - const mimeType = this.getSupportedAudioMimeType() - const options: MediaRecorderOptions = { - audioBitsPerSecond: AUDIO_BITRATE, - ...(mimeType ? { mimeType } : {}), - } - - const recorder = new MediaRecorder(stream, options) - const chunks: Blob[] = [] - - const recordedBlobPromise = new Promise((resolve, reject) => { - recorder.ondataavailable = (event: BlobEvent) => { - if (event.data && event.data.size > 0) { - chunks.push(event.data) - } - } - recorder.onerror = () => { - reject(new Error('MediaRecorder failed while capturing speed-adjusted audio')) - } - recorder.onstop = () => { - const type = mimeType || chunks[0]?.type || 'audio/webm' - resolve(new Blob(chunks, { type })) - } - }) - - recorder.start() - return { recorder, recordedBlobPromise } - } - - private getSupportedAudioMimeType(): string | undefined { - const candidates = ['audio/webm;codecs=opus', 'audio/webm'] - for (const candidate of candidates) { - if (MediaRecorder.isTypeSupported(candidate)) { - return candidate - } - } - return undefined - } - - private waitForLoadedMetadata(media: HTMLMediaElement): Promise { - if (Number.isFinite(media.duration) && media.readyState >= HTMLMediaElement.HAVE_METADATA) { - return Promise.resolve() - } - - return new Promise((resolve, reject) => { - let timeoutId: ReturnType | null = null - - const onLoaded = () => { - cleanup() - resolve() - } - const onError = () => { - cleanup() - reject(new Error('Failed to load media metadata for speed-adjusted audio')) - } - const onTimeout = () => { - cleanup() - reject(new Error('Timed out waiting for media metadata (30s)')) - } - const cleanup = () => { - if (timeoutId !== null) { - clearTimeout(timeoutId) - timeoutId = null - } - media.removeEventListener('loadedmetadata', onLoaded) - media.removeEventListener('error', onError) - } - - timeoutId = setTimeout(onTimeout, 30_000) - media.addEventListener('loadedmetadata', onLoaded) - media.addEventListener('error', onError, { once: true }) - }) - } - - private seekTo(media: HTMLMediaElement, targetSec: number): Promise { - if (Math.abs(media.currentTime - targetSec) < 0.0001) { - return Promise.resolve() - } - - return new Promise((resolve, reject) => { - let timeoutId: ReturnType | null = null - - const onSeeked = () => { - cleanup() - resolve() - } - const onError = () => { - cleanup() - reject(new Error('Failed to seek media for speed-adjusted audio')) - } - const onTimeout = () => { - cleanup() - reject(new Error('Timed out waiting for media seek (30s)')) - } - const cleanup = () => { - if (timeoutId !== null) { - clearTimeout(timeoutId) - timeoutId = null - } - media.removeEventListener('seeked', onSeeked) - media.removeEventListener('error', onError) - } - - timeoutId = setTimeout(onTimeout, 30_000) - media.addEventListener('seeked', onSeeked, { once: true }) - media.addEventListener('error', onError, { once: true }) - media.currentTime = targetSec - }) - } - - private findActiveTrimRegion( - currentTimeMs: number, - trimRegions: TrimLikeRegion[], - ): TrimLikeRegion | null { - return ( - trimRegions.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ) || null - ) - } - - private findActiveSpeedRegion( - currentTimeMs: number, - speedRegions: SpeedRegion[], - ): SpeedRegion | null { - return ( - speedRegions.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ) || null - ) - } - - private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { - const isPlanar = src.format?.includes('planar') ?? false - const numPlanes = isPlanar ? src.numberOfChannels : 1 - - let totalSize = 0 - for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { - totalSize += src.allocationSize({ planeIndex }) - } - - const buffer = new ArrayBuffer(totalSize) - let offset = 0 - - for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { - const planeSize = src.allocationSize({ planeIndex }) - src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex }) - offset += planeSize - } - - return new AudioData({ - format: src.format!, - sampleRate: src.sampleRate, - numberOfFrames: src.numberOfFrames, - numberOfChannels: src.numberOfChannels, - timestamp: newTimestamp, - data: buffer, - }) - } - - private cloneEncodedAudioChunkWithTimestamp( - src: EncodedAudioChunk, - newTimestamp: number, - ): EncodedAudioChunk { - const data = new Uint8Array(src.byteLength) - src.copyTo(data) - - return new EncodedAudioChunk({ - type: src.type, - timestamp: newTimestamp, - duration: src.duration ?? undefined, - data, - }) - } - - private isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) { - return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs) - } - - private computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) { - let offset = 0 - for (const trim of trims) { - if (trim.endMs <= timestampMs) { - offset += trim.endMs - trim.startMs - } - } - return offset - } - - cancel() { - this.cancelled = true - } + private cancelled = false; + private onProgress?: (progress: number) => void; + + private isPassthroughAudioCodec(codec: string | undefined): boolean { + if (!codec) { + return false; + } + + const normalizedCodec = codec.toLowerCase(); + return ( + normalizedCodec === MP4_AUDIO_CODEC || + normalizedCodec === "aac" || + normalizedCodec.startsWith("mp4a.40.2") + ); + } + + private async passthroughAudioStream( + audioStream: ReadableStream, + audioConfig: AudioDecoderConfig, + muxer: VideoMuxer, + ): Promise { + if (!this.isPassthroughAudioCodec(audioConfig.codec)) { + return false; + } + + let reader: ReadableStreamDefaultReader | null = null; + let wroteAudio = false; + let passthroughTimestampOffsetUs: number | null = null; + + try { + reader = audioStream.getReader(); + while (!this.cancelled) { + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + + if (passthroughTimestampOffsetUs === null) { + passthroughTimestampOffsetUs = chunk.timestamp; + } + + const normalizedTimestamp = Math.max( + 0, + chunk.timestamp - passthroughTimestampOffsetUs, + ); + const outputChunk = + passthroughTimestampOffsetUs === 0 + ? chunk + : this.cloneEncodedAudioChunkWithTimestamp(chunk, normalizedTimestamp); + + await muxer.addAudioChunk( + outputChunk, + wroteAudio + ? undefined + : { + decoderConfig: audioConfig, + }, + ); + wroteAudio = true; + } + } finally { + if (reader) { + try { + await reader.cancel(); + } catch { + // reader already closed + } + } + } + + return wroteAudio; + } + + /** + * Audio export has two modes: + * 1) no speed regions -> fast WebCodecs trim-only pipeline + * 2) speed regions present -> pitch-preserving rendered timeline pipeline + */ + setOnProgress(callback: (progress: number) => void) { + this.onProgress = callback; + } + + async process( + demuxer: WebDemuxer | null, + muxer: VideoMuxer, + videoUrl: string, + trimRegions?: TrimLikeRegion[], + speedRegions?: SpeedRegion[], + readEndSec?: number, + audioRegions?: AudioRegion[], + sourceAudioFallbackPaths?: string[], + ): Promise { + const sortedTrims = trimRegions + ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) + : []; + const sortedSpeedRegions = speedRegions + ? [...speedRegions] + .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) + .sort((a, b) => a.startMs - b.startMs) + : []; + const sortedAudioRegions = audioRegions + ? [...audioRegions].sort((a, b) => a.startMs - b.startMs) + : []; + const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths + ? sourceAudioFallbackPaths.filter( + (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, + ) + : []; + + // When speed edits, audio regions, or multiple audio sources need mixing, use AudioContext mixing path. + // Note: real-time rendering is required here; it plays audio at 1x speed via HTMLMediaElement. + if ( + sortedSpeedRegions.length > 0 || + sortedAudioRegions.length > 0 || + sortedSourceAudioFallbackPaths.length > 1 + ) { + const renderedAudioBlob = await this.renderMixedTimelineAudio( + videoUrl, + sortedTrims, + sortedSpeedRegions, + sortedAudioRegions, + sortedSourceAudioFallbackPaths, + ); + if (!this.cancelled) { + await this.muxRenderedAudioBlob(renderedAudioBlob, muxer); + } + return; + } + + // Single sidecar audio with no speed/audio edits: demux directly (skips slow real-time rendering). + if (sortedSourceAudioFallbackPaths.length === 1) { + const sidecarDemuxer = await this.loadAudioFileDemuxer( + sortedSourceAudioFallbackPaths[0], + ); + if (sidecarDemuxer) { + try { + await this.processTrimOnlyAudio(sidecarDemuxer, muxer, sortedTrims); + } finally { + try { + sidecarDemuxer.destroy(); + } catch { + /* cleanup */ + } + } + return; + } + // Fallback to real-time rendering if demuxer creation failed + console.warn( + "[AudioProcessor] Fast sidecar demux failed, falling back to real-time rendering", + ); + const renderedAudioBlob = await this.renderMixedTimelineAudio( + videoUrl, + sortedTrims, + sortedSpeedRegions, + sortedAudioRegions, + sortedSourceAudioFallbackPaths, + ); + if (!this.cancelled) { + await this.muxRenderedAudioBlob(renderedAudioBlob, muxer); + } + return; + } + + // No speed edits or audio regions: keep the original demux/decode/encode path with trim timestamp remap. + if (!demuxer) { + console.warn("[AudioProcessor] No demuxer available, skipping audio"); + return; + } + + if (sortedTrims.length === 0) { + let audioConfig: AudioDecoderConfig; + try { + audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; + } catch { + console.warn("[AudioProcessor] No audio track found, skipping"); + return; + } + + const audioStream = + typeof readEndSec === "number" + ? demuxer.read("audio", 0, readEndSec) + : demuxer.read("audio"); + + const copiedSourceAudio = await this.passthroughAudioStream( + audioStream as ReadableStream, + audioConfig, + muxer, + ); + + if (copiedSourceAudio) { + return; + } + } + + await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec); + } + + async renderEditedAudioTrack( + videoUrl: string, + trimRegions?: TrimLikeRegion[], + speedRegions?: SpeedRegion[], + audioRegions?: AudioRegion[], + sourceAudioFallbackPaths?: string[], + ): Promise { + const sortedTrims = trimRegions + ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) + : []; + const sortedSpeedRegions = speedRegions + ? [...speedRegions] + .filter((region) => region.endMs - region.startMs > MIN_SPEED_REGION_DELTA_MS) + .sort((a, b) => a.startMs - b.startMs) + : []; + const sortedAudioRegions = audioRegions + ? [...audioRegions].sort((a, b) => a.startMs - b.startMs) + : []; + const sortedSourceAudioFallbackPaths = sourceAudioFallbackPaths + ? sourceAudioFallbackPaths.filter( + (audioPath) => typeof audioPath === "string" && audioPath.trim().length > 0, + ) + : []; + + return this.renderMixedTimelineAudio( + videoUrl, + sortedTrims, + sortedSpeedRegions, + sortedAudioRegions, + sortedSourceAudioFallbackPaths, + ); + } + + // Legacy trim-only path used when no speed regions are configured. + private async processTrimOnlyAudio( + demuxer: WebDemuxer, + muxer: VideoMuxer, + sortedTrims: TrimLikeRegion[], + readEndSec?: number, + ): Promise { + let audioConfig: AudioDecoderConfig; + try { + audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; + } catch { + console.warn("[AudioProcessor] No audio track found, skipping"); + return; + } + + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); + return; + } + + const audioStream = + typeof readEndSec === "number" + ? demuxer.read("audio", 0, readEndSec) + : demuxer.read("audio"); + + let sourceTimestampOffsetUs: number | null = null; + + await this.transcodeAudioStream( + audioStream as ReadableStream, + audioConfig, + muxer, + { + observeChunkTimestampUs: (timestampUs) => { + if (sourceTimestampOffsetUs === null) { + sourceTimestampOffsetUs = timestampUs; + } + }, + shouldSkipChunk: (timestampMs) => this.isInTrimRegion(timestampMs, sortedTrims), + transformAudioData: (data) => { + const timestampMs = data.timestamp / 1000; + const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); + const adjustedTimestampUs = + data.timestamp - (sourceTimestampOffsetUs ?? 0) - trimOffsetMs * 1000; + return this.cloneWithTimestamp(data, Math.max(0, adjustedTimestampUs)); + }, + }, + ); + } + + private async transcodeAudioStream( + audioStream: ReadableStream, + audioConfig: AudioDecoderConfig, + muxer: VideoMuxer, + options: { + observeChunkTimestampUs?: (timestampUs: number) => void; + shouldSkipChunk?: (timestampMs: number) => boolean; + transformAudioData?: (data: AudioData) => AudioData | null; + } = {}, + ): Promise { + const pendingFrames: AudioData[] = []; + let decodeError: Error | null = null; + let encodeError: Error | null = null; + let muxError: Error | null = null; + let pendingMuxing = Promise.resolve(); + + const failIfNeeded = () => { + if (decodeError) throw decodeError; + if (encodeError) throw encodeError; + if (muxError) throw muxError; + }; + + const pumpEncodedFrames = () => { + while (!this.cancelled && pendingFrames.length > 0) { + if (encodeError || muxError) { + break; + } + if (encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) { + break; + } + + const frame = pendingFrames.shift(); + if (!frame) { + break; + } + + encoder.encode(frame); + frame.close(); + } + }; + + const cleanupPendingFrames = () => { + for (const frame of pendingFrames) { + frame.close(); + } + pendingFrames.length = 0; + }; + + const sampleRate = audioConfig.sampleRate || 48_000; + const channels = audioConfig.numberOfChannels || 2; + const encodeConfig: AudioEncoderConfig = { + codec: MP4_AUDIO_CODEC, + sampleRate, + numberOfChannels: channels, + bitrate: AUDIO_BITRATE, + }; + + const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); + if (!encodeSupport.supported) { + console.warn("[AudioProcessor] AAC encoding not supported, skipping audio"); + return; + } + + const encoder = new AudioEncoder({ + output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { + pendingMuxing = pendingMuxing + .then(async () => { + if (this.cancelled) { + return; + } + await muxer.addAudioChunk(chunk, meta); + }) + .catch((error) => { + muxError = error instanceof Error ? error : new Error(String(error)); + }); + }, + error: (error: DOMException) => { + encodeError = new Error(`[AudioProcessor] Encode error: ${error.message}`); + }, + }); + + encoder.configure(encodeConfig); + + const decoder = new AudioDecoder({ + output: (data: AudioData) => { + if (this.cancelled || encodeError || muxError) { + data.close(); + return; + } + + const transformed = options.transformAudioData + ? options.transformAudioData(data) + : data; + + if (transformed !== data) { + data.close(); + } + + if (!transformed) { + return; + } + + pendingFrames.push(transformed); + }, + error: (error: DOMException) => { + decodeError = new Error(`[AudioProcessor] Decode error: ${error.message}`); + }, + }); + decoder.configure(audioConfig); + + let reader: ReadableStreamDefaultReader | null = null; + + try { + reader = audioStream.getReader(); + while (!this.cancelled) { + failIfNeeded(); + + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + + options.observeChunkTimestampUs?.(chunk.timestamp); + const timestampMs = chunk.timestamp / 1000; + if (options.shouldSkipChunk?.(timestampMs)) continue; + + decoder.decode(chunk); + pumpEncodedFrames(); + + while ( + !this.cancelled && + (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT || + pendingFrames.length > DECODE_BACKPRESSURE_LIMIT || + encoder.encodeQueueSize >= ENCODE_BACKPRESSURE_LIMIT) + ) { + failIfNeeded(); + pumpEncodedFrames(); + await new Promise((resolve) => setTimeout(resolve, 1)); + } + } + + if (decoder.state === "configured") { + await decoder.flush(); + } + + while (!this.cancelled && (pendingFrames.length > 0 || encoder.encodeQueueSize > 0)) { + failIfNeeded(); + pumpEncodedFrames(); + if (pendingFrames.length > 0 || encoder.encodeQueueSize > 0) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + } + + failIfNeeded(); + + if (encoder.state === "configured") { + await encoder.flush(); + } + + await pendingMuxing; + failIfNeeded(); + } finally { + if (reader) { + try { + await reader.cancel(); + } catch { + // reader already closed + } + } + + cleanupPendingFrames(); + + if (decoder.state === "configured") { + decoder.close(); + } + + if (encoder.state === "configured") { + encoder.close(); + } + } + + if (this.cancelled) { + return; + } + } + + // Renders mixed audio: original video audio (with speed/trim) + external audio regions. + // Uses AudioContext to mix all sources into a single recorded stream. + private async renderMixedTimelineAudio( + videoUrl: string, + trimRegions: TrimLikeRegion[], + speedRegions: SpeedRegion[], + audioRegions: AudioRegion[], + sourceAudioFallbackPaths: string[] = [], + ): Promise { + const timelineMediaSource = await resolveMediaElementSource(videoUrl); + const timelineMedia = document.createElement("video"); + timelineMedia.src = timelineMediaSource.src; + timelineMedia.preload = "auto"; + timelineMedia.playsInline = true; + + const pitchMedia = timelineMedia as HTMLMediaElement & { + preservesPitch?: boolean; + mozPreservesPitch?: boolean; + webkitPreservesPitch?: boolean; + }; + pitchMedia.preservesPitch = true; + pitchMedia.mozPreservesPitch = true; + pitchMedia.webkitPreservesPitch = true; + + let audioContext: AudioContext | null = null; + let destinationNode: MediaStreamAudioDestinationNode | null = null; + let timelineAudioSourceNode: MediaElementAudioSourceNode | null = null; + + const sourceAudioElements: { + media: HTMLAudioElement; + sourceNode: MediaElementAudioSourceNode; + startDelaySeconds: number; + cleanup: () => void; + }[] = []; + + // Prepare external audio region elements + const audioRegionElements: { + media: HTMLAudioElement; + sourceNode: MediaElementAudioSourceNode; + gainNode: GainNode; + region: AudioRegion; + cleanup: () => void; + }[] = []; + + let recorder: MediaRecorder | null = null; + let recordedBlobPromise: Promise | null = null; + let tickTimerId: ReturnType | null = null; + + try { + await this.waitForLoadedMetadata(timelineMedia); + if (this.cancelled) { + throw new Error("Export cancelled"); + } + + audioContext = new AudioContext(); + const currentDestinationNode = audioContext.createMediaStreamDestination(); + destinationNode = currentDestinationNode; + + if (sourceAudioFallbackPaths.length === 0) { + timelineAudioSourceNode = audioContext.createMediaElementSource(timelineMedia); + timelineAudioSourceNode.connect(currentDestinationNode); + } + + for (const sourceAudioPath of sourceAudioFallbackPaths) { + const sourceFileSource = await resolveMediaElementSource(sourceAudioPath); + const audioEl = document.createElement("audio"); + audioEl.src = sourceFileSource.src; + audioEl.preload = "auto"; + try { + await this.waitForLoadedMetadata(audioEl); + } catch { + sourceFileSource.revoke(); + console.warn( + "[AudioProcessor] Failed to load source audio fallback:", + sourceAudioPath, + ); + continue; + } + if (this.cancelled) throw new Error("Export cancelled"); + + const sourceNode = audioContext.createMediaElementSource(audioEl); + sourceNode.connect(currentDestinationNode); + + sourceAudioElements.push({ + media: audioEl, + sourceNode, + startDelaySeconds: estimateCompanionAudioStartDelaySeconds( + timelineMedia.duration, + audioEl.duration, + ), + cleanup: sourceFileSource.revoke, + }); + } + + for (const region of audioRegions) { + const regionFileSource = await resolveMediaElementSource(region.audioPath); + const audioEl = document.createElement("audio"); + audioEl.src = regionFileSource.src; + audioEl.preload = "auto"; + try { + await this.waitForLoadedMetadata(audioEl); + } catch { + regionFileSource.revoke(); + console.warn("[AudioProcessor] Failed to load audio region:", region.audioPath); + continue; + } + if (this.cancelled) throw new Error("Export cancelled"); + + const regionSourceNode = audioContext.createMediaElementSource(audioEl); + const gainNode = audioContext.createGain(); + gainNode.gain.value = Math.max(0, Math.min(1, region.volume)); + regionSourceNode.connect(gainNode); + gainNode.connect(currentDestinationNode); + + audioRegionElements.push({ + media: audioEl, + sourceNode: regionSourceNode, + gainNode, + region, + cleanup: regionFileSource.revoke, + }); + } + + const recording = this.startAudioRecording(currentDestinationNode.stream); + recorder = recording.recorder; + recordedBlobPromise = recording.recordedBlobPromise; + + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + await this.seekTo(timelineMedia, 0); + await timelineMedia.play(); + + const totalDurationMs = (timelineMedia.duration || 0) * 1000; + let lastProgressReport = 0; + + await new Promise((resolve, reject) => { + const cleanup = () => { + if (tickTimerId !== null) { + clearTimeout(tickTimerId); + tickTimerId = null; + } + timelineMedia.removeEventListener("error", onError); + timelineMedia.removeEventListener("ended", onEnded); + }; + + const onError = () => { + cleanup(); + reject(new Error("Failed while rendering mixed audio timeline")); + }; + + const onEnded = () => { + cleanup(); + resolve(); + }; + + const tick = () => { + if (this.cancelled) { + cleanup(); + resolve(); + return; + } + + // Report audio rendering progress + if (this.onProgress && totalDurationMs > 0) { + const now = performance.now(); + if (now - lastProgressReport > 250) { + lastProgressReport = now; + const progress = Math.min( + (timelineMedia.currentTime * 1000) / totalDurationMs, + 1, + ); + this.onProgress(progress); + } + } + + let currentTimeMs = timelineMedia.currentTime * 1000; + const activeTrimRegion = this.findActiveTrimRegion(currentTimeMs, trimRegions); + + if (activeTrimRegion && !timelineMedia.paused && !timelineMedia.ended) { + const skipToTime = activeTrimRegion.endMs / 1000; + if (skipToTime >= timelineMedia.duration) { + timelineMedia.pause(); + cleanup(); + resolve(); + return; + } + timelineMedia.currentTime = skipToTime; + currentTimeMs = skipToTime * 1000; + } + + const activeSpeedRegion = this.findActiveSpeedRegion( + currentTimeMs, + speedRegions, + ); + const playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + if (Math.abs(timelineMedia.playbackRate - playbackRate) > 0.0001) { + timelineMedia.playbackRate = playbackRate; + } + + for (const entry of sourceAudioElements) { + const audioEl = entry.media; + const audioDuration = Number.isFinite(audioEl.duration) + ? audioEl.duration + : null; + const beforeAudioStart = currentTimeMs + 1 < entry.startDelaySeconds * 1000; + const targetTimeSec = clampMediaTimeToDuration( + currentTimeMs / 1000 - entry.startDelaySeconds, + audioDuration, + ); + + const atEnd = audioDuration !== null && targetTimeSec >= audioDuration; + if (beforeAudioStart || atEnd) { + if (!audioEl.paused) { + audioEl.pause(); + } + continue; + } + + if (Math.abs(audioEl.currentTime - targetTimeSec) > 0.3) { + audioEl.currentTime = targetTimeSec; + } + + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: playbackRate, + currentTime: audioEl.currentTime, + targetTime: targetTimeSec, + }); + if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { + audioEl.playbackRate = syncedPlaybackRate; + } + + if (audioEl.paused) { + audioEl.currentTime = targetTimeSec; + audioEl.play().catch(() => undefined); + } + } + + // Sync external audio regions with the video timeline position + for (const entry of audioRegionElements) { + const { media: audioEl, region } = entry; + const isInRegion = + currentTimeMs >= region.startMs && currentTimeMs < region.endMs; + + if (isInRegion) { + const audioOffset = (currentTimeMs - region.startMs) / 1000; + if (Math.abs(audioEl.currentTime - audioOffset) > 0.3) { + audioEl.currentTime = audioOffset; + } + + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: playbackRate, + currentTime: audioEl.currentTime, + targetTime: audioOffset, + }); + if (Math.abs(audioEl.playbackRate - syncedPlaybackRate) > 0.0001) { + audioEl.playbackRate = syncedPlaybackRate; + } + + if (audioEl.paused) { + audioEl.currentTime = audioOffset; + audioEl.play().catch(() => undefined); + } + } else { + if (!audioEl.paused) { + audioEl.pause(); + } + } + } + + if (!timelineMedia.paused && !timelineMedia.ended) { + tickTimerId = setTimeout(tick, 16); + } else { + cleanup(); + resolve(); + } + }; + + timelineMedia.addEventListener("error", onError, { once: true }); + timelineMedia.addEventListener("ended", onEnded, { once: true }); + tickTimerId = setTimeout(tick, 16); + }); + } finally { + if (tickTimerId !== null) { + clearTimeout(tickTimerId); + } + timelineMedia.pause(); + timelineAudioSourceNode?.disconnect(); + timelineMedia.src = ""; + timelineMedia.load(); + timelineMediaSource.revoke(); + for (const entry of sourceAudioElements) { + entry.media.pause(); + entry.sourceNode.disconnect(); + entry.media.src = ""; + entry.media.load(); + entry.cleanup(); + } + for (const entry of audioRegionElements) { + entry.media.pause(); + entry.sourceNode.disconnect(); + entry.gainNode.disconnect(); + entry.media.src = ""; + entry.media.load(); + entry.cleanup(); + } + if (recorder && recorder.state !== "inactive") { + recorder.stop(); + } + destinationNode?.stream.getTracks().forEach((track) => track.stop()); + destinationNode?.disconnect(); + if (audioContext && audioContext.state !== "closed") { + try { + await audioContext.close(); + } catch { + // Ignore teardown failures during export cleanup. + } + } + } + + if (!recordedBlobPromise) { + throw new Error("Failed to record mixed timeline audio"); + } + + const recordedBlob = await recordedBlobPromise; + if (this.cancelled) { + throw new Error("Export cancelled"); + } + return recordedBlob; + } + + // Demuxes the rendered speed-adjusted blob, decodes it, and re-encodes it to AAC for MP4 output. + private async muxRenderedAudioBlob(blob: Blob, muxer: VideoMuxer): Promise { + if (this.cancelled) return; + + const file = new File([blob], "speed-audio.webm", { type: blob.type || "audio/webm" }); + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; + const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); + + try { + await demuxer.load(file); + const audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + console.warn( + "[AudioProcessor] Rendered audio codec not supported:", + audioConfig.codec, + ); + return; + } + + await this.transcodeAudioStream( + demuxer.read("audio") as ReadableStream, + audioConfig, + muxer, + ); + } finally { + try { + demuxer.destroy(); + } catch { + // ignore + } + } + } + + // Loads a sidecar audio file into a WebDemuxer for direct transcoding (avoiding real-time rendering). + private async loadAudioFileDemuxer(audioPath: string): Promise { + try { + const source = await resolveMediaElementSource(audioPath); + try { + const response = await fetch(source.src); + const blob = await response.blob(); + const filename = audioPath.split("/").pop() || "sidecar-audio"; + const file = new File([blob], filename, { type: blob.type || "audio/webm" }); + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; + const demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); + await demuxer.load(file); + return demuxer; + } finally { + source.revoke(); + } + } catch (error) { + console.warn("[AudioProcessor] Failed to create demuxer for sidecar audio:", error); + return null; + } + } + + private startAudioRecording(stream: MediaStream): { + recorder: MediaRecorder; + recordedBlobPromise: Promise; + } { + const mimeType = this.getSupportedAudioMimeType(); + const options: MediaRecorderOptions = { + audioBitsPerSecond: AUDIO_BITRATE, + ...(mimeType ? { mimeType } : {}), + }; + + const recorder = new MediaRecorder(stream, options); + const chunks: Blob[] = []; + + const recordedBlobPromise = new Promise((resolve, reject) => { + recorder.ondataavailable = (event: BlobEvent) => { + if (event.data && event.data.size > 0) { + chunks.push(event.data); + } + }; + recorder.onerror = () => { + reject(new Error("MediaRecorder failed while capturing speed-adjusted audio")); + }; + recorder.onstop = () => { + const type = mimeType || chunks[0]?.type || "audio/webm"; + resolve(new Blob(chunks, { type })); + }; + }); + + recorder.start(); + return { recorder, recordedBlobPromise }; + } + + private getSupportedAudioMimeType(): string | undefined { + const candidates = ["audio/webm;codecs=opus", "audio/webm"]; + for (const candidate of candidates) { + if (MediaRecorder.isTypeSupported(candidate)) { + return candidate; + } + } + return undefined; + } + + private waitForLoadedMetadata(media: HTMLMediaElement): Promise { + if (Number.isFinite(media.duration) && media.readyState >= HTMLMediaElement.HAVE_METADATA) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + let timeoutId: ReturnType | null = null; + + const onLoaded = () => { + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + reject(new Error("Failed to load media metadata for speed-adjusted audio")); + }; + const onTimeout = () => { + cleanup(); + reject(new Error("Timed out waiting for media metadata (30s)")); + }; + const cleanup = () => { + if (timeoutId !== null) { + clearTimeout(timeoutId); + timeoutId = null; + } + media.removeEventListener("loadedmetadata", onLoaded); + media.removeEventListener("error", onError); + }; + + timeoutId = setTimeout(onTimeout, 30_000); + media.addEventListener("loadedmetadata", onLoaded); + media.addEventListener("error", onError, { once: true }); + }); + } + + private seekTo(media: HTMLMediaElement, targetSec: number): Promise { + if (Math.abs(media.currentTime - targetSec) < 0.0001) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + let timeoutId: ReturnType | null = null; + + const onSeeked = () => { + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + reject(new Error("Failed to seek media for speed-adjusted audio")); + }; + const onTimeout = () => { + cleanup(); + reject(new Error("Timed out waiting for media seek (30s)")); + }; + const cleanup = () => { + if (timeoutId !== null) { + clearTimeout(timeoutId); + timeoutId = null; + } + media.removeEventListener("seeked", onSeeked); + media.removeEventListener("error", onError); + }; + + timeoutId = setTimeout(onTimeout, 30_000); + media.addEventListener("seeked", onSeeked, { once: true }); + media.addEventListener("error", onError, { once: true }); + media.currentTime = targetSec; + }); + } + + private findActiveTrimRegion( + currentTimeMs: number, + trimRegions: TrimLikeRegion[], + ): TrimLikeRegion | null { + return ( + trimRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ) || null + ); + } + + private findActiveSpeedRegion( + currentTimeMs: number, + speedRegions: SpeedRegion[], + ): SpeedRegion | null { + return ( + speedRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ) || null + ); + } + + private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { + const isPlanar = src.format?.includes("planar") ?? false; + const numPlanes = isPlanar ? src.numberOfChannels : 1; + + let totalSize = 0; + for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { + totalSize += src.allocationSize({ planeIndex }); + } + + const buffer = new ArrayBuffer(totalSize); + let offset = 0; + + for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { + const planeSize = src.allocationSize({ planeIndex }); + src.copyTo(new Uint8Array(buffer, offset, planeSize), { planeIndex }); + offset += planeSize; + } + + return new AudioData({ + format: src.format!, + sampleRate: src.sampleRate, + numberOfFrames: src.numberOfFrames, + numberOfChannels: src.numberOfChannels, + timestamp: newTimestamp, + data: buffer, + }); + } + + private cloneEncodedAudioChunkWithTimestamp( + src: EncodedAudioChunk, + newTimestamp: number, + ): EncodedAudioChunk { + const data = new Uint8Array(src.byteLength); + src.copyTo(data); + + return new EncodedAudioChunk({ + type: src.type, + timestamp: newTimestamp, + duration: src.duration ?? undefined, + data, + }); + } + + private isInTrimRegion(timestampMs: number, trims: TrimLikeRegion[]) { + return trims.some((trim) => timestampMs >= trim.startMs && timestampMs < trim.endMs); + } + + private computeTrimOffset(timestampMs: number, trims: TrimLikeRegion[]) { + let offset = 0; + for (const trim of trims) { + if (trim.endMs <= timestampMs) { + offset += trim.endMs - trim.startMs; + } + } + return offset; + } + + cancel() { + this.cancelled = true; + } } diff --git a/src/lib/exporter/captionRenderer.ts b/src/lib/exporter/captionRenderer.ts index 8cbf4586e..1459eb7cb 100644 --- a/src/lib/exporter/captionRenderer.ts +++ b/src/lib/exporter/captionRenderer.ts @@ -1,104 +1,107 @@ +import { buildActiveCaptionLayout } from "@/components/video-editor/captionLayout"; import { - getDefaultCaptionFontFamily, - type AutoCaptionSettings, - type CaptionCue, + CAPTION_FONT_WEIGHT, + CAPTION_LINE_HEIGHT, + getCaptionPadding, + getCaptionScaledFontSize, + getCaptionScaledRadius, + getCaptionTextMaxWidth, + getCaptionWordVisualState, +} from "@/components/video-editor/captionStyle"; +import { + type AutoCaptionSettings, + type CaptionCue, + getDefaultCaptionFontFamily, } from "@/components/video-editor/types"; import { drawSquircleOnCanvas } from "@/lib/geometry/squircle"; -import { - buildActiveCaptionLayout, -} from "@/components/video-editor/captionLayout"; -import { - CAPTION_FONT_WEIGHT, - CAPTION_LINE_HEIGHT, - getCaptionPadding, - getCaptionScaledRadius, - getCaptionScaledFontSize, - getCaptionTextMaxWidth, - getCaptionWordVisualState, -} from "@/components/video-editor/captionStyle"; export function renderCaptions( - ctx: CanvasRenderingContext2D, - cues: CaptionCue[], - settings: AutoCaptionSettings, - width: number, - height: number, - timeMs: number, + ctx: CanvasRenderingContext2D, + cues: CaptionCue[], + settings: AutoCaptionSettings, + width: number, + height: number, + timeMs: number, ) { - if (!settings.enabled || cues.length === 0) { - return; - } + if (!settings.enabled || cues.length === 0) { + return; + } - ctx.save(); + ctx.save(); - const fontSize = getCaptionScaledFontSize(settings.fontSize, width, settings.maxWidth); - ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`; - const padding = getCaptionPadding(fontSize); + const fontSize = getCaptionScaledFontSize(settings.fontSize, width, settings.maxWidth); + ctx.font = `${CAPTION_FONT_WEIGHT} ${fontSize}px ${getDefaultCaptionFontFamily()}`; + const padding = getCaptionPadding(fontSize); - const activeCaptionLayout = buildActiveCaptionLayout({ - cues, - timeMs, - settings, - maxWidthPx: getCaptionTextMaxWidth(width, settings.maxWidth, fontSize), - measureText: (text) => ctx.measureText(text).width, - }); - if (!activeCaptionLayout) { - ctx.restore(); - return; - } + const activeCaptionLayout = buildActiveCaptionLayout({ + cues, + timeMs, + settings, + maxWidthPx: getCaptionTextMaxWidth(width, settings.maxWidth, fontSize), + measureText: (text) => ctx.measureText(text).width, + }); + if (!activeCaptionLayout) { + ctx.restore(); + return; + } - const lineHeight = fontSize * CAPTION_LINE_HEIGHT; - const paddingX = padding.x; - const paddingY = padding.y; - const textBlockHeight = activeCaptionLayout.visibleLines.length * lineHeight; - const boxHeight = textBlockHeight + paddingY * 2; - const centerX = width / 2; - const centerY = height - (height * settings.bottomOffset) / 100 - boxHeight / 2; - const maxMeasuredWidth = activeCaptionLayout.visibleLines.reduce( - (largest, line) => Math.max(largest, line.width), - 0, - ); - const boxWidth = Math.min(width * (settings.maxWidth / 100) + paddingX * 2, maxMeasuredWidth + paddingX * 2); + const lineHeight = fontSize * CAPTION_LINE_HEIGHT; + const paddingX = padding.x; + const paddingY = padding.y; + const textBlockHeight = activeCaptionLayout.visibleLines.length * lineHeight; + const boxHeight = textBlockHeight + paddingY * 2; + const centerX = width / 2; + const centerY = height - (height * settings.bottomOffset) / 100 - boxHeight / 2; + const maxMeasuredWidth = activeCaptionLayout.visibleLines.reduce( + (largest, line) => Math.max(largest, line.width), + 0, + ); + const boxWidth = Math.min( + width * (settings.maxWidth / 100) + paddingX * 2, + maxMeasuredWidth + paddingX * 2, + ); - ctx.translate(centerX, centerY + activeCaptionLayout.translateY); - ctx.scale(activeCaptionLayout.scale, activeCaptionLayout.scale); - ctx.globalAlpha = activeCaptionLayout.opacity; + ctx.translate(centerX, centerY + activeCaptionLayout.translateY); + ctx.scale(activeCaptionLayout.scale, activeCaptionLayout.scale); + ctx.globalAlpha = activeCaptionLayout.opacity; - ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`; - drawSquircleOnCanvas(ctx, { - x: -boxWidth / 2, - y: -boxHeight / 2, - width: boxWidth, - height: boxHeight, - radius: getCaptionScaledRadius(settings.boxRadius, fontSize), - }); - ctx.fill(); + ctx.fillStyle = `rgba(0, 0, 0, ${settings.backgroundOpacity})`; + drawSquircleOnCanvas(ctx, { + x: -boxWidth / 2, + y: -boxHeight / 2, + width: boxWidth, + height: boxHeight, + radius: getCaptionScaledRadius(settings.boxRadius, fontSize), + }); + ctx.fill(); - ctx.textAlign = "left"; - ctx.textBaseline = "middle"; + ctx.textAlign = "left"; + ctx.textBaseline = "middle"; - activeCaptionLayout.visibleLines.forEach((line, lineIndex) => { - let cursorX = -line.width / 2; - const lineY = -boxHeight / 2 + paddingY + lineHeight * lineIndex + lineHeight / 2; + activeCaptionLayout.visibleLines.forEach((line, lineIndex) => { + let cursorX = -line.width / 2; + const lineY = -boxHeight / 2 + paddingY + lineHeight * lineIndex + lineHeight / 2; - line.words.forEach((word) => { - const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`; - const segmentWidth = ctx.measureText(segmentText).width; - const visualState = getCaptionWordVisualState( - activeCaptionLayout.hasWordTimings, - word.state, - ); + line.words.forEach((word) => { + const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`; + const segmentWidth = ctx.measureText(segmentText).width; + const visualState = getCaptionWordVisualState( + activeCaptionLayout.hasWordTimings, + word.state, + ); - ctx.save(); - ctx.translate(cursorX, lineY); - ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor; - ctx.globalAlpha = activeCaptionLayout.opacity * visualState.opacity; - ctx.fillText(segmentText, 0, 0); - ctx.restore(); + ctx.save(); + ctx.translate(cursorX, lineY); + ctx.fillStyle = visualState.isInactive + ? settings.inactiveTextColor + : settings.textColor; + ctx.globalAlpha = activeCaptionLayout.opacity * visualState.opacity; + ctx.fillText(segmentText, 0, 0); + ctx.restore(); - cursorX += segmentWidth; - }); - }); + cursorX += segmentWidth; + }); + }); - ctx.restore(); -} \ No newline at end of file + ctx.restore(); +} diff --git a/src/lib/exporter/exportTuning.test.ts b/src/lib/exporter/exportTuning.test.ts index 8af95de96..990124e19 100644 --- a/src/lib/exporter/exportTuning.test.ts +++ b/src/lib/exporter/exportTuning.test.ts @@ -80,4 +80,4 @@ describe("exportTuning", () => { expect(breezeHeavyProfile.maxDecodeQueue).toBe(6); expect(breezeHeavyProfile.maxPendingFrames).toBe(12); }); -}); \ No newline at end of file +}); diff --git a/src/lib/exporter/exportTuning.ts b/src/lib/exporter/exportTuning.ts index 9d6fed3ec..1ec03b59d 100644 --- a/src/lib/exporter/exportTuning.ts +++ b/src/lib/exporter/exportTuning.ts @@ -4,10 +4,7 @@ const DEFAULT_ENCODING_MODE: ExportEncodingMode = "balanced"; type WebCodecsLatencyMode = "quality" | "realtime"; const BASELINE_PIXELS_PER_SECOND = 1280 * 720 * 60; -const LATENCY_MODE_PREFERENCES: Record< - ExportEncodingMode, - readonly WebCodecsLatencyMode[] -> = { +const LATENCY_MODE_PREFERENCES: Record = { fast: ["realtime", "quality"], balanced: ["realtime", "quality"], quality: ["quality", "realtime"], @@ -62,7 +59,10 @@ function getEffectiveHardwareConcurrency(hardwareConcurrency?: number): number { } function getRelativePixelRate(width: number, height: number, frameRate: number): number { - return (Math.max(1, width) * Math.max(1, height) * Math.max(1, frameRate)) / BASELINE_PIXELS_PER_SECOND; + return ( + (Math.max(1, width) * Math.max(1, height) * Math.max(1, frameRate)) / + BASELINE_PIXELS_PER_SECOND + ); } export interface ExportBackpressureProfile { @@ -123,10 +123,7 @@ export function getExportBackpressureProfile( const isHighCoreSystem = hardwareConcurrency >= 8; const isHeavyWorkload = relativePixelRate >= 1.5; const isExtremeWorkload = relativePixelRate >= 3; - const maxEncodeQueue = getWebCodecsEncodeQueueLimit( - options.frameRate, - options.encodingMode, - ); + const maxEncodeQueue = getWebCodecsEncodeQueueLimit(options.frameRate, options.encodingMode); if (options.encodeBackend === "ffmpeg") { if (isLowCoreSystem || isExtremeWorkload) { @@ -185,4 +182,4 @@ export function getExportBackpressureProfile( maxPendingFrames: 24, maxInFlightNativeWrites: 1, }; -} \ No newline at end of file +} diff --git a/src/lib/exporter/forwardFrameSource.ts b/src/lib/exporter/forwardFrameSource.ts index 19ec5bd38..c719c62e3 100644 --- a/src/lib/exporter/forwardFrameSource.ts +++ b/src/lib/exporter/forwardFrameSource.ts @@ -6,13 +6,13 @@ const DEFAULT_MAX_DECODE_QUEUE = 12; const DEFAULT_MAX_PENDING_FRAMES = 32; export interface ForwardFrameSourceMetadata { - width: number; - height: number; - duration: number; - mediaStartTime?: number; - streamStartTime?: number; - streamDuration?: number; - codec: string; + width: number; + height: number; + duration: number; + mediaStartTime?: number; + streamStartTime?: number; + streamDuration?: number; + codec: string; } /** @@ -22,380 +22,404 @@ export interface ForwardFrameSourceMetadata { * the nearest decoded frame for increasing target timestamps. */ export class ForwardFrameSource { - private demuxer: WebDemuxer | null = null; - private decoder: VideoDecoder | null = null; - private cancelled = false; - private metadata: ForwardFrameSourceMetadata | null = null; - private pendingFrames: VideoFrame[] = []; - private frameResolve: ((frame: VideoFrame | null) => void) | null = null; - private decodeError: Error | null = null; - private decodeDone = false; - private feedPromise: Promise | null = null; - private reader: ReadableStreamDefaultReader | null = null; - private heldFrame: VideoFrame | null = null; - private heldFrameSec = 0; - private lastTargetTimeSec = 0; - private firstFrameTimestampUs: number | null = null; - private frameTimelineOffsetUs = 0; - - private toLocalFilePath(resourceUrl: string): string | null { - if (!resourceUrl.startsWith("file:")) { - return null; - } - - try { - const url = new URL(resourceUrl); - let filePath = decodeURIComponent(url.pathname); - if (/^\/[A-Za-z]:/.test(filePath)) { - filePath = filePath.slice(1); - } - return filePath; - } catch { - return resourceUrl.replace(/^file:\/\//, ""); - } - } - - private inferMimeType(fileName: string): string { - const extension = fileName.split(".").pop()?.toLowerCase(); - switch (extension) { - case "mov": - return "video/quicktime"; - case "webm": - return "video/webm"; - case "mkv": - return "video/x-matroska"; - case "avi": - return "video/x-msvideo"; - case "mp4": - default: - return "video/mp4"; - } - } - - private async loadVideoFile(resourceUrl: string): Promise { - const filename = resourceUrl.split("/").pop() || "video"; - const localFilePath = this.toLocalFilePath(resourceUrl); - - if (localFilePath) { - const result = await window.electronAPI.readLocalFile(localFilePath); - if (!result.success || !result.data) { - throw new Error(result.error || "Failed to read local video file"); - } - - const bytes = - result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data); - const arrayBuffer = bytes.buffer.slice( - bytes.byteOffset, - bytes.byteOffset + bytes.byteLength, - ) as ArrayBuffer; - return new File([arrayBuffer], filename, { - type: this.inferMimeType(filename), - }); - } - - const response = await fetch(resourceUrl); - if (!response.ok) { - throw new Error( - `Failed to load video resource: ${response.status} ${response.statusText}`, - ); - } - - const blob = await response.blob(); - return new File([blob], filename, { - type: blob.type || this.inferMimeType(filename), - }); - } - - private resolveVideoResourceUrl(videoUrl: string): string { - if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) { - return videoUrl; - } - - if (videoUrl.startsWith("/")) { - return `file://${encodeURI(videoUrl)}`; - } - - return videoUrl; - } - - async initialize(videoUrl: string): Promise { - const resourceUrl = this.resolveVideoResourceUrl(videoUrl); - const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; - this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); - - const file = await this.loadVideoFile(resourceUrl); - await this.demuxer.load(file); - - const mediaInfo = await this.demuxer.getMediaInfo(); - const videoStream = mediaInfo.streams.find( - (stream) => stream.codec_type_string === "video", - ); - const mediaStartTime = - typeof mediaInfo.start_time === "number" && Number.isFinite(mediaInfo.start_time) - ? mediaInfo.start_time - : 0; - const streamStartTime = - typeof videoStream?.start_time === "number" && Number.isFinite(videoStream.start_time) - ? videoStream.start_time - : mediaStartTime; - - this.metadata = { - width: videoStream?.width || 0, - height: videoStream?.height || 0, - duration: mediaInfo.duration, - mediaStartTime, - streamStartTime, - streamDuration: - typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration) - ? videoStream.duration - : undefined, - codec: videoStream?.codec_string || "unknown", - }; - - await this.startDecoder(); - return this.metadata; - } - - private async startDecoder(): Promise { - if (!this.demuxer || !this.metadata) { - throw new Error("Must call initialize() before starting decoder"); - } - - const decoderConfig = await this.demuxer.getDecoderConfig("video"); - const codec = this.metadata.codec.toLowerCase(); - const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1"); - - this.decoder = new VideoDecoder({ - output: (frame: VideoFrame) => { - if (this.frameResolve) { - const resolve = this.frameResolve; - this.frameResolve = null; - resolve(frame); - } else { - this.pendingFrames.push(frame); - } - }, - error: (error: DOMException) => { - this.decodeError = new Error(`VideoDecoder error: ${error.message}`); - if (this.frameResolve) { - const resolve = this.frameResolve; - this.frameResolve = null; - resolve(null); - } - }, - }); - - const preferredDecoderConfig = shouldPreferSoftwareDecode - ? { - ...decoderConfig, - hardwareAcceleration: "prefer-software" as const, - } - : decoderConfig; - - try { - this.decoder.configure(preferredDecoderConfig); - } catch (error) { - if (!shouldPreferSoftwareDecode) { - throw error; - } - this.decoder.configure(decoderConfig); - } - - const readEndSec = Math.max( - this.metadata.duration + (this.metadata.mediaStartTime ?? 0), - (this.metadata.streamDuration ?? this.metadata.duration) + - (this.metadata.streamStartTime ?? this.metadata.mediaStartTime ?? 0), - ) + 0.5; - this.reader = this.demuxer.read("video", 0, readEndSec).getReader(); - - this.feedPromise = (async () => { - try { - while (!this.cancelled) { - const { done, value: chunk } = await this.reader!.read(); - if (done || !chunk) { - break; - } - - while ( - ( - this.decoder!.decodeQueueSize > DEFAULT_MAX_DECODE_QUEUE || - this.pendingFrames.length > DEFAULT_MAX_PENDING_FRAMES - ) && - !this.cancelled - ) { - await new Promise((resolve) => setTimeout(resolve, 1)); - } - - if (this.cancelled) { - break; - } - - this.decoder!.decode(chunk); - } - - if (!this.cancelled && this.decoder?.state === "configured") { - await this.decoder.flush(); - } - } catch (error) { - this.decodeError = - error instanceof Error ? error : new Error(String(error)); - } finally { - this.decodeDone = true; - if (this.frameResolve) { - const resolve = this.frameResolve; - this.frameResolve = null; - resolve(null); - } - } - })(); - } - - private getNextFrame(): Promise { - if (this.decodeError) { - throw this.decodeError; - } - - if (this.pendingFrames.length > 0) { - return Promise.resolve(this.pendingFrames.shift()!); - } - - if (this.decodeDone) { - return Promise.resolve(null); - } - - return new Promise((resolve) => { - this.frameResolve = resolve; - }); - } - - async getFrameAtTime(targetTimeSec: number): Promise { - if (!this.metadata) { - throw new Error("Frame source not initialized"); - } - - const clampedTargetTime = Math.max( - 0, - Math.min( - targetTimeSec, - getEffectiveVideoStreamDurationSeconds({ - duration: this.metadata.duration, - streamDuration: this.metadata.streamDuration, - }) || targetTimeSec, - ), - ); - if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) { - throw new Error("ForwardFrameSource only supports increasing timestamps"); - } - this.lastTargetTimeSec = clampedTargetTime; - - if (!this.heldFrame) { - const firstFrame = await this.getNextFrame(); - if (!firstFrame) { - return null; - } - this.firstFrameTimestampUs = firstFrame.timestamp; - this.frameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs( - firstFrame.timestamp, - this.metadata, - ); - this.heldFrame = firstFrame; - this.heldFrameSec = Math.max(0, this.frameTimelineOffsetUs / 1_000_000); - } - - while (!this.cancelled) { - const nextFrame = await this.getNextFrame(); - if (!nextFrame) { - return new VideoFrame(this.heldFrame, { - timestamp: this.heldFrame.timestamp, - }); - } - - const nextFrameSec = Math.max( - this.heldFrameSec, - Math.max( - 0, - ( - nextFrame.timestamp - (this.firstFrameTimestampUs ?? nextFrame.timestamp) + - this.frameTimelineOffsetUs - ) / 1_000_000, - ), - ); - const handoffBoundarySec = (this.heldFrameSec + nextFrameSec) / 2; - if (clampedTargetTime <= handoffBoundarySec) { - this.pendingFrames.unshift(nextFrame); - return new VideoFrame(this.heldFrame, { - timestamp: this.heldFrame.timestamp, - }); - } - - this.heldFrame.close(); - this.heldFrame = nextFrame; - this.heldFrameSec = nextFrameSec; - } - - return null; - } - - cancel(): void { - this.cancelled = true; - } - - async destroy(): Promise { - this.cancelled = true; - - if (this.reader) { - try { - await this.reader.cancel(); - } catch { - // Ignore cancellation errors during shutdown. - } - this.reader = null; - } - - if (this.feedPromise) { - try { - await this.feedPromise; - } catch { - // Decoder errors are already surfaced through getFrameAtTime. - } - this.feedPromise = null; - } - - if (this.heldFrame) { - this.heldFrame.close(); - this.heldFrame = null; - } - - for (const frame of this.pendingFrames) { - frame.close(); - } - this.pendingFrames = []; - - if (this.decoder) { - try { - if (this.decoder.state === "configured") { - this.decoder.close(); - } - } catch { - // Ignore decoder shutdown errors. - } - this.decoder = null; - } - - if (this.demuxer) { - try { - this.demuxer.destroy(); - } catch { - // Ignore demuxer shutdown errors. - } - this.demuxer = null; - } - - this.metadata = null; - this.decodeDone = false; - this.decodeError = null; - this.lastTargetTimeSec = 0; - this.firstFrameTimestampUs = null; - this.frameTimelineOffsetUs = 0; - } + private demuxer: WebDemuxer | null = null; + private decoder: VideoDecoder | null = null; + private cancelled = false; + private metadata: ForwardFrameSourceMetadata | null = null; + private pendingFrames: VideoFrame[] = []; + private frameResolve: ((frame: VideoFrame | null) => void) | null = null; + private decodeError: Error | null = null; + private decodeDone = false; + private feedPromise: Promise | null = null; + private reader: ReadableStreamDefaultReader | null = null; + private heldFrame: VideoFrame | null = null; + private heldFrameSec = 0; + private lastTargetTimeSec = 0; + private firstFrameTimestampUs: number | null = null; + private frameTimelineOffsetUs = 0; + + private toLocalFilePath(resourceUrl: string): string | null { + if (!resourceUrl.startsWith("file:")) { + return null; + } + + try { + const url = new URL(resourceUrl); + let filePath = decodeURIComponent(url.pathname); + if (url.host && url.host !== "localhost") { + return `\\\\${url.host}${filePath.replace(/\//g, "\\")}`; + } + if (/^\/[A-Za-z]:/.test(filePath)) { + filePath = filePath.slice(1); + } + return filePath; + } catch { + const uncMatch = resourceUrl.match(/^file:\/\/([^/]+)(\/.*)$/i); + if (uncMatch && uncMatch[1].toLowerCase() !== "localhost") { + return `\\\\${uncMatch[1]}${decodeURIComponent(uncMatch[2]).replace(/\//g, "\\")}`; + } + return resourceUrl.replace(/^file:\/\//, ""); + } + } + + private inferMimeType(fileName: string): string { + const extension = fileName.split(".").pop()?.toLowerCase(); + switch (extension) { + case "mov": + return "video/quicktime"; + case "webm": + return "video/webm"; + case "mkv": + return "video/x-matroska"; + case "avi": + return "video/x-msvideo"; + case "mp4": + default: + return "video/mp4"; + } + } + + private async loadVideoFile(resourceUrl: string): Promise { + const filename = resourceUrl.split("/").pop() || "video"; + const localFilePath = this.toLocalFilePath(resourceUrl); + + if (localFilePath) { + const result = await window.electronAPI.readLocalFile(localFilePath); + if (!result.success || !result.data) { + throw new Error(result.error || "Failed to read local video file"); + } + + const bytes = + result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data); + const arrayBuffer = bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ) as ArrayBuffer; + return new File([arrayBuffer], filename, { + type: this.inferMimeType(filename), + }); + } + + const response = await fetch(resourceUrl); + if (!response.ok) { + throw new Error( + `Failed to load video resource: ${response.status} ${response.statusText}`, + ); + } + + const blob = await response.blob(); + return new File([blob], filename, { + type: blob.type || this.inferMimeType(filename), + }); + } + + private resolveVideoResourceUrl(videoUrl: string): string { + if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) { + return videoUrl; + } + + if (/^[A-Za-z]:[\\/]/.test(videoUrl)) { + const normalized = videoUrl.replace(/\\/g, "/"); + return `file:///${encodeURI(normalized)}`; + } + + if (videoUrl.startsWith("/")) { + return `file://${encodeURI(videoUrl)}`; + } + + return videoUrl; + } + + async initialize(videoUrl: string): Promise { + const resourceUrl = this.resolveVideoResourceUrl(videoUrl); + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; + this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); + + const file = await this.loadVideoFile(resourceUrl); + await this.demuxer.load(file); + + const mediaInfo = await this.demuxer.getMediaInfo(); + const videoStream = mediaInfo.streams.find( + (stream) => stream.codec_type_string === "video", + ); + const mediaStartTime = + typeof mediaInfo.start_time === "number" && Number.isFinite(mediaInfo.start_time) + ? mediaInfo.start_time + : 0; + const streamStartTime = + typeof videoStream?.start_time === "number" && Number.isFinite(videoStream.start_time) + ? videoStream.start_time + : mediaStartTime; + + this.metadata = { + width: videoStream?.width || 0, + height: videoStream?.height || 0, + duration: mediaInfo.duration, + mediaStartTime, + streamStartTime, + streamDuration: + typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration) + ? videoStream.duration + : undefined, + codec: videoStream?.codec_string || "unknown", + }; + + await this.startDecoder(); + return this.metadata; + } + + private async startDecoder(): Promise { + if (!this.demuxer || !this.metadata) { + throw new Error("Must call initialize() before starting decoder"); + } + + const decoderConfig = await this.demuxer.getDecoderConfig("video"); + const codec = this.metadata.codec.toLowerCase(); + const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1"); + + this.decoder = new VideoDecoder({ + output: (frame: VideoFrame) => { + if (this.frameResolve) { + const resolve = this.frameResolve; + this.frameResolve = null; + resolve(frame); + } else { + this.pendingFrames.push(frame); + } + }, + error: (error: DOMException) => { + this.decodeError = new Error(`VideoDecoder error: ${error.message}`); + if (this.frameResolve) { + const resolve = this.frameResolve; + this.frameResolve = null; + resolve(null); + } + }, + }); + + const preferredDecoderConfig = shouldPreferSoftwareDecode + ? { + ...decoderConfig, + hardwareAcceleration: "prefer-software" as const, + } + : decoderConfig; + + try { + this.decoder.configure(preferredDecoderConfig); + } catch (error) { + if (!shouldPreferSoftwareDecode) { + throw error; + } + this.decoder.configure(decoderConfig); + } + + const readEndSec = + Math.max( + this.metadata.duration + (this.metadata.mediaStartTime ?? 0), + (this.metadata.streamDuration ?? this.metadata.duration) + + (this.metadata.streamStartTime ?? this.metadata.mediaStartTime ?? 0), + ) + 0.5; + this.reader = this.demuxer.read("video", 0, readEndSec).getReader(); + + this.feedPromise = (async () => { + try { + while (!this.cancelled) { + const { done, value: chunk } = await this.reader!.read(); + if (done || !chunk) { + break; + } + + while ( + (this.decoder!.decodeQueueSize > DEFAULT_MAX_DECODE_QUEUE || + this.pendingFrames.length > DEFAULT_MAX_PENDING_FRAMES) && + !this.cancelled + ) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + + if (this.cancelled) { + break; + } + + this.decoder!.decode(chunk); + } + + if (!this.cancelled && this.decoder?.state === "configured") { + await this.decoder.flush(); + } + } catch (error) { + this.decodeError = error instanceof Error ? error : new Error(String(error)); + } finally { + this.decodeDone = true; + if (this.frameResolve) { + const resolve = this.frameResolve; + this.frameResolve = null; + resolve(null); + } + } + })(); + } + + private getNextFrame(): Promise { + if (this.decodeError) { + throw this.decodeError; + } + + if (this.pendingFrames.length > 0) { + return Promise.resolve(this.pendingFrames.shift()!); + } + + if (this.decodeDone) { + return Promise.resolve(null); + } + + if (this.frameResolve) { + throw new Error("Concurrent getFrameAtTime() calls are not supported"); + } + + return new Promise((resolve) => { + this.frameResolve = resolve; + }); + } + + async getFrameAtTime(targetTimeSec: number): Promise { + if (!this.metadata) { + throw new Error("Frame source not initialized"); + } + + const clampedTargetTime = Math.max( + 0, + Math.min( + targetTimeSec, + getEffectiveVideoStreamDurationSeconds({ + duration: this.metadata.duration, + streamDuration: this.metadata.streamDuration, + }) || targetTimeSec, + ), + ); + if (clampedTargetTime + 0.001 < this.lastTargetTimeSec) { + throw new Error("ForwardFrameSource only supports increasing timestamps"); + } + this.lastTargetTimeSec = clampedTargetTime; + + if (!this.heldFrame) { + const firstFrame = await this.getNextFrame(); + if (!firstFrame) { + return null; + } + this.firstFrameTimestampUs = firstFrame.timestamp; + this.frameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs( + firstFrame.timestamp, + this.metadata, + ); + this.heldFrame = firstFrame; + this.heldFrameSec = Math.max(0, this.frameTimelineOffsetUs / 1_000_000); + } + + while (!this.cancelled) { + const nextFrame = await this.getNextFrame(); + if (!nextFrame) { + return new VideoFrame(this.heldFrame, { + timestamp: this.heldFrame.timestamp, + }); + } + + const nextFrameSec = Math.max( + this.heldFrameSec, + Math.max( + 0, + (nextFrame.timestamp - + (this.firstFrameTimestampUs ?? nextFrame.timestamp) + + this.frameTimelineOffsetUs) / + 1_000_000, + ), + ); + const handoffBoundarySec = (this.heldFrameSec + nextFrameSec) / 2; + if (clampedTargetTime <= handoffBoundarySec) { + this.pendingFrames.unshift(nextFrame); + return new VideoFrame(this.heldFrame, { + timestamp: this.heldFrame.timestamp, + }); + } + + this.heldFrame.close(); + this.heldFrame = nextFrame; + this.heldFrameSec = nextFrameSec; + } + + return null; + } + + cancel(): void { + this.cancelled = true; + if (this.frameResolve) { + const resolve = this.frameResolve; + this.frameResolve = null; + resolve(null); + } + if (this.reader) { + void this.reader.cancel().catch(() => { + // Ignore cancellation errors during shutdown. + }); + } + } + + async destroy(): Promise { + this.cancelled = true; + + if (this.reader) { + try { + await this.reader.cancel(); + } catch { + // Ignore cancellation errors during shutdown. + } + this.reader = null; + } + + if (this.feedPromise) { + try { + await this.feedPromise; + } catch { + // Decoder errors are already surfaced through getFrameAtTime. + } + this.feedPromise = null; + } + + if (this.heldFrame) { + this.heldFrame.close(); + this.heldFrame = null; + } + + for (const frame of this.pendingFrames) { + frame.close(); + } + this.pendingFrames = []; + + if (this.decoder) { + try { + if (this.decoder.state === "configured") { + this.decoder.close(); + } + } catch { + // Ignore decoder shutdown errors. + } + this.decoder = null; + } + + if (this.demuxer) { + try { + this.demuxer.destroy(); + } catch { + // Ignore demuxer shutdown errors. + } + this.demuxer = null; + } + + this.metadata = null; + this.decodeDone = false; + this.decodeError = null; + this.lastTargetTimeSec = 0; + this.firstFrameTimestampUs = null; + this.frameTimelineOffsetUs = 0; + } } diff --git a/src/lib/exporter/frameRenderer.test.ts b/src/lib/exporter/frameRenderer.test.ts index c43427a17..3d6629d78 100644 --- a/src/lib/exporter/frameRenderer.test.ts +++ b/src/lib/exporter/frameRenderer.test.ts @@ -1,361 +1,408 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { DEFAULT_WEBCAM_OVERLAY } from '@/components/video-editor/types'; - -vi.mock('pixi.js', () => ({ - Application: class {}, - Container: class {}, - Sprite: class {}, - Graphics: class {}, - BlurFilter: class {}, - Texture: { - from: vi.fn(() => ({ destroy: vi.fn() })), - }, +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_WEBCAM_OVERLAY } from "../../components/video-editor/types"; + +vi.mock("pixi.js", () => ({ + Application: vi.fn(), + Container: vi.fn(), + Sprite: vi.fn(), + Graphics: vi.fn(), + BlurFilter: vi.fn(), + Texture: { + from: vi.fn(() => ({ destroy: vi.fn() })), + }, })); -vi.mock('pixi-filters/motion-blur', () => ({ - MotionBlurFilter: class {}, +vi.mock("pixi-filters/motion-blur", () => ({ + MotionBlurFilter: vi.fn(), })); -vi.mock('@/lib/assetPath', () => ({ - getAssetPath: vi.fn(async (value: string) => value), - getRenderableAssetUrl: vi.fn((value: string) => value), +vi.mock("@/lib/assetPath", () => ({ + getAssetPath: vi.fn(async (value: string) => value), + getRenderableAssetUrl: vi.fn((value: string) => value), })); -vi.mock('@/components/video-editor/videoPlayback/zoomRegionUtils', () => ({ - findDominantRegion: vi.fn(() => ({ - region: null, - strength: 0, - blendedScale: 1, - transition: null, - })), +vi.mock("@/components/video-editor/videoPlayback/zoomRegionUtils", () => ({ + findDominantRegion: vi.fn(() => ({ + region: null, + strength: 0, + blendedScale: 1, + transition: null, + })), })); -vi.mock('@/components/video-editor/videoPlayback/zoomTransform', () => ({ - applyZoomTransform: vi.fn(), - computeFocusFromTransform: vi.fn(() => ({ cx: 0.5, cy: 0.5 })), - computeZoomTransform: vi.fn(() => ({ scale: 1, x: 0, y: 0 })), - createMotionBlurState: vi.fn(() => ({})), +vi.mock("@/components/video-editor/videoPlayback/zoomTransform", () => ({ + applyZoomTransform: vi.fn(), + computeFocusFromTransform: vi.fn(() => ({ cx: 0.5, cy: 0.5 })), + computeZoomTransform: vi.fn(() => ({ scale: 1, x: 0, y: 0 })), + createMotionBlurState: vi.fn(() => ({})), })); -vi.mock('./annotationRenderer', () => ({ - renderAnnotations: vi.fn(), +vi.mock("./annotationRenderer", () => ({ + renderAnnotations: vi.fn(), })); -vi.mock('@/components/video-editor/videoPlayback/cursorRenderer', () => ({ - PixiCursorOverlay: class { - container = {}; - update = vi.fn(); - destroy = vi.fn(); - }, - DEFAULT_CURSOR_CONFIG: { - dotRadius: 28, - smoothingFactor: 0.18, - motionBlur: 0, - clickBounce: 1, - sway: 0, - }, - preloadCursorAssets: vi.fn(async () => undefined), +vi.mock("@/components/video-editor/videoPlayback/cursorRenderer", () => ({ + PixiCursorOverlay: class { + container = {}; + update = vi.fn(); + destroy = vi.fn(); + }, + DEFAULT_CURSOR_CONFIG: { + dotRadius: 28, + smoothingFactor: 0.18, + motionBlur: 0, + clickBounce: 1, + sway: 0, + }, + preloadCursorAssets: vi.fn(async () => undefined), })); -import { FrameRenderer } from './frameRenderer'; +import { FrameRenderer } from "./frameRenderer"; + +type MockFunction = ReturnType; +type MockContext = { + beginPath: MockFunction; + moveTo: MockFunction; + lineTo: MockFunction; + closePath: MockFunction; + clip: MockFunction; + drawImage: MockFunction; + save: MockFunction; + restore: MockFunction; + translate: MockFunction; + scale: MockFunction; + clearRect: MockFunction; + filter: string; +}; +type MockCanvas = ReturnType; +type FrameRendererTestAccess = { + webcamVideoElement: FakeVideoElement | null; + webcamSeekPromise: Promise | null; + webcamFrameCacheCanvas: MockCanvas | null; + webcamFrameCacheCtx: CanvasRenderingContext2D | null; + lastSyncedWebcamTime: number | null; + currentVideoTime: number; + animationState: { appliedScale: number }; + syncWebcamFrame: (targetTimeSec: number) => Promise; + drawWebcamOverlay: ( + outputCtx: CanvasRenderingContext2D, + outputWidth: number, + outputHeight: number, + ) => void; +}; type Listener = { - callback: () => void; - once: boolean; + callback: () => void; + once: boolean; }; class FakeVideoElement { - duration: number; - readyState: number; - seeking = false; - videoWidth: number; - videoHeight: number; - muted = true; - preload = 'auto'; - playsInline = true; - src = ''; - - private currentTimeValue: number; - private listeners = new Map(); - - constructor({ - duration = 5, - currentTime = 0, - readyState = 2, - videoWidth = 1280, - videoHeight = 720, - }: { - duration?: number; - currentTime?: number; - readyState?: number; - videoWidth?: number; - videoHeight?: number; - } = {}) { - this.duration = duration; - this.currentTimeValue = currentTime; - this.readyState = readyState; - this.videoWidth = videoWidth; - this.videoHeight = videoHeight; - } - - get currentTime() { - return this.currentTimeValue; - } - - set currentTime(next: number) { - this.currentTimeValue = next; - this.seeking = true; - queueMicrotask(() => { - this.seeking = false; - this.dispatch('seeked'); - }); - } - - addEventListener(name: string, callback: () => void, options?: boolean | AddEventListenerOptions) { - const listeners = this.listeners.get(name) ?? []; - listeners.push({ - callback, - once: !!(typeof options === 'object' && options?.once), - }); - this.listeners.set(name, listeners); - } - - removeEventListener(name: string, callback: () => void) { - const listeners = this.listeners.get(name) ?? []; - this.listeners.set( - name, - listeners.filter((listener) => listener.callback !== callback), - ); - } - - load() {} - - pause() {} - - private dispatch(name: string) { - const listeners = [...(this.listeners.get(name) ?? [])]; - if (listeners.length === 0) { - return; - } - - for (const listener of listeners) { - listener.callback(); - if (listener.once) { - this.removeEventListener(name, listener.callback); - } - } - } + duration: number; + readyState: number; + seeking = false; + videoWidth: number; + videoHeight: number; + muted = true; + preload = "auto"; + playsInline = true; + src = ""; + + private currentTimeValue: number; + private listeners = new Map(); + + constructor({ + duration = 5, + currentTime = 0, + readyState = 2, + videoWidth = 1280, + videoHeight = 720, + }: { + duration?: number; + currentTime?: number; + readyState?: number; + videoWidth?: number; + videoHeight?: number; + } = {}) { + this.duration = duration; + this.currentTimeValue = currentTime; + this.readyState = readyState; + this.videoWidth = videoWidth; + this.videoHeight = videoHeight; + } + + get currentTime() { + return this.currentTimeValue; + } + + set currentTime(next: number) { + this.currentTimeValue = next; + this.seeking = true; + queueMicrotask(() => { + this.seeking = false; + this.dispatch("seeked"); + }); + } + + addEventListener( + name: string, + callback: () => void, + options?: boolean | AddEventListenerOptions, + ) { + const listeners = this.listeners.get(name) ?? []; + listeners.push({ + callback, + once: !!(typeof options === "object" && options?.once), + }); + this.listeners.set(name, listeners); + } + + removeEventListener(name: string, callback: () => void) { + const listeners = this.listeners.get(name) ?? []; + this.listeners.set( + name, + listeners.filter((listener) => listener.callback !== callback), + ); + } + + load() { + // Intentional no-op for the mock video element. + } + + pause() { + // Intentional no-op for the mock video element. + } + + private dispatch(name: string) { + const listeners = [...(this.listeners.get(name) ?? [])]; + if (listeners.length === 0) { + return; + } + + for (const listener of listeners) { + listener.callback(); + if (listener.once) { + this.removeEventListener(name, listener.callback); + } + } + } } function createMockContext() { - return { - beginPath: vi.fn(), - moveTo: vi.fn(), - lineTo: vi.fn(), - closePath: vi.fn(), - clip: vi.fn(), - drawImage: vi.fn(), - save: vi.fn(), - restore: vi.fn(), - translate: vi.fn(), - scale: vi.fn(), - clearRect: vi.fn(), - filter: '', - } as unknown as CanvasRenderingContext2D; + return { + beginPath: vi.fn(), + moveTo: vi.fn(), + lineTo: vi.fn(), + closePath: vi.fn(), + clip: vi.fn(), + drawImage: vi.fn(), + save: vi.fn(), + restore: vi.fn(), + translate: vi.fn(), + scale: vi.fn(), + clearRect: vi.fn(), + filter: "", + }; } function createMockCanvas() { - const context = createMockContext(); - return { - width: 0, - height: 0, - context, - getContext: vi.fn(() => context), - }; + const context = createMockContext(); + return { + width: 0, + height: 0, + context, + getContext: vi.fn((_type?: string) => context as unknown as CanvasRenderingContext2D), + }; } function createRenderer() { - return new FrameRenderer({ - width: 1920, - height: 1080, - wallpaper: '#000000', - zoomRegions: [], - showShadow: false, - shadowIntensity: 0, - backgroundBlur: 0, - cropRegion: { x: 0, y: 0, width: 1, height: 1 }, - webcam: { - ...DEFAULT_WEBCAM_OVERLAY, - enabled: true, - mirror: false, - shadow: 0, - }, - webcamUrl: 'file:///tmp/webcam.webm', - videoWidth: 1920, - videoHeight: 1080, - }); + return new FrameRenderer({ + width: 1920, + height: 1080, + wallpaper: "#000000", + zoomRegions: [], + showShadow: false, + shadowIntensity: 0, + backgroundBlur: 0, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + webcam: { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: true, + mirror: false, + shadow: 0, + }, + webcamUrl: "file:///tmp/webcam.webm", + videoWidth: 1920, + videoHeight: 1080, + }); } -describe('FrameRenderer webcam export path', () => { - const createdCanvases: ReturnType[] = []; - - beforeEach(() => { - createdCanvases.length = 0; - - Object.assign(globalThis, { - window: globalThis, - requestAnimationFrame: (callback: FrameRequestCallback) => { - callback(0); - return 1; - }, - cancelAnimationFrame: vi.fn(), - HTMLMediaElement: { - HAVE_CURRENT_DATA: 2, - }, - document: { - createElement: vi.fn((tag: string) => { - if (tag !== 'canvas') { - throw new Error(`Unexpected element requested in test: ${tag}`); - } - - const canvas = createMockCanvas(); - createdCanvases.push(canvas); - return canvas; - }), - }, - }); - }); - - it('clamps webcam sync seeks to the media duration', async () => { - const renderer = createRenderer() as any; - const webcamVideo = new FakeVideoElement({ duration: 4.5, currentTime: 0.25 }); - renderer.webcamVideoElement = webcamVideo; - - await renderer.syncWebcamFrame(12); - - expect(webcamVideo.currentTime).toBe(4.5); - expect(renderer.lastSyncedWebcamTime).toBe(4.5); - expect(renderer.webcamSeekPromise).toBeNull(); - }); - - it('falls back to animation frame when requestVideoFrameCallback does not fire', async () => { - const renderer = createRenderer() as any; - const webcamVideo = new FakeVideoElement({ duration: 4.5, currentTime: 0.25 }) as FakeVideoElement & { - requestVideoFrameCallback?: (callback: () => void) => number; - cancelVideoFrameCallback?: (handle: number) => void; - }; - webcamVideo.requestVideoFrameCallback = vi.fn(() => 7); - webcamVideo.cancelVideoFrameCallback = vi.fn(); - renderer.webcamVideoElement = webcamVideo; - - await renderer.syncWebcamFrame(1.5); - - expect(webcamVideo.currentTime).toBe(1.5); - expect(renderer.lastSyncedWebcamTime).toBe(1.5); - expect(webcamVideo.requestVideoFrameCallback).toHaveBeenCalledTimes(1); - expect(webcamVideo.cancelVideoFrameCallback).not.toHaveBeenCalled(); - expect(renderer.webcamSeekPromise).toBeNull(); - }); - - it('uses the cached webcam frame when the live video is out of sync', () => { - const renderer = createRenderer() as any; - const outputContext = createMockContext(); - const webcamVideo = new FakeVideoElement({ - currentTime: 2, - readyState: 2, - videoWidth: 640, - videoHeight: 360, - }); - const cachedFrameCanvas = createMockCanvas(); - cachedFrameCanvas.width = 640; - cachedFrameCanvas.height = 360; - - renderer.webcamVideoElement = webcamVideo; - renderer.webcamFrameCacheCanvas = cachedFrameCanvas; - renderer.webcamFrameCacheCtx = cachedFrameCanvas.getContext('2d'); - renderer.lastSyncedWebcamTime = 1.5; - renderer.currentVideoTime = 2; - renderer.animationState.appliedScale = 1; - - renderer.drawWebcamOverlay(outputContext, 1280, 720); - - const bubbleCanvas = createdCanvases[0]; - expect(bubbleCanvas).toBeDefined(); - expect((bubbleCanvas.context.drawImage as any).mock.calls[0][0]).toBe(cachedFrameCanvas); - expect((outputContext.drawImage as any).mock.calls[0][0]).toBe(bubbleCanvas); - }); - - it('keeps drawing the cached webcam frame when the live element temporarily has no current data', () => { - const renderer = createRenderer() as any; - const outputContext = createMockContext(); - const webcamVideo = new FakeVideoElement({ - currentTime: 2, - readyState: 0, - videoWidth: 640, - videoHeight: 360, - }); - const cachedFrameCanvas = createMockCanvas(); - cachedFrameCanvas.width = 640; - cachedFrameCanvas.height = 360; - - renderer.webcamVideoElement = webcamVideo; - renderer.webcamFrameCacheCanvas = cachedFrameCanvas; - renderer.webcamFrameCacheCtx = cachedFrameCanvas.getContext('2d'); - renderer.lastSyncedWebcamTime = 2; - renderer.currentVideoTime = 2; - renderer.animationState.appliedScale = 1; - - renderer.drawWebcamOverlay(outputContext, 1280, 720); - - const bubbleCanvas = createdCanvases[0]; - expect(bubbleCanvas).toBeDefined(); - expect((bubbleCanvas.context.drawImage as any).mock.calls[0][0]).toBe(cachedFrameCanvas); - expect((outputContext.drawImage as any).mock.calls[0][0]).toBe(bubbleCanvas); - }); - - it('uses the live webcam frame and refreshes the cache when the video is synchronized', () => { - const renderer = createRenderer() as any; - const outputContext = createMockContext(); - const webcamVideo = new FakeVideoElement({ - currentTime: 2, - readyState: 2, - videoWidth: 800, - videoHeight: 600, - }); - - renderer.webcamVideoElement = webcamVideo; - renderer.lastSyncedWebcamTime = 2; - renderer.currentVideoTime = 2; - renderer.animationState.appliedScale = 1; - - renderer.drawWebcamOverlay(outputContext, 1280, 720); - - const bubbleCanvas = createdCanvases[0]; - const cacheCanvas = createdCanvases[1]; - expect(cacheCanvas).toBeDefined(); - expect((cacheCanvas.context.drawImage as any).mock.calls[0][0]).toBe(webcamVideo); - expect((bubbleCanvas.context.drawImage as any).mock.calls[0][0]).toBe(cacheCanvas); - expect((outputContext.drawImage as any).mock.calls[0][0]).toBe(bubbleCanvas); - }); - - it('reuses the webcam bubble canvas across frames', () => { - const renderer = createRenderer() as any; - const outputContext = createMockContext(); - const webcamVideo = new FakeVideoElement({ - currentTime: 2, - readyState: 2, - videoWidth: 800, - videoHeight: 600, - }); - - renderer.webcamVideoElement = webcamVideo; - renderer.lastSyncedWebcamTime = 2; - renderer.currentVideoTime = 2; - renderer.animationState.appliedScale = 1; - - renderer.drawWebcamOverlay(outputContext, 1280, 720); - renderer.drawWebcamOverlay(outputContext, 1280, 720); - - expect(createdCanvases).toHaveLength(2); - }); -}); \ No newline at end of file +describe("FrameRenderer webcam export path", () => { + const createdCanvases: ReturnType[] = []; + + beforeEach(() => { + createdCanvases.length = 0; + + Object.assign(globalThis, { + window: globalThis, + requestAnimationFrame: (callback: FrameRequestCallback) => { + callback(0); + return 1; + }, + cancelAnimationFrame: vi.fn(), + HTMLMediaElement: { + HAVE_CURRENT_DATA: 2, + }, + document: { + createElement: vi.fn((tag: string) => { + if (tag !== "canvas") { + throw new Error(`Unexpected element requested in test: ${tag}`); + } + + const canvas = createMockCanvas(); + createdCanvases.push(canvas); + return canvas; + }), + }, + }); + }); + + it("clamps webcam sync seeks to the media duration", async () => { + const renderer = createRenderer() as unknown as FrameRendererTestAccess; + const webcamVideo = new FakeVideoElement({ duration: 4.5, currentTime: 0.25 }); + renderer.webcamVideoElement = webcamVideo; + + await renderer.syncWebcamFrame(12); + + expect(webcamVideo.currentTime).toBe(4.5); + expect(renderer.lastSyncedWebcamTime).toBe(4.5); + expect(renderer.webcamSeekPromise).toBeNull(); + }); + + it("falls back to animation frame when requestVideoFrameCallback does not fire", async () => { + const renderer = createRenderer() as unknown as FrameRendererTestAccess; + const webcamVideo = new FakeVideoElement({ + duration: 4.5, + currentTime: 0.25, + }) as FakeVideoElement & { + requestVideoFrameCallback?: (callback: () => void) => number; + cancelVideoFrameCallback?: (handle: number) => void; + }; + webcamVideo.requestVideoFrameCallback = vi.fn(() => 7); + webcamVideo.cancelVideoFrameCallback = vi.fn(); + renderer.webcamVideoElement = webcamVideo; + + await renderer.syncWebcamFrame(1.5); + + expect(webcamVideo.currentTime).toBe(1.5); + expect(renderer.lastSyncedWebcamTime).toBe(1.5); + expect(webcamVideo.requestVideoFrameCallback).toHaveBeenCalledTimes(1); + expect(webcamVideo.cancelVideoFrameCallback).not.toHaveBeenCalled(); + expect(renderer.webcamSeekPromise).toBeNull(); + }); + + it("uses the cached webcam frame when the live video is out of sync", () => { + const renderer = createRenderer() as unknown as FrameRendererTestAccess; + const outputContext = createMockContext(); + const webcamVideo = new FakeVideoElement({ + currentTime: 2, + readyState: 2, + videoWidth: 640, + videoHeight: 360, + }); + const cachedFrameCanvas = createMockCanvas(); + cachedFrameCanvas.width = 640; + cachedFrameCanvas.height = 360; + + renderer.webcamVideoElement = webcamVideo; + renderer.webcamFrameCacheCanvas = cachedFrameCanvas; + renderer.webcamFrameCacheCtx = cachedFrameCanvas.getContext("2d"); + renderer.lastSyncedWebcamTime = 1.5; + renderer.currentVideoTime = 2; + renderer.animationState.appliedScale = 1; + + renderer.drawWebcamOverlay(outputContext as unknown as CanvasRenderingContext2D, 1280, 720); + + const bubbleCanvas = createdCanvases[0]; + expect(bubbleCanvas).toBeDefined(); + expect((bubbleCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe( + cachedFrameCanvas, + ); + expect((outputContext as MockContext).drawImage.mock.calls[0][0]).toBe(bubbleCanvas); + }); + + it("keeps drawing the cached webcam frame when the live element temporarily has no current data", () => { + const renderer = createRenderer() as unknown as FrameRendererTestAccess; + const outputContext = createMockContext(); + const webcamVideo = new FakeVideoElement({ + currentTime: 2, + readyState: 0, + videoWidth: 640, + videoHeight: 360, + }); + const cachedFrameCanvas = createMockCanvas(); + cachedFrameCanvas.width = 640; + cachedFrameCanvas.height = 360; + + renderer.webcamVideoElement = webcamVideo; + renderer.webcamFrameCacheCanvas = cachedFrameCanvas; + renderer.webcamFrameCacheCtx = cachedFrameCanvas.getContext("2d"); + renderer.lastSyncedWebcamTime = 2; + renderer.currentVideoTime = 2; + renderer.animationState.appliedScale = 1; + + renderer.drawWebcamOverlay(outputContext as unknown as CanvasRenderingContext2D, 1280, 720); + + const bubbleCanvas = createdCanvases[0]; + expect(bubbleCanvas).toBeDefined(); + expect((bubbleCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe( + cachedFrameCanvas, + ); + expect((outputContext as MockContext).drawImage.mock.calls[0][0]).toBe(bubbleCanvas); + }); + + it("uses the live webcam frame and refreshes the cache when the video is synchronized", () => { + const renderer = createRenderer() as unknown as FrameRendererTestAccess; + const outputContext = createMockContext(); + const webcamVideo = new FakeVideoElement({ + currentTime: 2, + readyState: 2, + videoWidth: 800, + videoHeight: 600, + }); + + renderer.webcamVideoElement = webcamVideo; + renderer.lastSyncedWebcamTime = 2; + renderer.currentVideoTime = 2; + renderer.animationState.appliedScale = 1; + + renderer.drawWebcamOverlay(outputContext as unknown as CanvasRenderingContext2D, 1280, 720); + + const bubbleCanvas = createdCanvases[0]; + const cacheCanvas = createdCanvases[1]; + expect(cacheCanvas).toBeDefined(); + expect((cacheCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe(webcamVideo); + expect((bubbleCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe(cacheCanvas); + expect((outputContext as MockContext).drawImage.mock.calls[0][0]).toBe(bubbleCanvas); + }); + + it("reuses the webcam bubble canvas across frames", () => { + const renderer = createRenderer() as unknown as FrameRendererTestAccess; + const outputContext = createMockContext(); + const webcamVideo = new FakeVideoElement({ + currentTime: 2, + readyState: 2, + videoWidth: 800, + videoHeight: 600, + }); + + renderer.webcamVideoElement = webcamVideo; + renderer.lastSyncedWebcamTime = 2; + renderer.currentVideoTime = 2; + renderer.animationState.appliedScale = 1; + + renderer.drawWebcamOverlay(outputContext as unknown as CanvasRenderingContext2D, 1280, 720); + renderer.drawWebcamOverlay(outputContext as unknown as CanvasRenderingContext2D, 1280, 720); + + expect(createdCanvases).toHaveLength(2); + }); +}); diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index c22872a99..392978c93 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -1,1746 +1,1764 @@ -import { - Application, - Container, - Sprite, - Graphics, - BlurFilter, - Texture, - TextureSource, -} from "pixi.js"; +import { Application, BlurFilter, Container, Graphics, Sprite, Texture } from "pixi.js"; import { MotionBlurFilter } from "pixi-filters/motion-blur"; import type { - AutoCaptionSettings, - CaptionCue, - ZoomRegion, - CropRegion, - AnnotationRegion, - SpeedRegion, - CursorStyle, - CursorTelemetryPoint, - WebcamOverlaySettings, - ZoomTransitionEasing, + AnnotationRegion, + AutoCaptionSettings, + CaptionCue, + CropRegion, + CursorStyle, + CursorTelemetryPoint, + SpeedRegion, + WebcamOverlaySettings, + ZoomRegion, + ZoomTransitionEasing, } from "@/components/video-editor/types"; -import { ZOOM_DEPTH_SCALES, BASE_PREVIEW_WIDTH, BASE_PREVIEW_HEIGHT } from "@/components/video-editor/types"; - -import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath"; -import { extensionHost } from "@/lib/extensions"; -import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils"; import { - type CursorFollowCameraState, - createCursorFollowCameraState, - computeCursorFollowFocus, - SNAP_TO_EDGES_RATIO_AUTO, -} from "@/components/video-editor/videoPlayback/cursorFollowCamera"; + BASE_PREVIEW_HEIGHT, + BASE_PREVIEW_WIDTH, + ZOOM_DEPTH_SCALES, +} from "@/components/video-editor/types"; +import { DEFAULT_FOCUS } from "@/components/video-editor/videoPlayback/constants"; import { - applyZoomTransform, - computeFocusFromTransform, - computeZoomTransform, - createMotionBlurState, - type MotionBlurState, -} from "@/components/video-editor/videoPlayback/zoomTransform"; + type CursorFollowCameraState, + computeCursorFollowFocus, + createCursorFollowCameraState, + SNAP_TO_EDGES_RATIO_AUTO, +} from "@/components/video-editor/videoPlayback/cursorFollowCamera"; import { - DEFAULT_FOCUS, -} from "@/components/video-editor/videoPlayback/constants"; + DEFAULT_CURSOR_CONFIG, + PixiCursorOverlay, + preloadCursorAssets, +} from "@/components/video-editor/videoPlayback/cursorRenderer"; import { - type SpringState, - createSpringState, - stepSpringValue, - resetSpringState, - getZoomSpringConfig, + createSpringState, + getZoomSpringConfig, + resetSpringState, + type SpringState, + stepSpringValue, } from "@/components/video-editor/videoPlayback/motionSmoothing"; -import { renderAnnotations } from "./annotationRenderer"; -import { renderCaptions } from "./captionRenderer"; +import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils"; import { - executeExtensionRenderHooks, - executeExtensionCursorEffects, - notifyCursorInteraction, -} from "@/lib/extensions/renderHooks"; + applyZoomTransform, + computeFocusFromTransform, + computeZoomTransform, + createMotionBlurState, + type MotionBlurState, +} from "@/components/video-editor/videoPlayback/zoomTransform"; +import { + getWebcamOverlayPosition, + getWebcamOverlaySizePx, +} from "@/components/video-editor/webcamOverlay"; +import { getAssetPath, getRenderableAssetUrl } from "@/lib/assetPath"; +import { extensionHost } from "@/lib/extensions"; import { - mapCursorToCanvasNormalized, - mapSmoothedCursorToCanvasNormalized, + mapCursorToCanvasNormalized, + mapSmoothedCursorToCanvasNormalized, } from "@/lib/extensions/cursorCoordinates"; -import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform"; import { - PixiCursorOverlay, - DEFAULT_CURSOR_CONFIG, - preloadCursorAssets, -} from "@/components/video-editor/videoPlayback/cursorRenderer"; -import { clampMediaTimeToDuration } from "@/lib/mediaTiming"; -import { getWebcamOverlayPosition, getWebcamOverlaySizePx } from "@/components/video-editor/webcamOverlay"; + executeExtensionCursorEffects, + executeExtensionRenderHooks, + notifyCursorInteraction, +} from "@/lib/extensions/renderHooks"; +import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform"; import { drawSquircleOnCanvas, drawSquircleOnGraphics } from "@/lib/geometry/squircle"; +import { clampMediaTimeToDuration } from "@/lib/mediaTiming"; +import { isVideoWallpaperSource } from "@/lib/wallpapers"; +import { renderAnnotations } from "./annotationRenderer"; +import { renderCaptions } from "./captionRenderer"; import { ForwardFrameSource } from "./forwardFrameSource"; import { resolveMediaElementSource } from "./localMediaSource"; -import { isVideoWallpaperSource } from "@/lib/wallpapers"; interface FrameRenderConfig { - width: number; - height: number; - preferredRenderBackend?: "webgl" | "webgpu"; - wallpaper: string; - zoomRegions: ZoomRegion[]; - showShadow: boolean; - shadowIntensity: number; - backgroundBlur: number; - zoomMotionBlur?: number; - connectZooms?: boolean; - zoomInDurationMs?: number; - zoomInOverlapMs?: number; - zoomOutDurationMs?: number; - connectedZoomGapMs?: number; - connectedZoomDurationMs?: number; - zoomInEasing?: ZoomTransitionEasing; - zoomOutEasing?: ZoomTransitionEasing; - connectedZoomEasing?: ZoomTransitionEasing; - borderRadius?: number; - padding?: number; - cropRegion: CropRegion; - webcam?: WebcamOverlaySettings; - webcamUrl?: string | null; - videoWidth: number; - videoHeight: number; - annotationRegions?: AnnotationRegion[]; - autoCaptions?: CaptionCue[]; - autoCaptionSettings?: AutoCaptionSettings; - speedRegions?: SpeedRegion[]; - previewWidth?: number; - previewHeight?: number; - cursorTelemetry?: CursorTelemetryPoint[]; - showCursor?: boolean; - cursorStyle?: CursorStyle; - cursorSize?: number; - cursorSmoothing?: number; - zoomSmoothness?: number; - zoomClassicMode?: boolean; - cursorMotionBlur?: number; - cursorClickBounce?: number; - cursorClickBounceDuration?: number; - cursorSway?: number; - frame?: string | null; + width: number; + height: number; + preferredRenderBackend?: "webgl" | "webgpu"; + wallpaper: string; + zoomRegions: ZoomRegion[]; + showShadow: boolean; + shadowIntensity: number; + backgroundBlur: number; + zoomMotionBlur?: number; + connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; + borderRadius?: number; + padding?: number; + cropRegion: CropRegion; + webcam?: WebcamOverlaySettings; + webcamUrl?: string | null; + videoWidth: number; + videoHeight: number; + annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; + speedRegions?: SpeedRegion[]; + previewWidth?: number; + previewHeight?: number; + cursorTelemetry?: CursorTelemetryPoint[]; + showCursor?: boolean; + cursorStyle?: CursorStyle; + cursorSize?: number; + cursorSmoothing?: number; + zoomSmoothness?: number; + zoomClassicMode?: boolean; + cursorMotionBlur?: number; + cursorClickBounce?: number; + cursorClickBounceDuration?: number; + cursorSway?: number; + frame?: string | null; } interface AnimationState { - scale: number; - appliedScale: number; - focusX: number; - focusY: number; - progress: number; - x: number; - y: number; + scale: number; + appliedScale: number; + focusX: number; + focusY: number; + progress: number; + x: number; + y: number; +} + +interface VideoTextureSource { + resource: VideoFrame | CanvasImageSource; + update: () => void; +} + +type PixiTextureInput = Parameters[0]; + +interface LayoutCache { + stageSize: { width: number; height: number }; + videoSize: { width: number; height: number }; + baseScale: number; + baseOffset: { x: number; y: number }; + maskRect: { + x: number; + y: number; + width: number; + height: number; + sourceCrop: CropRegion; + }; } function createAnimationState(): AnimationState { - return { - scale: 1, - appliedScale: 1, - focusX: DEFAULT_FOCUS.cx, - focusY: DEFAULT_FOCUS.cy, - progress: 0, - x: 0, - y: 0, - }; + return { + scale: 1, + appliedScale: 1, + focusX: DEFAULT_FOCUS.cx, + focusY: DEFAULT_FOCUS.cy, + progress: 0, + x: 0, + y: 0, + }; } function configureHighQuality2DContext( - context: CanvasRenderingContext2D | null, + context: CanvasRenderingContext2D | null, ): CanvasRenderingContext2D | null { - if (!context) { - return null; - } + if (!context) { + return null; + } - context.imageSmoothingEnabled = true; - context.imageSmoothingQuality = "high"; + context.imageSmoothingEnabled = true; + context.imageSmoothingQuality = "high"; - return context; + return context; } // Renders video frames with all effects (background, zoom, crop, blur, shadow) to an offscreen canvas for export. export class FrameRenderer { - private app: Application | null = null; - private cameraContainer: Container | null = null; - private videoContainer: Container | null = null; - private cursorContainer: Container | null = null; - private videoSprite: Sprite | null = null; - private videoTextureSource: TextureSource | null = null; - private backgroundSprite: Sprite | null = null; - private maskGraphics: Graphics | null = null; - private blurFilter: BlurFilter | null = null; - private motionBlurFilter: MotionBlurFilter | null = null; - private shadowCanvas: HTMLCanvasElement | null = null; - private shadowCtx: CanvasRenderingContext2D | null = null; - private compositeCanvas: HTMLCanvasElement | null = null; - private compositeCtx: CanvasRenderingContext2D | null = null; - private backgroundVideoElement: HTMLVideoElement | null = null; - private backgroundCtx: CanvasRenderingContext2D | null = null; - private cleanupBackgroundSource: (() => void) | null = null; - private config: FrameRenderConfig; - private animationState: AnimationState; - private motionBlurState: MotionBlurState; - private layoutCache: any = null; - private currentVideoTime = 0; - private lastMotionVector = { x: 0, y: 0 }; - private springScale: SpringState; - private springX: SpringState; - private springY: SpringState; - private cursorFollowCamera: CursorFollowCameraState; - private lastContentTimeMs: number | null = null; - private cursorOverlay: PixiCursorOverlay | null = null; - private webcamForwardFrameSource: ForwardFrameSource | null = null; - private webcamDecodedFrame: VideoFrame | null = null; - private webcamVideoElement: HTMLVideoElement | null = null; - private webcamSeekPromise: Promise | null = null; - private webcamFrameCacheCanvas: HTMLCanvasElement | null = null; - private webcamFrameCacheCtx: CanvasRenderingContext2D | null = null; - private webcamBubbleCanvas: HTMLCanvasElement | null = null; - private webcamBubbleCtx: CanvasRenderingContext2D | null = null; - private lastSyncedWebcamTime: number | null = null; - private cleanupWebcamSource: (() => void) | null = null; - private frameImage: HTMLImageElement | null = null; - private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null; - private frameDraw: ((ctx: CanvasRenderingContext2D, w: number, h: number) => void) | null = null; - - constructor(config: FrameRenderConfig) { - this.config = config; - this.animationState = createAnimationState(); - this.motionBlurState = createMotionBlurState(); - this.springScale = createSpringState(1); - this.springX = createSpringState(0); - this.springY = createSpringState(0); - this.cursorFollowCamera = createCursorFollowCameraState(); - } - - async initialize(): Promise { - let cursorOverlayEnabled = true; - try { - await preloadCursorAssets(); - } catch (error) { - cursorOverlayEnabled = false; - console.warn( - "[FrameRenderer] Native cursor assets are unavailable; continuing export without cursor overlay.", - error, - ); - } - - // Create canvas for rendering - const canvas = document.createElement("canvas"); - canvas.width = this.config.width; - canvas.height = this.config.height; - - // Try to set colorSpace if supported (may not be available on all platforms) - try { - if (canvas && "colorSpace" in canvas) { - // @ts-ignore - canvas.colorSpace = "srgb"; - } - } catch (error) { - // Silently ignore colorSpace errors on platforms that don't support it - console.warn( - "[FrameRenderer] colorSpace not supported on this platform:", - error, - ); - } - - // Initialize PixiJS with optimized settings for export performance - this.app = new Application(); - await this.app.init({ - canvas, - width: this.config.width, - height: this.config.height, - backgroundAlpha: 0, - antialias: true, - failIfMajorPerformanceCaveat: false, - resolution: 1, - autoDensity: true, - autoStart: false, - sharedTicker: false, - powerPreference: "high-performance", - ...(this.config.preferredRenderBackend - ? { preference: this.config.preferredRenderBackend } - : {}), - }); - - // Setup containers - this.cameraContainer = new Container(); - this.videoContainer = new Container(); - this.cursorContainer = new Container(); - this.app.stage.addChild(this.cameraContainer); - this.cameraContainer.addChild(this.videoContainer); - this.cameraContainer.addChild(this.cursorContainer); - - if (cursorOverlayEnabled) { - this.cursorOverlay = new PixiCursorOverlay({ - dotRadius: - DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4), - style: this.config.cursorStyle ?? "tahoe", - smoothingFactor: - this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor, - motionBlur: this.config.cursorMotionBlur ?? 0, - clickBounce: - this.config.cursorClickBounce ?? DEFAULT_CURSOR_CONFIG.clickBounce, - clickBounceDuration: - this.config.cursorClickBounceDuration ?? DEFAULT_CURSOR_CONFIG.clickBounceDuration, - sway: this.config.cursorSway ?? DEFAULT_CURSOR_CONFIG.sway, - }); - } - - // Setup background (render separately, not in PixiJS) - await this.setupBackground(); - await this.setupWebcamSource(); - await this.setupFrame(); - - // Setup blur filter for video container - this.blurFilter = new BlurFilter(); - this.blurFilter.quality = 5; - this.blurFilter.resolution = this.app.renderer.resolution; - this.blurFilter.blur = 0; - this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); - this.videoContainer.filters = [this.blurFilter, this.motionBlurFilter]; - - // Setup composite canvas for final output with shadows - this.compositeCanvas = document.createElement("canvas"); - this.compositeCanvas.width = this.config.width; - this.compositeCanvas.height = this.config.height; - this.compositeCtx = configureHighQuality2DContext( - this.compositeCanvas.getContext("2d", { - willReadFrequently: false, - }), - ); - - if (!this.compositeCtx) { - throw new Error("Failed to get 2D context for composite canvas"); - } - - // Setup shadow canvas if needed - if (this.config.showShadow) { - this.shadowCanvas = document.createElement("canvas"); - this.shadowCanvas.width = this.config.width; - this.shadowCanvas.height = this.config.height; - this.shadowCtx = configureHighQuality2DContext( - this.shadowCanvas.getContext("2d", { - willReadFrequently: false, - }), - ); - - if (!this.shadowCtx) { - throw new Error("Failed to get 2D context for shadow canvas"); - } - } - - // Setup mask - this.maskGraphics = new Graphics(); - this.videoContainer.addChild(this.maskGraphics); - this.videoContainer.mask = this.maskGraphics; - if (this.cursorOverlay) { - this.cursorContainer.addChild(this.cursorOverlay.container); - } - } - - private async setupBackground(): Promise { - const wallpaper = await this.resolveWallpaperForExport( - this.config.wallpaper, - ); - - // Create background canvas for separate rendering (not affected by zoom) - const bgCanvas = document.createElement("canvas"); - bgCanvas.width = this.config.width; - bgCanvas.height = this.config.height; - const bgCtx = configureHighQuality2DContext(bgCanvas.getContext("2d")); - - if (!bgCtx) { - throw new Error("Failed to get 2D context for background canvas"); - } - - this.backgroundCtx = bgCtx; - - try { - // Check for video wallpaper first - if (isVideoWallpaperSource(wallpaper)) { - let videoSrc = wallpaper; - if (wallpaper.startsWith("/") && !wallpaper.startsWith("//")) { - videoSrc = await getAssetPath(wallpaper.replace(/^\//, "")); - } - - const video = document.createElement("video"); - video.muted = true; - video.loop = true; - video.playsInline = true; - video.preload = "auto"; - video.src = videoSrc; - - await new Promise((resolve, reject) => { - video.onloadeddata = () => resolve(); - video.onerror = () => reject(new Error(`Failed to load video wallpaper: ${wallpaper}`)); - }); - - this.backgroundVideoElement = video; - this.drawVideoFrameToBackground(); - this.backgroundSprite = bgCanvas as any; - return; - } - - // Render background based on type - if ( - wallpaper.startsWith("file://") || - wallpaper.startsWith("data:") || - wallpaper.startsWith("/") || - wallpaper.startsWith("http") - ) { - // Image background - const img = new Image(); - const imageUrl = await this.resolveWallpaperImageUrl(wallpaper); - // Don't set crossOrigin for same-origin images to avoid CORS taint. - if ( - imageUrl.startsWith("http") && - window.location.origin && - !imageUrl.startsWith(window.location.origin) - ) { - img.crossOrigin = "anonymous"; - } - - await new Promise((resolve, reject) => { - img.onload = () => resolve(); - img.onerror = (err) => { - console.error( - "[FrameRenderer] Failed to load background image:", - imageUrl, - err, - ); - reject(new Error(`Failed to load background image: ${imageUrl}`)); - }; - img.src = imageUrl; - }); - - // Draw the image using cover and center positioning - const imgAspect = img.width / img.height; - const canvasAspect = this.config.width / this.config.height; - - let drawWidth, drawHeight, drawX, drawY; - - if (imgAspect > canvasAspect) { - drawHeight = this.config.height; - drawWidth = drawHeight * imgAspect; - drawX = (this.config.width - drawWidth) / 2; - drawY = 0; - } else { - drawWidth = this.config.width; - drawHeight = drawWidth / imgAspect; - drawX = 0; - drawY = (this.config.height - drawHeight) / 2; - } - - bgCtx.drawImage(img, drawX, drawY, drawWidth, drawHeight); - } else if (wallpaper.startsWith("#")) { - bgCtx.fillStyle = wallpaper; - bgCtx.fillRect(0, 0, this.config.width, this.config.height); - } else if ( - wallpaper.startsWith("linear-gradient") || - wallpaper.startsWith("radial-gradient") - ) { - const gradientMatch = wallpaper.match( - /(linear|radial)-gradient\((.+)\)/, - ); - if (gradientMatch) { - const [, type, params] = gradientMatch; - const parts = params.split(",").map((s) => s.trim()); - - let gradient: CanvasGradient; - - if (type === "linear") { - gradient = bgCtx.createLinearGradient(0, 0, 0, this.config.height); - parts.forEach((part, index) => { - if (part.startsWith("to ") || part.includes("deg")) return; - - const colorMatch = part.match( - /^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/, - ); - if (colorMatch) { - const color = colorMatch[1]; - const position = index / (parts.length - 1); - gradient.addColorStop(position, color); - } - }); - } else { - const cx = this.config.width / 2; - const cy = this.config.height / 2; - const radius = Math.max(this.config.width, this.config.height) / 2; - gradient = bgCtx.createRadialGradient(cx, cy, 0, cx, cy, radius); - - parts.forEach((part, index) => { - const colorMatch = part.match( - /^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/, - ); - if (colorMatch) { - const color = colorMatch[1]; - const position = index / (parts.length - 1); - gradient.addColorStop(position, color); - } - }); - } - - bgCtx.fillStyle = gradient; - bgCtx.fillRect(0, 0, this.config.width, this.config.height); - } else { - console.warn( - "[FrameRenderer] Could not parse gradient, using black fallback", - ); - bgCtx.fillStyle = "#000000"; - bgCtx.fillRect(0, 0, this.config.width, this.config.height); - } - } else { - bgCtx.fillStyle = wallpaper; - bgCtx.fillRect(0, 0, this.config.width, this.config.height); - } - } catch (error) { - console.error( - "[FrameRenderer] Error setting up background, using fallback:", - error, - ); - bgCtx.fillStyle = "#000000"; - bgCtx.fillRect(0, 0, this.config.width, this.config.height); - } - - // Store the background canvas for compositing - this.backgroundSprite = bgCanvas as any; - } - - private drawVideoFrameToBackground(): void { - const video = this.backgroundVideoElement; - const ctx = this.backgroundCtx; - if (!video || !ctx) return; - - const w = this.config.width; - const h = this.config.height; - const videoAspect = video.videoWidth / video.videoHeight; - const canvasAspect = w / h; - - let drawWidth: number, drawHeight: number, drawX: number, drawY: number; - if (videoAspect > canvasAspect) { - drawHeight = h; - drawWidth = drawHeight * videoAspect; - drawX = (w - drawWidth) / 2; - drawY = 0; - } else { - drawWidth = w; - drawHeight = drawWidth / videoAspect; - drawX = 0; - drawY = (h - drawHeight) / 2; - } - - ctx.clearRect(0, 0, w, h); - ctx.drawImage(video, drawX, drawY, drawWidth, drawHeight); - } - - private async syncBackgroundVideo(timeSeconds: number): Promise { - const video = this.backgroundVideoElement; - if (!video) return; - - if (video.duration && Number.isFinite(video.duration)) { - const targetTime = timeSeconds % video.duration; - if (Math.abs(video.currentTime - targetTime) > 0.01) { - video.currentTime = targetTime; - await new Promise((resolve) => { - const onSeeked = () => { - video.removeEventListener("seeked", onSeeked); - resolve(); - }; - video.addEventListener("seeked", onSeeked); - }); - } - } - this.drawVideoFrameToBackground(); - } - - private async resolveWallpaperImageUrl(wallpaper: string): Promise { - if ( - wallpaper.startsWith("file://") || - wallpaper.startsWith("data:") || - wallpaper.startsWith("http") - ) { - return wallpaper; - } - - const resolved = await getAssetPath(wallpaper.replace(/^\/+/, "")); - if ( - resolved.startsWith("/") && - window.location.protocol.startsWith("http") - ) { - return `${window.location.origin}${resolved}`; - } - - return resolved; - } - - private async resolveWallpaperForExport(wallpaper: string): Promise { - if (!wallpaper) { - return wallpaper; - } - - if ( - wallpaper.startsWith("#") || - wallpaper.startsWith("linear-gradient") || - wallpaper.startsWith("radial-gradient") - ) { - return wallpaper; - } - - const looksLikeAbsoluteFilePath = - wallpaper.startsWith("/") && - !wallpaper.startsWith("//") && - !wallpaper.startsWith("/wallpapers/") && - !wallpaper.startsWith("/app-icons/"); - - const wallpaperAsset = looksLikeAbsoluteFilePath - ? `file://${encodeURI(wallpaper)}` - : wallpaper; - - return getRenderableAssetUrl(wallpaperAsset); - } - - private async setupWebcamSource(): Promise { - const webcamUrl = this.config.webcamUrl; - if (!this.config.webcam?.enabled || !webcamUrl) { - this.webcamForwardFrameSource?.cancel(); - void this.webcamForwardFrameSource?.destroy(); - this.webcamForwardFrameSource = null; - this.closeWebcamDecodedFrame(); - this.cleanupWebcamSource?.(); - this.cleanupWebcamSource = null; - this.webcamVideoElement = null; - this.webcamFrameCacheCanvas = null; - this.webcamFrameCacheCtx = null; - this.lastSyncedWebcamTime = null; - return; - } - - this.webcamForwardFrameSource?.cancel(); - void this.webcamForwardFrameSource?.destroy(); - this.webcamForwardFrameSource = null; - this.closeWebcamDecodedFrame(); - this.cleanupWebcamSource?.(); - this.cleanupWebcamSource = null; - - try { - const frameSource = new ForwardFrameSource(); - await frameSource.initialize(webcamUrl); - this.webcamForwardFrameSource = frameSource; - this.webcamVideoElement = null; - this.webcamSeekPromise = null; - this.webcamFrameCacheCanvas = null; - this.webcamFrameCacheCtx = null; - this.lastSyncedWebcamTime = null; - return; - } catch (error) { - console.warn( - "[FrameRenderer] Decoder-backed webcam source unavailable during export; falling back to media element sync:", - error, - ); - } - - const webcamSource = await resolveMediaElementSource(webcamUrl); - this.cleanupWebcamSource = webcamSource.revoke; - - const video = document.createElement("video"); - video.src = webcamSource.src; - video.muted = true; - video.preload = "auto"; - video.playsInline = true; - video.load(); - - await new Promise((resolve, reject) => { - const onReady = () => { - if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { - return; - } - cleanup(); - resolve(); - }; - const onError = () => { - cleanup(); - reject(new Error("Failed to load webcam source for export")); - }; - const cleanup = () => { - video.removeEventListener("loadeddata", onReady); - video.removeEventListener("canplay", onReady); - video.removeEventListener("canplaythrough", onReady); - video.removeEventListener("error", onError); - }; - if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { - resolve(); - return; - } - video.addEventListener("loadeddata", onReady, { once: true }); - video.addEventListener("canplay", onReady, { once: true }); - video.addEventListener("canplaythrough", onReady, { once: true }); - video.addEventListener("error", onError, { once: true }); - }).catch((error) => { - console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error); - this.webcamVideoElement = null; - }); - - if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { - this.webcamVideoElement = video; - return; - } - - this.webcamVideoElement = null; - this.webcamFrameCacheCanvas = null; - this.webcamFrameCacheCtx = null; - this.lastSyncedWebcamTime = null; - } - - private async syncWebcamFrame(targetTime: number): Promise { - if (this.webcamForwardFrameSource) { - const clampedTime = clampMediaTimeToDuration(targetTime, null); - const decodedFrame = await this.webcamForwardFrameSource.getFrameAtTime(clampedTime); - this.closeWebcamDecodedFrame(); - this.webcamDecodedFrame = decodedFrame; - if (decodedFrame) { - this.lastSyncedWebcamTime = clampedTime; - } - return; - } - - const webcamVideo = this.webcamVideoElement; - if (!webcamVideo) { - return; - } - - const clampedTime = clampMediaTimeToDuration( - targetTime, - Number.isFinite(webcamVideo.duration) ? webcamVideo.duration : null, - ); - - if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.008) { - this.lastSyncedWebcamTime = clampedTime; - return; - } - - if (this.webcamSeekPromise) { - await this.webcamSeekPromise; - } - - this.webcamSeekPromise = new Promise((resolve) => { - let settled = false; - let fallbackTimeout: number | null = null; - let animationFrameRequestId: number | null = null; - let videoFrameRequestId: number | null = null; - const waitForPresentedFrame = () => { - const requestVideoFrameCallback = ( - webcamVideo as HTMLVideoElement & { - requestVideoFrameCallback?: ( - callback: (now: DOMHighResTimeStamp, metadata: VideoFrameCallbackMetadata) => void, - ) => number; - cancelVideoFrameCallback?: (handle: number) => void; - } - ).requestVideoFrameCallback; - - const scheduleAnimationFrameFinish = () => { - animationFrameRequestId = requestAnimationFrame(() => { - animationFrameRequestId = null; - finish(); - }); - }; - - scheduleAnimationFrameFinish(); - - if (typeof requestVideoFrameCallback === "function") { - videoFrameRequestId = requestVideoFrameCallback.call( - webcamVideo, - () => { - videoFrameRequestId = null; - finish(); - }, - ); - return; - } - }; - const finish = () => { - if (settled) { - return; - } - settled = true; - if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.02) { - this.lastSyncedWebcamTime = clampedTime; - } - cleanup(); - resolve(); - }; - const handleMediaReady = () => { - if ( - !webcamVideo.seeking && - Math.abs(webcamVideo.currentTime - clampedTime) <= 0.01 && - webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA - ) { - waitForPresentedFrame(); - } - }; - const cleanup = () => { - webcamVideo.removeEventListener("seeked", waitForPresentedFrame); - webcamVideo.removeEventListener("loadeddata", handleMediaReady); - webcamVideo.removeEventListener("canplay", handleMediaReady); - webcamVideo.removeEventListener("error", finish); - if (animationFrameRequestId !== null) { - cancelAnimationFrame(animationFrameRequestId); - animationFrameRequestId = null; - } - if ( - videoFrameRequestId !== null && - typeof ( - webcamVideo as HTMLVideoElement & { - cancelVideoFrameCallback?: (handle: number) => void; - } - ).cancelVideoFrameCallback === "function" - ) { - ( - webcamVideo as HTMLVideoElement & { - cancelVideoFrameCallback: (handle: number) => void; - } - ).cancelVideoFrameCallback(videoFrameRequestId); - videoFrameRequestId = null; - } - if (fallbackTimeout !== null) { - window.clearTimeout(fallbackTimeout); - } - }; - - webcamVideo.addEventListener("seeked", waitForPresentedFrame, { - once: true, - }); - webcamVideo.addEventListener("loadeddata", handleMediaReady, { - once: true, - }); - webcamVideo.addEventListener("canplay", handleMediaReady, { - once: true, - }); - webcamVideo.addEventListener("error", finish, { - once: true, - }); - fallbackTimeout = window.setTimeout(() => { - finish(); - }, 50); - - try { - webcamVideo.currentTime = clampedTime; - } catch { - finish(); - return; - } - - if ( - !webcamVideo.seeking && - Math.abs(webcamVideo.currentTime - clampedTime) <= 0.001 && - webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA - ) { - waitForPresentedFrame(); - } - }); - - try { - await this.webcamSeekPromise; - } finally { - this.webcamSeekPromise = null; - } - } - - private async setupFrame(): Promise { - const frameId = this.config.frame; - if (!frameId) return; - - const { extensionHost } = await import("@/lib/extensions/extensionHost"); - const frames = extensionHost.getFrames(); - const frame = frames.find((f) => f.id === frameId); - if (!frame) { - console.warn(`[FrameRenderer] Device frame "${frameId}" not found`); - return; - } - - this.frameInsets = frame.screenInsets; - - if (frame.draw) { - // Prefer draw function — renders at export resolution, no bitmap scaling - this.frameDraw = frame.draw; - return; - } - - const img = new Image(); - img.crossOrigin = "anonymous"; - await new Promise((resolve, reject) => { - img.onload = () => resolve(); - img.onerror = () => reject(new Error(`Failed to load device frame image: ${frameId}`)); - img.src = frame.filePath; - }); - - this.frameImage = img; - } - - async renderFrame( - videoFrame: VideoFrame, - timestamp: number, - cursorTimestamp = timestamp, - ): Promise { - if (!this.app || !this.videoContainer || !this.cameraContainer) { - throw new Error("Renderer not initialized"); - } - - this.currentVideoTime = timestamp / 1000000; - - if (this.webcamForwardFrameSource || this.webcamVideoElement) { - const targetTime = Math.max(0, this.currentVideoTime); - await this.syncWebcamFrame(targetTime); - } - - // Sync video wallpaper frame - if (this.backgroundVideoElement) { - await this.syncBackgroundVideo(this.currentVideoTime); - } - - // Create or update video sprite from VideoFrame - if (!this.videoSprite) { - const texture = Texture.from(videoFrame as any); - this.videoSprite = new Sprite(texture); - this.videoTextureSource = texture.source as TextureSource; - this.videoContainer.addChild(this.videoSprite); - if (this.cursorOverlay && this.cursorContainer) { - this.cursorContainer.addChild(this.cursorOverlay.container); - } - if (this.maskGraphics) { - this.videoContainer.addChild(this.maskGraphics); - } - } else { - this.videoTextureSource ??= this.videoSprite.texture.source as TextureSource; - this.videoTextureSource.resource = videoFrame as any; - this.videoTextureSource.update(); - } - - // Apply layout - this.updateLayout(); - - const timeMs = this.currentVideoTime * 1000; - const cursorTimeMs = cursorTimestamp / 1000; - - if (this.cursorOverlay) { - this.cursorOverlay.update( - this.config.cursorTelemetry ?? [], - cursorTimeMs, - this.layoutCache.maskRect, - this.config.showCursor ?? true, - false, - ); - } - - const smoothedCursor = mapSmoothedCursorToCanvasNormalized( - this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null, - { - maskRect: this.layoutCache.maskRect, - canvasWidth: this.config.width, - canvasHeight: this.config.height, - }, - ); - extensionHost.setSmoothedCursor( - smoothedCursor - ? { - timeMs, - cx: smoothedCursor.cx, - cy: smoothedCursor.cy, - trail: smoothedCursor.trail, - } - : null, - ); - - const TICKS_PER_FRAME = 1; - - let maxMotionIntensity = 0; - for (let i = 0; i < TICKS_PER_FRAME; i++) { - const motionIntensity = this.updateAnimationState(timeMs); - maxMotionIntensity = Math.max(maxMotionIntensity, motionIntensity); - } - - // Apply transform once with maximum motion intensity from all ticks - applyZoomTransform({ - cameraContainer: this.cameraContainer, - blurFilter: this.blurFilter, - motionBlurFilter: this.motionBlurFilter, - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: this.animationState.scale, - zoomProgress: this.animationState.progress, - focusX: this.animationState.focusX, - focusY: this.animationState.focusY, - motionIntensity: maxMotionIntensity, - motionVector: this.lastMotionVector, - isPlaying: true, - motionBlurAmount: this.config.zoomMotionBlur ?? 0, - transformOverride: { - scale: this.animationState.appliedScale, - x: this.animationState.x, - y: this.animationState.y, - }, - motionBlurState: this.motionBlurState, - frameTimeMs: timeMs, - }); - - // Render the PixiJS stage to its canvas (video only, transparent background) - this.app.renderer.render(this.app.stage); - - // Composite with shadows to final output canvas - this.compositeWithShadows(); - - // Draw device frame overlay on top of video content - this.drawFrame(); - - // Render annotations on top if present - if ( - this.config.annotationRegions && - this.config.annotationRegions.length > 0 && - this.compositeCtx - ) { - // Calculate scale factor based on export vs preview dimensions - const scaleX = this.config.width / BASE_PREVIEW_WIDTH; - const scaleY = this.config.height / BASE_PREVIEW_HEIGHT; - const scaleFactor = (scaleX + scaleY) / 2; - - - await renderAnnotations( - this.compositeCtx, - this.config.annotationRegions, - this.config.width, - this.config.height, - timeMs, - scaleFactor, - ); - } - - if ( - this.config.autoCaptions && - this.config.autoCaptions.length > 0 && - this.config.autoCaptionSettings && - this.compositeCtx - ) { - renderCaptions( - this.compositeCtx, - this.config.autoCaptions, - this.config.autoCaptionSettings, - this.config.width, - this.config.height, - timeMs, - ); - } - - // Extension render hooks — run after all built-in rendering - if (this.compositeCtx) { - const maskRect = this.layoutCache?.maskRect; - const hookParams = { - width: this.config.width, - height: this.config.height, - timeMs, - durationMs: 0, - cursor: smoothedCursor - ? { - cx: smoothedCursor.cx, - cy: smoothedCursor.cy, - interactionType: this.getCursorPosition(cursorTimeMs)?.interactionType, - } - : this.getCursorPosition(cursorTimeMs), - smoothedCursor, - videoLayout: maskRect ? { - maskRect: { x: maskRect.x, y: maskRect.y, width: maskRect.width, height: maskRect.height }, - borderRadius: this.config.borderRadius ?? 0, - padding: this.config.padding ?? 0, - } : undefined, - zoom: { - scale: this.animationState.scale, - focusX: this.animationState.focusX, - focusY: this.animationState.focusY, - progress: this.animationState.progress, - }, - shadow: { - enabled: this.config.showShadow, - intensity: this.config.shadowIntensity, - }, - sceneTransform: { - scale: this.animationState.appliedScale, - x: this.animationState.x, - y: this.animationState.y, - }, - }; - - this.compositeCtx.save(); - applyCanvasSceneTransform(this.compositeCtx, { - scale: this.animationState.appliedScale, - x: this.animationState.x, - y: this.animationState.y, - }); - executeExtensionRenderHooks('post-video', this.compositeCtx, hookParams); - executeExtensionRenderHooks('post-zoom', this.compositeCtx, hookParams); - executeExtensionRenderHooks('post-cursor', this.compositeCtx, hookParams); - - // Cursor click effects - this.emitCursorInteractions(cursorTimeMs); - executeExtensionCursorEffects( - this.compositeCtx, - timeMs, - this.config.width, - this.config.height, - { - zoom: hookParams.zoom, - sceneTransform: hookParams.sceneTransform, - videoLayout: hookParams.videoLayout, - }, - ); - this.compositeCtx.restore(); - - executeExtensionRenderHooks('post-webcam', this.compositeCtx, hookParams); - executeExtensionRenderHooks('post-annotations', this.compositeCtx, hookParams); - - executeExtensionRenderHooks('final', this.compositeCtx, hookParams); - } - } - - /** - * Get the cursor position (normalized 0-1) at the given time. - */ - private getCursorPosition(timeMs: number): { cx: number; cy: number; interactionType?: string } | null { - const telemetry = this.config.cursorTelemetry; - if (!telemetry || telemetry.length === 0) return null; - - // Find the closest telemetry point - let closest = telemetry[0]; - let minDist = Math.abs(telemetry[0].timeMs - timeMs); - for (let i = 1; i < telemetry.length; i++) { - const dist = Math.abs(telemetry[i].timeMs - timeMs); - if (dist < minDist) { - minDist = dist; - closest = telemetry[i]; - } - if (telemetry[i].timeMs > timeMs) break; - } - - return mapCursorToCanvasNormalized( - { cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType }, - { - maskRect: this.layoutCache?.maskRect, - canvasWidth: this.config.width, - canvasHeight: this.config.height, - }, - ); - } - - /** - * Emit cursor interaction events for extensions based on telemetry clicks. - */ - private lastEmittedClickTimeMs = -1; - - private emitCursorInteractions(timeMs: number): void { - const telemetry = this.config.cursorTelemetry; - if (!telemetry || telemetry.length === 0) return; - - // Find click events near this time - for (const point of telemetry) { - if (point.timeMs > timeMs) break; - if (point.timeMs < timeMs - 100) continue; - if (!point.interactionType || point.interactionType === 'move') continue; - if (point.timeMs === this.lastEmittedClickTimeMs) continue; - - const mappedCursor = mapCursorToCanvasNormalized( - { cx: point.cx, cy: point.cy, interactionType: point.interactionType }, - { - maskRect: this.layoutCache?.maskRect, - canvasWidth: this.config.width, - canvasHeight: this.config.height, - }, - ); - if (!mappedCursor) continue; - - this.lastEmittedClickTimeMs = point.timeMs; - notifyCursorInteraction( - point.timeMs, - mappedCursor.cx, - mappedCursor.cy, - point.interactionType, - ); - } - } - - private updateLayout(): void { - if ( - !this.app || - !this.videoSprite || - !this.maskGraphics || - !this.videoContainer - ) - return; - - const { width, height } = this.config; - const { cropRegion, borderRadius = 0, padding = 0 } = this.config; - const videoWidth = this.config.videoWidth; - const videoHeight = this.config.videoHeight; - - // Calculate cropped video dimensions - const cropStartX = cropRegion.x; - const cropStartY = cropRegion.y; - const cropEndX = cropRegion.x + cropRegion.width; - const cropEndY = cropRegion.y + cropRegion.height; - - const croppedVideoWidth = videoWidth * (cropEndX - cropStartX); - const croppedVideoHeight = videoHeight * (cropEndY - cropStartY); - - const paddingScale = 1.0 - (padding / 100) * 0.4; - const viewportWidth = width * paddingScale; - const viewportHeight = height * paddingScale; - - // When a device frame is active, scale to fit the ENTIRE frame (video + bezels) - const insets = this.frameInsets; - const screenFracW = insets ? (1 - insets.left - insets.right) : 1; - const screenFracH = insets ? (1 - insets.top - insets.bottom) : 1; - const fullFrameVideoW = croppedVideoWidth / screenFracW; - const fullFrameVideoH = croppedVideoHeight / screenFracH; - - const scale = Math.min( - viewportWidth / fullFrameVideoW, - viewportHeight / fullFrameVideoH, - ); - - this.videoSprite.scale.set(scale); - - const fullVideoDisplayWidth = videoWidth * scale; - const fullVideoDisplayHeight = videoHeight * scale; - const croppedDisplayWidth = croppedVideoWidth * scale; - const croppedDisplayHeight = croppedVideoHeight * scale; - - // Center the full frame (video + bezels) in the output canvas - const fullFrameDisplayW = fullFrameVideoW * scale; - const fullFrameDisplayH = fullFrameVideoH * scale; - const frameCenterX = (width - fullFrameDisplayW) / 2; - const frameCenterY = (height - fullFrameDisplayH) / 2; - const centerOffsetX = insets - ? frameCenterX + insets.left * fullFrameDisplayW - : (width - croppedDisplayWidth) / 2; - const centerOffsetY = insets - ? frameCenterY + insets.top * fullFrameDisplayH - : (height - croppedDisplayHeight) / 2; - - const spriteX = centerOffsetX - cropRegion.x * fullVideoDisplayWidth; - const spriteY = centerOffsetY - cropRegion.y * fullVideoDisplayHeight; - this.videoSprite.position.set(spriteX, spriteY); - - this.videoContainer.position.set(0, 0); - - const canvasScaleFactor = Math.min( - width / BASE_PREVIEW_WIDTH, - height / BASE_PREVIEW_HEIGHT, - ); - - const scaledBorderRadius = borderRadius * canvasScaleFactor; - - this.maskGraphics.clear(); - drawSquircleOnGraphics(this.maskGraphics, { - x: centerOffsetX, - y: centerOffsetY, - width: croppedDisplayWidth, - height: croppedDisplayHeight, - radius: scaledBorderRadius, - }); - this.maskGraphics.fill({ color: 0xffffff }); - - // Cache layout info - this.layoutCache = { - stageSize: { width, height }, - videoSize: { width: croppedVideoWidth, height: croppedVideoHeight }, - baseScale: scale, - baseOffset: { x: spriteX, y: spriteY }, - maskRect: { - x: centerOffsetX, - y: centerOffsetY, - width: croppedDisplayWidth, - height: croppedDisplayHeight, - sourceCrop: cropRegion, - }, - }; - } - - private updateAnimationState(timeMs: number): number { - if (!this.cameraContainer || !this.layoutCache) return 0; - - const { region, strength, blendedScale, transition } = findDominantRegion( - this.config.zoomRegions, - timeMs, - { - connectZooms: this.config.connectZooms, - }, - ); - - const defaultFocus = DEFAULT_FOCUS; - let targetScaleFactor = 1; - let targetFocus = { ...defaultFocus }; - let targetProgress = 0; - - if (region && strength > 0) { - const zoomScale = blendedScale ?? ZOOM_DEPTH_SCALES[region.depth]; - - // Cursor follow: use cursor-follow camera for non-manual zoom regions - let regionFocus = region.focus; - if (!this.config.zoomClassicMode && region.mode !== 'manual' && this.config.cursorTelemetry && this.config.cursorTelemetry.length > 0) { - regionFocus = computeCursorFollowFocus( - this.cursorFollowCamera, - this.config.cursorTelemetry, - timeMs, - zoomScale, - strength, - region.focus, - { snapToEdgesRatio: SNAP_TO_EDGES_RATIO_AUTO }, - ); - } - - targetScaleFactor = zoomScale; - targetFocus = regionFocus; - targetProgress = strength; - - if (transition) { - const startTransform = computeZoomTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: transition.startScale, - zoomProgress: 1, - focusX: transition.startFocus.cx, - focusY: transition.startFocus.cy, - }); - const endTransform = computeZoomTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: transition.endScale, - zoomProgress: 1, - focusX: transition.endFocus.cx, - focusY: transition.endFocus.cy, - }); - - const interpolatedTransform = { - scale: - startTransform.scale + - (endTransform.scale - startTransform.scale) * transition.progress, - x: - startTransform.x + - (endTransform.x - startTransform.x) * transition.progress, - y: - startTransform.y + - (endTransform.y - startTransform.y) * transition.progress, - }; - - targetScaleFactor = interpolatedTransform.scale; - targetFocus = computeFocusFromTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: interpolatedTransform.scale, - x: interpolatedTransform.x, - y: interpolatedTransform.y, - }); - targetProgress = 1; - } - } - - const state = this.animationState; - - const prevScale = state.appliedScale; - const prevX = state.x; - const prevY = state.y; - - state.scale = targetScaleFactor; - state.focusX = targetFocus.cx; - state.focusY = targetFocus.cy; - state.progress = targetProgress; - - const projectedTransform = computeZoomTransform({ - stageSize: this.layoutCache.stageSize, - baseMask: this.layoutCache.maskRect, - zoomScale: state.scale, - zoomProgress: state.progress, - focusX: state.focusX, - focusY: state.focusY, - }); - - // Spring-driven zoom animation for export — use content time, not wall-clock, - // so the spring advances at the same rate as the video regardless of render speed. - const deltaMs = this.lastContentTimeMs !== null - ? timeMs - this.lastContentTimeMs - : 1000 / 60; - this.lastContentTimeMs = timeMs; - - const zoomSpringConfig = getZoomSpringConfig(this.config.zoomSmoothness); - - if (this.config.zoomClassicMode) { - state.appliedScale = projectedTransform.scale; - state.x = projectedTransform.x; - state.y = projectedTransform.y; - resetSpringState(this.springScale, state.appliedScale); - resetSpringState(this.springX, state.x); - resetSpringState(this.springY, state.y); - } else { - state.appliedScale = stepSpringValue(this.springScale, projectedTransform.scale, deltaMs, zoomSpringConfig); - state.x = stepSpringValue(this.springX, projectedTransform.x, deltaMs, zoomSpringConfig); - state.y = stepSpringValue(this.springY, projectedTransform.y, deltaMs, zoomSpringConfig); - } - - this.lastMotionVector = { - x: state.x - prevX, - y: state.y - prevY, - }; - - return Math.max( - Math.abs(state.appliedScale - prevScale), - Math.abs(state.x - prevX) / Math.max(1, this.layoutCache.stageSize.width), - Math.abs(state.y - prevY) / - Math.max(1, this.layoutCache.stageSize.height), - ); - } - - private compositeWithShadows(): void { - if (!this.compositeCanvas || !this.compositeCtx || !this.app) return; - - const videoCanvas = this.app.canvas as HTMLCanvasElement; - const ctx = this.compositeCtx; - const w = this.compositeCanvas.width; - const h = this.compositeCanvas.height; - - // Clear composite canvas - ctx.clearRect(0, 0, w, h); - ctx.imageSmoothingEnabled = true; - ctx.imageSmoothingQuality = "high"; - - // Step 1: Draw background layer (with optional blur, not affected by zoom) - if (this.backgroundSprite) { - const bgCanvas = this.backgroundSprite as any as HTMLCanvasElement; - - if (this.config.backgroundBlur > 0) { - ctx.save(); - ctx.filter = `blur(${this.config.backgroundBlur * 3}px)`; - ctx.drawImage(bgCanvas, 0, 0, w, h); - ctx.restore(); - } else { - ctx.drawImage(bgCanvas, 0, 0, w, h); - } - } else { - console.warn( - "[FrameRenderer] No background sprite found during compositing!", - ); - } - - // Draw video layer with shadows on top of background - if ( - this.config.showShadow && - this.config.shadowIntensity > 0 && - this.shadowCanvas && - this.shadowCtx - ) { - const shadowCtx = this.shadowCtx; - shadowCtx.clearRect(0, 0, w, h); - shadowCtx.imageSmoothingEnabled = true; - shadowCtx.imageSmoothingQuality = "high"; - shadowCtx.save(); - - // Calculate shadow parameters based on intensity (0-1) - const intensity = this.config.shadowIntensity; - const baseBlur1 = 48 * intensity; - const baseBlur2 = 16 * intensity; - const baseBlur3 = 8 * intensity; - const baseAlpha1 = 0.7 * intensity; - const baseAlpha2 = 0.5 * intensity; - const baseAlpha3 = 0.3 * intensity; - const baseOffset = 12 * intensity; - - shadowCtx.filter = `drop-shadow(0 ${baseOffset}px ${baseBlur1}px rgba(0,0,0,${baseAlpha1})) drop-shadow(0 ${baseOffset / 3}px ${baseBlur2}px rgba(0,0,0,${baseAlpha2})) drop-shadow(0 ${baseOffset / 6}px ${baseBlur3}px rgba(0,0,0,${baseAlpha3}))`; - shadowCtx.drawImage(videoCanvas, 0, 0, w, h); - shadowCtx.restore(); - ctx.drawImage(this.shadowCanvas, 0, 0, w, h); - } else { - ctx.drawImage(videoCanvas, 0, 0, w, h); - } - - this.drawWebcamOverlay(ctx, w, h); - } - - private drawFrame(): void { - if ((!this.frameImage && !this.frameDraw) || !this.compositeCtx || !this.layoutCache) return; - - const ctx = this.compositeCtx; - const maskRect = this.layoutCache.maskRect; - const insets = this.frameInsets; - - if (!insets) { - // No insets: draw frame spanning entire mask area - if (this.frameDraw) { - const c = document.createElement('canvas'); - c.width = Math.round(maskRect.width); - c.height = Math.round(maskRect.height); - const dCtx = c.getContext('2d'); - if (dCtx) this.frameDraw(dCtx, c.width, c.height); - ctx.drawImage(c, maskRect.x, maskRect.y, maskRect.width, maskRect.height); - } else { - ctx.drawImage( - this.frameImage!, - maskRect.x, - maskRect.y, - maskRect.width, - maskRect.height, - ); - } - return; - } - - // Calculate frame dimensions from insets - const screenW = maskRect.width; - const screenH = maskRect.height; - const frameW = screenW / (1 - insets.left - insets.right); - const frameH = screenH / (1 - insets.top - insets.bottom); - const frameX = maskRect.x - insets.left * frameW; - const frameY = maskRect.y - insets.top * frameH; - - if (this.frameDraw) { - // Draw at the exact export resolution — no bitmap scaling - const c = document.createElement('canvas'); - c.width = Math.round(frameW); - c.height = Math.round(frameH); - const dCtx = c.getContext('2d'); - if (dCtx) this.frameDraw(dCtx, c.width, c.height); - ctx.drawImage(c, frameX, frameY, frameW, frameH); - } else { - ctx.drawImage(this.frameImage!, frameX, frameY, frameW, frameH); - } - } - - private drawWebcamOverlay( - ctx: CanvasRenderingContext2D, - width: number, - height: number, - ): void { - const webcam = this.config.webcam; - const webcamDecodedFrame = this.webcamDecodedFrame; - const webcamVideo = this.webcamVideoElement; - if (!webcam?.enabled || (!webcamDecodedFrame && !webcamVideo)) { - return; - } - - const hasCachedWebcamFrame = Boolean( - this.webcamFrameCacheCanvas && - this.webcamFrameCacheCanvas.width > 0 && - this.webcamFrameCacheCanvas.height > 0, - ); - const hasLiveWebcamFrame = - webcamDecodedFrame - ? webcamDecodedFrame.displayWidth > 0 && webcamDecodedFrame.displayHeight > 0 - : Boolean( - webcamVideo && - webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && - webcamVideo.videoWidth > 0 && - webcamVideo.videoHeight > 0, - ); - if (!hasLiveWebcamFrame && !hasCachedWebcamFrame) { - return; - } - - const margin = webcam.margin ?? 24; - const size = getWebcamOverlaySizePx({ - containerWidth: width, - containerHeight: height, - sizePercent: webcam.size ?? 50, - margin, - zoomScale: this.animationState.appliedScale || 1, - reactToZoom: webcam.reactToZoom ?? true, - }); - const { x, y } = getWebcamOverlayPosition({ - containerWidth: width, - containerHeight: height, - size, - margin, - positionPreset: webcam.positionPreset ?? webcam.corner, - positionX: webcam.positionX ?? 1, - positionY: webcam.positionY ?? 1, - legacyCorner: webcam.corner, - }); - const radius = Math.max(0, webcam.cornerRadius ?? 18); - - const bubbleCanvas = this.webcamBubbleCanvas ?? document.createElement("canvas"); - const bubbleSize = Math.max(1, Math.ceil(size)); - if (bubbleCanvas.width !== bubbleSize || bubbleCanvas.height !== bubbleSize) { - bubbleCanvas.width = bubbleSize; - bubbleCanvas.height = bubbleSize; - } - this.webcamBubbleCanvas = bubbleCanvas; - const bubbleCtx = this.webcamBubbleCtx ?? configureHighQuality2DContext(bubbleCanvas.getContext("2d")); - if (!bubbleCtx) { - return; - } - this.webcamBubbleCtx = bubbleCtx; - bubbleCtx.clearRect(0, 0, bubbleCanvas.width, bubbleCanvas.height); - bubbleCtx.imageSmoothingEnabled = true; - bubbleCtx.imageSmoothingQuality = "high"; - - const canRefreshCache = - hasLiveWebcamFrame && - this.lastSyncedWebcamTime !== null && - Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime) <= 0.02 && - (webcamDecodedFrame - ? true - : Boolean( - webcamVideo && - !webcamVideo.seeking && - Math.abs(webcamVideo.currentTime - this.currentVideoTime) <= 0.02 && - webcamVideo.videoWidth > 0 && - webcamVideo.videoHeight > 0, - )); - - if (canRefreshCache) { - const liveFrameWidth = webcamDecodedFrame?.displayWidth ?? webcamVideo?.videoWidth ?? 0; - const liveFrameHeight = webcamDecodedFrame?.displayHeight ?? webcamVideo?.videoHeight ?? 0; - if ( - !this.webcamFrameCacheCanvas || - this.webcamFrameCacheCanvas.width !== liveFrameWidth || - this.webcamFrameCacheCanvas.height !== liveFrameHeight - ) { - this.webcamFrameCacheCanvas = document.createElement("canvas"); - this.webcamFrameCacheCanvas.width = liveFrameWidth; - this.webcamFrameCacheCanvas.height = liveFrameHeight; - this.webcamFrameCacheCtx = configureHighQuality2DContext( - this.webcamFrameCacheCanvas.getContext("2d"), - ); - } - - this.webcamFrameCacheCtx?.clearRect( - 0, - 0, - this.webcamFrameCacheCanvas!.width, - this.webcamFrameCacheCanvas!.height, - ); - this.webcamFrameCacheCtx?.drawImage( - webcamDecodedFrame ?? webcamVideo!, - 0, - 0, - this.webcamFrameCacheCanvas!.width, - this.webcamFrameCacheCanvas!.height, - ); - } - - const webcamFrameSource = this.webcamFrameCacheCanvas ?? (hasLiveWebcamFrame ? (webcamDecodedFrame ?? webcamVideo) : null); - if (!webcamFrameSource) { - return; - } - - const sourceWidth = - ("displayWidth" in webcamFrameSource - ? webcamFrameSource.displayWidth - : "videoWidth" in webcamFrameSource - ? webcamFrameSource.videoWidth - : webcamFrameSource.width) || size; - const sourceHeight = - ("displayHeight" in webcamFrameSource - ? webcamFrameSource.displayHeight - : "videoHeight" in webcamFrameSource - ? webcamFrameSource.videoHeight - : webcamFrameSource.height) || size; - const coverScale = Math.max(size / sourceWidth, size / sourceHeight); - const drawWidth = sourceWidth * coverScale; - const drawHeight = sourceHeight * coverScale; - const drawX = (size - drawWidth) / 2; - const drawY = (size - drawHeight) / 2; - - bubbleCtx.save(); - drawSquircleOnCanvas(bubbleCtx, { x: 0, y: 0, width: size, height: size, radius }); - bubbleCtx.clip(); - if (webcam.mirror) { - bubbleCtx.save(); - bubbleCtx.translate(size, 0); - bubbleCtx.scale(-1, 1); - bubbleCtx.drawImage(webcamFrameSource, drawX, drawY, drawWidth, drawHeight); - bubbleCtx.restore(); - } else { - bubbleCtx.drawImage(webcamFrameSource, drawX, drawY, drawWidth, drawHeight); - } - bubbleCtx.restore(); - - if ((webcam.shadow ?? 0) > 0) { - const shadow = Math.max(0, Math.min(1, webcam.shadow)); - ctx.save(); - ctx.filter = `drop-shadow(0 ${Math.round(size * 0.06)}px ${Math.round(size * 0.22)}px rgba(0,0,0,${shadow}))`; - ctx.drawImage(bubbleCanvas, x, y, size, size); - ctx.restore(); - return; - } - - ctx.drawImage(bubbleCanvas, x, y, size, size); - } - - private closeWebcamDecodedFrame(): void { - if (!this.webcamDecodedFrame) { - return; - } - - this.webcamDecodedFrame.close(); - this.webcamDecodedFrame = null; - } - - getCanvas(): HTMLCanvasElement { - if (!this.compositeCanvas) { - throw new Error("Renderer not initialized"); - } - return this.compositeCanvas; - } - - destroy(): void { - if (this.videoSprite) { - const videoTexture = this.videoSprite.texture; - this.videoSprite.destroy({ texture: false, textureSource: false }); - videoTexture?.destroy(true); - this.videoSprite = null; - this.videoTextureSource = null; - } - this.backgroundSprite = null; - if (this.app) { - this.app.destroy(true, { - children: true, - texture: false, - textureSource: false, - }); - this.app = null; - } - this.cameraContainer = null; - this.videoContainer = null; - this.maskGraphics = null; - this.blurFilter = null; - this.motionBlurFilter = null; - if (this.cursorOverlay) { - this.cursorOverlay.destroy(); - this.cursorOverlay = null; - } - this.shadowCanvas = null; - this.shadowCtx = null; - this.compositeCanvas = null; - this.compositeCtx = null; - this.backgroundCtx = null; - if (this.backgroundVideoElement) { - this.backgroundVideoElement.pause(); - this.backgroundVideoElement.src = ""; - this.backgroundVideoElement.load(); - this.backgroundVideoElement = null; - } - this.cleanupBackgroundSource?.(); - this.cleanupBackgroundSource = null; - if (this.webcamVideoElement) { - this.webcamVideoElement.pause(); - this.webcamVideoElement.src = ""; - this.webcamVideoElement.load(); - this.webcamVideoElement = null; - } - this.webcamForwardFrameSource?.cancel(); - void this.webcamForwardFrameSource?.destroy(); - this.webcamForwardFrameSource = null; - this.closeWebcamDecodedFrame(); - this.cleanupWebcamSource?.(); - this.cleanupWebcamSource = null; - this.webcamFrameCacheCanvas = null; - this.webcamFrameCacheCtx = null; - this.webcamBubbleCanvas = null; - this.webcamBubbleCtx = null; - this.lastSyncedWebcamTime = null; - this.frameImage = null; - this.frameInsets = null; - this.frameDraw = null; - } + private app: Application | null = null; + private cameraContainer: Container | null = null; + private videoContainer: Container | null = null; + private cursorContainer: Container | null = null; + private videoSprite: Sprite | null = null; + private videoTextureSource: VideoTextureSource | null = null; + private backgroundSprite: HTMLCanvasElement | null = null; + private maskGraphics: Graphics | null = null; + private blurFilter: BlurFilter | null = null; + private motionBlurFilter: MotionBlurFilter | null = null; + private shadowCanvas: HTMLCanvasElement | null = null; + private shadowCtx: CanvasRenderingContext2D | null = null; + private compositeCanvas: HTMLCanvasElement | null = null; + private compositeCtx: CanvasRenderingContext2D | null = null; + private backgroundVideoElement: HTMLVideoElement | null = null; + private backgroundCtx: CanvasRenderingContext2D | null = null; + private cleanupBackgroundSource: (() => void) | null = null; + private config: FrameRenderConfig; + private animationState: AnimationState; + private motionBlurState: MotionBlurState; + private layoutCache: LayoutCache | null = null; + private currentVideoTime = 0; + private lastMotionVector = { x: 0, y: 0 }; + private springScale: SpringState; + private springX: SpringState; + private springY: SpringState; + private cursorFollowCamera: CursorFollowCameraState; + private lastContentTimeMs: number | null = null; + private cursorOverlay: PixiCursorOverlay | null = null; + private webcamForwardFrameSource: ForwardFrameSource | null = null; + private webcamDecodedFrame: VideoFrame | null = null; + private webcamVideoElement: HTMLVideoElement | null = null; + private webcamSeekPromise: Promise | null = null; + private webcamFrameCacheCanvas: HTMLCanvasElement | null = null; + private webcamFrameCacheCtx: CanvasRenderingContext2D | null = null; + private webcamBubbleCanvas: HTMLCanvasElement | null = null; + private webcamBubbleCtx: CanvasRenderingContext2D | null = null; + private lastSyncedWebcamTime: number | null = null; + private cleanupWebcamSource: (() => void) | null = null; + private frameImage: HTMLImageElement | null = null; + private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null; + private frameDraw: ((ctx: CanvasRenderingContext2D, w: number, h: number) => void) | null = + null; + + constructor(config: FrameRenderConfig) { + this.config = config; + this.animationState = createAnimationState(); + this.motionBlurState = createMotionBlurState(); + this.springScale = createSpringState(1); + this.springX = createSpringState(0); + this.springY = createSpringState(0); + this.cursorFollowCamera = createCursorFollowCameraState(); + } + + async initialize(): Promise { + let cursorOverlayEnabled = true; + try { + await preloadCursorAssets(); + } catch (error) { + cursorOverlayEnabled = false; + console.warn( + "[FrameRenderer] Native cursor assets are unavailable; continuing export without cursor overlay.", + error, + ); + } + + // Create canvas for rendering + const canvas = document.createElement("canvas"); + canvas.width = this.config.width; + canvas.height = this.config.height; + + // Try to set colorSpace if supported (may not be available on all platforms) + try { + if (canvas && "colorSpace" in canvas) { + canvas.colorSpace = "srgb"; + } + } catch (error) { + // Silently ignore colorSpace errors on platforms that don't support it + console.warn("[FrameRenderer] colorSpace not supported on this platform:", error); + } + + // Initialize PixiJS with optimized settings for export performance + this.app = new Application(); + await this.app.init({ + canvas, + width: this.config.width, + height: this.config.height, + backgroundAlpha: 0, + antialias: true, + failIfMajorPerformanceCaveat: false, + resolution: 1, + autoDensity: true, + autoStart: false, + sharedTicker: false, + powerPreference: "high-performance", + ...(this.config.preferredRenderBackend + ? { preference: this.config.preferredRenderBackend } + : {}), + }); + + // Setup containers + this.cameraContainer = new Container(); + this.videoContainer = new Container(); + this.cursorContainer = new Container(); + this.app.stage.addChild(this.cameraContainer); + this.cameraContainer.addChild(this.videoContainer); + this.cameraContainer.addChild(this.cursorContainer); + + if (cursorOverlayEnabled) { + this.cursorOverlay = new PixiCursorOverlay({ + dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4), + style: this.config.cursorStyle ?? "tahoe", + smoothingFactor: + this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor, + motionBlur: this.config.cursorMotionBlur ?? 0, + clickBounce: this.config.cursorClickBounce ?? DEFAULT_CURSOR_CONFIG.clickBounce, + clickBounceDuration: + this.config.cursorClickBounceDuration ?? + DEFAULT_CURSOR_CONFIG.clickBounceDuration, + sway: this.config.cursorSway ?? DEFAULT_CURSOR_CONFIG.sway, + }); + } + + // Setup background (render separately, not in PixiJS) + await this.setupBackground(); + await this.setupWebcamSource(); + await this.setupFrame(); + + // Setup blur filter for video container + this.blurFilter = new BlurFilter(); + this.blurFilter.quality = 5; + this.blurFilter.resolution = this.app.renderer.resolution; + this.blurFilter.blur = 0; + this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); + this.videoContainer.filters = [this.blurFilter, this.motionBlurFilter]; + + // Setup composite canvas for final output with shadows + this.compositeCanvas = document.createElement("canvas"); + this.compositeCanvas.width = this.config.width; + this.compositeCanvas.height = this.config.height; + this.compositeCtx = configureHighQuality2DContext( + this.compositeCanvas.getContext("2d", { + willReadFrequently: false, + }), + ); + + if (!this.compositeCtx) { + throw new Error("Failed to get 2D context for composite canvas"); + } + + // Setup shadow canvas if needed + if (this.config.showShadow) { + this.shadowCanvas = document.createElement("canvas"); + this.shadowCanvas.width = this.config.width; + this.shadowCanvas.height = this.config.height; + this.shadowCtx = configureHighQuality2DContext( + this.shadowCanvas.getContext("2d", { + willReadFrequently: false, + }), + ); + + if (!this.shadowCtx) { + throw new Error("Failed to get 2D context for shadow canvas"); + } + } + + // Setup mask + this.maskGraphics = new Graphics(); + this.videoContainer.addChild(this.maskGraphics); + this.videoContainer.mask = this.maskGraphics; + if (this.cursorOverlay) { + this.cursorContainer.addChild(this.cursorOverlay.container); + } + } + + private async setupBackground(): Promise { + const wallpaper = await this.resolveWallpaperForExport(this.config.wallpaper); + + // Create background canvas for separate rendering (not affected by zoom) + const bgCanvas = document.createElement("canvas"); + bgCanvas.width = this.config.width; + bgCanvas.height = this.config.height; + const bgCtx = configureHighQuality2DContext(bgCanvas.getContext("2d")); + + if (!bgCtx) { + throw new Error("Failed to get 2D context for background canvas"); + } + + this.backgroundCtx = bgCtx; + + try { + // Check for video wallpaper first + if (isVideoWallpaperSource(wallpaper)) { + let videoSrc = wallpaper; + if (wallpaper.startsWith("/") && !wallpaper.startsWith("//")) { + videoSrc = await getAssetPath(wallpaper.replace(/^\//, "")); + } + + const video = document.createElement("video"); + video.muted = true; + video.loop = true; + video.playsInline = true; + video.preload = "auto"; + video.src = videoSrc; + + await new Promise((resolve, reject) => { + video.onloadeddata = () => resolve(); + video.onerror = () => + reject(new Error(`Failed to load video wallpaper: ${wallpaper}`)); + }); + + this.backgroundVideoElement = video; + this.drawVideoFrameToBackground(); + this.backgroundSprite = bgCanvas; + return; + } + + // Render background based on type + if ( + wallpaper.startsWith("file://") || + wallpaper.startsWith("data:") || + wallpaper.startsWith("/") || + wallpaper.startsWith("http") + ) { + // Image background + const img = new Image(); + const imageUrl = await this.resolveWallpaperImageUrl(wallpaper); + // Don't set crossOrigin for same-origin images to avoid CORS taint. + if ( + imageUrl.startsWith("http") && + window.location.origin && + !imageUrl.startsWith(window.location.origin) + ) { + img.crossOrigin = "anonymous"; + } + + await new Promise((resolve, reject) => { + img.onload = () => resolve(); + img.onerror = (err) => { + console.error( + "[FrameRenderer] Failed to load background image:", + imageUrl, + err, + ); + reject(new Error(`Failed to load background image: ${imageUrl}`)); + }; + img.src = imageUrl; + }); + + // Draw the image using cover and center positioning + const imgAspect = img.width / img.height; + const canvasAspect = this.config.width / this.config.height; + + let drawWidth, drawHeight, drawX, drawY; + + if (imgAspect > canvasAspect) { + drawHeight = this.config.height; + drawWidth = drawHeight * imgAspect; + drawX = (this.config.width - drawWidth) / 2; + drawY = 0; + } else { + drawWidth = this.config.width; + drawHeight = drawWidth / imgAspect; + drawX = 0; + drawY = (this.config.height - drawHeight) / 2; + } + + bgCtx.drawImage(img, drawX, drawY, drawWidth, drawHeight); + } else if (wallpaper.startsWith("#")) { + bgCtx.fillStyle = wallpaper; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } else if ( + wallpaper.startsWith("linear-gradient") || + wallpaper.startsWith("radial-gradient") + ) { + const gradientMatch = wallpaper.match(/(linear|radial)-gradient\((.+)\)/); + if (gradientMatch) { + const [, type, params] = gradientMatch; + const parts = params.split(",").map((s) => s.trim()); + + let gradient: CanvasGradient; + + if (type === "linear") { + gradient = bgCtx.createLinearGradient(0, 0, 0, this.config.height); + parts.forEach((part, index) => { + if (part.startsWith("to ") || part.includes("deg")) return; + + const colorMatch = part.match( + /^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/, + ); + if (colorMatch) { + const color = colorMatch[1]; + const position = index / (parts.length - 1); + gradient.addColorStop(position, color); + } + }); + } else { + const cx = this.config.width / 2; + const cy = this.config.height / 2; + const radius = Math.max(this.config.width, this.config.height) / 2; + gradient = bgCtx.createRadialGradient(cx, cy, 0, cx, cy, radius); + + parts.forEach((part, index) => { + const colorMatch = part.match( + /^(#[0-9a-fA-F]{3,8}|rgba?\([^)]+\)|[a-z]+)/, + ); + if (colorMatch) { + const color = colorMatch[1]; + const position = index / (parts.length - 1); + gradient.addColorStop(position, color); + } + }); + } + + bgCtx.fillStyle = gradient; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } else { + console.warn("[FrameRenderer] Could not parse gradient, using black fallback"); + bgCtx.fillStyle = "#000000"; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } + } else { + bgCtx.fillStyle = wallpaper; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } + } catch (error) { + console.error("[FrameRenderer] Error setting up background, using fallback:", error); + bgCtx.fillStyle = "#000000"; + bgCtx.fillRect(0, 0, this.config.width, this.config.height); + } + + // Store the background canvas for compositing + this.backgroundSprite = bgCanvas; + } + + private drawVideoFrameToBackground(): void { + const video = this.backgroundVideoElement; + const ctx = this.backgroundCtx; + if (!video || !ctx) return; + + const w = this.config.width; + const h = this.config.height; + const videoAspect = video.videoWidth / video.videoHeight; + const canvasAspect = w / h; + + let drawWidth: number, drawHeight: number, drawX: number, drawY: number; + if (videoAspect > canvasAspect) { + drawHeight = h; + drawWidth = drawHeight * videoAspect; + drawX = (w - drawWidth) / 2; + drawY = 0; + } else { + drawWidth = w; + drawHeight = drawWidth / videoAspect; + drawX = 0; + drawY = (h - drawHeight) / 2; + } + + ctx.clearRect(0, 0, w, h); + ctx.drawImage(video, drawX, drawY, drawWidth, drawHeight); + } + + private async syncBackgroundVideo(timeSeconds: number): Promise { + const video = this.backgroundVideoElement; + if (!video) return; + + if (video.duration && Number.isFinite(video.duration)) { + const targetTime = timeSeconds % video.duration; + if (Math.abs(video.currentTime - targetTime) > 0.01) { + video.currentTime = targetTime; + await new Promise((resolve) => { + const onSeeked = () => { + video.removeEventListener("seeked", onSeeked); + resolve(); + }; + video.addEventListener("seeked", onSeeked); + }); + } + } + this.drawVideoFrameToBackground(); + } + + private async resolveWallpaperImageUrl(wallpaper: string): Promise { + if ( + wallpaper.startsWith("file://") || + wallpaper.startsWith("data:") || + wallpaper.startsWith("http") + ) { + return wallpaper; + } + + const resolved = await getAssetPath(wallpaper.replace(/^\/+/, "")); + if (resolved.startsWith("/") && window.location.protocol.startsWith("http")) { + return `${window.location.origin}${resolved}`; + } + + return resolved; + } + + private async resolveWallpaperForExport(wallpaper: string): Promise { + if (!wallpaper) { + return wallpaper; + } + + if ( + wallpaper.startsWith("#") || + wallpaper.startsWith("linear-gradient") || + wallpaper.startsWith("radial-gradient") + ) { + return wallpaper; + } + + const looksLikeAbsoluteFilePath = + wallpaper.startsWith("/") && + !wallpaper.startsWith("//") && + !wallpaper.startsWith("/wallpapers/") && + !wallpaper.startsWith("/app-icons/"); + + const wallpaperAsset = looksLikeAbsoluteFilePath + ? `file://${encodeURI(wallpaper)}` + : wallpaper; + + return getRenderableAssetUrl(wallpaperAsset); + } + + private async setupWebcamSource(): Promise { + const webcamUrl = this.config.webcamUrl; + if (!this.config.webcam?.enabled || !webcamUrl) { + this.webcamForwardFrameSource?.cancel(); + void this.webcamForwardFrameSource?.destroy(); + this.webcamForwardFrameSource = null; + this.closeWebcamDecodedFrame(); + this.cleanupWebcamSource?.(); + this.cleanupWebcamSource = null; + this.webcamVideoElement = null; + this.webcamFrameCacheCanvas = null; + this.webcamFrameCacheCtx = null; + this.lastSyncedWebcamTime = null; + return; + } + + this.webcamForwardFrameSource?.cancel(); + void this.webcamForwardFrameSource?.destroy(); + this.webcamForwardFrameSource = null; + this.closeWebcamDecodedFrame(); + this.cleanupWebcamSource?.(); + this.cleanupWebcamSource = null; + + try { + const frameSource = new ForwardFrameSource(); + await frameSource.initialize(webcamUrl); + this.webcamForwardFrameSource = frameSource; + this.webcamVideoElement = null; + this.webcamSeekPromise = null; + this.webcamFrameCacheCanvas = null; + this.webcamFrameCacheCtx = null; + this.lastSyncedWebcamTime = null; + return; + } catch (error) { + console.warn( + "[FrameRenderer] Decoder-backed webcam source unavailable during export; falling back to media element sync:", + error, + ); + } + + const webcamSource = await resolveMediaElementSource(webcamUrl); + this.cleanupWebcamSource = webcamSource.revoke; + + const video = document.createElement("video"); + video.src = webcamSource.src; + video.muted = true; + video.preload = "auto"; + video.playsInline = true; + video.load(); + + await new Promise((resolve, reject) => { + const onReady = () => { + if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { + return; + } + cleanup(); + resolve(); + }; + const onError = () => { + cleanup(); + reject(new Error("Failed to load webcam source for export")); + }; + const cleanup = () => { + video.removeEventListener("loadeddata", onReady); + video.removeEventListener("canplay", onReady); + video.removeEventListener("canplaythrough", onReady); + video.removeEventListener("error", onError); + }; + if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + resolve(); + return; + } + video.addEventListener("loadeddata", onReady, { once: true }); + video.addEventListener("canplay", onReady, { once: true }); + video.addEventListener("canplaythrough", onReady, { once: true }); + video.addEventListener("error", onError, { once: true }); + }).catch((error) => { + console.warn("[FrameRenderer] Webcam overlay unavailable during export:", error); + this.webcamVideoElement = null; + }); + + if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + this.webcamVideoElement = video; + return; + } + + this.webcamVideoElement = null; + this.webcamFrameCacheCanvas = null; + this.webcamFrameCacheCtx = null; + this.lastSyncedWebcamTime = null; + } + + private async syncWebcamFrame(targetTime: number): Promise { + if (this.webcamForwardFrameSource) { + const clampedTime = clampMediaTimeToDuration(targetTime, null); + const decodedFrame = await this.webcamForwardFrameSource.getFrameAtTime(clampedTime); + this.closeWebcamDecodedFrame(); + this.webcamDecodedFrame = decodedFrame; + if (decodedFrame) { + this.lastSyncedWebcamTime = clampedTime; + } + return; + } + + const webcamVideo = this.webcamVideoElement; + if (!webcamVideo) { + return; + } + + const clampedTime = clampMediaTimeToDuration( + targetTime, + Number.isFinite(webcamVideo.duration) ? webcamVideo.duration : null, + ); + + if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.008) { + this.lastSyncedWebcamTime = clampedTime; + return; + } + + if (this.webcamSeekPromise) { + await this.webcamSeekPromise; + } + + this.webcamSeekPromise = new Promise((resolve) => { + let settled = false; + let fallbackTimeout: number | null = null; + let animationFrameRequestId: number | null = null; + let videoFrameRequestId: number | null = null; + const waitForPresentedFrame = () => { + const requestVideoFrameCallback = ( + webcamVideo as HTMLVideoElement & { + requestVideoFrameCallback?: ( + callback: ( + now: DOMHighResTimeStamp, + metadata: VideoFrameCallbackMetadata, + ) => void, + ) => number; + cancelVideoFrameCallback?: (handle: number) => void; + } + ).requestVideoFrameCallback; + + const scheduleAnimationFrameFinish = () => { + animationFrameRequestId = requestAnimationFrame(() => { + animationFrameRequestId = null; + finish(); + }); + }; + + scheduleAnimationFrameFinish(); + + if (typeof requestVideoFrameCallback === "function") { + videoFrameRequestId = requestVideoFrameCallback.call(webcamVideo, () => { + videoFrameRequestId = null; + finish(); + }); + return; + } + }; + const finish = () => { + if (settled) { + return; + } + settled = true; + if (Math.abs(webcamVideo.currentTime - clampedTime) <= 0.02) { + this.lastSyncedWebcamTime = clampedTime; + } + cleanup(); + resolve(); + }; + const handleMediaReady = () => { + if ( + !webcamVideo.seeking && + Math.abs(webcamVideo.currentTime - clampedTime) <= 0.01 && + webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA + ) { + waitForPresentedFrame(); + } + }; + const cleanup = () => { + webcamVideo.removeEventListener("seeked", waitForPresentedFrame); + webcamVideo.removeEventListener("loadeddata", handleMediaReady); + webcamVideo.removeEventListener("canplay", handleMediaReady); + webcamVideo.removeEventListener("error", finish); + if (animationFrameRequestId !== null) { + cancelAnimationFrame(animationFrameRequestId); + animationFrameRequestId = null; + } + if ( + videoFrameRequestId !== null && + typeof ( + webcamVideo as HTMLVideoElement & { + cancelVideoFrameCallback?: (handle: number) => void; + } + ).cancelVideoFrameCallback === "function" + ) { + ( + webcamVideo as HTMLVideoElement & { + cancelVideoFrameCallback: (handle: number) => void; + } + ).cancelVideoFrameCallback(videoFrameRequestId); + videoFrameRequestId = null; + } + if (fallbackTimeout !== null) { + window.clearTimeout(fallbackTimeout); + } + }; + + webcamVideo.addEventListener("seeked", waitForPresentedFrame, { + once: true, + }); + webcamVideo.addEventListener("loadeddata", handleMediaReady, { + once: true, + }); + webcamVideo.addEventListener("canplay", handleMediaReady, { + once: true, + }); + webcamVideo.addEventListener("error", finish, { + once: true, + }); + fallbackTimeout = window.setTimeout(() => { + finish(); + }, 50); + + try { + webcamVideo.currentTime = clampedTime; + } catch { + finish(); + return; + } + + if ( + !webcamVideo.seeking && + Math.abs(webcamVideo.currentTime - clampedTime) <= 0.001 && + webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA + ) { + waitForPresentedFrame(); + } + }); + + try { + await this.webcamSeekPromise; + } finally { + this.webcamSeekPromise = null; + } + } + + private async setupFrame(): Promise { + const frameId = this.config.frame; + if (!frameId) return; + + const { extensionHost } = await import("@/lib/extensions/extensionHost"); + const frames = extensionHost.getFrames(); + const frame = frames.find((f) => f.id === frameId); + if (!frame) { + console.warn(`[FrameRenderer] Device frame "${frameId}" not found`); + return; + } + + this.frameInsets = frame.screenInsets; + + if (frame.draw) { + // Prefer draw function — renders at export resolution, no bitmap scaling + this.frameDraw = frame.draw; + return; + } + + const img = new Image(); + img.crossOrigin = "anonymous"; + await new Promise((resolve, reject) => { + img.onload = () => resolve(); + img.onerror = () => reject(new Error(`Failed to load device frame image: ${frameId}`)); + img.src = frame.filePath; + }); + + this.frameImage = img; + } + + async renderFrame( + videoFrame: VideoFrame, + timestamp: number, + cursorTimestamp = timestamp, + ): Promise { + if (!this.app || !this.videoContainer || !this.cameraContainer) { + throw new Error("Renderer not initialized"); + } + + this.currentVideoTime = timestamp / 1000000; + + if (this.webcamForwardFrameSource || this.webcamVideoElement) { + const targetTime = Math.max(0, this.currentVideoTime); + await this.syncWebcamFrame(targetTime); + } + + // Sync video wallpaper frame + if (this.backgroundVideoElement) { + await this.syncBackgroundVideo(this.currentVideoTime); + } + + // Create or update video sprite from VideoFrame + if (!this.videoSprite) { + const texture = Texture.from(videoFrame as unknown as PixiTextureInput); + this.videoSprite = new Sprite(texture); + this.videoTextureSource = texture.source as unknown as VideoTextureSource; + this.videoContainer.addChild(this.videoSprite); + if (this.cursorOverlay && this.cursorContainer) { + this.cursorContainer.addChild(this.cursorOverlay.container); + } + if (this.maskGraphics) { + this.videoContainer.addChild(this.maskGraphics); + } + } else { + this.videoTextureSource ??= this.videoSprite.texture + .source as unknown as VideoTextureSource; + this.videoTextureSource.resource = videoFrame; + this.videoTextureSource.update(); + } + + // Apply layout + this.updateLayout(); + const layoutCache = this.layoutCache; + if (!layoutCache) { + return; + } + + const timeMs = this.currentVideoTime * 1000; + const cursorTimeMs = cursorTimestamp / 1000; + + if (this.cursorOverlay) { + this.cursorOverlay.update( + this.config.cursorTelemetry ?? [], + cursorTimeMs, + layoutCache.maskRect, + this.config.showCursor ?? true, + false, + ); + } + + const smoothedCursor = mapSmoothedCursorToCanvasNormalized( + this.cursorOverlay?.getSmoothedCursorSnapshot() ?? null, + { + maskRect: layoutCache.maskRect, + canvasWidth: this.config.width, + canvasHeight: this.config.height, + }, + ); + extensionHost.setSmoothedCursor( + smoothedCursor + ? { + timeMs, + cx: smoothedCursor.cx, + cy: smoothedCursor.cy, + trail: smoothedCursor.trail, + } + : null, + ); + + const TICKS_PER_FRAME = 1; + + let maxMotionIntensity = 0; + for (let i = 0; i < TICKS_PER_FRAME; i++) { + const motionIntensity = this.updateAnimationState(timeMs); + maxMotionIntensity = Math.max(maxMotionIntensity, motionIntensity); + } + + // Apply transform once with maximum motion intensity from all ticks + applyZoomTransform({ + cameraContainer: this.cameraContainer, + blurFilter: this.blurFilter, + motionBlurFilter: this.motionBlurFilter, + stageSize: layoutCache.stageSize, + baseMask: layoutCache.maskRect, + zoomScale: this.animationState.scale, + zoomProgress: this.animationState.progress, + focusX: this.animationState.focusX, + focusY: this.animationState.focusY, + motionIntensity: maxMotionIntensity, + motionVector: this.lastMotionVector, + isPlaying: true, + motionBlurAmount: this.config.zoomMotionBlur ?? 0, + transformOverride: { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }, + motionBlurState: this.motionBlurState, + frameTimeMs: timeMs, + }); + + // Render the PixiJS stage to its canvas (video only, transparent background) + this.app.renderer.render(this.app.stage); + + // Composite with shadows to final output canvas + this.compositeWithShadows(); + + // Draw device frame overlay on top of video content + this.drawFrame(); + + // Render annotations on top if present + if ( + this.config.annotationRegions && + this.config.annotationRegions.length > 0 && + this.compositeCtx + ) { + // Calculate scale factor based on export vs preview dimensions + const scaleX = this.config.width / BASE_PREVIEW_WIDTH; + const scaleY = this.config.height / BASE_PREVIEW_HEIGHT; + const scaleFactor = (scaleX + scaleY) / 2; + + await renderAnnotations( + this.compositeCtx, + this.config.annotationRegions, + this.config.width, + this.config.height, + timeMs, + scaleFactor, + ); + } + + if ( + this.config.autoCaptions && + this.config.autoCaptions.length > 0 && + this.config.autoCaptionSettings && + this.compositeCtx + ) { + renderCaptions( + this.compositeCtx, + this.config.autoCaptions, + this.config.autoCaptionSettings, + this.config.width, + this.config.height, + timeMs, + ); + } + + // Extension render hooks — run after all built-in rendering + if (this.compositeCtx) { + const maskRect = this.layoutCache?.maskRect; + const hookParams = { + width: this.config.width, + height: this.config.height, + timeMs, + durationMs: 0, + cursor: smoothedCursor + ? { + cx: smoothedCursor.cx, + cy: smoothedCursor.cy, + interactionType: this.getCursorPosition(cursorTimeMs)?.interactionType, + } + : this.getCursorPosition(cursorTimeMs), + smoothedCursor, + videoLayout: maskRect + ? { + maskRect: { + x: maskRect.x, + y: maskRect.y, + width: maskRect.width, + height: maskRect.height, + }, + borderRadius: this.config.borderRadius ?? 0, + padding: this.config.padding ?? 0, + } + : undefined, + zoom: { + scale: this.animationState.scale, + focusX: this.animationState.focusX, + focusY: this.animationState.focusY, + progress: this.animationState.progress, + }, + shadow: { + enabled: this.config.showShadow, + intensity: this.config.shadowIntensity, + }, + sceneTransform: { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }, + }; + + this.compositeCtx.save(); + applyCanvasSceneTransform(this.compositeCtx, { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }); + executeExtensionRenderHooks("post-video", this.compositeCtx, hookParams); + executeExtensionRenderHooks("post-zoom", this.compositeCtx, hookParams); + executeExtensionRenderHooks("post-cursor", this.compositeCtx, hookParams); + + // Cursor click effects + this.emitCursorInteractions(cursorTimeMs); + executeExtensionCursorEffects( + this.compositeCtx, + timeMs, + this.config.width, + this.config.height, + { + zoom: hookParams.zoom, + sceneTransform: hookParams.sceneTransform, + videoLayout: hookParams.videoLayout, + }, + ); + this.compositeCtx.restore(); + + executeExtensionRenderHooks("post-webcam", this.compositeCtx, hookParams); + executeExtensionRenderHooks("post-annotations", this.compositeCtx, hookParams); + + executeExtensionRenderHooks("final", this.compositeCtx, hookParams); + } + } + + /** + * Get the cursor position (normalized 0-1) at the given time. + */ + private getCursorPosition( + timeMs: number, + ): { cx: number; cy: number; interactionType?: string } | null { + const telemetry = this.config.cursorTelemetry; + if (!telemetry || telemetry.length === 0) return null; + + // Find the closest telemetry point + let closest = telemetry[0]; + let minDist = Math.abs(telemetry[0].timeMs - timeMs); + for (let i = 1; i < telemetry.length; i++) { + const dist = Math.abs(telemetry[i].timeMs - timeMs); + if (dist < minDist) { + minDist = dist; + closest = telemetry[i]; + } + if (telemetry[i].timeMs > timeMs) break; + } + + return mapCursorToCanvasNormalized( + { cx: closest.cx, cy: closest.cy, interactionType: closest.interactionType }, + { + maskRect: this.layoutCache?.maskRect, + canvasWidth: this.config.width, + canvasHeight: this.config.height, + }, + ); + } + + /** + * Emit cursor interaction events for extensions based on telemetry clicks. + */ + private lastEmittedClickTimeMs = -1; + + private emitCursorInteractions(timeMs: number): void { + const telemetry = this.config.cursorTelemetry; + if (!telemetry || telemetry.length === 0) return; + + // Find click events near this time + for (const point of telemetry) { + if (point.timeMs > timeMs) break; + if (point.timeMs < timeMs - 100) continue; + if (!point.interactionType || point.interactionType === "move") continue; + if (point.timeMs === this.lastEmittedClickTimeMs) continue; + + const mappedCursor = mapCursorToCanvasNormalized( + { cx: point.cx, cy: point.cy, interactionType: point.interactionType }, + { + maskRect: this.layoutCache?.maskRect, + canvasWidth: this.config.width, + canvasHeight: this.config.height, + }, + ); + if (!mappedCursor) continue; + + this.lastEmittedClickTimeMs = point.timeMs; + notifyCursorInteraction( + point.timeMs, + mappedCursor.cx, + mappedCursor.cy, + point.interactionType, + ); + } + } + + private updateLayout(): void { + if (!this.app || !this.videoSprite || !this.maskGraphics || !this.videoContainer) return; + + const { width, height } = this.config; + const { cropRegion, borderRadius = 0, padding = 0 } = this.config; + const videoWidth = this.config.videoWidth; + const videoHeight = this.config.videoHeight; + + // Calculate cropped video dimensions + const cropStartX = cropRegion.x; + const cropStartY = cropRegion.y; + const cropEndX = cropRegion.x + cropRegion.width; + const cropEndY = cropRegion.y + cropRegion.height; + + const croppedVideoWidth = videoWidth * (cropEndX - cropStartX); + const croppedVideoHeight = videoHeight * (cropEndY - cropStartY); + + const paddingScale = 1.0 - (padding / 100) * 0.4; + const viewportWidth = width * paddingScale; + const viewportHeight = height * paddingScale; + + // When a device frame is active, scale to fit the ENTIRE frame (video + bezels) + const insets = this.frameInsets; + const screenFracW = insets ? 1 - insets.left - insets.right : 1; + const screenFracH = insets ? 1 - insets.top - insets.bottom : 1; + const fullFrameVideoW = croppedVideoWidth / screenFracW; + const fullFrameVideoH = croppedVideoHeight / screenFracH; + + const scale = Math.min(viewportWidth / fullFrameVideoW, viewportHeight / fullFrameVideoH); + + this.videoSprite.scale.set(scale); + + const fullVideoDisplayWidth = videoWidth * scale; + const fullVideoDisplayHeight = videoHeight * scale; + const croppedDisplayWidth = croppedVideoWidth * scale; + const croppedDisplayHeight = croppedVideoHeight * scale; + + // Center the full frame (video + bezels) in the output canvas + const fullFrameDisplayW = fullFrameVideoW * scale; + const fullFrameDisplayH = fullFrameVideoH * scale; + const frameCenterX = (width - fullFrameDisplayW) / 2; + const frameCenterY = (height - fullFrameDisplayH) / 2; + const centerOffsetX = insets + ? frameCenterX + insets.left * fullFrameDisplayW + : (width - croppedDisplayWidth) / 2; + const centerOffsetY = insets + ? frameCenterY + insets.top * fullFrameDisplayH + : (height - croppedDisplayHeight) / 2; + + const spriteX = centerOffsetX - cropRegion.x * fullVideoDisplayWidth; + const spriteY = centerOffsetY - cropRegion.y * fullVideoDisplayHeight; + this.videoSprite.position.set(spriteX, spriteY); + + this.videoContainer.position.set(0, 0); + + const canvasScaleFactor = Math.min( + width / BASE_PREVIEW_WIDTH, + height / BASE_PREVIEW_HEIGHT, + ); + + const scaledBorderRadius = borderRadius * canvasScaleFactor; + + this.maskGraphics.clear(); + drawSquircleOnGraphics(this.maskGraphics, { + x: centerOffsetX, + y: centerOffsetY, + width: croppedDisplayWidth, + height: croppedDisplayHeight, + radius: scaledBorderRadius, + }); + this.maskGraphics.fill({ color: 0xffffff }); + + // Cache layout info + this.layoutCache = { + stageSize: { width, height }, + videoSize: { width: croppedVideoWidth, height: croppedVideoHeight }, + baseScale: scale, + baseOffset: { x: spriteX, y: spriteY }, + maskRect: { + x: centerOffsetX, + y: centerOffsetY, + width: croppedDisplayWidth, + height: croppedDisplayHeight, + sourceCrop: cropRegion, + }, + }; + } + + private updateAnimationState(timeMs: number): number { + if (!this.cameraContainer || !this.layoutCache) return 0; + + const { region, strength, blendedScale, transition } = findDominantRegion( + this.config.zoomRegions, + timeMs, + { + connectZooms: this.config.connectZooms, + }, + ); + + const defaultFocus = DEFAULT_FOCUS; + let targetScaleFactor = 1; + let targetFocus = { ...defaultFocus }; + let targetProgress = 0; + + if (region && strength > 0) { + const zoomScale = blendedScale ?? ZOOM_DEPTH_SCALES[region.depth]; + + // Cursor follow: use cursor-follow camera for non-manual zoom regions + let regionFocus = region.focus; + if ( + !this.config.zoomClassicMode && + region.mode !== "manual" && + this.config.cursorTelemetry && + this.config.cursorTelemetry.length > 0 + ) { + regionFocus = computeCursorFollowFocus( + this.cursorFollowCamera, + this.config.cursorTelemetry, + timeMs, + zoomScale, + strength, + region.focus, + { snapToEdgesRatio: SNAP_TO_EDGES_RATIO_AUTO }, + ); + } + + targetScaleFactor = zoomScale; + targetFocus = regionFocus; + targetProgress = strength; + + if (transition) { + const startTransform = computeZoomTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: transition.startScale, + zoomProgress: 1, + focusX: transition.startFocus.cx, + focusY: transition.startFocus.cy, + }); + const endTransform = computeZoomTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: transition.endScale, + zoomProgress: 1, + focusX: transition.endFocus.cx, + focusY: transition.endFocus.cy, + }); + + const interpolatedTransform = { + scale: + startTransform.scale + + (endTransform.scale - startTransform.scale) * transition.progress, + x: startTransform.x + (endTransform.x - startTransform.x) * transition.progress, + y: startTransform.y + (endTransform.y - startTransform.y) * transition.progress, + }; + + targetScaleFactor = interpolatedTransform.scale; + targetFocus = computeFocusFromTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: interpolatedTransform.scale, + x: interpolatedTransform.x, + y: interpolatedTransform.y, + }); + targetProgress = 1; + } + } + + const state = this.animationState; + + const prevScale = state.appliedScale; + const prevX = state.x; + const prevY = state.y; + + state.scale = targetScaleFactor; + state.focusX = targetFocus.cx; + state.focusY = targetFocus.cy; + state.progress = targetProgress; + + const projectedTransform = computeZoomTransform({ + stageSize: this.layoutCache.stageSize, + baseMask: this.layoutCache.maskRect, + zoomScale: state.scale, + zoomProgress: state.progress, + focusX: state.focusX, + focusY: state.focusY, + }); + + // Spring-driven zoom animation for export — use content time, not wall-clock, + // so the spring advances at the same rate as the video regardless of render speed. + const deltaMs = + this.lastContentTimeMs !== null ? timeMs - this.lastContentTimeMs : 1000 / 60; + this.lastContentTimeMs = timeMs; + + const zoomSpringConfig = getZoomSpringConfig(this.config.zoomSmoothness); + + if (this.config.zoomClassicMode) { + state.appliedScale = projectedTransform.scale; + state.x = projectedTransform.x; + state.y = projectedTransform.y; + resetSpringState(this.springScale, state.appliedScale); + resetSpringState(this.springX, state.x); + resetSpringState(this.springY, state.y); + } else { + state.appliedScale = stepSpringValue( + this.springScale, + projectedTransform.scale, + deltaMs, + zoomSpringConfig, + ); + state.x = stepSpringValue( + this.springX, + projectedTransform.x, + deltaMs, + zoomSpringConfig, + ); + state.y = stepSpringValue( + this.springY, + projectedTransform.y, + deltaMs, + zoomSpringConfig, + ); + } + + this.lastMotionVector = { + x: state.x - prevX, + y: state.y - prevY, + }; + + return Math.max( + Math.abs(state.appliedScale - prevScale), + Math.abs(state.x - prevX) / Math.max(1, this.layoutCache.stageSize.width), + Math.abs(state.y - prevY) / Math.max(1, this.layoutCache.stageSize.height), + ); + } + + private compositeWithShadows(): void { + if (!this.compositeCanvas || !this.compositeCtx || !this.app) return; + + const videoCanvas = this.app.canvas as HTMLCanvasElement; + const ctx = this.compositeCtx; + const w = this.compositeCanvas.width; + const h = this.compositeCanvas.height; + + // Clear composite canvas + ctx.clearRect(0, 0, w, h); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = "high"; + + // Step 1: Draw background layer (with optional blur, not affected by zoom) + if (this.backgroundSprite) { + const bgCanvas = this.backgroundSprite; + + if (this.config.backgroundBlur > 0) { + ctx.save(); + ctx.filter = `blur(${this.config.backgroundBlur * 3}px)`; + ctx.drawImage(bgCanvas, 0, 0, w, h); + ctx.restore(); + } else { + ctx.drawImage(bgCanvas, 0, 0, w, h); + } + } else { + console.warn("[FrameRenderer] No background sprite found during compositing!"); + } + + // Draw video layer with shadows on top of background + if ( + this.config.showShadow && + this.config.shadowIntensity > 0 && + this.shadowCanvas && + this.shadowCtx + ) { + const shadowCtx = this.shadowCtx; + shadowCtx.clearRect(0, 0, w, h); + shadowCtx.imageSmoothingEnabled = true; + shadowCtx.imageSmoothingQuality = "high"; + shadowCtx.save(); + + // Calculate shadow parameters based on intensity (0-1) + const intensity = this.config.shadowIntensity; + const baseBlur1 = 48 * intensity; + const baseBlur2 = 16 * intensity; + const baseBlur3 = 8 * intensity; + const baseAlpha1 = 0.7 * intensity; + const baseAlpha2 = 0.5 * intensity; + const baseAlpha3 = 0.3 * intensity; + const baseOffset = 12 * intensity; + + shadowCtx.filter = `drop-shadow(0 ${baseOffset}px ${baseBlur1}px rgba(0,0,0,${baseAlpha1})) drop-shadow(0 ${baseOffset / 3}px ${baseBlur2}px rgba(0,0,0,${baseAlpha2})) drop-shadow(0 ${baseOffset / 6}px ${baseBlur3}px rgba(0,0,0,${baseAlpha3}))`; + shadowCtx.drawImage(videoCanvas, 0, 0, w, h); + shadowCtx.restore(); + ctx.drawImage(this.shadowCanvas, 0, 0, w, h); + } else { + ctx.drawImage(videoCanvas, 0, 0, w, h); + } + + this.drawWebcamOverlay(ctx, w, h); + } + + private drawFrame(): void { + if ((!this.frameImage && !this.frameDraw) || !this.compositeCtx || !this.layoutCache) + return; + + const ctx = this.compositeCtx; + const maskRect = this.layoutCache.maskRect; + const insets = this.frameInsets; + + if (!insets) { + // No insets: draw frame spanning entire mask area + if (this.frameDraw) { + const c = document.createElement("canvas"); + c.width = Math.round(maskRect.width); + c.height = Math.round(maskRect.height); + const dCtx = c.getContext("2d"); + if (dCtx) this.frameDraw(dCtx, c.width, c.height); + ctx.drawImage(c, maskRect.x, maskRect.y, maskRect.width, maskRect.height); + } else { + ctx.drawImage( + this.frameImage!, + maskRect.x, + maskRect.y, + maskRect.width, + maskRect.height, + ); + } + return; + } + + // Calculate frame dimensions from insets + const screenW = maskRect.width; + const screenH = maskRect.height; + const frameW = screenW / (1 - insets.left - insets.right); + const frameH = screenH / (1 - insets.top - insets.bottom); + const frameX = maskRect.x - insets.left * frameW; + const frameY = maskRect.y - insets.top * frameH; + + if (this.frameDraw) { + // Draw at the exact export resolution — no bitmap scaling + const c = document.createElement("canvas"); + c.width = Math.round(frameW); + c.height = Math.round(frameH); + const dCtx = c.getContext("2d"); + if (dCtx) this.frameDraw(dCtx, c.width, c.height); + ctx.drawImage(c, frameX, frameY, frameW, frameH); + } else { + ctx.drawImage(this.frameImage!, frameX, frameY, frameW, frameH); + } + } + + private drawWebcamOverlay(ctx: CanvasRenderingContext2D, width: number, height: number): void { + const webcam = this.config.webcam; + const webcamDecodedFrame = this.webcamDecodedFrame; + const webcamVideo = this.webcamVideoElement; + if (!webcam?.enabled || (!webcamDecodedFrame && !webcamVideo)) { + return; + } + + const hasCachedWebcamFrame = Boolean( + this.webcamFrameCacheCanvas && + this.webcamFrameCacheCanvas.width > 0 && + this.webcamFrameCacheCanvas.height > 0, + ); + const hasLiveWebcamFrame = webcamDecodedFrame + ? webcamDecodedFrame.displayWidth > 0 && webcamDecodedFrame.displayHeight > 0 + : Boolean( + webcamVideo && + webcamVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && + webcamVideo.videoWidth > 0 && + webcamVideo.videoHeight > 0, + ); + if (!hasLiveWebcamFrame && !hasCachedWebcamFrame) { + return; + } + + const margin = webcam.margin ?? 24; + const size = getWebcamOverlaySizePx({ + containerWidth: width, + containerHeight: height, + sizePercent: webcam.size ?? 50, + margin, + zoomScale: this.animationState.appliedScale || 1, + reactToZoom: webcam.reactToZoom ?? true, + }); + const { x, y } = getWebcamOverlayPosition({ + containerWidth: width, + containerHeight: height, + size, + margin, + positionPreset: webcam.positionPreset ?? webcam.corner, + positionX: webcam.positionX ?? 1, + positionY: webcam.positionY ?? 1, + legacyCorner: webcam.corner, + }); + const radius = Math.max(0, webcam.cornerRadius ?? 18); + + const bubbleCanvas = this.webcamBubbleCanvas ?? document.createElement("canvas"); + const bubbleSize = Math.max(1, Math.ceil(size)); + if (bubbleCanvas.width !== bubbleSize || bubbleCanvas.height !== bubbleSize) { + bubbleCanvas.width = bubbleSize; + bubbleCanvas.height = bubbleSize; + } + this.webcamBubbleCanvas = bubbleCanvas; + const bubbleCtx = + this.webcamBubbleCtx ?? configureHighQuality2DContext(bubbleCanvas.getContext("2d")); + if (!bubbleCtx) { + return; + } + this.webcamBubbleCtx = bubbleCtx; + bubbleCtx.clearRect(0, 0, bubbleCanvas.width, bubbleCanvas.height); + bubbleCtx.imageSmoothingEnabled = true; + bubbleCtx.imageSmoothingQuality = "high"; + + const canRefreshCache = + hasLiveWebcamFrame && + this.lastSyncedWebcamTime !== null && + Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime) <= 0.02 && + (webcamDecodedFrame + ? true + : Boolean( + webcamVideo && + !webcamVideo.seeking && + Math.abs(webcamVideo.currentTime - this.currentVideoTime) <= 0.02 && + webcamVideo.videoWidth > 0 && + webcamVideo.videoHeight > 0, + )); + + if (canRefreshCache) { + const liveFrameWidth = webcamDecodedFrame?.displayWidth ?? webcamVideo?.videoWidth ?? 0; + const liveFrameHeight = + webcamDecodedFrame?.displayHeight ?? webcamVideo?.videoHeight ?? 0; + if ( + !this.webcamFrameCacheCanvas || + this.webcamFrameCacheCanvas.width !== liveFrameWidth || + this.webcamFrameCacheCanvas.height !== liveFrameHeight + ) { + this.webcamFrameCacheCanvas = document.createElement("canvas"); + this.webcamFrameCacheCanvas.width = liveFrameWidth; + this.webcamFrameCacheCanvas.height = liveFrameHeight; + this.webcamFrameCacheCtx = configureHighQuality2DContext( + this.webcamFrameCacheCanvas.getContext("2d"), + ); + } + + this.webcamFrameCacheCtx?.clearRect( + 0, + 0, + this.webcamFrameCacheCanvas!.width, + this.webcamFrameCacheCanvas!.height, + ); + this.webcamFrameCacheCtx?.drawImage( + webcamDecodedFrame ?? webcamVideo!, + 0, + 0, + this.webcamFrameCacheCanvas!.width, + this.webcamFrameCacheCanvas!.height, + ); + } + + const webcamFrameSource = + this.webcamFrameCacheCanvas ?? + (hasLiveWebcamFrame ? (webcamDecodedFrame ?? webcamVideo) : null); + if (!webcamFrameSource) { + return; + } + + const sourceWidth = + ("displayWidth" in webcamFrameSource + ? webcamFrameSource.displayWidth + : "videoWidth" in webcamFrameSource + ? webcamFrameSource.videoWidth + : webcamFrameSource.width) || size; + const sourceHeight = + ("displayHeight" in webcamFrameSource + ? webcamFrameSource.displayHeight + : "videoHeight" in webcamFrameSource + ? webcamFrameSource.videoHeight + : webcamFrameSource.height) || size; + const coverScale = Math.max(size / sourceWidth, size / sourceHeight); + const drawWidth = sourceWidth * coverScale; + const drawHeight = sourceHeight * coverScale; + const drawX = (size - drawWidth) / 2; + const drawY = (size - drawHeight) / 2; + + bubbleCtx.save(); + drawSquircleOnCanvas(bubbleCtx, { x: 0, y: 0, width: size, height: size, radius }); + bubbleCtx.clip(); + if (webcam.mirror) { + bubbleCtx.save(); + bubbleCtx.translate(size, 0); + bubbleCtx.scale(-1, 1); + bubbleCtx.drawImage(webcamFrameSource, drawX, drawY, drawWidth, drawHeight); + bubbleCtx.restore(); + } else { + bubbleCtx.drawImage(webcamFrameSource, drawX, drawY, drawWidth, drawHeight); + } + bubbleCtx.restore(); + + if ((webcam.shadow ?? 0) > 0) { + const shadow = Math.max(0, Math.min(1, webcam.shadow)); + ctx.save(); + ctx.filter = `drop-shadow(0 ${Math.round(size * 0.06)}px ${Math.round(size * 0.22)}px rgba(0,0,0,${shadow}))`; + ctx.drawImage(bubbleCanvas, x, y, size, size); + ctx.restore(); + return; + } + + ctx.drawImage(bubbleCanvas, x, y, size, size); + } + + private closeWebcamDecodedFrame(): void { + if (!this.webcamDecodedFrame) { + return; + } + + this.webcamDecodedFrame.close(); + this.webcamDecodedFrame = null; + } + + getCanvas(): HTMLCanvasElement { + if (!this.compositeCanvas) { + throw new Error("Renderer not initialized"); + } + return this.compositeCanvas; + } + + destroy(): void { + if (this.videoSprite) { + const videoTexture = this.videoSprite.texture; + this.videoSprite.destroy({ texture: false, textureSource: false }); + videoTexture?.destroy(true); + this.videoSprite = null; + this.videoTextureSource = null; + } + this.backgroundSprite = null; + if (this.app) { + this.app.destroy(true, { + children: true, + texture: false, + textureSource: false, + }); + this.app = null; + } + this.cameraContainer = null; + this.videoContainer = null; + this.maskGraphics = null; + this.blurFilter = null; + this.motionBlurFilter = null; + if (this.cursorOverlay) { + this.cursorOverlay.destroy(); + this.cursorOverlay = null; + } + this.shadowCanvas = null; + this.shadowCtx = null; + this.compositeCanvas = null; + this.compositeCtx = null; + this.backgroundCtx = null; + if (this.backgroundVideoElement) { + this.backgroundVideoElement.pause(); + this.backgroundVideoElement.src = ""; + this.backgroundVideoElement.load(); + this.backgroundVideoElement = null; + } + this.cleanupBackgroundSource?.(); + this.cleanupBackgroundSource = null; + if (this.webcamVideoElement) { + this.webcamVideoElement.pause(); + this.webcamVideoElement.src = ""; + this.webcamVideoElement.load(); + this.webcamVideoElement = null; + } + this.webcamForwardFrameSource?.cancel(); + void this.webcamForwardFrameSource?.destroy(); + this.webcamForwardFrameSource = null; + this.closeWebcamDecodedFrame(); + this.cleanupWebcamSource?.(); + this.cleanupWebcamSource = null; + this.webcamFrameCacheCanvas = null; + this.webcamFrameCacheCtx = null; + this.webcamBubbleCanvas = null; + this.webcamBubbleCtx = null; + this.lastSyncedWebcamTime = null; + this.frameImage = null; + this.frameInsets = null; + this.frameDraw = null; + } } diff --git a/src/lib/exporter/gifExporter.test.ts b/src/lib/exporter/gifExporter.test.ts index 84ca1bb66..174112e89 100644 --- a/src/lib/exporter/gifExporter.test.ts +++ b/src/lib/exporter/gifExporter.test.ts @@ -1,475 +1,470 @@ -import { describe, it, expect } from 'vitest'; -import * as fc from 'fast-check'; -import { calculateOutputDimensions } from './gifExporter'; -import { GIF_SIZE_PRESETS, GifSizePreset } from './types'; +import * as fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { calculateOutputDimensions, getGifRepeat } from "./gifExporter"; +import { GIF_SIZE_PRESETS, GifSizePreset } from "./types"; /** * Property 2: Loop Encoding Correctness - * - * *For any* GIF export configuration, when loop is enabled the output GIF SHALL - * have a loop count of 0 (infinite), and when loop is disabled the output GIF + * + * *For any* GIF export configuration, when loop is enabled the output GIF SHALL + * have a loop count of 0 (infinite), and when loop is disabled the output GIF * SHALL have a loop count of 1 (play once). - * + * * **Validates: Requirements 3.2, 3.3** - * + * * Feature: gif-export, Property 2: Loop Encoding Correctness */ -describe('GIF Exporter', () => { - describe('Property 2: Loop Encoding Correctness', () => { - /** - * Test the loop configuration mapping logic. - * In gif.js: repeat=0 means infinite loop, repeat=1 means play once (no loop) - */ - it('should map loop=true to repeat=0 (infinite) and loop=false to repeat=1 (once)', () => { - fc.assert( - fc.property( - fc.boolean(), - (loopEnabled: boolean) => { - // This is the logic used in GifExporter constructor - const repeat = loopEnabled ? 0 : 1; - - if (loopEnabled) { - // When loop is enabled, repeat should be 0 (infinite loop) - expect(repeat).toBe(0); - } else { - // When loop is disabled, repeat should be 1 (play once) - expect(repeat).toBe(1); - } - } - ), - { numRuns: 100 } - ); - }); - - it('should always produce valid repeat values (0 or 1)', () => { - fc.assert( - fc.property( - fc.boolean(), - (loopEnabled: boolean) => { - const repeat = loopEnabled ? 0 : 1; - expect([0, 1]).toContain(repeat); - } - ), - { numRuns: 100 } - ); - }); - }); - - /** - * Property 4: Aspect Ratio Preservation - * - * *For any* source video with aspect ratio R and any size preset, the exported - * GIF SHALL have an aspect ratio within 0.01 of R. - * - * **Validates: Requirements 4.4** - * - * Feature: gif-export, Property 4: Aspect Ratio Preservation - */ - describe('Property 4: Aspect Ratio Preservation', () => { - const sizePresets: GifSizePreset[] = ['medium', 'large', 'original']; - - it('should preserve aspect ratio within 0.01 tolerance for all size presets', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 4000 }), // sourceWidth - fc.integer({ min: 100, max: 4000 }), // sourceHeight - fc.constantFrom(...sizePresets), - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const originalAspectRatio = sourceWidth / sourceHeight; - - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - const outputAspectRatio = width / height; - - // Aspect ratio should be preserved within 0.01 tolerance - // (small deviation allowed due to rounding to even numbers) - expect(Math.abs(originalAspectRatio - outputAspectRatio)).toBeLessThan(0.02); - } - ), - { numRuns: 100 } - ); - }); - - it('should return original dimensions when source is smaller than preset max height', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 400 }), // sourceWidth (small) - fc.integer({ min: 100, max: 400 }), // sourceHeight (small, less than 720p) - (sourceWidth: number, sourceHeight: number) => { - // For 'medium' preset with maxHeight 720, if source is smaller, use original - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - 'medium', - GIF_SIZE_PRESETS - ); - - expect(width).toBe(sourceWidth); - expect(height).toBe(sourceHeight); - } - ), - { numRuns: 100 } - ); - }); - - it('should return original dimensions for "original" preset regardless of size', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 4000 }), - fc.integer({ min: 100, max: 4000 }), - (sourceWidth: number, sourceHeight: number) => { - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - 'original', - GIF_SIZE_PRESETS - ); - - expect(width).toBe(sourceWidth); - expect(height).toBe(sourceHeight); - } - ), - { numRuns: 100 } - ); - }); - - it('should scale down to preset max height when source is larger', () => { - fc.assert( - fc.property( - fc.integer({ min: 1000, max: 4000 }), // sourceWidth (large) - fc.integer({ min: 800, max: 2000 }), // sourceHeight (larger than 720p) - (sourceWidth: number, sourceHeight: number) => { - // For 'medium' preset with maxHeight 720 - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - 'medium', - GIF_SIZE_PRESETS - ); - - // Height should be at most 720 (or 722 due to even rounding) - expect(height).toBeLessThanOrEqual(722); - // Width should be scaled proportionally - expect(width).toBeLessThan(sourceWidth); - } - ), - { numRuns: 100 } - ); - }); - }); -}); +describe("GIF Exporter", () => { + describe("Property 2: Loop Encoding Correctness", () => { + /** + * Test the loop configuration mapping logic. + * In gif.js: repeat=0 means infinite loop, repeat=1 means play once (no loop) + */ + it("should map loop=true to repeat=0 (infinite) and loop=false to repeat=1 (once)", () => { + fc.assert( + fc.property(fc.boolean(), (loopEnabled: boolean) => { + const repeat = getGifRepeat(loopEnabled); + + if (loopEnabled) { + // When loop is enabled, repeat should be 0 (infinite loop) + expect(repeat).toBe(0); + } else { + // When loop is disabled, repeat should be 1 (play once) + expect(repeat).toBe(1); + } + }), + { numRuns: 100 }, + ); + }); + + it("should always produce valid repeat values (0 or 1)", () => { + fc.assert( + fc.property(fc.boolean(), (loopEnabled: boolean) => { + const repeat = getGifRepeat(loopEnabled); + expect([0, 1]).toContain(repeat); + }), + { numRuns: 100 }, + ); + }); + }); + + /** + * Property 4: Aspect Ratio Preservation + * + * *For any* source video with aspect ratio R and any size preset, the exported + * GIF SHALL have an aspect ratio within 0.01 of R. + * + * **Validates: Requirements 4.4** + * + * Feature: gif-export, Property 4: Aspect Ratio Preservation + */ + describe("Property 4: Aspect Ratio Preservation", () => { + const sizePresets: GifSizePreset[] = ["medium", "large", "original"]; + + it("should preserve aspect ratio within 0.01 tolerance for all size presets", () => { + fc.assert( + fc.property( + fc.integer({ min: 100, max: 4000 }), // sourceWidth + fc.integer({ min: 100, max: 4000 }), // sourceHeight + fc.constantFrom(...sizePresets), + (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { + const originalAspectRatio = sourceWidth / sourceHeight; + + const { width, height } = calculateOutputDimensions( + sourceWidth, + sourceHeight, + sizePreset, + GIF_SIZE_PRESETS, + ); + + const outputAspectRatio = width / height; + + // Aspect ratio should be preserved within 0.01 tolerance + // (small deviation allowed due to rounding to even numbers) + expect(Math.abs(originalAspectRatio - outputAspectRatio)).toBeLessThan( + 0.01, + ); + }, + ), + { numRuns: 100 }, + ); + }); + + it("should return original dimensions when source is smaller than preset max height", () => { + fc.assert( + fc.property( + fc.integer({ min: 100, max: 400 }), // sourceWidth (small) + fc.integer({ min: 100, max: 400 }), // sourceHeight (small, less than 720p) + (sourceWidth: number, sourceHeight: number) => { + // For 'medium' preset with maxHeight 720, if source is smaller, use original + const { width, height } = calculateOutputDimensions( + sourceWidth, + sourceHeight, + "medium", + GIF_SIZE_PRESETS, + ); + + expect(width).toBe(sourceWidth); + expect(height).toBe(sourceHeight); + }, + ), + { numRuns: 100 }, + ); + }); + + it('should return original dimensions for "original" preset regardless of size', () => { + fc.assert( + fc.property( + fc.integer({ min: 100, max: 4000 }), + fc.integer({ min: 100, max: 4000 }), + (sourceWidth: number, sourceHeight: number) => { + const { width, height } = calculateOutputDimensions( + sourceWidth, + sourceHeight, + "original", + GIF_SIZE_PRESETS, + ); + expect(width).toBe(sourceWidth); + expect(height).toBe(sourceHeight); + }, + ), + { numRuns: 100 }, + ); + }); + + it("should scale down to preset max height when source is larger", () => { + fc.assert( + fc.property( + fc.integer({ min: 1000, max: 4000 }), // sourceWidth (large) + fc.integer({ min: 800, max: 2000 }), // sourceHeight (larger than 720p) + (sourceWidth: number, sourceHeight: number) => { + // For 'medium' preset with maxHeight 720 + const { width, height } = calculateOutputDimensions( + sourceWidth, + sourceHeight, + "medium", + GIF_SIZE_PRESETS, + ); + + // Height should be at most 720 (or 722 due to even rounding) + expect(height).toBeLessThanOrEqual(722); + // Width should be scaled proportionally + expect(width).toBeLessThan(sourceWidth); + }, + ), + { numRuns: 100 }, + ); + }); + }); +}); /** * Property 3: Size Preset Resolution Mapping - * - * *For any* valid size preset and source video dimensions, the GIF_Exporter SHALL - * produce output with height matching the preset's max height (or source height if smaller), + * + * *For any* valid size preset and source video dimensions, the GIF_Exporter SHALL + * produce output with height matching the preset's max height (or source height if smaller), * with width calculated to maintain aspect ratio. - * + * * **Validates: Requirements 4.2** - * + * * Feature: gif-export, Property 3: Size Preset Resolution Mapping */ -describe('Property 3: Size Preset Resolution Mapping', () => { - it('should map size presets to correct max heights', () => { - fc.assert( - fc.property( - fc.integer({ min: 800, max: 4000 }), // sourceWidth (large enough to trigger scaling) - fc.integer({ min: 800, max: 2000 }), // sourceHeight (larger than all presets except original) - fc.constantFrom('medium', 'large') as fc.Arbitrary, - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const { height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - const expectedMaxHeight = GIF_SIZE_PRESETS[sizePreset].maxHeight; - - // Height should be at or below the preset's max height - // (allowing +2 for even number rounding) - expect(height).toBeLessThanOrEqual(expectedMaxHeight + 2); - } - ), - { numRuns: 100 } - ); - }); - - it('should use source dimensions when smaller than preset', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 400 }), // sourceWidth - fc.integer({ min: 100, max: 400 }), // sourceHeight (smaller than 720p 'medium' preset) - fc.constantFrom('medium', 'large', 'original') as fc.Arbitrary, - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - // When source is smaller than preset, use original dimensions - expect(width).toBe(sourceWidth); - expect(height).toBe(sourceHeight); - } - ), - { numRuns: 100 } - ); - }); - - it('should produce even dimensions for encoder compatibility', () => { - fc.assert( - fc.property( - fc.integer({ min: 100, max: 4000 }), - fc.integer({ min: 100, max: 4000 }), - fc.constantFrom('medium', 'large', 'original') as fc.Arbitrary, - (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { - const { width, height } = calculateOutputDimensions( - sourceWidth, - sourceHeight, - sizePreset, - GIF_SIZE_PRESETS - ); - - // When scaling occurs, dimensions should be even - // (original dimensions are passed through as-is) - if (sourceHeight > GIF_SIZE_PRESETS[sizePreset].maxHeight && sizePreset !== 'original') { - expect(width % 2).toBe(0); - expect(height % 2).toBe(0); - } - } - ), - { numRuns: 100 } - ); - }); +describe("Property 3: Size Preset Resolution Mapping", () => { + it("should map size presets to correct max heights", () => { + fc.assert( + fc.property( + fc.integer({ min: 800, max: 4000 }), // sourceWidth (large enough to trigger scaling) + fc.integer({ min: 800, max: 2000 }), // sourceHeight (larger than all presets except original) + fc.constantFrom("medium", "large") as fc.Arbitrary, + (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { + const { height } = calculateOutputDimensions( + sourceWidth, + sourceHeight, + sizePreset, + GIF_SIZE_PRESETS, + ); + + const expectedMaxHeight = GIF_SIZE_PRESETS[sizePreset].maxHeight; + + // Height should be at or below the preset's max height + // (allowing +2 for even number rounding) + expect(height).toBeLessThanOrEqual(expectedMaxHeight + 2); + }, + ), + { numRuns: 100 }, + ); + }); + + it("should use source dimensions when smaller than preset", () => { + fc.assert( + fc.property( + fc.integer({ min: 100, max: 400 }), // sourceWidth + fc.integer({ min: 100, max: 400 }), // sourceHeight (smaller than 720p 'medium' preset) + fc.constantFrom("medium", "large", "original") as fc.Arbitrary, + (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { + const { width, height } = calculateOutputDimensions( + sourceWidth, + sourceHeight, + sizePreset, + GIF_SIZE_PRESETS, + ); + + // When source is smaller than preset, use original dimensions + expect(width).toBe(sourceWidth); + expect(height).toBe(sourceHeight); + }, + ), + { numRuns: 100 }, + ); + }); + + it("should produce even dimensions for encoder compatibility", () => { + fc.assert( + fc.property( + fc.integer({ min: 100, max: 4000 }), + fc.integer({ min: 100, max: 4000 }), + fc.constantFrom("medium", "large", "original") as fc.Arbitrary, + (sourceWidth: number, sourceHeight: number, sizePreset: GifSizePreset) => { + const { width, height } = calculateOutputDimensions( + sourceWidth, + sourceHeight, + sizePreset, + GIF_SIZE_PRESETS, + ); + + // When scaling occurs, dimensions should be even + // (original dimensions are passed through as-is) + if ( + sourceHeight > GIF_SIZE_PRESETS[sizePreset].maxHeight && + sizePreset !== "original" + ) { + expect(width % 2).toBe(0); + expect(height % 2).toBe(0); + } + }, + ), + { numRuns: 100 }, + ); + }); }); /** * Property 6: Frame Count Consistency - * - * *For any* video with effective duration D (excluding trim regions) and frame rate F, + * + * *For any* video with effective duration D (excluding trim regions) and frame rate F, * the exported GIF SHALL contain approximately D × F frames (within ±1 frame tolerance). - * + * * **Validates: Requirements 5.1** - * + * * Feature: gif-export, Property 6: Frame Count Consistency */ -describe('Property 6: Frame Count Consistency', () => { - // Helper function to calculate expected frame count - const calculateExpectedFrameCount = (durationSeconds: number, frameRate: number): number => { - return Math.ceil(durationSeconds * frameRate); - }; - - it('should calculate correct frame count for duration and frame rate', () => { - fc.assert( - fc.property( - fc.float({ min: 0.5, max: 60, noNaN: true }), // duration in seconds - fc.constantFrom(10, 15, 20, 25, 30), // valid frame rates - (duration: number, frameRate: number) => { - const expectedFrames = calculateExpectedFrameCount(duration, frameRate); - - // Frame count should be positive - expect(expectedFrames).toBeGreaterThan(0); - - // Frame count should be approximately duration * frameRate - const approximateFrames = duration * frameRate; - expect(Math.abs(expectedFrames - approximateFrames)).toBeLessThanOrEqual(1); - } - ), - { numRuns: 100 } - ); - }); - - it('should produce more frames with higher frame rates', () => { - fc.assert( - fc.property( - fc.float({ min: 1, max: 30, noNaN: true }), // duration in seconds - (duration: number) => { - const frames10fps = calculateExpectedFrameCount(duration, 10); - const frames30fps = calculateExpectedFrameCount(duration, 30); - - // 30fps should produce approximately 3x more frames than 10fps - expect(frames30fps).toBeGreaterThan(frames10fps); - expect(frames30fps / frames10fps).toBeCloseTo(3, 0); - } - ), - { numRuns: 100 } - ); - }); - - it('should handle trim regions by reducing effective duration', () => { - fc.assert( - fc.property( - fc.float({ min: 5, max: 60, noNaN: true }), // total duration - fc.float({ min: 0.5, max: 2, noNaN: true }), // trim duration (smaller than total) - fc.constantFrom(10, 15, 20, 25, 30), - (totalDuration: number, trimDuration: number, frameRate: number) => { - const effectiveDuration = totalDuration - trimDuration; - const framesWithTrim = calculateExpectedFrameCount(effectiveDuration, frameRate); - const framesWithoutTrim = calculateExpectedFrameCount(totalDuration, frameRate); - - // Trimmed video should have fewer frames - expect(framesWithTrim).toBeLessThan(framesWithoutTrim); - } - ), - { numRuns: 100 } - ); - }); -}); +describe("Property 6: Frame Count Consistency", () => { + // Helper function to calculate expected frame count + const calculateExpectedFrameCount = (durationSeconds: number, frameRate: number): number => { + return Math.ceil(durationSeconds * frameRate); + }; + + it("should calculate correct frame count for duration and frame rate", () => { + fc.assert( + fc.property( + fc.float({ min: 0.5, max: 60, noNaN: true }), // duration in seconds + fc.constantFrom(10, 15, 20, 25, 30), // valid frame rates + (duration: number, frameRate: number) => { + const expectedFrames = calculateExpectedFrameCount(duration, frameRate); + // Frame count should be positive + expect(expectedFrames).toBeGreaterThan(0); + + // Frame count should be approximately duration * frameRate + const approximateFrames = duration * frameRate; + expect(Math.abs(expectedFrames - approximateFrames)).toBeLessThanOrEqual(1); + }, + ), + { numRuns: 100 }, + ); + }); + + it("should produce more frames with higher frame rates", () => { + fc.assert( + fc.property( + fc.float({ min: 1, max: 30, noNaN: true }), // duration in seconds + (duration: number) => { + const frames10fps = calculateExpectedFrameCount(duration, 10); + const frames30fps = calculateExpectedFrameCount(duration, 30); + + // 30fps should produce approximately 3x more frames than 10fps + expect(frames30fps).toBeGreaterThan(frames10fps); + expect(frames30fps / frames10fps).toBeCloseTo(3, 0); + }, + ), + { numRuns: 100 }, + ); + }); + + it("should handle trim regions by reducing effective duration", () => { + fc.assert( + fc.property( + fc.float({ min: 5, max: 60, noNaN: true }), // total duration + fc.float({ min: 0.5, max: 2, noNaN: true }), // trim duration (smaller than total) + fc.constantFrom(10, 15, 20, 25, 30), + (totalDuration: number, trimDuration: number, frameRate: number) => { + const effectiveDuration = totalDuration - trimDuration; + const framesWithTrim = calculateExpectedFrameCount( + effectiveDuration, + frameRate, + ); + const framesWithoutTrim = calculateExpectedFrameCount(totalDuration, frameRate); + + // Trimmed video should have fewer frames + expect(framesWithTrim).toBeLessThan(framesWithoutTrim); + }, + ), + { numRuns: 100 }, + ); + }); +}); /** * Property 5: Valid GIF Output (Configuration Validation) - * + * * *For any* successful GIF export, the output blob SHALL be a valid GIF file. * This test validates the GIF configuration parameters are correctly set up. - * + * * **Validates: Requirements 5.3** - * + * * Feature: gif-export, Property 5: Valid GIF Output - * + * * Note: Full GIF encoding validation requires browser environment with video. * This test validates configuration correctness. */ -describe('Property 5: Valid GIF Output (Configuration)', () => { - it('should generate valid GIF configuration for all frame rates', () => { - fc.assert( - fc.property( - fc.constantFrom(10, 15, 20, 25, 30), - fc.integer({ min: 100, max: 1920 }), - fc.integer({ min: 100, max: 1080 }), - fc.boolean(), - (frameRate: number, width: number, height: number, loop: boolean) => { - // Validate frame delay calculation (gif.js uses milliseconds) - const frameDelay = Math.round(1000 / frameRate); - - // Frame delay should be positive and reasonable - expect(frameDelay).toBeGreaterThan(0); - expect(frameDelay).toBeLessThanOrEqual(100); // 10fps = 100ms delay - - // Loop configuration - const repeat = loop ? 0 : 1; - expect([0, 1]).toContain(repeat); - - // Dimensions should be positive - expect(width).toBeGreaterThan(0); - expect(height).toBeGreaterThan(0); - } - ), - { numRuns: 100 } - ); - }); - - it('should calculate correct frame delays for each frame rate', () => { - const expectedDelays: Record = { - 10: 100, // 1000ms / 10fps = 100ms - 15: 67, // 1000ms / 15fps ≈ 67ms - 20: 50, // 1000ms / 20fps = 50ms - 25: 40, // 1000ms / 25fps = 40ms - 30: 33, // 1000ms / 30fps ≈ 33ms - }; - - for (const [fps, expectedDelay] of Object.entries(expectedDelays)) { - const frameRate = Number(fps); - const actualDelay = Math.round(1000 / frameRate); - expect(actualDelay).toBe(expectedDelay); - } - }); +describe("Property 5: Valid GIF Output (Configuration)", () => { + it("should generate valid GIF configuration for all frame rates", () => { + fc.assert( + fc.property( + fc.constantFrom(10, 15, 20, 25, 30), + fc.integer({ min: 100, max: 1920 }), + fc.integer({ min: 100, max: 1080 }), + fc.boolean(), + (frameRate: number, width: number, height: number, loop: boolean) => { + // Validate frame delay calculation (gif.js uses milliseconds) + const frameDelay = Math.round(1000 / frameRate); + + // Frame delay should be positive and reasonable + expect(frameDelay).toBeGreaterThan(0); + expect(frameDelay).toBeLessThanOrEqual(100); // 10fps = 100ms delay + + // Loop configuration + const repeat = loop ? 0 : 1; + expect([0, 1]).toContain(repeat); + + // Dimensions should be positive + expect(width).toBeGreaterThan(0); + expect(height).toBeGreaterThan(0); + }, + ), + { numRuns: 100 }, + ); + }); + + it("should calculate correct frame delays for each frame rate", () => { + const expectedDelays: Record = { + 10: 100, // 1000ms / 10fps = 100ms + 15: 67, // 1000ms / 15fps ≈ 67ms + 20: 50, // 1000ms / 20fps = 50ms + 25: 40, // 1000ms / 25fps = 40ms + 30: 33, // 1000ms / 30fps ≈ 33ms + }; + + for (const [fps, expectedDelay] of Object.entries(expectedDelays)) { + const frameRate = Number(fps); + const actualDelay = Math.round(1000 / frameRate); + expect(actualDelay).toBe(expectedDelay); + } + }); }); /** * Property 7: MP4 Export Regression - * - * *For any* valid MP4 export configuration that worked before this feature, + * + * *For any* valid MP4 export configuration that worked before this feature, * the Video_Exporter SHALL continue to produce valid MP4 output. - * + * * **Validates: Requirements 7.2** - * + * * Feature: gif-export, Property 7: MP4 Export Regression - * + * * Note: This test validates that MP4 export configuration remains unchanged. */ -describe('Property 7: MP4 Export Regression', () => { - it('should maintain valid MP4 quality presets', () => { - const qualityPresets = ['medium', 'good', 'source']; - - fc.assert( - fc.property( - fc.constantFrom(...qualityPresets), - (quality: string) => { - // Quality presets should be valid - expect(['medium', 'good', 'source']).toContain(quality); - } - ), - { numRuns: 100 } - ); - }); - - it('should calculate valid MP4 export dimensions', () => { - fc.assert( - fc.property( - fc.integer({ min: 640, max: 3840 }), // sourceWidth - fc.integer({ min: 480, max: 2160 }), // sourceHeight - fc.constantFrom('medium', 'good', 'source'), - (sourceWidth: number, sourceHeight: number, quality: string) => { - let exportWidth: number; - let exportHeight: number; - const aspectRatio = sourceWidth / sourceHeight; - - if (quality === 'source') { - // Source quality uses original dimensions (may be odd) - exportWidth = sourceWidth; - exportHeight = sourceHeight; - - // Dimensions should be positive - expect(exportWidth).toBeGreaterThan(0); - expect(exportHeight).toBeGreaterThan(0); - } else { - const targetHeight = quality === 'medium' ? 720 : 1080; - exportHeight = Math.floor(targetHeight / 2) * 2; - exportWidth = Math.floor((exportHeight * aspectRatio) / 2) * 2; - - // Dimensions should be positive and even for non-source quality - expect(exportWidth).toBeGreaterThan(0); - expect(exportHeight).toBeGreaterThan(0); - expect(exportWidth % 2).toBe(0); - expect(exportHeight % 2).toBe(0); - } - } - ), - { numRuns: 100 } - ); - }); - - it('should maintain aspect ratio in MP4 export', () => { - fc.assert( - fc.property( - fc.integer({ min: 640, max: 3840 }), - fc.integer({ min: 480, max: 2160 }), - fc.constantFrom('medium', 'good'), - (sourceWidth: number, sourceHeight: number, quality: string) => { - const originalAspectRatio = sourceWidth / sourceHeight; - const targetHeight = quality === 'medium' ? 720 : 1080; - - const exportHeight = Math.floor(targetHeight / 2) * 2; - const exportWidth = Math.floor((exportHeight * originalAspectRatio) / 2) * 2; - - const exportAspectRatio = exportWidth / exportHeight; - - // Aspect ratio should be preserved within tolerance (due to even rounding) - expect(Math.abs(originalAspectRatio - exportAspectRatio)).toBeLessThan(0.05); - } - ), - { numRuns: 100 } - ); - }); -}); +describe("Property 7: MP4 Export Regression", () => { + it("should maintain valid MP4 quality presets", () => { + const qualityPresets = ["medium", "good", "source"]; + + fc.assert( + fc.property(fc.constantFrom(...qualityPresets), (quality: string) => { + // Quality presets should be valid + expect(["medium", "good", "source"]).toContain(quality); + }), + { numRuns: 100 }, + ); + }); + + it("should calculate valid MP4 export dimensions", () => { + fc.assert( + fc.property( + fc.integer({ min: 640, max: 3840 }), // sourceWidth + fc.integer({ min: 480, max: 2160 }), // sourceHeight + fc.constantFrom("medium", "good", "source"), + (sourceWidth: number, sourceHeight: number, quality: string) => { + let exportWidth: number; + let exportHeight: number; + const aspectRatio = sourceWidth / sourceHeight; + + if (quality === "source") { + // Source quality uses original dimensions (may be odd) + exportWidth = sourceWidth; + exportHeight = sourceHeight; + + // Dimensions should be positive + expect(exportWidth).toBeGreaterThan(0); + expect(exportHeight).toBeGreaterThan(0); + } else { + const targetHeight = quality === "medium" ? 720 : 1080; + exportHeight = Math.floor(targetHeight / 2) * 2; + exportWidth = Math.floor((exportHeight * aspectRatio) / 2) * 2; + // Dimensions should be positive and even for non-source quality + expect(exportWidth).toBeGreaterThan(0); + expect(exportHeight).toBeGreaterThan(0); + expect(exportWidth % 2).toBe(0); + expect(exportHeight % 2).toBe(0); + } + }, + ), + { numRuns: 100 }, + ); + }); + + it("should maintain aspect ratio in MP4 export", () => { + fc.assert( + fc.property( + fc.integer({ min: 640, max: 3840 }), + fc.integer({ min: 480, max: 2160 }), + fc.constantFrom("medium", "good"), + (sourceWidth: number, sourceHeight: number, quality: string) => { + const originalAspectRatio = sourceWidth / sourceHeight; + const targetHeight = quality === "medium" ? 720 : 1080; + + const exportHeight = Math.floor(targetHeight / 2) * 2; + const exportWidth = Math.floor((exportHeight * originalAspectRatio) / 2) * 2; + + const exportAspectRatio = exportWidth / exportHeight; + + // Aspect ratio should be preserved within tolerance (due to even rounding) + expect(Math.abs(originalAspectRatio - exportAspectRatio)).toBeLessThan(0.05); + }, + ), + { numRuns: 100 }, + ); + }); +}); diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index dfd68e2db..789e9dff3 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -1,84 +1,81 @@ import GIF from "gif.js"; import type { - ExportProgress, - ExportResult, - GifFrameRate, - GifSizePreset, - GIF_SIZE_PRESETS, -} from "./types"; -import { StreamingVideoDecoder } from "./streamingDecoder"; + AnnotationRegion, + AutoCaptionSettings, + CaptionCue, + CropRegion, + CursorStyle, + CursorTelemetryPoint, + SpeedRegion, + TrimRegion, + WebcamOverlaySettings, + ZoomRegion, + ZoomTransitionEasing, +} from "@/components/video-editor/types"; import { FrameRenderer } from "./frameRenderer"; +import { StreamingVideoDecoder } from "./streamingDecoder"; import type { - AutoCaptionSettings, - ZoomRegion, - CropRegion, - TrimRegion, - AnnotationRegion, - CaptionCue, - SpeedRegion, - CursorStyle, - CursorTelemetryPoint, - WebcamOverlaySettings, - ZoomTransitionEasing, -} from "@/components/video-editor/types"; + ExportProgress, + ExportResult, + GIF_SIZE_PRESETS, + GifFrameRate, + GifSizePreset, +} from "./types"; -const GIF_WORKER_URL = new URL( - "gif.js/dist/gif.worker.js", - import.meta.url, -).toString(); +const GIF_WORKER_URL = new URL("gif.js/dist/gif.worker.js", import.meta.url).toString(); const PROGRESS_SAMPLE_WINDOW_MS = 1_000; interface GifExporterConfig { - videoUrl: string; - width: number; - height: number; - frameRate: GifFrameRate; - loop: boolean; - sizePreset: GifSizePreset; - wallpaper: string; - zoomRegions: ZoomRegion[]; - trimRegions?: TrimRegion[]; - speedRegions?: SpeedRegion[]; - showShadow: boolean; - shadowIntensity: number; - backgroundBlur: number; - zoomMotionBlur?: number; - connectZooms?: boolean; - zoomInDurationMs?: number; - zoomInOverlapMs?: number; - zoomOutDurationMs?: number; - connectedZoomGapMs?: number; - connectedZoomDurationMs?: number; - zoomInEasing?: ZoomTransitionEasing; - zoomOutEasing?: ZoomTransitionEasing; - connectedZoomEasing?: ZoomTransitionEasing; - borderRadius?: number; - padding?: number; - videoPadding?: number; - cropRegion: CropRegion; - webcam?: WebcamOverlaySettings; - webcamUrl?: string | null; - annotationRegions?: AnnotationRegion[]; - autoCaptions?: CaptionCue[]; - autoCaptionSettings?: AutoCaptionSettings; - cursorTelemetry?: CursorTelemetryPoint[]; - showCursor?: boolean; - cursorStyle?: CursorStyle; - cursorSize?: number; - cursorSmoothing?: number; - zoomSmoothness?: number; - zoomClassicMode?: boolean; - cursorMotionBlur?: number; - cursorClickBounce?: number; - cursorClickBounceDuration?: number; - cursorSway?: number; - frame?: string | null; - previewWidth?: number; - previewHeight?: number; - maxDecodeQueue?: number; - maxPendingFrames?: number; - onProgress?: (progress: ExportProgress) => void; + videoUrl: string; + width: number; + height: number; + frameRate: GifFrameRate; + loop: boolean; + sizePreset: GifSizePreset; + wallpaper: string; + zoomRegions: ZoomRegion[]; + trimRegions?: TrimRegion[]; + speedRegions?: SpeedRegion[]; + showShadow: boolean; + shadowIntensity: number; + backgroundBlur: number; + zoomMotionBlur?: number; + connectZooms?: boolean; + zoomInDurationMs?: number; + zoomInOverlapMs?: number; + zoomOutDurationMs?: number; + connectedZoomGapMs?: number; + connectedZoomDurationMs?: number; + zoomInEasing?: ZoomTransitionEasing; + zoomOutEasing?: ZoomTransitionEasing; + connectedZoomEasing?: ZoomTransitionEasing; + borderRadius?: number; + padding?: number; + videoPadding?: number; + cropRegion: CropRegion; + webcam?: WebcamOverlaySettings; + webcamUrl?: string | null; + annotationRegions?: AnnotationRegion[]; + autoCaptions?: CaptionCue[]; + autoCaptionSettings?: AutoCaptionSettings; + cursorTelemetry?: CursorTelemetryPoint[]; + showCursor?: boolean; + cursorStyle?: CursorStyle; + cursorSize?: number; + cursorSmoothing?: number; + zoomSmoothness?: number; + zoomClassicMode?: boolean; + cursorMotionBlur?: number; + cursorClickBounce?: number; + cursorClickBounceDuration?: number; + cursorSway?: number; + frame?: string | null; + previewWidth?: number; + previewHeight?: number; + maxDecodeQueue?: number; + maxPendingFrames?: number; + onProgress?: (progress: ExportProgress) => void; } /** @@ -90,298 +87,297 @@ interface GifExporterConfig { * @returns The calculated output dimensions */ export function calculateOutputDimensions( - sourceWidth: number, - sourceHeight: number, - sizePreset: GifSizePreset, - sizePresets: typeof GIF_SIZE_PRESETS, + sourceWidth: number, + sourceHeight: number, + sizePreset: GifSizePreset, + sizePresets: typeof GIF_SIZE_PRESETS, ): { width: number; height: number } { - const preset = sizePresets[sizePreset]; - const maxHeight = preset.maxHeight; - - // If original is smaller than max height or preset is 'original', use source dimensions - if (sourceHeight <= maxHeight || sizePreset === "original") { - return { width: sourceWidth, height: sourceHeight }; - } - - // Calculate scaled dimensions preserving aspect ratio - const aspectRatio = sourceWidth / sourceHeight; - const newHeight = maxHeight; - const newWidth = Math.round(newHeight * aspectRatio); - - // Ensure dimensions are even (required for some encoders) - return { - width: newWidth % 2 === 0 ? newWidth : newWidth + 1, - height: newHeight % 2 === 0 ? newHeight : newHeight + 1, - }; + const preset = sizePresets[sizePreset]; + const maxHeight = preset.maxHeight; + + // If original is smaller than max height or preset is 'original', use source dimensions + if (sourceHeight <= maxHeight || sizePreset === "original") { + return { width: sourceWidth, height: sourceHeight }; + } + + // Calculate scaled dimensions preserving aspect ratio + const aspectRatio = sourceWidth / sourceHeight; + const newHeight = maxHeight; + const newWidth = Math.round(newHeight * aspectRatio); + + // Ensure dimensions are even (required for some encoders) + return { + width: newWidth % 2 === 0 ? newWidth : newWidth + 1, + height: newHeight % 2 === 0 ? newHeight : newHeight + 1, + }; +} + +export function getGifRepeat(loop: boolean): 0 | 1 { + return loop ? 0 : 1; } export class GifExporter { - private config: GifExporterConfig; - private streamingDecoder: StreamingVideoDecoder | null = null; - private renderer: FrameRenderer | null = null; - private gif: GIF | null = null; - private cancelled = false; - private exportStartTimeMs = 0; - private progressSampleStartTimeMs = 0; - private progressSampleStartFrame = 0; - private lastRenderFps: number | undefined; - - constructor(config: GifExporterConfig) { - this.config = config; - } - - async export(): Promise { - try { - this.cleanup(); - this.cancelled = false; - this.exportStartTimeMs = this.getNowMs(); - this.progressSampleStartTimeMs = this.exportStartTimeMs; - this.progressSampleStartFrame = 0; - this.lastRenderFps = undefined; - - // Initialize streaming decoder and load video metadata - this.streamingDecoder = new StreamingVideoDecoder({ - maxDecodeQueue: this.config.maxDecodeQueue, - maxPendingFrames: this.config.maxPendingFrames, - }); - const videoInfo = await this.streamingDecoder.loadMetadata( - this.config.videoUrl, - ); - - // Initialize frame renderer - this.renderer = new FrameRenderer({ - width: this.config.width, - height: this.config.height, - wallpaper: this.config.wallpaper, - zoomRegions: this.config.zoomRegions, - showShadow: this.config.showShadow, - shadowIntensity: this.config.shadowIntensity, - backgroundBlur: this.config.backgroundBlur, - zoomMotionBlur: this.config.zoomMotionBlur, - connectZooms: this.config.connectZooms, - zoomInDurationMs: this.config.zoomInDurationMs, - zoomInOverlapMs: this.config.zoomInOverlapMs, - zoomOutDurationMs: this.config.zoomOutDurationMs, - connectedZoomGapMs: this.config.connectedZoomGapMs, - connectedZoomDurationMs: this.config.connectedZoomDurationMs, - zoomInEasing: this.config.zoomInEasing, - zoomOutEasing: this.config.zoomOutEasing, - connectedZoomEasing: this.config.connectedZoomEasing, - borderRadius: this.config.borderRadius, - padding: this.config.padding, - cropRegion: this.config.cropRegion, - webcam: this.config.webcam, - webcamUrl: this.config.webcamUrl, - videoWidth: videoInfo.width, - videoHeight: videoInfo.height, - annotationRegions: this.config.annotationRegions, - autoCaptions: this.config.autoCaptions, - autoCaptionSettings: this.config.autoCaptionSettings, - speedRegions: this.config.speedRegions, - previewWidth: this.config.previewWidth, - previewHeight: this.config.previewHeight, - cursorTelemetry: this.config.cursorTelemetry, - showCursor: this.config.showCursor, - cursorStyle: this.config.cursorStyle, - cursorSize: this.config.cursorSize, - cursorSmoothing: this.config.cursorSmoothing, - zoomSmoothness: this.config.zoomSmoothness, - zoomClassicMode: this.config.zoomClassicMode, - cursorMotionBlur: this.config.cursorMotionBlur, - cursorClickBounce: this.config.cursorClickBounce, - cursorClickBounceDuration: this.config.cursorClickBounceDuration, - cursorSway: this.config.cursorSway, - frame: this.config.frame, - }); - await this.renderer.initialize(); - - // Initialize GIF encoder - // Loop: 0 = infinite loop, 1 = play once (no loop) - const repeat = this.config.loop ? 0 : 1; - const cores = navigator.hardwareConcurrency || 4; - const WORKER_COUNT = Math.max(1, Math.min(8, cores - 1)); - - this.gif = new GIF({ - workers: WORKER_COUNT, - quality: 10, - width: this.config.width, - height: this.config.height, - workerScript: GIF_WORKER_URL, - repeat, - background: "#000000", - transparent: null, - dither: "FloydSteinberg", - }); - - // Calculate effective duration and frame count (excluding trim regions) - const effectiveDuration = this.streamingDecoder.getEffectiveDuration( - this.config.trimRegions, - this.config.speedRegions, - ); - const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); - - // Calculate frame delay in milliseconds (gif.js uses ms) - const frameDelay = Math.round(1000 / this.config.frameRate); - - console.log("[GifExporter] Original duration:", videoInfo.duration, "s"); - console.log("[GifExporter] Effective duration:", effectiveDuration, "s"); - console.log("[GifExporter] Total frames to export:", totalFrames); - console.log("[GifExporter] Frame rate:", this.config.frameRate, "FPS"); - console.log("[GifExporter] Frame delay:", frameDelay, "ms"); - console.log( - "[GifExporter] Loop:", - this.config.loop ? "infinite" : "once", - ); - console.log( - "[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)", - ); - - let frameIndex = 0; - - // Stream decode and process frames — no seeking! - await this.streamingDecoder.decodeAll( - this.config.frameRate, - this.config.trimRegions, - this.config.speedRegions, - async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => { - if (this.cancelled) { - videoFrame.close(); - return; - } - - const sourceTimestampUs = sourceTimestampMs * 1000; - const cursorTimestampUs = cursorTimestampMs * 1000; - await this.renderer!.renderFrame(videoFrame, sourceTimestampUs, cursorTimestampUs); - videoFrame.close(); - - this.addRenderedGifFrame(frameDelay); - frameIndex++; - this.reportProgress(frameIndex, totalFrames); - }, - ); - - if (this.cancelled) { - return { success: false, error: "Export cancelled" }; - } - - // Update progress to show we're now in the finalizing phase - if (this.config.onProgress) { - this.config.onProgress({ - currentFrame: totalFrames, - totalFrames, - percentage: 100, - estimatedTimeRemaining: 0, - phase: "finalizing", - renderFps: this.lastRenderFps, - }); - } - - // Render the GIF - const blob = await new Promise((resolve, _reject) => { - this.gif!.on("finished", (blob: Blob) => { - resolve(blob); - }); - - // Track rendering progress - this.gif!.on("progress", (progress: number) => { - if (this.config.onProgress) { - this.config.onProgress({ - currentFrame: totalFrames, - totalFrames, - percentage: 100, - estimatedTimeRemaining: 0, - phase: "finalizing", - renderFps: this.lastRenderFps, - renderProgress: Math.round(progress * 100), - }); - } - }); - - // gif.js doesn't have a typed 'error' event, but we can catch errors in the try/catch - this.gif!.render(); - }); - - return { success: true, blob }; - } catch (error) { - console.error("GIF Export error:", error); - return { - success: false, - error: error instanceof Error ? error.message : String(error), - }; - } finally { - this.cleanup(); - } - } - - private addRenderedGifFrame(frameDelay: number) { - const canvas = this.renderer!.getCanvas(); - this.gif!.addFrame(canvas, { delay: frameDelay, copy: true }); - } - - private reportProgress(currentFrame: number, totalFrames: number) { - const nowMs = this.getNowMs(); - const elapsedSeconds = Math.max( - (nowMs - this.exportStartTimeMs) / 1000, - 0.001, - ); - const averageRenderFps = currentFrame / elapsedSeconds; - const sampleElapsedMs = Math.max(nowMs - this.progressSampleStartTimeMs, 1); - const sampleFrameDelta = Math.max(currentFrame - this.progressSampleStartFrame, 0); - const renderFps = (sampleFrameDelta * 1000) / sampleElapsedMs; - const remainingFrames = Math.max(totalFrames - currentFrame, 0); - const estimatedTimeRemaining = averageRenderFps > 0 ? remainingFrames / averageRenderFps : 0; - this.lastRenderFps = renderFps; - - if (sampleElapsedMs >= PROGRESS_SAMPLE_WINDOW_MS) { - this.progressSampleStartTimeMs = nowMs; - this.progressSampleStartFrame = currentFrame; - } - - if (this.config.onProgress) { - this.config.onProgress({ - currentFrame, - totalFrames, - percentage: totalFrames > 0 ? (currentFrame / totalFrames) * 100 : 100, - estimatedTimeRemaining, - renderFps, - }); - } - } - - private getNowMs(): number { - return typeof performance !== "undefined" ? performance.now() : Date.now(); - } - - cancel(): void { - this.cancelled = true; - if (this.streamingDecoder) { - this.streamingDecoder.cancel(); - } - if (this.gif) { - this.gif.abort(); - } - this.cleanup(); - } - - private cleanup(): void { - if (this.streamingDecoder) { - try { - this.streamingDecoder.destroy(); - } catch (e) { - console.warn("Error destroying streaming decoder:", e); - } - this.streamingDecoder = null; - } - - if (this.renderer) { - try { - this.renderer.destroy(); - } catch (e) { - console.warn("Error destroying renderer:", e); - } - this.renderer = null; - } - - this.gif = null; - } + private config: GifExporterConfig; + private streamingDecoder: StreamingVideoDecoder | null = null; + private renderer: FrameRenderer | null = null; + private gif: GIF | null = null; + private cancelled = false; + private exportStartTimeMs = 0; + private progressSampleStartTimeMs = 0; + private progressSampleStartFrame = 0; + private lastRenderFps: number | undefined; + + constructor(config: GifExporterConfig) { + this.config = config; + } + + async export(): Promise { + try { + this.cleanup(); + this.cancelled = false; + this.exportStartTimeMs = this.getNowMs(); + this.progressSampleStartTimeMs = this.exportStartTimeMs; + this.progressSampleStartFrame = 0; + this.lastRenderFps = undefined; + + // Initialize streaming decoder and load video metadata + this.streamingDecoder = new StreamingVideoDecoder({ + maxDecodeQueue: this.config.maxDecodeQueue, + maxPendingFrames: this.config.maxPendingFrames, + }); + const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl); + + // Initialize frame renderer + this.renderer = new FrameRenderer({ + width: this.config.width, + height: this.config.height, + wallpaper: this.config.wallpaper, + zoomRegions: this.config.zoomRegions, + showShadow: this.config.showShadow, + shadowIntensity: this.config.shadowIntensity, + backgroundBlur: this.config.backgroundBlur, + zoomMotionBlur: this.config.zoomMotionBlur, + connectZooms: this.config.connectZooms, + zoomInDurationMs: this.config.zoomInDurationMs, + zoomInOverlapMs: this.config.zoomInOverlapMs, + zoomOutDurationMs: this.config.zoomOutDurationMs, + connectedZoomGapMs: this.config.connectedZoomGapMs, + connectedZoomDurationMs: this.config.connectedZoomDurationMs, + zoomInEasing: this.config.zoomInEasing, + zoomOutEasing: this.config.zoomOutEasing, + connectedZoomEasing: this.config.connectedZoomEasing, + borderRadius: this.config.borderRadius, + padding: this.config.padding, + cropRegion: this.config.cropRegion, + webcam: this.config.webcam, + webcamUrl: this.config.webcamUrl, + videoWidth: videoInfo.width, + videoHeight: videoInfo.height, + annotationRegions: this.config.annotationRegions, + autoCaptions: this.config.autoCaptions, + autoCaptionSettings: this.config.autoCaptionSettings, + speedRegions: this.config.speedRegions, + previewWidth: this.config.previewWidth, + previewHeight: this.config.previewHeight, + cursorTelemetry: this.config.cursorTelemetry, + showCursor: this.config.showCursor, + cursorStyle: this.config.cursorStyle, + cursorSize: this.config.cursorSize, + cursorSmoothing: this.config.cursorSmoothing, + zoomSmoothness: this.config.zoomSmoothness, + zoomClassicMode: this.config.zoomClassicMode, + cursorMotionBlur: this.config.cursorMotionBlur, + cursorClickBounce: this.config.cursorClickBounce, + cursorClickBounceDuration: this.config.cursorClickBounceDuration, + cursorSway: this.config.cursorSway, + frame: this.config.frame, + }); + await this.renderer.initialize(); + + // Initialize GIF encoder + // Loop: 0 = infinite loop, 1 = play once (no loop) + const repeat = getGifRepeat(this.config.loop); + const cores = navigator.hardwareConcurrency || 4; + const WORKER_COUNT = Math.max(1, Math.min(8, cores - 1)); + + this.gif = new GIF({ + workers: WORKER_COUNT, + quality: 10, + width: this.config.width, + height: this.config.height, + workerScript: GIF_WORKER_URL, + repeat, + background: "#000000", + transparent: null, + dither: "FloydSteinberg", + }); + + // Calculate effective duration and frame count (excluding trim regions) + const effectiveDuration = this.streamingDecoder.getEffectiveDuration( + this.config.trimRegions, + this.config.speedRegions, + ); + const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); + + // Calculate frame delay in milliseconds (gif.js uses ms) + const frameDelay = Math.round(1000 / this.config.frameRate); + + console.log("[GifExporter] Original duration:", videoInfo.duration, "s"); + console.log("[GifExporter] Effective duration:", effectiveDuration, "s"); + console.log("[GifExporter] Total frames to export:", totalFrames); + console.log("[GifExporter] Frame rate:", this.config.frameRate, "FPS"); + console.log("[GifExporter] Frame delay:", frameDelay, "ms"); + console.log("[GifExporter] Loop:", this.config.loop ? "infinite" : "once"); + console.log("[GifExporter] Using streaming decode (web-demuxer + VideoDecoder)"); + + let frameIndex = 0; + + // Stream decode and process frames — no seeking! + await this.streamingDecoder.decodeAll( + this.config.frameRate, + this.config.trimRegions, + this.config.speedRegions, + async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => { + if (this.cancelled) { + videoFrame.close(); + return; + } + + const sourceTimestampUs = sourceTimestampMs * 1000; + const cursorTimestampUs = cursorTimestampMs * 1000; + await this.renderer!.renderFrame( + videoFrame, + sourceTimestampUs, + cursorTimestampUs, + ); + videoFrame.close(); + + this.addRenderedGifFrame(frameDelay); + frameIndex++; + this.reportProgress(frameIndex, totalFrames); + }, + ); + + if (this.cancelled) { + return { success: false, error: "Export cancelled" }; + } + + // Update progress to show we're now in the finalizing phase + if (this.config.onProgress) { + this.config.onProgress({ + currentFrame: totalFrames, + totalFrames, + percentage: 100, + estimatedTimeRemaining: 0, + phase: "finalizing", + renderFps: this.lastRenderFps, + }); + } + + // Render the GIF + const blob = await new Promise((resolve, _reject) => { + this.gif!.on("finished", (blob: Blob) => { + resolve(blob); + }); + + // Track rendering progress + this.gif!.on("progress", (progress: number) => { + if (this.config.onProgress) { + this.config.onProgress({ + currentFrame: totalFrames, + totalFrames, + percentage: 100, + estimatedTimeRemaining: 0, + phase: "finalizing", + renderFps: this.lastRenderFps, + renderProgress: Math.round(progress * 100), + }); + } + }); + + // gif.js doesn't have a typed 'error' event, but we can catch errors in the try/catch + this.gif!.render(); + }); + + return { success: true, blob }; + } catch (error) { + console.error("GIF Export error:", error); + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } finally { + this.cleanup(); + } + } + + private addRenderedGifFrame(frameDelay: number) { + const canvas = this.renderer!.getCanvas(); + this.gif!.addFrame(canvas, { delay: frameDelay, copy: true }); + } + + private reportProgress(currentFrame: number, totalFrames: number) { + const nowMs = this.getNowMs(); + const elapsedSeconds = Math.max((nowMs - this.exportStartTimeMs) / 1000, 0.001); + const averageRenderFps = currentFrame / elapsedSeconds; + const sampleElapsedMs = Math.max(nowMs - this.progressSampleStartTimeMs, 1); + const sampleFrameDelta = Math.max(currentFrame - this.progressSampleStartFrame, 0); + const renderFps = (sampleFrameDelta * 1000) / sampleElapsedMs; + const remainingFrames = Math.max(totalFrames - currentFrame, 0); + const estimatedTimeRemaining = + averageRenderFps > 0 ? remainingFrames / averageRenderFps : 0; + this.lastRenderFps = renderFps; + + if (sampleElapsedMs >= PROGRESS_SAMPLE_WINDOW_MS) { + this.progressSampleStartTimeMs = nowMs; + this.progressSampleStartFrame = currentFrame; + } + + if (this.config.onProgress) { + this.config.onProgress({ + currentFrame, + totalFrames, + percentage: totalFrames > 0 ? (currentFrame / totalFrames) * 100 : 100, + estimatedTimeRemaining, + renderFps, + }); + } + } + + private getNowMs(): number { + return typeof performance !== "undefined" ? performance.now() : Date.now(); + } + + cancel(): void { + this.cancelled = true; + if (this.streamingDecoder) { + this.streamingDecoder.cancel(); + } + if (this.gif) { + this.gif.abort(); + } + this.cleanup(); + } + + private cleanup(): void { + if (this.streamingDecoder) { + try { + this.streamingDecoder.destroy(); + } catch (e) { + console.warn("Error destroying streaming decoder:", e); + } + this.streamingDecoder = null; + } + + if (this.renderer) { + try { + this.renderer.destroy(); + } catch (e) { + console.warn("Error destroying renderer:", e); + } + this.renderer = null; + } + + this.gif = null; + } } diff --git a/src/lib/exporter/index.ts b/src/lib/exporter/index.ts index 550aa54f5..3561cb3a1 100644 --- a/src/lib/exporter/index.ts +++ b/src/lib/exporter/index.ts @@ -1,46 +1,44 @@ -export { VideoExporter } from './videoExporter'; -export { ModernVideoExporter } from './modernVideoExporter'; -export { VideoFileDecoder } from './videoDecoder'; -export { StreamingVideoDecoder } from './streamingDecoder'; -export { FrameRenderer } from './frameRenderer'; -export { VideoMuxer } from './muxer'; -export { GifExporter, calculateOutputDimensions } from './gifExporter'; +export { FrameRenderer } from "./frameRenderer"; +export { calculateOutputDimensions, GifExporter } from "./gifExporter"; +export { ModernVideoExporter } from "./modernVideoExporter"; +export type { + SupportedMp4Dimensions, + SupportedMp4EncoderPath, +} from "./mp4Support"; export { - DEFAULT_MP4_CODEC, - MP4_CODEC_FALLBACK_LIST, - probeSupportedMp4Dimensions, - resolveSupportedMp4EncoderPath, -} from './mp4Support'; -export type { - ExportConfig, - ExportProgress, - ExportResult, - ExportMetrics, - VideoFrameData, - ExportRenderBackend, - ExportEncodeBackend, - ExportBackendPreference, - ExportPipelineModel, - ExportEncodingMode, - ExportQuality, - ExportMp4FrameRate, - ExportFormat, - GifFrameRate, - GifSizePreset, - GifExportConfig, - ExportSettings, -} from './types'; + DEFAULT_MP4_CODEC, + MP4_CODEC_FALLBACK_LIST, + probeSupportedMp4Dimensions, + resolveSupportedMp4EncoderPath, +} from "./mp4Support"; +export { VideoMuxer } from "./muxer"; +export { StreamingVideoDecoder } from "./streamingDecoder"; export type { - SupportedMp4Dimensions, - SupportedMp4EncoderPath, -} from './mp4Support'; -export { - MP4_FRAME_RATES, - isValidMp4FrameRate, - GIF_SIZE_PRESETS, - GIF_FRAME_RATES, - VALID_GIF_FRAME_RATES, - isValidGifFrameRate -} from './types'; - - + ExportBackendPreference, + ExportConfig, + ExportEncodeBackend, + ExportEncodingMode, + ExportFormat, + ExportMetrics, + ExportMp4FrameRate, + ExportPipelineModel, + ExportProgress, + ExportQuality, + ExportRenderBackend, + ExportResult, + ExportSettings, + GifExportConfig, + GifFrameRate, + GifSizePreset, + VideoFrameData, +} from "./types"; +export { + GIF_FRAME_RATES, + GIF_SIZE_PRESETS, + isValidGifFrameRate, + isValidMp4FrameRate, + MP4_FRAME_RATES, + VALID_GIF_FRAME_RATES, +} from "./types"; +export { VideoFileDecoder } from "./videoDecoder"; +export { VideoExporter } from "./videoExporter"; diff --git a/src/lib/exporter/localMediaSource.test.ts b/src/lib/exporter/localMediaSource.test.ts index 2882efb8a..3a7f2a08c 100644 --- a/src/lib/exporter/localMediaSource.test.ts +++ b/src/lib/exporter/localMediaSource.test.ts @@ -57,4 +57,4 @@ describe("resolveMediaElementSource", () => { expect(result.src).toBe("https://example.com/video.mp4"); expect(readLocalFile).not.toHaveBeenCalled(); }); -}); \ No newline at end of file +}); diff --git a/src/lib/exporter/localMediaSource.ts b/src/lib/exporter/localMediaSource.ts index 6e5c3710e..b4a7dd58d 100644 --- a/src/lib/exporter/localMediaSource.ts +++ b/src/lib/exporter/localMediaSource.ts @@ -1,6 +1,6 @@ import { fromFileUrl, toFileUrl } from "@/components/video-editor/projectPersistence"; -const NOOP = () => {}; +const NOOP = () => undefined; const REMOTE_MEDIA_URL_PATTERN = /^(https?:|blob:|data:)/i; function isAbsoluteLocalPath(resource: string) { @@ -67,8 +67,7 @@ export async function resolveMediaElementSource(resource: string): Promise<{ return { src: normalizedResource, revoke: NOOP }; } - const bytes = - result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data); + const bytes = result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data); const blob = new Blob([Uint8Array.from(bytes)], { type: inferMimeType(localFilePath) }); const objectUrl = URL.createObjectURL(blob); diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index 592110939..cc6e6ab79 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -184,4 +184,4 @@ describe("ModernFrameRenderer blur export path", () => { expect(renderer.getCanvas()).not.toBe(sourceCanvas); expect(renderer.capturePixelsForNativeExport()).toBeNull(); }); -}); \ No newline at end of file +}); diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 7fe8fe77b..40e60cc1d 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -23,28 +23,26 @@ import type { ZoomTransitionEasing, } from "@/components/video-editor/types"; import { getDefaultCaptionFontFamily, ZOOM_DEPTH_SCALES } from "@/components/video-editor/types"; +import { DEFAULT_FOCUS } from "@/components/video-editor/videoPlayback/constants"; import { - DEFAULT_FOCUS, -} from "@/components/video-editor/videoPlayback/constants"; + type CursorFollowCameraState, + computeCursorFollowFocus, + createCursorFollowCameraState, + SNAP_TO_EDGES_RATIO_AUTO, +} from "@/components/video-editor/videoPlayback/cursorFollowCamera"; import { DEFAULT_CURSOR_CONFIG, PixiCursorOverlay, preloadCursorAssets, } from "@/components/video-editor/videoPlayback/cursorRenderer"; -import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils"; -import { - type CursorFollowCameraState, - createCursorFollowCameraState, - computeCursorFollowFocus, - SNAP_TO_EDGES_RATIO_AUTO, -} from "@/components/video-editor/videoPlayback/cursorFollowCamera"; import { - type SpringState, createSpringState, - stepSpringValue, - resetSpringState, getZoomSpringConfig, + resetSpringState, + type SpringState, + stepSpringValue, } from "@/components/video-editor/videoPlayback/motionSmoothing"; +import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils"; import { applyZoomTransform, computeFocusFromTransform, @@ -62,14 +60,14 @@ import { mapCursorToCanvasNormalized, mapSmoothedCursorToCanvasNormalized, } from "@/lib/extensions/cursorCoordinates"; -import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform"; -import { drawSquircleOnCanvas, drawSquircleOnGraphics } from "@/lib/geometry/squircle"; -import { clampMediaTimeToDuration } from "@/lib/mediaTiming"; import { executeExtensionCursorEffects, executeExtensionRenderHooks, notifyCursorInteraction, } from "@/lib/extensions/renderHooks"; +import { applyCanvasSceneTransform } from "@/lib/extensions/sceneTransform"; +import { drawSquircleOnCanvas, drawSquircleOnGraphics } from "@/lib/geometry/squircle"; +import { clampMediaTimeToDuration } from "@/lib/mediaTiming"; import { isVideoWallpaperSource } from "@/lib/wallpapers"; import { type AnnotationRenderAssets, @@ -90,7 +88,6 @@ interface FrameRenderConfig { width: number; height: number; preferredRenderBackend?: ExportRenderBackend; - nativeReadbackMode?: "canvas" | "pixels"; wallpaper: string; zoomRegions: ZoomRegion[]; showShadow: boolean; @@ -372,8 +369,6 @@ export class FrameRenderer { private webcamLayoutCache: WebcamLayoutCache | null = null; private videoTextureUsesStartupStaging = false; private webcamTextureUsesStartupStaging = false; - private nativePixelReadbackWarningShown = false; - private nativeReadbackBuffer: Uint8Array | null = null; private compositeCanvas: HTMLCanvasElement | null = null; private compositeCtx: CanvasRenderingContext2D | null = null; private lastEmittedClickTimeMs = -1; @@ -398,9 +393,10 @@ export class FrameRenderer { return; } - const activeFilters = this.shouldUseZoomMotionBlur() && this.motionBlurFilter - ? [this.motionBlurFilter] - : null; + const activeFilters = + this.shouldUseZoomMotionBlur() && this.motionBlurFilter + ? [this.motionBlurFilter] + : null; this.videoEffectsContainer.filters = activeFilters; } @@ -480,11 +476,13 @@ export class FrameRenderer { this.cursorOverlay = new PixiCursorOverlay({ dotRadius: DEFAULT_CURSOR_CONFIG.dotRadius * (this.config.cursorSize ?? 1.4), style: this.config.cursorStyle ?? "tahoe", - smoothingFactor: this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor, + smoothingFactor: + this.config.cursorSmoothing ?? DEFAULT_CURSOR_CONFIG.smoothingFactor, motionBlur: this.config.cursorMotionBlur ?? 0, clickBounce: this.config.cursorClickBounce ?? DEFAULT_CURSOR_CONFIG.clickBounce, clickBounceDuration: - this.config.cursorClickBounceDuration ?? DEFAULT_CURSOR_CONFIG.clickBounceDuration, + this.config.cursorClickBounceDuration ?? + DEFAULT_CURSOR_CONFIG.clickBounceDuration, sway: this.config.cursorSway ?? DEFAULT_CURSOR_CONFIG.sway, }); this.cursorContainer.addChild(this.cursorOverlay.container); @@ -713,7 +711,9 @@ export class FrameRenderer { const targetWidth = Math.max(1, Math.ceil(width)); const targetHeight = Math.max(1, Math.ceil(height)); const currentCanvas = - kind === "scene" ? this.sceneVideoFrameStagingCanvas : this.webcamVideoFrameStagingCanvas; + kind === "scene" + ? this.sceneVideoFrameStagingCanvas + : this.webcamVideoFrameStagingCanvas; const currentContext = kind === "scene" ? this.sceneVideoFrameStagingCtx : this.webcamVideoFrameStagingCtx; @@ -861,7 +861,8 @@ export class FrameRenderer { await new Promise((resolve, reject) => { video.onloadeddata = () => resolve(); - video.onerror = () => reject(new Error(`Failed to load video wallpaper: ${wallpaper}`)); + video.onerror = () => + reject(new Error(`Failed to load video wallpaper: ${wallpaper}`)); }); this.backgroundVideoElement = video; @@ -897,7 +898,11 @@ export class FrameRenderer { await new Promise((resolve, reject) => { img.onload = () => resolve(); img.onerror = (err) => { - console.error("[FrameRenderer] Failed to load background image:", imageUrl, err); + console.error( + "[FrameRenderer] Failed to load background image:", + imageUrl, + err, + ); reject(new Error(`Failed to load background image: ${imageUrl}`)); }; img.src = imageUrl; @@ -960,7 +965,9 @@ export class FrameRenderer { } const blurredCanvas = - this.config.backgroundBlur > 0 ? this.createPreblurredBackgroundCanvas(bgCanvas) : null; + this.config.backgroundBlur > 0 + ? this.createPreblurredBackgroundCanvas(bgCanvas) + : null; const backgroundSource = blurredCanvas ?? bgCanvas; const backgroundTexture = Texture.from(backgroundSource); this.backgroundSprite = new Sprite(backgroundTexture); @@ -1210,7 +1217,8 @@ export class FrameRenderer { private updateAnnotationLayer(currentTimeMs: number): void { for (const entry of this.annotationSprites) { entry.sprite.visible = - currentTimeMs >= entry.annotation.startMs && currentTimeMs <= entry.annotation.endMs; + currentTimeMs >= entry.annotation.startMs && + currentTimeMs <= entry.annotation.endMs; } } @@ -1357,11 +1365,16 @@ export class FrameRenderer { line.words.forEach((word) => { const segmentText = `${word.leadingSpace ? " " : ""}${word.text}`; const segmentWidth = ctx.measureText(segmentText).width; - const visualState = getCaptionWordVisualState(state.layout.hasWordTimings, word.state); + const visualState = getCaptionWordVisualState( + state.layout.hasWordTimings, + word.state, + ); ctx.save(); ctx.translate(cursorX, lineY); - ctx.fillStyle = visualState.isInactive ? settings.inactiveTextColor : settings.textColor; + ctx.fillStyle = visualState.isInactive + ? settings.inactiveTextColor + : settings.textColor; ctx.globalAlpha = visualState.opacity; ctx.fillText(segmentText, 0, 0); ctx.restore(); @@ -1484,7 +1497,9 @@ export class FrameRenderer { !wallpaper.startsWith("/wallpapers/") && !wallpaper.startsWith("/app-icons/"); - const wallpaperAsset = looksLikeAbsoluteFilePath ? `file://${encodeURI(wallpaper)}` : wallpaper; + const wallpaperAsset = looksLikeAbsoluteFilePath + ? `file://${encodeURI(wallpaper)}` + : wallpaper; return getRenderableAssetUrl(wallpaperAsset); } @@ -1875,7 +1890,10 @@ export class FrameRenderer { const requestVideoFrameCallback = ( webcamVideo as HTMLVideoElement & { requestVideoFrameCallback?: ( - callback: (now: DOMHighResTimeStamp, metadata: VideoFrameCallbackMetadata) => void, + callback: ( + now: DOMHighResTimeStamp, + metadata: VideoFrameCallbackMetadata, + ) => void, ) => number; cancelVideoFrameCallback?: (handle: number) => void; } @@ -2107,7 +2125,10 @@ export class FrameRenderer { this.videoContainer.addChildAt(this.videoSprite, 0); this.videoTextureUsesStartupStaging = usesStartupStaging; } else if (this.videoTextureUsesStartupStaging !== usesStartupStaging) { - this.videoTextureSource = this.replaceSpriteTexture(this.videoSprite, resolvedVideoSource); + this.videoTextureSource = this.replaceSpriteTexture( + this.videoSprite, + resolvedVideoSource, + ); this.videoTextureUsesStartupStaging = usesStartupStaging; } else if (this.videoTextureSource) { this.videoTextureSource.resource = resolvedVideoSource; @@ -2229,11 +2250,11 @@ export class FrameRenderer { extensionHost.setSmoothedCursor( smoothedCursor ? { - timeMs, - cx: smoothedCursor.cx, - cy: smoothedCursor.cy, - trail: smoothedCursor.trail, - } + timeMs, + cx: smoothedCursor.cx, + cy: smoothedCursor.cy, + trail: smoothedCursor.trail, + } : null, ); const rawCursor = this.getCursorPosition(cursorTimeMs); @@ -2244,23 +2265,23 @@ export class FrameRenderer { durationMs: 0, cursor: smoothedCursor ? { - cx: smoothedCursor.cx, - cy: smoothedCursor.cy, - interactionType: rawCursor?.interactionType, - } + cx: smoothedCursor.cx, + cy: smoothedCursor.cy, + interactionType: rawCursor?.interactionType, + } : rawCursor, smoothedCursor, videoLayout: maskRect ? { - maskRect: { - x: maskRect.x, - y: maskRect.y, - width: maskRect.width, - height: maskRect.height, - }, - borderRadius: this.config.borderRadius ?? 0, - padding: this.config.padding ?? 0, - } + maskRect: { + x: maskRect.x, + y: maskRect.y, + width: maskRect.width, + height: maskRect.height, + }, + borderRadius: this.config.borderRadius ?? 0, + padding: this.config.padding ?? 0, + } : undefined, zoom: { scale: this.animationState.scale, @@ -2401,7 +2422,10 @@ export class FrameRenderer { const paddingScale = 1.0 - (padding / 100) * 0.4; const viewportWidth = width * paddingScale; const viewportHeight = height * paddingScale; - const scale = Math.min(viewportWidth / croppedVideoWidth, viewportHeight / croppedVideoHeight); + const scale = Math.min( + viewportWidth / croppedVideoWidth, + viewportHeight / croppedVideoHeight, + ); this.videoSprite.scale.set(scale); @@ -2504,7 +2528,12 @@ export class FrameRenderer { // Cursor follow: use cursor-follow camera for non-manual zoom regions let regionFocus = region.focus; - if (!this.config.zoomClassicMode && region.mode !== 'manual' && this.config.cursorTelemetry && this.config.cursorTelemetry.length > 0) { + if ( + !this.config.zoomClassicMode && + region.mode !== "manual" && + this.config.cursorTelemetry && + this.config.cursorTelemetry.length > 0 + ) { regionFocus = computeCursorFollowFocus( this.cursorFollowCamera, this.config.cursorTelemetry, @@ -2579,9 +2608,8 @@ export class FrameRenderer { // Spring-driven zoom animation for export — use content time, not wall-clock, // so the spring advances at the same rate as the video regardless of render speed. - const deltaMs = this.lastContentTimeMs !== null - ? timeMs - this.lastContentTimeMs - : 1000 / 60; + const deltaMs = + this.lastContentTimeMs !== null ? timeMs - this.lastContentTimeMs : 1000 / 60; this.lastContentTimeMs = timeMs; const zoomSpringConfig = getZoomSpringConfig(this.config.zoomSmoothness); @@ -2594,9 +2622,24 @@ export class FrameRenderer { resetSpringState(this.springX, state.x); resetSpringState(this.springY, state.y); } else { - state.appliedScale = stepSpringValue(this.springScale, projectedTransform.scale, deltaMs, zoomSpringConfig); - state.x = stepSpringValue(this.springX, projectedTransform.x, deltaMs, zoomSpringConfig); - state.y = stepSpringValue(this.springY, projectedTransform.y, deltaMs, zoomSpringConfig); + state.appliedScale = stepSpringValue( + this.springScale, + projectedTransform.scale, + deltaMs, + zoomSpringConfig, + ); + state.x = stepSpringValue( + this.springX, + projectedTransform.x, + deltaMs, + zoomSpringConfig, + ); + state.y = stepSpringValue( + this.springY, + projectedTransform.y, + deltaMs, + zoomSpringConfig, + ); } this.lastMotionVector = { @@ -2620,117 +2663,6 @@ export class FrameRenderer { this.webcamDecodedFrame = null; } - private getNativeReadbackBuffer(byteLength: number): Uint8Array { - if (!this.nativeReadbackBuffer || this.nativeReadbackBuffer.byteLength !== byteLength) { - this.nativeReadbackBuffer = new Uint8Array(byteLength); - } - - return this.nativeReadbackBuffer; - } - - private capturePixelsFromWebGL(): Uint8Array | null { - if (!this.app) { - throw new Error("Renderer not initialized"); - } - - const renderer = this.app.renderer as typeof this.app.renderer & { - gl?: WebGLRenderingContext | WebGL2RenderingContext; - }; - const gl = renderer.gl; - if (!gl) { - return null; - } - - const width = this.config.width; - const height = this.config.height; - const pixelByteLength = width * height * 4; - const pixelBuffer = this.getNativeReadbackBuffer(pixelByteLength); - while (gl.getError() !== gl.NO_ERROR) { - // Clear stale GL errors from earlier work before checking this readback attempt. - } - - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.pixelStorei(gl.PACK_ALIGNMENT, 1); - gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixelBuffer); - - let glError = gl.getError(); - if (glError !== gl.NO_ERROR) { - // ANGLE/Metal may reject RGBA+UNSIGNED_BYTE on the default framebuffer. - // Blit to an FBO with explicit RGBA8 attachment where this is guaranteed. - const gl2 = gl as WebGL2RenderingContext; - if (typeof gl2.blitFramebuffer !== "function") { - throw new Error(`[FrameRenderer] WebGL readPixels failed with error code ${glError}`); - } - - while (gl2.getError() !== gl2.NO_ERROR) { - // drain - } - - const readFb = gl2.createFramebuffer(); - const rb = gl2.createRenderbuffer(); - try { - gl2.bindRenderbuffer(gl2.RENDERBUFFER, rb); - gl2.renderbufferStorage(gl2.RENDERBUFFER, gl2.RGBA8, width, height); - gl2.bindFramebuffer(gl2.DRAW_FRAMEBUFFER, readFb); - gl2.framebufferRenderbuffer( - gl2.DRAW_FRAMEBUFFER, - gl2.COLOR_ATTACHMENT0, - gl2.RENDERBUFFER, - rb, - ); - gl2.bindFramebuffer(gl2.READ_FRAMEBUFFER, null); - gl2.blitFramebuffer( - 0, 0, width, height, - 0, 0, width, height, - gl2.COLOR_BUFFER_BIT, - gl2.NEAREST, - ); - gl2.bindFramebuffer(gl2.FRAMEBUFFER, readFb); - gl2.readPixels(0, 0, width, height, gl2.RGBA, gl2.UNSIGNED_BYTE, pixelBuffer); - glError = gl2.getError(); - if (glError !== gl2.NO_ERROR) { - throw new Error( - `[FrameRenderer] WebGL FBO readPixels failed with error code ${glError}`, - ); - } - } finally { - gl2.bindFramebuffer(gl2.FRAMEBUFFER, null); - gl2.deleteFramebuffer(readFb); - gl2.deleteRenderbuffer(rb); - } - } - - return pixelBuffer; - } - - capturePixelsForNativeExport(): Uint8Array | null { - if (!this.app) { - throw new Error("Renderer not initialized"); - } - - if (this.outputCanvasOverride || this.shouldCompositeExtensionFrame()) { - return null; - } - - if (this.config.nativeReadbackMode !== "pixels") { - return null; - } - - try { - return this.capturePixelsFromWebGL(); - } catch (error) { - if (!this.nativePixelReadbackWarningShown) { - this.nativePixelReadbackWarningShown = true; - console.warn( - "[FrameRenderer] Direct WebGL readback unavailable for native export; falling back to canvas capture.", - error, - ); - } - - return null; - } - } - getCanvas(): HTMLCanvasElement { if (!this.app) { throw new Error("Renderer not initialized"); @@ -2872,7 +2804,6 @@ export class FrameRenderer { this.captionRenderKey = null; this.exportCompositeCanvas = null; this.outputCanvasOverride = null; - this.nativeReadbackBuffer = null; this.annotationScaleFactor = 1; this.lastSyncedWebcamTime = null; diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 3a36d1d94..e2cc41cc4 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -12,20 +12,22 @@ import type { ZoomRegion, ZoomTransitionEasing, } from "@/components/video-editor/types"; +import { extensionHost } from "@/lib/extensions"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; +import { normalizeLightningRuntimePlatform } from "./backendPolicy"; import { - normalizeLightningRuntimePlatform, - shouldPreferNativeAutoBackend, - type LightningRuntimePlatform, -} from "./backendPolicy"; + type ExportBackpressureProfile, + getExportBackpressureProfile, + getPreferredWebCodecsLatencyModes, + getWebCodecsEncodeQueueLimit, + getWebCodecsKeyFrameInterval, +} from "./exportTuning"; import { FrameRenderer as ModernFrameRenderer } from "./modernFrameRenderer"; import { getOrderedSupportedMp4EncoderCandidates, type SupportedMp4EncoderPath, } from "./mp4Support"; import { VideoMuxer } from "./muxer"; -import { captureCanvasFrameForNativeExport } from "./nativeFrameCapture"; -import { extensionHost } from "@/lib/extensions"; import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder"; import type { ExportConfig, @@ -35,13 +37,6 @@ import type { ExportRenderBackend, ExportResult, } from "./types"; -import { - type ExportBackpressureProfile, - getExportBackpressureProfile, - getPreferredWebCodecsLatencyModes, - getWebCodecsEncodeQueueLimit, - getWebCodecsKeyFrameInterval, -} from "./exportTuning"; interface VideoExporterConfig extends ExportConfig { videoUrl: string; @@ -107,8 +102,9 @@ type NativeAudioPlan = const NATIVE_EXPORT_ENGINE_NAME = "Breeze"; const LIGHTNING_PIPELINE_NAME = "Lightning (Beta)"; - export class ModernVideoExporter { + private static readonly NATIVE_ENCODER_QUEUE_LIMIT = 32; + private config: VideoExporterConfig; private streamingDecoder: StreamingVideoDecoder | null = null; private renderer: ModernFrameRenderer | null = null; @@ -130,10 +126,12 @@ export class ModernVideoExporter { private encoderName: string | null = null; private backpressureProfile: ExportBackpressureProfile | null = null; private nativeExportSessionId: string | null = null; - private nativeWritePromises: Promise[] = []; + private nativeWritePromises = new Set>(); private nativeWriteError: Error | null = null; private maxNativeWriteInFlight = 1; private lastNativeExportError: string | null = null; + private nativeH264Encoder: VideoEncoder | null = null; + private nativeEncoderError: Error | null = null; private readonly FINALIZATION_TIMEOUT_MS = 600_000; private totalExportStartTimeMs = 0; private metadataLoadTimeMs = 0; @@ -163,18 +161,12 @@ export class ModernVideoExporter { this.cleanup(); this.cancelled = false; this.encoderError = null; - this.totalExportStartTimeMs = this.getNowMs(); - - let stageStartedAt = this.getNowMs(); + this.nativeEncoderError = null; const backendPreference = this.config.backendPreference ?? "auto"; - const runtimePlatform = - backendPreference === "auto" ? await this.getRuntimePlatform() : "unknown"; - const preferNativeFirstInAuto = - backendPreference === "auto" && shouldPreferNativeAutoBackend(runtimePlatform); let useNativeEncoder = false; this.lastNativeExportError = null; - stageStartedAt = this.getNowMs(); + let stageStartedAt = this.getNowMs(); if (backendPreference === "breeze") { useNativeEncoder = await this.tryStartNativeVideoExport(); this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; @@ -185,71 +177,42 @@ export class ModernVideoExporter { ); } } else { - if (preferNativeFirstInAuto) { - stageStartedAt = this.getNowMs(); - useNativeEncoder = await this.tryStartNativeVideoExport(); - this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; - if (useNativeEncoder) { - console.log( - `[VideoExporter] Auto backend preferred ${NATIVE_EXPORT_ENGINE_NAME} first on ${runtimePlatform}; skipping WebCodecs startup.`, - ); - } else { - console.log( - `[VideoExporter] Auto backend could not start ${NATIVE_EXPORT_ENGINE_NAME} on ${runtimePlatform}; falling back to WebCodecs.`, - ); - } - } - - if (!useNativeEncoder) { - try { - const configuredWebCodecsPath = await this.initializeEncoder(); - if ( - !preferNativeFirstInAuto && - backendPreference === "auto" && - configuredWebCodecsPath.hardwareAcceleration === "prefer-software" - ) { - console.warn( - "[VideoExporter] Auto backend resolved to a software WebCodecs encoder; trying Breeze native export instead.", - ); - stageStartedAt = this.getNowMs(); - useNativeEncoder = await this.tryStartNativeVideoExport(); - this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; - if (useNativeEncoder) { - this.disposeEncoder(); - } - } else if ( - preferNativeFirstInAuto && - backendPreference === "auto" && - configuredWebCodecsPath.hardwareAcceleration === "prefer-software" - ) { - console.warn( - `[VideoExporter] Auto backend fell back to a software WebCodecs encoder after ${NATIVE_EXPORT_ENGINE_NAME} startup was unavailable on ${runtimePlatform}.`, - ); - } - } catch (error) { - const webCodecsError = error instanceof Error ? error : new Error(String(error)); - if (backendPreference === "webcodecs") { - throw webCodecsError; - } - - if (preferNativeFirstInAuto) { - throw webCodecsError; - } - + try { + const configuredWebCodecsPath = await this.initializeEncoder(); + if ( + backendPreference === "auto" && + configuredWebCodecsPath.hardwareAcceleration === "prefer-software" + ) { console.warn( - `[VideoExporter] WebCodecs encoder unavailable, trying ${NATIVE_EXPORT_ENGINE_NAME} native export fallback`, - webCodecsError, + "[VideoExporter] Auto backend resolved to a software WebCodecs encoder; trying Breeze native export instead.", ); - this.disposeEncoder(); - stageStartedAt = this.getNowMs(); useNativeEncoder = await this.tryStartNativeVideoExport(); this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; - - if (!useNativeEncoder) { - throw webCodecsError; + if (useNativeEncoder) { + this.disposeEncoder(); } } + } catch (error) { + const webCodecsError = + error instanceof Error ? error : new Error(String(error)); + if (backendPreference === "webcodecs") { + throw webCodecsError; + } + + console.warn( + `[VideoExporter] WebCodecs encoder unavailable, trying ${NATIVE_EXPORT_ENGINE_NAME} native export fallback`, + webCodecsError, + ); + this.disposeEncoder(); + + stageStartedAt = this.getNowMs(); + useNativeEncoder = await this.tryStartNativeVideoExport(); + this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + + if (!useNativeEncoder) { + throw webCodecsError; + } } } @@ -262,12 +225,12 @@ export class ModernVideoExporter { }); this.maxNativeWriteInFlight = useNativeEncoder ? Math.max( - 1, - Math.floor( - this.config.maxInFlightNativeWrites ?? - this.backpressureProfile.maxInFlightNativeWrites, - ), - ) + 1, + Math.floor( + this.config.maxInFlightNativeWrites ?? + this.backpressureProfile.maxInFlightNativeWrites, + ), + ) : 1; console.log("[VideoExporter] Backpressure profile", { @@ -307,7 +270,6 @@ export class ModernVideoExporter { width: this.config.width, height: this.config.height, preferredRenderBackend: useNativeEncoder ? "webgl" : undefined, - nativeReadbackMode: useNativeEncoder ? "pixels" : "canvas", wallpaper: this.config.wallpaper, zoomRegions: this.config.zoomRegions, showShadow: this.config.showShadow, @@ -390,7 +352,11 @@ export class ModernVideoExporter { const sourceTimestampUs = sourceTimestampMs * 1000; const cursorTimestampUs = cursorTimestampMs * 1000; const renderStartedAt = this.getNowMs(); - await this.renderer!.renderFrame(videoFrame, sourceTimestampUs, cursorTimestampUs); + await this.renderer!.renderFrame( + videoFrame, + sourceTimestampUs, + cursorTimestampUs, + ); this.renderFrameTimeMs += this.getNowMs() - renderStartedAt; videoFrame.close(); @@ -399,7 +365,7 @@ export class ModernVideoExporter { } if (useNativeEncoder) { - await this.encodeRenderedFrameNative(timestamp); + await this.encodeRenderedFrameNative(timestamp, frameDuration, frameIndex); } else { await this.encodeRenderedFrame(timestamp, frameDuration, frameIndex); } @@ -407,7 +373,10 @@ export class ModernVideoExporter { frameIndex++; this.processedFrameCount = frameIndex; this.reportProgress(frameIndex, totalFrames, "extracting"); - extensionHost.emitEvent({ type: 'export:frame', data: { frameIndex, totalFrames } }); + extensionHost.emitEvent({ + type: "export:frame", + data: { frameIndex, totalFrames }, + }); }, ); this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt; @@ -421,7 +390,11 @@ export class ModernVideoExporter { }; } - return { success: false, error: "Export cancelled", metrics: this.buildExportMetrics() }; + return { + success: false, + error: "Export cancelled", + metrics: this.buildExportMetrics(), + }; } this.reportFinalizingProgress(totalFrames, 96); @@ -429,6 +402,9 @@ export class ModernVideoExporter { if (useNativeEncoder) { stageStartedAt = this.getNowMs(); this.reportFinalizingProgress(totalFrames, 99); + if (this.nativeH264Encoder) { + await this.nativeH264Encoder.flush(); + } const finishResult = await this.finishNativeVideoExport(nativeAudioPlan); this.finalizationTimeMs = this.getNowMs() - stageStartedAt; if (!finishResult.success || !finishResult.blob) { @@ -453,7 +429,10 @@ export class ModernVideoExporter { } this.reportFinalizingProgress(totalFrames, 98); - await this.awaitWithFinalizationTimeout(this.pendingMuxing, "muxing queued video chunks"); + await this.awaitWithFinalizationTimeout( + this.pendingMuxing, + "muxing queued video chunks", + ); if (nativeAudioPlan.audioMode !== "none" && !shouldUseFfmpegAudioFallback && !this.cancelled) { const demuxer = this.streamingDecoder.getDemuxer(); @@ -484,7 +463,10 @@ export class ModernVideoExporter { } this.reportFinalizingProgress(totalFrames, 99); - const blob = await this.awaitWithFinalizationTimeout(this.muxer!.finalize(), "muxer finalization"); + const blob = await this.awaitWithFinalizationTimeout( + this.muxer!.finalize(), + "muxer finalization", + ); this.finalizationTimeMs = this.getNowMs() - stageStartedAt; if (shouldUseFfmpegAudioFallback) { @@ -510,7 +492,11 @@ export class ModernVideoExporter { return { success: true, blob, metrics: this.buildExportMetrics() }; } catch (error) { if (this.cancelled && !this.encoderError) { - return { success: false, error: "Export cancelled", metrics: this.buildExportMetrics() }; + return { + success: false, + error: "Export cancelled", + metrics: this.buildExportMetrics(), + }; } const resolvedError = this.encoderError ?? error; @@ -522,29 +508,12 @@ export class ModernVideoExporter { }; } finally { if (this.totalExportStartTimeMs > 0) { - console.log(`[VideoExporter] Final metrics ${JSON.stringify(this.buildExportMetrics())}`); - } - this.cleanup(); - } - } - - private async getRuntimePlatform(): Promise { - if (typeof window !== "undefined" && window.electronAPI?.getPlatform) { - try { - return normalizeLightningRuntimePlatform(await window.electronAPI.getPlatform()); - } catch (error) { - console.warn( - "[VideoExporter] Failed to read runtime platform from Electron API; falling back to navigator hints.", - error, + console.log( + `[VideoExporter] Final metrics ${JSON.stringify(this.buildExportMetrics())}`, ); } + this.cleanup(); } - - if (typeof navigator === "undefined") { - return "unknown"; - } - - return normalizeLightningRuntimePlatform(navigator.platform || navigator.userAgent || ""); } private getPlatformLabel(): string { @@ -555,11 +524,11 @@ export class ModernVideoExporter { const platformHint = navigator.platform || navigator.userAgent || ""; switch (normalizeLightningRuntimePlatform(platformHint)) { case "win32": - return "Windows"; + return "Windows"; case "linux": - return "Linux"; + return "Linux"; case "darwin": - return "macOS"; + return "macOS"; default: return platformHint || "Unknown"; } @@ -574,12 +543,20 @@ export class ModernVideoExporter { ); if (/even output dimensions/i.test(message)) { - guidance.add("Use an export size with even width and height. Switching quality presets usually fixes this automatically."); + guidance.add( + "Use an export size with even width and height. Switching quality presets usually fixes this automatically.", + ); } - if (/not supported on this system|H\.264 encoding|encoder path .* is not supported|Video encoding/i.test(message)) { + if ( + /not supported on this system|H\.264 encoding|encoder path .* is not supported|Video encoding/i.test( + message, + ) + ) { guidance.add("Try Good or Medium quality to reduce output resolution and bitrate."); - guidance.add("Update GPU and media drivers so system H.264 encoding paths are available."); + guidance.add( + "Update GPU and media drivers so system H.264 encoding paths are available.", + ); } if (this.lastNativeExportError) { @@ -703,7 +680,9 @@ export class ModernVideoExporter { } private buildNativeTrimSegments(durationMs: number): Array<{ startMs: number; endMs: number }> { - const trimRegions = [...(this.config.trimRegions ?? [])].sort((a, b) => a.startMs - b.startMs); + const trimRegions = [...(this.config.trimRegions ?? [])].sort( + (a, b) => a.startMs - b.startMs, + ); if (trimRegions.length === 0) { return [{ startMs: 0, endMs: Math.max(0, durationMs) }]; } @@ -735,13 +714,23 @@ export class ModernVideoExporter { ); const localVideoSourcePath = this.getNativeVideoSourcePath(); const primaryAudioSourcePath = - (videoInfo.hasAudio ? localVideoSourcePath : null) ?? sourceAudioFallbackPaths[0] ?? null; + (videoInfo.hasAudio ? localVideoSourcePath : null) ?? + sourceAudioFallbackPaths[0] ?? + null; - if (!videoInfo.hasAudio && sourceAudioFallbackPaths.length === 0 && audioRegions.length === 0) { + if ( + !videoInfo.hasAudio && + sourceAudioFallbackPaths.length === 0 && + audioRegions.length === 0 + ) { return { audioMode: "none" }; } - if (speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1) { + if ( + speedRegions.length > 0 || + audioRegions.length > 0 || + sourceAudioFallbackPaths.length > 1 + ) { return { audioMode: "edited-track" }; } @@ -781,106 +770,173 @@ export class ModernVideoExporter { } if (this.config.width % 2 !== 0 || this.config.height % 2 !== 0) { - this.lastNativeExportError = - `${NATIVE_EXPORT_ENGINE_NAME} export requires even output dimensions (${this.config.width}x${this.config.height}).`; + this.lastNativeExportError = `${NATIVE_EXPORT_ENGINE_NAME} export requires even output dimensions (${this.config.width}x${this.config.height}).`; console.warn( `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} export requires even output dimensions, falling back to WebCodecs (${this.config.width}x${this.config.height})`, ); return false; } + if (typeof VideoEncoder === "undefined" || typeof VideoEncoder.isConfigSupported !== "function") { + this.lastNativeExportError = `${NATIVE_EXPORT_ENGINE_NAME} export requires WebCodecs VideoEncoder support.`; + return false; + } + + const encoderConfig: VideoEncoderConfig = { + codec: "avc1.640034", + width: this.config.width, + height: this.config.height, + bitrate: this.config.bitrate, + framerate: this.config.frameRate, + hardwareAcceleration: "prefer-hardware", + avc: { format: "annexb" }, + }; + + try { + const support = await VideoEncoder.isConfigSupported(encoderConfig); + if (!support.supported) { + this.lastNativeExportError = `H.264 Annex B encoding is not supported at ${this.config.width}x${this.config.height}.`; + return false; + } + } catch (error) { + this.lastNativeExportError = + error instanceof Error ? error.message : String(error); + console.warn( + `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} encoder support check failed`, + error, + ); + return false; + } + const result = await window.electronAPI.nativeVideoExportStart({ width: this.config.width, height: this.config.height, frameRate: this.config.frameRate, bitrate: this.config.bitrate, encodingMode: this.config.encodingMode ?? "balanced", + inputMode: "h264-stream", }); if (!result.success || !result.sessionId) { this.lastNativeExportError = - result.error || `${NATIVE_EXPORT_ENGINE_NAME} export could not be started on this system.`; - console.warn(`[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} export unavailable`, result.error); + result.error || + `${NATIVE_EXPORT_ENGINE_NAME} export could not be started on this system.`; + console.warn( + `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} export unavailable`, + result.error, + ); return false; } this.nativeExportSessionId = result.sessionId; this.lastNativeExportError = null; this.encodeBackend = "ffmpeg"; - this.encoderName = result.encoderName ?? null; - console.log(`[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} session ready`, { - sessionId: result.sessionId, - encoderName: result.encoderName ?? "unknown", - encodingMode: this.config.encodingMode ?? "balanced", + this.encoderName = "h264-stream-copy"; + + const sessionId = result.sessionId; + const encoder = new VideoEncoder({ + output: (chunk) => { + if (this.cancelled || !this.nativeExportSessionId) { + return; + } + + const buffer = new ArrayBuffer(chunk.byteLength); + chunk.copyTo(buffer); + const writePromise = window.electronAPI + .nativeVideoExportWriteFrame(sessionId, new Uint8Array(buffer)) + .then((writeResult) => { + if (!writeResult.success && !this.cancelled) { + throw new Error( + writeResult.error || "Failed to write H.264 chunk to native encoder", + ); + } + }) + .catch((error) => { + if (!this.cancelled) { + const resolvedError = + error instanceof Error ? error : new Error(String(error)); + if (!this.nativeEncoderError) { + this.nativeEncoderError = resolvedError; + } + if (!this.nativeWriteError) { + this.nativeWriteError = resolvedError; + } + } + throw error; + }); + + this.trackNativeWritePromise(writePromise); + }, + error: (error) => { + this.nativeEncoderError = error; + }, }); - return true; - } - private async encodeRenderedFrameNative(timestamp: number): Promise { - const sessionId = this.nativeExportSessionId; - if (!sessionId) { - if (this.cancelled) { - return; + try { + encoder.configure(encoderConfig); + } catch (error) { + this.lastNativeExportError = + error instanceof Error ? error.message : String(error); + try { + encoder.close(); + } catch (closeError) { + console.debug( + "[VideoExporter] Ignoring error closing native H.264 encoder after startup failure:", + closeError, + ); } - - throw new Error(`${NATIVE_EXPORT_ENGINE_NAME} export session is not active`); + this.nativeExportSessionId = null; + await window.electronAPI.nativeVideoExportCancel?.(sessionId); + console.warn( + `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} encoder configure failed`, + error, + ); + return false; } - if (this.nativeWriteError) { - throw this.nativeWriteError; - } + this.nativeH264Encoder = encoder; - await this.waitForNativeWriteSlot(); + console.log(`[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} session ready (H264-stream)`, { + sessionId: result.sessionId, + }); + return true; + } - if (this.nativeWriteError) { - throw this.nativeWriteError; + private async encodeRenderedFrameNative( + timestamp: number, + frameDuration: number, + frameIndex: number, + ): Promise { + if (!this.nativeH264Encoder || !this.nativeExportSessionId) { + if (this.cancelled) return; + throw new Error(`${NATIVE_EXPORT_ENGINE_NAME} export session is not active`); } - - const captureStartedAt = this.getNowMs(); - const frameData = - this.renderer?.capturePixelsForNativeExport() ?? - (await captureCanvasFrameForNativeExport( - this.renderer!.getCanvas(), - timestamp, - true, - this.config.width, - this.config.height, - )); - this.nativeCaptureTimeMs += this.getNowMs() - captureStartedAt; - - if (this.cancelled) { - return; + if (this.nativeEncoderError) throw this.nativeEncoderError; + while (this.nativeWritePromises.size >= this.maxNativeWriteInFlight) { + await this.awaitOldestNativeWrite(); + if (this.cancelled) return; + if (this.nativeEncoderError) throw this.nativeEncoderError; } - - const writeStartedAt = this.getNowMs(); - const writePromise = window.electronAPI - .nativeVideoExportWriteFrame(sessionId, frameData) - .then((result) => { - this.nativeWriteTimeMs += this.getNowMs() - writeStartedAt; - - if (!result.success) { - if (this.cancelled || result.error === "Native video export session was cancelled") { - return; - } - - throw new Error(result.error || "Failed to write frame to native encoder"); - } - }) - .catch((error) => { - this.nativeWriteTimeMs += this.getNowMs() - writeStartedAt; - const resolvedError = error instanceof Error ? error : new Error(String(error)); - if (!this.cancelled && !this.nativeWriteError) { - this.nativeWriteError = resolvedError; - } - throw resolvedError; - }); - - this.trackNativeWritePromise(writePromise); + while ( + this.nativeH264Encoder.encodeQueueSize >= + ModernVideoExporter.NATIVE_ENCODER_QUEUE_LIMIT + ) { + await new Promise((r) => setTimeout(r, 2)); + if (this.cancelled) return; + if (this.nativeEncoderError) throw this.nativeEncoderError; + } + const canvas = this.renderer!.getCanvas(); + const frame = new VideoFrame(canvas, { timestamp, duration: frameDuration }); + this.nativeH264Encoder.encode(frame, { keyFrame: frameIndex % 300 === 0 }); + frame.close(); } private async finishNativeVideoExport(audioPlan: NativeAudioPlan): Promise { if (!this.nativeExportSessionId) { - return { success: false, error: `${NATIVE_EXPORT_ENGINE_NAME} export session is not active` }; + return { + success: false, + error: `${NATIVE_EXPORT_ENGINE_NAME} export session is not active`, + }; } let editedAudioBuffer: ArrayBuffer | undefined; @@ -922,7 +978,8 @@ export class ModernVideoExporter { audioPlan.audioMode === "copy-source" || audioPlan.audioMode === "trim-source" ? audioPlan.audioSourcePath : null, - trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + trimSegments: + audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), @@ -1014,7 +1071,11 @@ export class ModernVideoExporter { }; } - private async encodeRenderedFrame(timestamp: number, frameDuration: number, frameIndex: number) { + private async encodeRenderedFrame( + timestamp: number, + frameDuration: number, + frameIndex: number, + ) { const canvas = this.renderer!.getCanvas(); // @ts-expect-error - colorSpace not in TypeScript definitions but works at runtime @@ -1057,14 +1118,20 @@ export class ModernVideoExporter { this.encodeQueue, ); } else { - console.warn(`[Frame ${frameIndex}] Encoder not ready! State: ${this.encoder?.state}`); + console.warn( + `[Frame ${frameIndex}] Encoder not ready! State: ${this.encoder?.state}`, + ); } } finally { exportFrame.close(); } } - private reportFinalizingProgress(totalFrames: number, renderProgress: number, audioProgress?: number) { + private reportFinalizingProgress( + totalFrames: number, + renderProgress: number, + audioProgress?: number, + ) { this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress); } @@ -1082,21 +1149,23 @@ export class ModernVideoExporter { const sampleFrameDelta = Math.max(currentFrame - this.lastProgressSampleFrame, 0); const sampleRenderFps = (sampleFrameDelta * 1000) / sampleElapsedMs; const remainingFrames = Math.max(totalFrames - currentFrame, 0); - const estimatedTimeRemaining = averageRenderFps > 0 ? remainingFrames / averageRenderFps : 0; + const estimatedTimeRemaining = + averageRenderFps > 0 ? remainingFrames / averageRenderFps : 0; const safeRenderProgress = - phase === "finalizing" - ? Math.max(0, Math.min(renderProgress ?? 99, 99)) - : undefined; + phase === "finalizing" ? Math.max(0, Math.min(renderProgress ?? 99, 99)) : undefined; const percentage = phase === "finalizing" - ? safeRenderProgress ?? 99 + ? (safeRenderProgress ?? 99) : totalFrames > 0 ? (currentFrame / totalFrames) * 100 : 100; if (nowMs - this.lastThroughputLogTimeMs >= 1000 || currentFrame === totalFrames) { const safeFrameCount = Math.max(this.processedFrameCount, 1); - this.peakEncodeQueueSize = Math.max(this.peakEncodeQueueSize, this.getCurrentEncodeBacklog()); + this.peakEncodeQueueSize = Math.max( + this.peakEncodeQueueSize, + this.getCurrentEncodeBacklog(), + ); console.log( `[VideoExporter] Progress ${JSON.stringify({ phase, @@ -1112,11 +1181,17 @@ export class ModernVideoExporter { pendingEncodeQueue: this.encodeQueue, encodeBacklog: this.getCurrentEncodeBacklog(), peakEncodeQueueSize: this.peakEncodeQueueSize, - nativeWriteInFlight: this.nativeWritePromises.length, + nativeWriteInFlight: this.nativeWritePromises.size, peakNativeWriteInFlight: this.peakNativeWriteInFlight, - averageFrameCallbackMs: Number((this.frameCallbackTimeMs / safeFrameCount).toFixed(3)), - averageRenderFrameMs: Number((this.renderFrameTimeMs / safeFrameCount).toFixed(3)), - averageEncodeWaitMs: Number((this.encodeWaitTimeMs / safeFrameCount).toFixed(3)), + averageFrameCallbackMs: Number( + (this.frameCallbackTimeMs / safeFrameCount).toFixed(3), + ), + averageRenderFrameMs: Number( + (this.renderFrameTimeMs / safeFrameCount).toFixed(3), + ), + averageEncodeWaitMs: Number( + (this.encodeWaitTimeMs / safeFrameCount).toFixed(3), + ), averageNativeCaptureMs: this.nativeCaptureTimeMs > 0 ? Number((this.nativeCaptureTimeMs / safeFrameCount).toFixed(3)) @@ -1175,13 +1250,17 @@ export class ModernVideoExporter { encoderName: this.encoderName ?? undefined, backpressureProfile: this.backpressureProfile?.name, averageFrameCallbackMs: - this.processedFrameCount > 0 ? this.frameCallbackTimeMs / safeFrameCount : undefined, + this.processedFrameCount > 0 + ? this.frameCallbackTimeMs / safeFrameCount + : undefined, averageRenderFrameMs: this.processedFrameCount > 0 ? this.renderFrameTimeMs / safeFrameCount : undefined, averageEncodeWaitMs: this.processedFrameCount > 0 ? this.encodeWaitTimeMs / safeFrameCount : undefined, averageNativeCaptureMs: - this.processedFrameCount > 0 ? this.nativeCaptureTimeMs / safeFrameCount : undefined, + this.processedFrameCount > 0 + ? this.nativeCaptureTimeMs / safeFrameCount + : undefined, averageNativeWriteMs: this.processedFrameCount > 0 ? this.nativeWriteTimeMs / safeFrameCount : undefined, }; @@ -1192,27 +1271,19 @@ export class ModernVideoExporter { } private trackNativeWritePromise(writePromise: Promise): void { - const trackedPromise = writePromise.finally(() => { - this.nativeWritePromises = this.nativeWritePromises.filter( - (candidate) => candidate !== trackedPromise, - ); - }); - - this.nativeWritePromises.push(trackedPromise); + this.nativeWritePromises.add(writePromise); this.peakNativeWriteInFlight = Math.max( this.peakNativeWriteInFlight, - this.nativeWritePromises.length, + this.nativeWritePromises.size, ); - } - private async waitForNativeWriteSlot(): Promise { - while (!this.cancelled && this.nativeWritePromises.length >= this.maxNativeWriteInFlight) { - await this.awaitOldestNativeWrite(); - } + void writePromise.finally(() => { + this.nativeWritePromises.delete(writePromise); + }); } private async awaitOldestNativeWrite(): Promise { - const oldestWritePromise = this.nativeWritePromises[0]; + const oldestWritePromise = this.nativeWritePromises.values().next().value; if (!oldestWritePromise) { return; } @@ -1225,7 +1296,7 @@ export class ModernVideoExporter { } private async awaitPendingNativeWrites(): Promise { - while (this.nativeWritePromises.length > 0) { + while (this.nativeWritePromises.size > 0) { await this.awaitOldestNativeWrite(); } @@ -1234,6 +1305,20 @@ export class ModernVideoExporter { } } + private disposeNativeH264Encoder(): void { + if (!this.nativeH264Encoder) { + return; + } + + try { + this.nativeH264Encoder.close(); + } catch (error) { + console.debug("[VideoExporter] Ignoring error closing native H.264 encoder:", error); + } + + this.nativeH264Encoder = null; + } + private getNowMs(): number { if (typeof performance !== "undefined" && typeof performance.now === "function") { return performance.now(); @@ -1257,9 +1342,7 @@ export class ModernVideoExporter { let videoDescription: Uint8Array | undefined; const encoderCandidates = this.getEncoderCandidates(); - const latencyModePreferences = getPreferredWebCodecsLatencyModes( - this.config.encodingMode, - ); + const latencyModePreferences = getPreferredWebCodecsLatencyModes(this.config.encodingMode); let resolvedCodec: string | null = null; @@ -1330,7 +1413,10 @@ export class ModernVideoExporter { }, }); - const baseConfig: Omit = { + const baseConfig: Omit< + VideoEncoderConfig, + "codec" | "hardwareAcceleration" | "latencyMode" + > = { width: this.config.width, height: this.config.height, bitrate: this.config.bitrate, @@ -1414,6 +1500,7 @@ export class ModernVideoExporter { if (this.audioProcessor) { this.audioProcessor.cancel(); } + this.disposeNativeH264Encoder(); const nativeExportSessionId = this.nativeExportSessionId; this.nativeExportSessionId = null; @@ -1454,6 +1541,7 @@ export class ModernVideoExporter { this.muxer = null; this.audioProcessor?.cancel(); this.audioProcessor = null; + this.disposeNativeH264Encoder(); const nativeExportSessionId = this.nativeExportSessionId; this.nativeExportSessionId = null; if (nativeExportSessionId && typeof window !== "undefined") { @@ -1482,7 +1570,7 @@ export class ModernVideoExporter { this.processedFrameCount = 0; this.lastProgressSampleTimeMs = 0; this.lastProgressSampleFrame = 0; - this.nativeWritePromises = []; + this.nativeWritePromises = new Set(); this.nativeWriteError = null; this.maxNativeWriteInFlight = 1; this.videoDescription = undefined; diff --git a/src/lib/exporter/mp4Support.ts b/src/lib/exporter/mp4Support.ts index cbe2b23f0..e1f9a384e 100644 --- a/src/lib/exporter/mp4Support.ts +++ b/src/lib/exporter/mp4Support.ts @@ -52,7 +52,11 @@ function buildEncoderSupportCacheKey(options: ResolveMp4EncoderPathOptions): str ].join(":"); } -function scaleDimensions(width: number, height: number, scale: number): { width: number; height: number } { +function scaleDimensions( + width: number, + height: number, + scale: number, +): { width: number; height: number } { return { width: normalizeEvenDimension(width * scale), height: normalizeEvenDimension(height * scale), @@ -85,11 +89,9 @@ export function getOrderedSupportedMp4EncoderCandidates(options: { }): SupportedMp4EncoderPath[] { const orderedCodecs = Array.from( new Set( - [ - options.preferredEncoderPath?.codec, - options.codec, - ...MP4_CODEC_FALLBACK_LIST, - ].filter((value): value is string => typeof value === "string" && value.length > 0), + [options.preferredEncoderPath?.codec, options.codec, ...MP4_CODEC_FALLBACK_LIST].filter( + (value): value is string => typeof value === "string" && value.length > 0, + ), ), ); const candidates: SupportedMp4EncoderPath[] = []; @@ -165,14 +167,20 @@ export async function probeSupportedMp4Dimensions( const codec = options.codec ?? DEFAULT_MP4_CODEC; const normalizedWidth = normalizeEvenDimension(options.width); const normalizedHeight = normalizeEvenDimension(options.height); - const dimensionCacheKey = [codec, normalizedWidth, normalizedHeight, options.frameRate].join(":"); + const requestedBitrate = options.getBitrate(normalizedWidth, normalizedHeight); + const dimensionCacheKey = [ + codec, + normalizedWidth, + normalizedHeight, + options.frameRate, + requestedBitrate, + ].join(":"); const cachedResult = supportedDimensionCache.get(dimensionCacheKey); if (cachedResult) { return cachedResult; } - const requestedBitrate = options.getBitrate(normalizedWidth, normalizedHeight); const directPath = await resolveSupportedMp4EncoderPath({ width: normalizedWidth, height: normalizedHeight, @@ -212,7 +220,9 @@ export async function probeSupportedMp4Dimensions( bestResult = { width: candidateDimensions.width, height: candidateDimensions.height, - capped: candidateDimensions.width !== normalizedWidth || candidateDimensions.height !== normalizedHeight, + capped: + candidateDimensions.width !== normalizedWidth || + candidateDimensions.height !== normalizedHeight, encoderPath: candidatePath, }; low = mid + 1; @@ -232,4 +242,4 @@ export async function probeSupportedMp4Dimensions( supportedDimensionCache.set(dimensionCacheKey, result); return result; -} \ No newline at end of file +} diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index ae9423953..3b19cc8ac 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -1,97 +1,96 @@ -import type { ExportConfig } from './types'; -import { - Output, - Mp4OutputFormat, - BufferTarget, - EncodedVideoPacketSource, - EncodedAudioPacketSource, - EncodedPacket -} from 'mediabunny'; +import { + BufferTarget, + EncodedAudioPacketSource, + EncodedPacket, + EncodedVideoPacketSource, + Mp4OutputFormat, + Output, +} from "mediabunny"; +import type { ExportConfig } from "./types"; export class VideoMuxer { - private output: Output | null = null; - private videoSource: EncodedVideoPacketSource | null = null; - private audioSource: EncodedAudioPacketSource | null = null; - private hasAudio: boolean; - private target: BufferTarget | null = null; - private config: ExportConfig; - - constructor(config: ExportConfig, hasAudio = false) { - this.config = config; - this.hasAudio = hasAudio; - } - - async initialize(): Promise { - // Create the buffer target - this.target = new BufferTarget(); - - this.output = new Output({ - format: new Mp4OutputFormat({ - fastStart: false, - }), - target: this.target, - }); - - // Create video source - codec will be deduced from metadata - this.videoSource = new EncodedVideoPacketSource('avc'); - this.output.addVideoTrack(this.videoSource, { - frameRate: this.config.frameRate, - }); - - // Create audio source if needed - if (this.hasAudio) { - this.audioSource = new EncodedAudioPacketSource('aac'); - this.output.addAudioTrack(this.audioSource); - } - - // Start the output to begin accepting media data - await this.output.start(); - } - - async addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): Promise { - if (!this.videoSource) { - throw new Error('Muxer not initialized'); - } - - // Convert WebCodecs chunk to Mediabunny packet - const packet = EncodedPacket.fromEncodedChunk(chunk); - - // Add metadata with the first chunk - await this.videoSource.add(packet, meta); - } - - async addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): Promise { - if (!this.audioSource) { - throw new Error('Audio not configured for this muxer'); - } - - // Convert WebCodecs chunk to Mediabunny packet - const packet = EncodedPacket.fromEncodedChunk(chunk); - - // Add metadata with the first chunk - await this.audioSource.add(packet, meta); - } - - async finalize(): Promise { - if (!this.output || !this.target) { - throw new Error('Muxer not initialized'); - } - - await this.output.finalize(); - const buffer = this.target.buffer; - - if (!buffer) { - throw new Error('Failed to finalize output'); - } - - return new Blob([buffer], { type: 'video/mp4' }); - } - - destroy(): void { - this.output = null; - this.videoSource = null; - this.audioSource = null; - this.target = null; - } -} + private output: Output | null = null; + private videoSource: EncodedVideoPacketSource | null = null; + private audioSource: EncodedAudioPacketSource | null = null; + private hasAudio: boolean; + private target: BufferTarget | null = null; + private config: ExportConfig; + + constructor(config: ExportConfig, hasAudio = false) { + this.config = config; + this.hasAudio = hasAudio; + } + + async initialize(): Promise { + // Create the buffer target + this.target = new BufferTarget(); + + this.output = new Output({ + format: new Mp4OutputFormat({ + fastStart: false, + }), + target: this.target, + }); + + // Create video source - codec will be deduced from metadata + this.videoSource = new EncodedVideoPacketSource("avc"); + this.output.addVideoTrack(this.videoSource, { + frameRate: this.config.frameRate, + }); + + // Create audio source if needed + if (this.hasAudio) { + this.audioSource = new EncodedAudioPacketSource("aac"); + this.output.addAudioTrack(this.audioSource); + } + + // Start the output to begin accepting media data + await this.output.start(); + } + + async addVideoChunk(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): Promise { + if (!this.videoSource) { + throw new Error("Muxer not initialized"); + } + + // Convert WebCodecs chunk to Mediabunny packet + const packet = EncodedPacket.fromEncodedChunk(chunk); + // Add metadata with the first chunk + await this.videoSource.add(packet, meta); + } + + async addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): Promise { + if (!this.audioSource) { + throw new Error("Audio not configured for this muxer"); + } + + // Convert WebCodecs chunk to Mediabunny packet + const packet = EncodedPacket.fromEncodedChunk(chunk); + + // Add metadata with the first chunk + await this.audioSource.add(packet, meta); + } + + async finalize(): Promise { + if (!this.output || !this.target) { + throw new Error("Muxer not initialized"); + } + + await this.output.finalize(); + const buffer = this.target.buffer; + + if (!buffer) { + throw new Error("Failed to finalize output"); + } + + return new Blob([buffer], { type: "video/mp4" }); + } + + destroy(): void { + this.output = null; + this.videoSource = null; + this.audioSource = null; + this.target = null; + } +} diff --git a/src/lib/exporter/nativeFrameCapture.ts b/src/lib/exporter/nativeFrameCapture.ts index 2f26fc359..5d69f6978 100644 --- a/src/lib/exporter/nativeFrameCapture.ts +++ b/src/lib/exporter/nativeFrameCapture.ts @@ -1,22 +1,16 @@ -const RGBA_BYTES_PER_PIXEL = 4 +const RGBA_BYTES_PER_PIXEL = 4; -let nativeFrameCaptureMode: 'video-frame-rgba' | 'canvas-readback' | null = null -let nativeFrameCaptureFallbackWarned = false -let fallbackCanvas: - | HTMLCanvasElement - | OffscreenCanvas - | null = null -let fallbackContext: - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null = null +let nativeFrameCaptureMode: "video-frame-rgba" | "canvas-readback" | null = null; +let nativeFrameCaptureFallbackWarned = false; +let fallbackCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; +let fallbackContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null = null; function getRgbaLayout(width: number): PlaneLayout[] { - return [{ offset: 0, stride: width * RGBA_BYTES_PER_PIXEL }] + return [{ offset: 0, stride: width * RGBA_BYTES_PER_PIXEL }]; } function getRgbaByteSize(width: number, height: number): number { - return width * height * RGBA_BYTES_PER_PIXEL + return width * height * RGBA_BYTES_PER_PIXEL; } function getFallbackReadbackContext( @@ -24,53 +18,57 @@ function getFallbackReadbackContext( height: number, ): CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D { if (!fallbackCanvas || fallbackCanvas.width !== width || fallbackCanvas.height !== height) { - if (typeof OffscreenCanvas !== 'undefined') { - fallbackCanvas = new OffscreenCanvas(width, height) - fallbackContext = fallbackCanvas.getContext('2d', { + if (typeof OffscreenCanvas !== "undefined") { + fallbackCanvas = new OffscreenCanvas(width, height); + fallbackContext = fallbackCanvas.getContext("2d", { willReadFrequently: true, - }) - } else if (typeof document !== 'undefined') { - const canvas = document.createElement('canvas') - canvas.width = width - canvas.height = height - fallbackCanvas = canvas - fallbackContext = canvas.getContext('2d', { + }); + } else if (typeof document !== "undefined") { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + fallbackCanvas = canvas; + fallbackContext = canvas.getContext("2d", { willReadFrequently: true, - }) + }); } else { - throw new Error('No canvas readback path is available for native export') + throw new Error("No canvas readback path is available for native export"); } } if (!fallbackContext) { - throw new Error('Failed to initialize native export frame readback context') + throw new Error("Failed to initialize native export frame readback context"); } - return fallbackContext + return fallbackContext; } -function captureCanvasFrameWithReadback(canvas: HTMLCanvasElement, targetWidth?: number, targetHeight?: number): Uint8Array { - const outWidth = targetWidth ?? canvas.width - const outHeight = targetHeight ?? canvas.height - const context = getFallbackReadbackContext(outWidth, outHeight) - context.clearRect(0, 0, outWidth, outHeight) - context.drawImage(canvas, 0, 0, outWidth, outHeight) - const imageData = context.getImageData(0, 0, outWidth, outHeight) - return new Uint8Array(imageData.data) +function captureCanvasFrameWithReadback( + canvas: HTMLCanvasElement, + targetWidth?: number, + targetHeight?: number, +): Uint8Array { + const outWidth = targetWidth ?? canvas.width; + const outHeight = targetHeight ?? canvas.height; + const context = getFallbackReadbackContext(outWidth, outHeight); + context.clearRect(0, 0, outWidth, outHeight); + context.drawImage(canvas, 0, 0, outWidth, outHeight); + const imageData = context.getImageData(0, 0, outWidth, outHeight); + return new Uint8Array(imageData.data); } function flipRgbaRowsInPlace(buffer: Uint8Array, width: number, height: number): void { - const rowByteLength = width * RGBA_BYTES_PER_PIXEL - const scratchRow = new Uint8Array(rowByteLength) - const halfRows = Math.floor(height / 2) + const rowByteLength = width * RGBA_BYTES_PER_PIXEL; + const scratchRow = new Uint8Array(rowByteLength); + const halfRows = Math.floor(height / 2); for (let rowIndex = 0; rowIndex < halfRows; rowIndex += 1) { - const topOffset = rowIndex * rowByteLength - const bottomOffset = (height - rowIndex - 1) * rowByteLength + const topOffset = rowIndex * rowByteLength; + const bottomOffset = (height - rowIndex - 1) * rowByteLength; - scratchRow.set(buffer.subarray(topOffset, topOffset + rowByteLength)) - buffer.copyWithin(topOffset, bottomOffset, bottomOffset + rowByteLength) - buffer.set(scratchRow, bottomOffset) + scratchRow.set(buffer.subarray(topOffset, topOffset + rowByteLength)); + buffer.copyWithin(topOffset, bottomOffset, bottomOffset + rowByteLength); + buffer.set(scratchRow, bottomOffset); } } @@ -81,40 +79,40 @@ export async function captureCanvasFrameForNativeExport( targetWidth?: number, targetHeight?: number, ): Promise { - const outWidth = targetWidth ?? canvas.width - const outHeight = targetHeight ?? canvas.height + const outWidth = targetWidth ?? canvas.width; + const outHeight = targetHeight ?? canvas.height; - if (nativeFrameCaptureMode !== 'canvas-readback') { - const buffer = new Uint8Array(getRgbaByteSize(outWidth, outHeight)) - const frame = new VideoFrame(canvas, { timestamp }) + if (nativeFrameCaptureMode !== "canvas-readback") { + const buffer = new Uint8Array(getRgbaByteSize(outWidth, outHeight)); + const frame = new VideoFrame(canvas, { timestamp }); try { await frame.copyTo(buffer, { - format: 'RGBA', + format: "RGBA", layout: getRgbaLayout(outWidth), - }) + }); if (flipVertical) { - flipRgbaRowsInPlace(buffer, outWidth, outHeight) + flipRgbaRowsInPlace(buffer, outWidth, outHeight); } - nativeFrameCaptureMode = 'video-frame-rgba' - return buffer + nativeFrameCaptureMode = "video-frame-rgba"; + return buffer; } catch (error) { - nativeFrameCaptureMode = 'canvas-readback' + nativeFrameCaptureMode = "canvas-readback"; if (!nativeFrameCaptureFallbackWarned) { - nativeFrameCaptureFallbackWarned = true + nativeFrameCaptureFallbackWarned = true; console.warn( - '[native-export] VideoFrame RGBA copyTo failed, falling back to canvas readback', + "[native-export] VideoFrame RGBA copyTo failed, falling back to canvas readback", error, - ) + ); } } finally { - frame.close() + frame.close(); } } - const buffer = captureCanvasFrameWithReadback(canvas, outWidth, outHeight) + const buffer = captureCanvasFrameWithReadback(canvas, outWidth, outHeight); if (flipVertical) { - flipRgbaRowsInPlace(buffer, outWidth, outHeight) + flipRgbaRowsInPlace(buffer, outWidth, outHeight); } - return buffer -} \ No newline at end of file + return buffer; +} diff --git a/src/lib/exporter/shadowProfile.ts b/src/lib/exporter/shadowProfile.ts index f676311b4..61f555360 100644 --- a/src/lib/exporter/shadowProfile.ts +++ b/src/lib/exporter/shadowProfile.ts @@ -16,4 +16,4 @@ export const WEBCAM_SHADOW_LAYER_PROFILES: ReadonlyArray = O export function getShadowFilterPadding(blur: number, offsetY: number): number { return Math.ceil(Math.max(0, blur * 2 + Math.abs(offsetY))); -} \ No newline at end of file +} diff --git a/src/lib/exporter/streamingDecoder.test.ts b/src/lib/exporter/streamingDecoder.test.ts index 91c3b85db..d9a7326ec 100644 --- a/src/lib/exporter/streamingDecoder.test.ts +++ b/src/lib/exporter/streamingDecoder.test.ts @@ -1,61 +1,61 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it } from "vitest"; import { - getDecodedFrameStartupOffsetUs, - getDecodedFrameTimelineOffsetUs, -} from './streamingDecoder'; + getDecodedFrameStartupOffsetUs, + getDecodedFrameTimelineOffsetUs, +} from "./streamingDecoder"; -describe('getDecodedFrameStartupOffsetUs', () => { - it('ignores positive stream start metadata when the first decoded frame matches it', () => { - expect( - getDecodedFrameStartupOffsetUs(4_978_000, { - streamStartTime: 4.978, - }), - ).toBe(0); - }); +describe("getDecodedFrameStartupOffsetUs", () => { + it("ignores positive stream start metadata when the first decoded frame matches it", () => { + expect( + getDecodedFrameStartupOffsetUs(4_978_000, { + streamStartTime: 4.978, + }), + ).toBe(0); + }); - it('returns only the startup gap beyond the stream start timestamp', () => { - expect( - getDecodedFrameStartupOffsetUs(5_128_000, { - streamStartTime: 4.978, - }), - ).toBe(150_000); - }); + it("returns only the startup gap beyond the stream start timestamp", () => { + expect( + getDecodedFrameStartupOffsetUs(5_128_000, { + streamStartTime: 4.978, + }), + ).toBe(150_000); + }); - it('falls back to media start time and then zero when stream metadata is missing', () => { - expect( - getDecodedFrameStartupOffsetUs(250_000, { - mediaStartTime: 0.1, - }), - ).toBe(150_000); + it("falls back to media start time and then zero when stream metadata is missing", () => { + expect( + getDecodedFrameStartupOffsetUs(250_000, { + mediaStartTime: 0.1, + }), + ).toBe(150_000); - expect(getDecodedFrameStartupOffsetUs(250_000, {})).toBe(250_000); - }); + expect(getDecodedFrameStartupOffsetUs(250_000, {})).toBe(250_000); + }); }); -describe('getDecodedFrameTimelineOffsetUs', () => { - it('preserves a non-zero stream start time when decoded timestamps match the stream start', () => { - expect( - getDecodedFrameTimelineOffsetUs(6_741_667, { - mediaStartTime: 0, - streamStartTime: 6.741667, - }), - ).toBe(6_741_667); - }); +describe("getDecodedFrameTimelineOffsetUs", () => { + it("preserves a non-zero stream start time when decoded timestamps match the stream start", () => { + expect( + getDecodedFrameTimelineOffsetUs(6_741_667, { + mediaStartTime: 0, + streamStartTime: 6.741667, + }), + ).toBe(6_741_667); + }); - it('includes both the stream start offset and any startup gap beyond it', () => { - expect( - getDecodedFrameTimelineOffsetUs(5_128_000, { - mediaStartTime: 0, - streamStartTime: 4.978, - }), - ).toBe(5_128_000); - }); + it("includes both the stream start offset and any startup gap beyond it", () => { + expect( + getDecodedFrameTimelineOffsetUs(5_128_000, { + mediaStartTime: 0, + streamStartTime: 4.978, + }), + ).toBe(5_128_000); + }); - it('falls back to a media-relative startup gap when stream metadata is missing', () => { - expect( - getDecodedFrameTimelineOffsetUs(250_000, { - mediaStartTime: 0.1, - }), - ).toBe(150_000); - }); + it("falls back to a media-relative startup gap when stream metadata is missing", () => { + expect( + getDecodedFrameTimelineOffsetUs(250_000, { + mediaStartTime: 0.1, + }), + ).toBe(150_000); + }); }); diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index 9f1c43e89..cfc1a9d91 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -1,641 +1,700 @@ -import { WebDemuxer } from 'web-demuxer'; -import type { TrimRegion, SpeedRegion } from '@/components/video-editor/types'; -import { getEffectiveVideoStreamDurationSeconds } from '@/lib/mediaTiming'; - -const DEFAULT_MAX_DECODE_QUEUE = 12; -const DEFAULT_MAX_PENDING_FRAMES = 32; - -export interface DecodedVideoInfo { - width: number; - height: number; - duration: number; // seconds - mediaStartTime?: number; // seconds - streamStartTime?: number; // seconds - streamDuration?: number; // seconds - frameRate: number; - codec: string; - hasAudio: boolean; - audioCodec?: string; -} - -/** Caller must close the VideoFrame after use. */ -type OnFrameCallback = ( - frame: VideoFrame, - exportTimestampUs: number, - sourceTimestampMs: number, - cursorTimestampMs: number -) => Promise; - +import { WebDemuxer } from "web-demuxer"; +import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; +import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; + +const DEFAULT_MAX_DECODE_QUEUE = 12; +const DEFAULT_MAX_PENDING_FRAMES = 32; + +export interface DecodedVideoInfo { + width: number; + height: number; + duration: number; // seconds + mediaStartTime?: number; // seconds + streamStartTime?: number; // seconds + streamDuration?: number; // seconds + frameRate: number; + codec: string; + hasAudio: boolean; + audioCodec?: string; +} + +/** Caller must close the VideoFrame after use. */ +type OnFrameCallback = ( + frame: VideoFrame, + exportTimestampUs: number, + sourceTimestampMs: number, + cursorTimestampMs: number, +) => Promise; + export function getDecodedFrameStartupOffsetUs( - firstDecodedFrameTimestampUs: number, - metadata: Pick + firstDecodedFrameTimestampUs: number, + metadata: Pick, ): number { - const streamStartTimeUs = Math.round( - (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000 - ); + const streamStartTimeUs = Math.round( + (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000, + ); - return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs); + return Math.max(0, firstDecodedFrameTimestampUs - streamStartTimeUs); } export function getDecodedFrameTimelineOffsetUs( - firstDecodedFrameTimestampUs: number, - metadata: Pick + firstDecodedFrameTimestampUs: number, + metadata: Pick, ): number { - const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000); - const streamStartTimeUs = Math.round( - (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000 - ); - - return ( - Math.max(0, streamStartTimeUs - mediaStartTimeUs) + - getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata) - ); + const mediaStartTimeUs = Math.round((metadata.mediaStartTime ?? 0) * 1_000_000); + const streamStartTimeUs = Math.round( + (metadata.streamStartTime ?? metadata.mediaStartTime ?? 0) * 1_000_000, + ); + + return ( + Math.max(0, streamStartTimeUs - mediaStartTimeUs) + + getDecodedFrameStartupOffsetUs(firstDecodedFrameTimestampUs, metadata) + ); +} + +/** + * Decodes video frames via web-demuxer + VideoDecoder in a single forward pass. + * Way faster than seeking an HTMLVideoElement per frame. + * + * Frames in trimmed regions are decoded (needed for P/B-frame state) but discarded. + * Kept frames are resampled to the target frame rate in a streaming pass. + */ +export class StreamingVideoDecoder { + private demuxer: WebDemuxer | null = null; + private decoder: VideoDecoder | null = null; + private cancelled = false; + private metadata: DecodedVideoInfo | null = null; + private pendingFrames: VideoFrame[] = []; + private readonly maxDecodeQueue: number; + private readonly maxPendingFrames: number; + + constructor(options?: { maxDecodeQueue?: number; maxPendingFrames?: number }) { + this.maxDecodeQueue = Math.max( + 1, + Math.floor(options?.maxDecodeQueue ?? DEFAULT_MAX_DECODE_QUEUE), + ); + this.maxPendingFrames = Math.max( + 1, + Math.floor(options?.maxPendingFrames ?? DEFAULT_MAX_PENDING_FRAMES), + ); + } + + private toLocalFilePath(resourceUrl: string): string | null { + if (!resourceUrl.startsWith("file:")) { + return null; + } + + try { + const url = new URL(resourceUrl); + let filePath = decodeURIComponent(url.pathname); + if (/^\/[A-Za-z]:/.test(filePath)) { + filePath = filePath.slice(1); + } + return filePath; + } catch { + return resourceUrl.replace(/^file:\/\//, ""); + } + } + + private inferMimeType(fileName: string): string { + const extension = fileName.split(".").pop()?.toLowerCase(); + switch (extension) { + case "mov": + return "video/quicktime"; + case "webm": + return "video/webm"; + case "mkv": + return "video/x-matroska"; + case "avi": + return "video/x-msvideo"; + case "mp4": + default: + return "video/mp4"; + } + } + + private async loadVideoFile(resourceUrl: string): Promise { + const filename = resourceUrl.split("/").pop() || "video"; + const localFilePath = this.toLocalFilePath(resourceUrl); + + if (localFilePath) { + const result = await window.electronAPI.readLocalFile(localFilePath); + if (!result.success || !result.data) { + throw new Error(result.error || "Failed to read local video file"); + } + + const bytes = + result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data); + const arrayBuffer = bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ) as ArrayBuffer; + return new File([arrayBuffer], filename, { type: this.inferMimeType(filename) }); + } + + const response = await fetch(resourceUrl); + if (!response.ok) { + throw new Error( + `Failed to load video resource: ${response.status} ${response.statusText}`, + ); + } + + const blob = await response.blob(); + return new File([blob], filename, { type: blob.type || this.inferMimeType(filename) }); + } + + private resolveVideoResourceUrl(videoUrl: string): string { + if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) { + return videoUrl; + } + + if (videoUrl.startsWith("/")) { + return `file://${encodeURI(videoUrl)}`; + } + + return videoUrl; + } + + async loadMetadata(videoUrl: string): Promise { + if (this.decoder) { + try { + if (this.decoder.state === "configured") { + this.decoder.close(); + } + } catch { + // Ignore cleanup errors while reloading metadata. + } + this.decoder = null; + } + + if (this.demuxer) { + try { + this.demuxer.destroy(); + } catch { + // Ignore cleanup errors while reloading metadata. + } + this.demuxer = null; + } + + const resourceUrl = this.resolveVideoResourceUrl(videoUrl); + + // Relative URL so it resolves correctly in both dev (http) and packaged (file://) builds + const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; + this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); + const file = await this.loadVideoFile(resourceUrl); + await this.demuxer.load(file); + + const mediaInfo = await this.demuxer.getMediaInfo(); + const videoStream = mediaInfo.streams.find((s) => s.codec_type_string === "video"); + const audioStream = mediaInfo.streams.find((s) => s.codec_type_string === "audio"); + const mediaStartTime = + typeof mediaInfo.start_time === "number" && Number.isFinite(mediaInfo.start_time) + ? mediaInfo.start_time + : 0; + const streamStartTime = + typeof videoStream?.start_time === "number" && Number.isFinite(videoStream.start_time) + ? videoStream.start_time + : mediaStartTime; + + let frameRate = 60; + if (videoStream?.avg_frame_rate) { + const parts = videoStream.avg_frame_rate.split("/"); + if (parts.length === 2) { + const num = parseInt(parts[0], 10); + const den = parseInt(parts[1], 10); + if (den > 0 && num > 0) frameRate = num / den; + } + } + + this.metadata = { + width: videoStream?.width || 1920, + height: videoStream?.height || 1080, + duration: mediaInfo.duration, + mediaStartTime, + streamStartTime, + streamDuration: + typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration) + ? videoStream.duration + : undefined, + frameRate, + codec: videoStream?.codec_string || "unknown", + hasAudio: !!audioStream, + audioCodec: audioStream?.codec_string, + }; + + return this.metadata; + } + + async decodeAll( + targetFrameRate: number, + trimRegions: TrimRegion[] | undefined, + speedRegions: SpeedRegion[] | undefined, + onFrame: OnFrameCallback, + ): Promise { + if (!this.demuxer || !this.metadata) { + throw new Error("Must call loadMetadata() before decodeAll()"); + } + + const decoderConfig = await this.demuxer.getDecoderConfig("video"); + const codec = this.metadata.codec.toLowerCase(); + const shouldPreferSoftwareDecode = codec.includes("av01") || codec.includes("av1"); + const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({ + duration: this.metadata.duration, + streamDuration: this.metadata.streamDuration, + }); + const segments = this.splitBySpeed( + this.computeSegments(effectiveVideoDuration, trimRegions), + speedRegions, + ); + const segmentOutputFrameCounts = segments.map((segment) => + Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate), + ); + const expectedOutputFrames = segmentOutputFrameCounts.reduce( + (sum, count) => sum + count, + 0, + ); + const frameDurationUs = 1_000_000 / targetFrameRate; + const epsilonSec = 0.001; + const startupStabilizationSeconds = 3; + const startupFrameBudget = Math.max( + 1, + Math.round(targetFrameRate * startupStabilizationSeconds), + ); + let exportFrameIndex = 0; + let loggedSteadyStateBackpressure = false; + + console.log( + `[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`, + ); + + // Async frame queue — decoder pushes, consumer pulls + this.pendingFrames.length = 0; + const pendingFrames = this.pendingFrames; + let frameResolve: ((frame: VideoFrame | null) => void) | null = null; + let decodeError: Error | null = null; + let decodeDone = false; + let firstDecodedFrameTimestampUs: number | null = null; + let decodedFrameTimelineOffsetUs = 0; + + this.decoder = new VideoDecoder({ + output: (frame: VideoFrame) => { + if (frameResolve) { + const resolve = frameResolve; + frameResolve = null; + resolve(frame); + } else { + pendingFrames.push(frame); + } + }, + error: (e: DOMException) => { + decodeError = new Error(`VideoDecoder error: ${e.message}`); + if (frameResolve) { + const resolve = frameResolve; + frameResolve = null; + resolve(null); + } + }, + }); + const preferredDecoderConfig = shouldPreferSoftwareDecode + ? { + ...decoderConfig, + hardwareAcceleration: "prefer-software" as const, + } + : decoderConfig; + + try { + this.decoder.configure(preferredDecoderConfig); + } catch (error) { + if (!shouldPreferSoftwareDecode) { + throw error; + } + // Fall back to default decoder config if software preference is unsupported. + this.decoder.configure(decoderConfig); + } + + const getNextFrame = (): Promise => { + if (decodeError) throw decodeError; + if (pendingFrames.length > 0) return Promise.resolve(pendingFrames.shift()!); + if (decodeDone) return Promise.resolve(null); + return new Promise((resolve) => { + frameResolve = resolve; + }); + }; + + // One forward stream through the whole file. + // Pass explicit range because some containers are truncated when no end is provided. + const readEndSec = + Math.max( + this.metadata.duration + (this.metadata.mediaStartTime ?? 0), + (this.metadata.streamDuration ?? this.metadata.duration) + + (this.metadata.streamStartTime ?? this.metadata.mediaStartTime ?? 0), + ) + 0.5; + const reader = this.demuxer.read("video", 0, readEndSec).getReader(); + + // Feed chunks to decoder in background with backpressure + const feedPromise = (async () => { + try { + while (!this.cancelled) { + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + + if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) { + loggedSteadyStateBackpressure = true; + console.log( + "[StreamingVideoDecoder] Switched to steady-state decode backpressure", + ); + } + + const decodeQueueLimit = + exportFrameIndex < startupFrameBudget + ? Math.min(this.maxDecodeQueue, 10) + : this.maxDecodeQueue; + const pendingFrameLimit = + exportFrameIndex < startupFrameBudget + ? Math.min(this.maxPendingFrames, 24) + : this.maxPendingFrames; + + // Backpressure on both decode queue and decoded frame backlog. + while ( + (this.decoder!.decodeQueueSize > decodeQueueLimit || + pendingFrames.length > pendingFrameLimit) && + !this.cancelled + ) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + if (this.cancelled) break; + + this.decoder!.decode(chunk); + } + + if (!this.cancelled && this.decoder!.state === "configured") { + await this.decoder!.flush(); + } + } catch (e) { + decodeError = e instanceof Error ? e : new Error(String(e)); + } finally { + decodeDone = true; + if (frameResolve) { + const resolve = frameResolve; + frameResolve = null; + resolve(null); + } + } + })(); + + // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling + let segmentIdx = 0; + let segmentFrameIndex = 0; + let lastDecodedFrameSec: number | null = null; + let heldFrame: VideoFrame | null = null; + let heldFrameSec = 0; + + const emitHeldFrameForTarget = async (segment: { + startSec: number; + endSec: number; + speed: number; + }) => { + if (!heldFrame) return false; + const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; + if (segmentFrameIndex >= segmentFrameCount) return false; + + const segmentDurationSec = segment.endSec - segment.startSec; + const sourceTimeSec = + segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; + if (sourceTimeSec >= segment.endSec - epsilonSec) return false; + + const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp }); + const sourceTimestampMs = sourceTimeSec * 1000; + await onFrame( + clone, + exportFrameIndex * frameDurationUs, + sourceTimestampMs, + sourceTimestampMs, + ); + segmentFrameIndex++; + exportFrameIndex++; + return true; + }; + + while (!this.cancelled && segmentIdx < segments.length) { + const frame = await getNextFrame(); + if (!frame) break; + + if (firstDecodedFrameTimestampUs === null) { + firstDecodedFrameTimestampUs = frame.timestamp; + decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs( + firstDecodedFrameTimestampUs, + this.metadata, + ); + } + + const normalizedFrameTimeSec = Math.max( + 0, + (frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) / + 1_000_000, + ); + const frameTimeSec: number = + lastDecodedFrameSec === null + ? normalizedFrameTimeSec + : Math.max(lastDecodedFrameSec, normalizedFrameTimeSec); + lastDecodedFrameSec = frameTimeSec; + + // Finalize completed segments before handling this frame. + while ( + segmentIdx < segments.length && + frameTimeSec >= segments[segmentIdx].endSec - epsilonSec + ) { + const segment = segments[segmentIdx]; + while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { + // Keep emitting remaining output frames for this segment from the last known frame. + } + + segmentIdx++; + segmentFrameIndex = 0; + if ( + heldFrame && + segmentIdx < segments.length && + heldFrameSec < segments[segmentIdx].startSec - epsilonSec + ) { + heldFrame.close(); + heldFrame = null; + } + } + + if (segmentIdx >= segments.length) { + frame.close(); + continue; + } + + const currentSegment = segments[segmentIdx]; + + // Before current segment (trimmed region or pre-roll). + if (frameTimeSec < currentSegment.startSec - epsilonSec) { + frame.close(); + continue; + } + + if (!heldFrame) { + heldFrame = frame; + heldFrameSec = frameTimeSec; + continue; + } + + // Any target timestamp before this midpoint is closer to heldFrame than current frame. + const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2; + while (!this.cancelled) { + const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; + if (segmentFrameIndex >= segmentFrameCount) { + break; + } + + const segmentDurationSec = currentSegment.endSec - currentSegment.startSec; + const sourceTimeSec = + currentSegment.startSec + + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; + if (sourceTimeSec >= currentSegment.endSec - epsilonSec) { + break; + } + if (sourceTimeSec > handoffBoundarySec) { + break; + } + + const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp }); + const sourceTimestampMs = sourceTimeSec * 1000; + await onFrame( + clone, + exportFrameIndex * frameDurationUs, + sourceTimestampMs, + sourceTimestampMs, + ); + segmentFrameIndex++; + exportFrameIndex++; + } + + heldFrame.close(); + heldFrame = frame; + heldFrameSec = frameTimeSec; + } + + // Flush remaining output frames for the last decoded frame. + if (heldFrame && segmentIdx < segments.length) { + while (!this.cancelled && segmentIdx < segments.length) { + const segment = segments[segmentIdx]; + if (heldFrameSec < segment.startSec - epsilonSec) { + break; + } + + while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { + // Keep emitting output frames for the active segment. + } + + segmentIdx++; + segmentFrameIndex = 0; + if ( + segmentIdx < segments.length && + heldFrameSec < segments[segmentIdx].startSec - epsilonSec + ) { + break; + } + } + heldFrame.close(); + heldFrame = null; + } else if (heldFrame) { + heldFrame.close(); + heldFrame = null; + } + + // Drain leftover decoded frames + while (!decodeDone) { + const frame = await getNextFrame(); + if (!frame) break; + frame.close(); + } + + try { + reader.cancel(); + } catch { + /* already closed */ + } + await feedPromise; + for (const f of pendingFrames) f.close(); + pendingFrames.length = 0; + + if (this.decoder?.state === "configured") { + this.decoder.close(); + } + this.decoder = null; + + const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0; + if ( + !this.cancelled && + lastDecodedFrameSec !== null && + requiredEndSec - lastDecodedFrameSec > 1 && + exportFrameIndex < expectedOutputFrames + ) { + throw new Error( + `Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).`, + ); + } + } + + private computeSegments( + totalDuration: number, + trimRegions?: TrimRegion[], + ): Array<{ startSec: number; endSec: number }> { + if (!trimRegions || trimRegions.length === 0) { + return [{ startSec: 0, endSec: totalDuration }]; + } + + const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs); + const segments: Array<{ startSec: number; endSec: number }> = []; + let cursor = 0; + + for (const trim of sorted) { + const trimStart = trim.startMs / 1000; + const trimEnd = trim.endMs / 1000; + if (cursor < trimStart) { + segments.push({ startSec: cursor, endSec: trimStart }); + } + cursor = Math.max(cursor, trimEnd); + } + + if (cursor < totalDuration) { + segments.push({ startSec: cursor, endSec: totalDuration }); + } + + return segments; + } + + getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number { + if (!this.metadata) throw new Error("Must call loadMetadata() first"); + const trimSegments = this.computeSegments( + getEffectiveVideoStreamDurationSeconds({ + duration: this.metadata.duration, + streamDuration: this.metadata.streamDuration, + }), + trimRegions, + ); + const speedSegments = this.splitBySpeed(trimSegments, speedRegions); + return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0); + } + + private splitBySpeed( + segments: Array<{ startSec: number; endSec: number }>, + speedRegions?: SpeedRegion[], + ): Array<{ startSec: number; endSec: number; speed: number }> { + if (!speedRegions || speedRegions.length === 0) + return segments.map((s) => ({ ...s, speed: 1 })); + + const result: Array<{ startSec: number; endSec: number; speed: number }> = []; + for (const segment of segments) { + const overlapping = speedRegions + .filter( + (sr) => + sr.startMs / 1000 < segment.endSec && sr.endMs / 1000 > segment.startSec, + ) + .sort((a, b) => a.startMs - b.startMs); + + if (overlapping.length === 0) { + result.push({ ...segment, speed: 1 }); + continue; + } + + let cursor = segment.startSec; + for (const sr of overlapping) { + const srStart = Math.max(sr.startMs / 1000, segment.startSec); + const srEnd = Math.min(sr.endMs / 1000, segment.endSec); + if (cursor < srStart) { + result.push({ startSec: cursor, endSec: srStart, speed: 1 }); + } + const effectiveStart = Math.max(cursor, srStart); + if (srEnd > effectiveStart) { + result.push({ startSec: effectiveStart, endSec: srEnd, speed: sr.speed }); + } + cursor = Math.max(cursor, srEnd); + } + if (cursor < segment.endSec) + result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 }); + } + return result.filter((s) => s.endSec - s.startSec > 0.0001); + } + + cancel(): void { + this.cancelled = true; + } + + getDemuxer() { + return this.demuxer; + } + + destroy(): void { + this.cancelled = true; + + if (this.decoder) { + try { + if (this.decoder.state === "configured") this.decoder.close(); + } catch { + /* ignore */ + } + this.decoder = null; + } + + if (this.demuxer) { + try { + this.demuxer.destroy(); + } catch { + /* ignore */ + } + this.demuxer = null; + } + + for (const frame of this.pendingFrames) { + try { + frame.close(); + } catch { + /* ignore */ + } + } + this.pendingFrames.length = 0; + } } - -/** - * Decodes video frames via web-demuxer + VideoDecoder in a single forward pass. - * Way faster than seeking an HTMLVideoElement per frame. - * - * Frames in trimmed regions are decoded (needed for P/B-frame state) but discarded. - * Kept frames are resampled to the target frame rate in a streaming pass. - */ -export class StreamingVideoDecoder { - private demuxer: WebDemuxer | null = null; - private decoder: VideoDecoder | null = null; - private cancelled = false; - private metadata: DecodedVideoInfo | null = null; - private pendingFrames: VideoFrame[] = []; - private readonly maxDecodeQueue: number; - private readonly maxPendingFrames: number; - - constructor(options?: { maxDecodeQueue?: number; maxPendingFrames?: number }) { - this.maxDecodeQueue = Math.max(1, Math.floor(options?.maxDecodeQueue ?? DEFAULT_MAX_DECODE_QUEUE)); - this.maxPendingFrames = Math.max(1, Math.floor(options?.maxPendingFrames ?? DEFAULT_MAX_PENDING_FRAMES)); - } - - private toLocalFilePath(resourceUrl: string): string | null { - if (!resourceUrl.startsWith('file:')) { - return null; - } - - try { - const url = new URL(resourceUrl); - let filePath = decodeURIComponent(url.pathname); - if (/^\/[A-Za-z]:/.test(filePath)) { - filePath = filePath.slice(1); - } - return filePath; - } catch { - return resourceUrl.replace(/^file:\/\//, ''); - } - } - - private inferMimeType(fileName: string): string { - const extension = fileName.split('.').pop()?.toLowerCase(); - switch (extension) { - case 'mov': - return 'video/quicktime'; - case 'webm': - return 'video/webm'; - case 'mkv': - return 'video/x-matroska'; - case 'avi': - return 'video/x-msvideo'; - case 'mp4': - default: - return 'video/mp4'; - } - } - - private async loadVideoFile(resourceUrl: string): Promise { - const filename = resourceUrl.split('/').pop() || 'video'; - const localFilePath = this.toLocalFilePath(resourceUrl); - - if (localFilePath) { - const result = await window.electronAPI.readLocalFile(localFilePath); - if (!result.success || !result.data) { - throw new Error(result.error || 'Failed to read local video file'); - } - - const bytes = result.data instanceof Uint8Array ? result.data : new Uint8Array(result.data); - const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; - return new File([arrayBuffer], filename, { type: this.inferMimeType(filename) }); - } - - const response = await fetch(resourceUrl); - if (!response.ok) { - throw new Error(`Failed to load video resource: ${response.status} ${response.statusText}`); - } - - const blob = await response.blob(); - return new File([blob], filename, { type: blob.type || this.inferMimeType(filename) }); - } - - private resolveVideoResourceUrl(videoUrl: string): string { - if (/^(blob:|data:|https?:|file:)/i.test(videoUrl)) { - return videoUrl; - } - - if (videoUrl.startsWith('/')) { - return `file://${encodeURI(videoUrl)}`; - } - - return videoUrl; - } - - async loadMetadata(videoUrl: string): Promise { - const resourceUrl = this.resolveVideoResourceUrl(videoUrl); - - // Relative URL so it resolves correctly in both dev (http) and packaged (file://) builds - const wasmUrl = new URL('./wasm/web-demuxer.wasm', window.location.href).href; - this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); - const file = await this.loadVideoFile(resourceUrl); - await this.demuxer.load(file); - - const mediaInfo = await this.demuxer.getMediaInfo(); - const videoStream = mediaInfo.streams.find(s => s.codec_type_string === 'video'); - const audioStream = mediaInfo.streams.find(s => s.codec_type_string === 'audio'); - const mediaStartTime = - typeof mediaInfo.start_time === 'number' && Number.isFinite(mediaInfo.start_time) - ? mediaInfo.start_time - : 0; - const streamStartTime = - typeof videoStream?.start_time === 'number' && Number.isFinite(videoStream.start_time) - ? videoStream.start_time - : mediaStartTime; - - let frameRate = 60; - if (videoStream?.avg_frame_rate) { - const parts = videoStream.avg_frame_rate.split('/'); - if (parts.length === 2) { - const num = parseInt(parts[0], 10); - const den = parseInt(parts[1], 10); - if (den > 0 && num > 0) frameRate = num / den; - } - } - - this.metadata = { - width: videoStream?.width || 1920, - height: videoStream?.height || 1080, - duration: mediaInfo.duration, - mediaStartTime, - streamStartTime, - streamDuration: - typeof videoStream?.duration === 'number' && Number.isFinite(videoStream.duration) - ? videoStream.duration - : undefined, - frameRate, - codec: videoStream?.codec_string || 'unknown', - hasAudio: !!audioStream, - audioCodec: audioStream?.codec_string, - }; - - return this.metadata; - } - - async decodeAll( - targetFrameRate: number, - trimRegions: TrimRegion[] | undefined, - speedRegions: SpeedRegion[] | undefined, - onFrame: OnFrameCallback - ): Promise { - if (!this.demuxer || !this.metadata) { - throw new Error('Must call loadMetadata() before decodeAll()'); - } - - const decoderConfig = await this.demuxer.getDecoderConfig('video'); - const codec = this.metadata.codec.toLowerCase(); - const shouldPreferSoftwareDecode = codec.includes('av01') || codec.includes('av1'); - const effectiveVideoDuration = getEffectiveVideoStreamDurationSeconds({ - duration: this.metadata.duration, - streamDuration: this.metadata.streamDuration, - }); - const segments = this.splitBySpeed( - this.computeSegments(effectiveVideoDuration, trimRegions), - speedRegions - ); - const segmentOutputFrameCounts = segments.map(segment => - Math.ceil(((segment.endSec - segment.startSec) / segment.speed) * targetFrameRate) - ); - const expectedOutputFrames = segmentOutputFrameCounts.reduce((sum, count) => sum + count, 0); - const frameDurationUs = 1_000_000 / targetFrameRate; - const epsilonSec = 0.001; - const startupStabilizationSeconds = 3; - const startupFrameBudget = Math.max(1, Math.round(targetFrameRate * startupStabilizationSeconds)); - let exportFrameIndex = 0; - let loggedSteadyStateBackpressure = false; - - console.log( - `[StreamingVideoDecoder] Startup-safe decode backpressure active for first ${startupStabilizationSeconds}s (${startupFrameBudget} frames)`, - ); - - // Async frame queue — decoder pushes, consumer pulls - this.pendingFrames.length = 0 - const pendingFrames = this.pendingFrames - let frameResolve: ((frame: VideoFrame | null) => void) | null = null; - let decodeError: Error | null = null; - let decodeDone = false; - let firstDecodedFrameTimestampUs: number | null = null; - let decodedFrameTimelineOffsetUs = 0; - - this.decoder = new VideoDecoder({ - output: (frame: VideoFrame) => { - if (frameResolve) { - const resolve = frameResolve; - frameResolve = null; - resolve(frame); - } else { - pendingFrames.push(frame); - } - }, - error: (e: DOMException) => { - decodeError = new Error(`VideoDecoder error: ${e.message}`); - if (frameResolve) { - const resolve = frameResolve; - frameResolve = null; - resolve(null); - } - }, - }); - const preferredDecoderConfig = shouldPreferSoftwareDecode - ? { - ...decoderConfig, - hardwareAcceleration: 'prefer-software' as const, - } - : decoderConfig; - - try { - this.decoder.configure(preferredDecoderConfig); - } catch (error) { - if (!shouldPreferSoftwareDecode) { - throw error; - } - // Fall back to default decoder config if software preference is unsupported. - this.decoder.configure(decoderConfig); - } - - const getNextFrame = (): Promise => { - if (decodeError) throw decodeError; - if (pendingFrames.length > 0) return Promise.resolve(pendingFrames.shift()!); - if (decodeDone) return Promise.resolve(null); - return new Promise(resolve => { frameResolve = resolve; }); - }; - - // One forward stream through the whole file. - // Pass explicit range because some containers are truncated when no end is provided. - const readEndSec = Math.max( - this.metadata.duration + (this.metadata.mediaStartTime ?? 0), - (this.metadata.streamDuration ?? this.metadata.duration) + - (this.metadata.streamStartTime ?? this.metadata.mediaStartTime ?? 0) - ) + 0.5; - const reader = this.demuxer.read('video', 0, readEndSec).getReader(); - - // Feed chunks to decoder in background with backpressure - const feedPromise = (async () => { - try { - while (!this.cancelled) { - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; - - if (!loggedSteadyStateBackpressure && exportFrameIndex >= startupFrameBudget) { - loggedSteadyStateBackpressure = true; - console.log("[StreamingVideoDecoder] Switched to steady-state decode backpressure"); - } - - const decodeQueueLimit = - exportFrameIndex < startupFrameBudget - ? Math.min(this.maxDecodeQueue, 10) - : this.maxDecodeQueue; - const pendingFrameLimit = - exportFrameIndex < startupFrameBudget - ? Math.min(this.maxPendingFrames, 24) - : this.maxPendingFrames; - - // Backpressure on both decode queue and decoded frame backlog. - while ( - ( - this.decoder!.decodeQueueSize > decodeQueueLimit || - pendingFrames.length > pendingFrameLimit - ) && - !this.cancelled - ) { - await new Promise(resolve => setTimeout(resolve, 1)); - } - if (this.cancelled) break; - - this.decoder!.decode(chunk); - } - - if (!this.cancelled && this.decoder!.state === 'configured') { - await this.decoder!.flush(); - } - } catch (e) { - decodeError = e instanceof Error ? e : new Error(String(e)); - } finally { - decodeDone = true; - if (frameResolve) { - const resolve = frameResolve; - frameResolve = null; - resolve(null); - } - } - })(); - - // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling - let segmentIdx = 0; - let segmentFrameIndex = 0; - let lastDecodedFrameSec: number | null = null; - let heldFrame: VideoFrame | null = null; - let heldFrameSec = 0; - - const emitHeldFrameForTarget = async (segment: { - startSec: number; - endSec: number; - speed: number; - }) => { - if (!heldFrame) return false; - const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; - if (segmentFrameIndex >= segmentFrameCount) return false; - - const segmentDurationSec = segment.endSec - segment.startSec; - const sourceTimeSec = - segment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; - if (sourceTimeSec >= segment.endSec - epsilonSec) return false; - - const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp }); - const sourceTimestampMs = sourceTimeSec * 1000; - await onFrame( - clone, - exportFrameIndex * frameDurationUs, - sourceTimestampMs, - sourceTimestampMs, - ); - segmentFrameIndex++; - exportFrameIndex++; - return true; - }; - - while (!this.cancelled && segmentIdx < segments.length) { - const frame = await getNextFrame(); - if (!frame) break; - - if (firstDecodedFrameTimestampUs === null) { - firstDecodedFrameTimestampUs = frame.timestamp; - decodedFrameTimelineOffsetUs = getDecodedFrameTimelineOffsetUs( - firstDecodedFrameTimestampUs, - this.metadata - ); - } - - const normalizedFrameTimeSec = Math.max( - 0, - (frame.timestamp - firstDecodedFrameTimestampUs + decodedFrameTimelineOffsetUs) / - 1_000_000, - ); - const frameTimeSec: number = - lastDecodedFrameSec === null - ? normalizedFrameTimeSec - : Math.max(lastDecodedFrameSec, normalizedFrameTimeSec); - lastDecodedFrameSec = frameTimeSec; - - // Finalize completed segments before handling this frame. - while ( - segmentIdx < segments.length && - frameTimeSec >= segments[segmentIdx].endSec - epsilonSec - ) { - const segment = segments[segmentIdx]; - while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { - // Keep emitting remaining output frames for this segment from the last known frame. - } - - segmentIdx++; - segmentFrameIndex = 0; - if ( - heldFrame && - segmentIdx < segments.length && - heldFrameSec < segments[segmentIdx].startSec - epsilonSec - ) { - heldFrame.close(); - heldFrame = null; - } - } - - if (segmentIdx >= segments.length) { - frame.close(); - continue; - } - - const currentSegment = segments[segmentIdx]; - - // Before current segment (trimmed region or pre-roll). - if (frameTimeSec < currentSegment.startSec - epsilonSec) { - frame.close(); - continue; - } - - if (!heldFrame) { - heldFrame = frame; - heldFrameSec = frameTimeSec; - continue; - } - - // Any target timestamp before this midpoint is closer to heldFrame than current frame. - const handoffBoundarySec = (heldFrameSec + frameTimeSec) / 2; - while (!this.cancelled) { - const segmentFrameCount = segmentOutputFrameCounts[segmentIdx]; - if (segmentFrameIndex >= segmentFrameCount) { - break; - } - - const segmentDurationSec = currentSegment.endSec - currentSegment.startSec; - const sourceTimeSec = - currentSegment.startSec + (segmentFrameIndex / segmentFrameCount) * segmentDurationSec; - if (sourceTimeSec >= currentSegment.endSec - epsilonSec) { - break; - } - if (sourceTimeSec > handoffBoundarySec) { - break; - } - - const clone = new VideoFrame(heldFrame, { timestamp: heldFrame.timestamp }); - const sourceTimestampMs = sourceTimeSec * 1000; - await onFrame( - clone, - exportFrameIndex * frameDurationUs, - sourceTimestampMs, - sourceTimestampMs, - ); - segmentFrameIndex++; - exportFrameIndex++; - } - - heldFrame.close(); - heldFrame = frame; - heldFrameSec = frameTimeSec; - } - - // Flush remaining output frames for the last decoded frame. - if (heldFrame && segmentIdx < segments.length) { - while (!this.cancelled && segmentIdx < segments.length) { - const segment = segments[segmentIdx]; - if (heldFrameSec < segment.startSec - epsilonSec) { - break; - } - - while (!this.cancelled && (await emitHeldFrameForTarget(segment))) { - // Keep emitting output frames for the active segment. - } - - segmentIdx++; - segmentFrameIndex = 0; - if ( - segmentIdx < segments.length && - heldFrameSec < segments[segmentIdx].startSec - epsilonSec - ) { - break; - } - } - heldFrame.close(); - heldFrame = null; - } else if (heldFrame) { - heldFrame.close(); - heldFrame = null; - } - - // Drain leftover decoded frames - while (!decodeDone) { - const frame = await getNextFrame(); - if (!frame) break; - frame.close(); - } - - try { reader.cancel(); } catch { /* already closed */ } - await feedPromise; - for (const f of pendingFrames) f.close(); - pendingFrames.length = 0; - - if (this.decoder?.state === 'configured') { - this.decoder.close(); - } - this.decoder = null; - - const requiredEndSec = segments.length > 0 ? segments[segments.length - 1].endSec : 0; - if ( - !this.cancelled && - lastDecodedFrameSec !== null && - requiredEndSec - lastDecodedFrameSec > 1 && - exportFrameIndex < expectedOutputFrames - ) { - throw new Error( - `Video decode ended early at ${lastDecodedFrameSec.toFixed(3)}s (needed ${requiredEndSec.toFixed(3)}s; rendered ${exportFrameIndex}/${expectedOutputFrames} frames).` - ); - } - } - - private computeSegments( - totalDuration: number, - trimRegions?: TrimRegion[] - ): Array<{ startSec: number; endSec: number }> { - if (!trimRegions || trimRegions.length === 0) { - return [{ startSec: 0, endSec: totalDuration }]; - } - - const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs); - const segments: Array<{ startSec: number; endSec: number }> = []; - let cursor = 0; - - for (const trim of sorted) { - const trimStart = trim.startMs / 1000; - const trimEnd = trim.endMs / 1000; - if (cursor < trimStart) { - segments.push({ startSec: cursor, endSec: trimStart }); - } - cursor = trimEnd; - } - - if (cursor < totalDuration) { - segments.push({ startSec: cursor, endSec: totalDuration }); - } - - return segments; - } - - getEffectiveDuration(trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[]): number { - if (!this.metadata) throw new Error('Must call loadMetadata() first'); - const trimSegments = this.computeSegments( - getEffectiveVideoStreamDurationSeconds({ - duration: this.metadata.duration, - streamDuration: this.metadata.streamDuration, - }), - trimRegions, - ); - const speedSegments = this.splitBySpeed(trimSegments, speedRegions); - return speedSegments.reduce((sum, seg) => sum + (seg.endSec - seg.startSec) / seg.speed, 0); - } - - private splitBySpeed( - segments: Array<{ startSec: number; endSec: number }>, - speedRegions?: SpeedRegion[] - ): Array<{ startSec: number; endSec: number; speed: number }> { - if (!speedRegions || speedRegions.length === 0) - return segments.map(s => ({ ...s, speed: 1 })); - - const result: Array<{ startSec: number; endSec: number; speed: number }> = []; - for (const segment of segments) { - const overlapping = speedRegions - .filter(sr => (sr.startMs / 1000) < segment.endSec && (sr.endMs / 1000) > segment.startSec) - .sort((a, b) => a.startMs - b.startMs); - - if (overlapping.length === 0) { result.push({ ...segment, speed: 1 }); continue; } - - let cursor = segment.startSec; - for (const sr of overlapping) { - const srStart = Math.max(sr.startMs / 1000, segment.startSec); - const srEnd = Math.min(sr.endMs / 1000, segment.endSec); - if (cursor < srStart) result.push({ startSec: cursor, endSec: srStart, speed: 1 }); - result.push({ startSec: srStart, endSec: srEnd, speed: sr.speed }); - cursor = srEnd; - } - if (cursor < segment.endSec) result.push({ startSec: cursor, endSec: segment.endSec, speed: 1 }); - } - return result.filter(s => s.endSec - s.startSec > 0.0001); - } - - cancel(): void { - this.cancelled = true; - } - - getDemuxer() { - return this.demuxer; - } - - destroy(): void { - this.cancelled = true; - - if (this.decoder) { - try { - if (this.decoder.state === 'configured') this.decoder.close(); - } catch { /* ignore */ } - this.decoder = null; - } - - if (this.demuxer) { - try { - this.demuxer.destroy(); - } catch { - /* ignore */ - } - this.demuxer = null; - } - - for (const frame of this.pendingFrames) { - try { - frame.close() - } catch { - /* ignore */ - } - } - this.pendingFrames.length = 0 - } -} - diff --git a/src/lib/exporter/types.test.ts b/src/lib/exporter/types.test.ts index 200841933..81f9553c3 100644 --- a/src/lib/exporter/types.test.ts +++ b/src/lib/exporter/types.test.ts @@ -1,64 +1,56 @@ -import { describe, it, expect } from 'vitest'; -import * as fc from 'fast-check'; -import { - isValidGifFrameRate, - VALID_GIF_FRAME_RATES, - GifFrameRate -} from './types'; +import * as fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { GifFrameRate, isValidGifFrameRate, VALID_GIF_FRAME_RATES } from "./types"; /** * Property 1: Valid Frame Rate Acceptance - * - * *For any* frame rate value, the GIF_Exporter SHALL accept it if and only if - * it is one of the valid presets (15, 20, 25, 30 FPS). Invalid frame rates + * + * *For any* frame rate value, the GIF_Exporter SHALL accept it if and only if + * it is one of the valid presets (15, 20, 25, 30 FPS). Invalid frame rates * should be rejected with an error. - * + * * **Validates: Requirements 2.2** - * + * * Feature: gif-export, Property 1: Valid Frame Rate Acceptance */ -describe('GIF Export Types', () => { - describe('Property 1: Valid Frame Rate Acceptance', () => { - // Property test: Valid frame rates should be accepted - it('should accept all valid frame rates (15, 20, 25, 30)', () => { - fc.assert( - fc.property( - fc.constantFrom(...VALID_GIF_FRAME_RATES), - (frameRate: GifFrameRate) => { - expect(isValidGifFrameRate(frameRate)).toBe(true); - } - ), - { numRuns: 100 } - ); - }); +describe("GIF Export Types", () => { + describe("Property 1: Valid Frame Rate Acceptance", () => { + // Property test: Valid frame rates should be accepted + it("should accept all valid frame rates (15, 20, 25, 30)", () => { + fc.assert( + fc.property( + fc.constantFrom(...VALID_GIF_FRAME_RATES), + (frameRate: GifFrameRate) => { + expect(isValidGifFrameRate(frameRate)).toBe(true); + }, + ), + { numRuns: 100 }, + ); + }); - // Property test: Invalid frame rates should be rejected - it('should reject any frame rate not in the valid set', () => { - fc.assert( - fc.property( - fc.integer().filter(n => !VALID_GIF_FRAME_RATES.includes(n as GifFrameRate)), - (invalidFrameRate: number) => { - expect(isValidGifFrameRate(invalidFrameRate)).toBe(false); - } - ), - { numRuns: 100 } - ); - }); + // Property test: Invalid frame rates should be rejected + it("should reject any frame rate not in the valid set", () => { + fc.assert( + fc.property( + fc.integer().filter((n) => !VALID_GIF_FRAME_RATES.includes(n as GifFrameRate)), + (invalidFrameRate: number) => { + expect(isValidGifFrameRate(invalidFrameRate)).toBe(false); + }, + ), + { numRuns: 100 }, + ); + }); - // Property test: Frame rate validation is deterministic - it('should return consistent results for the same input', () => { - fc.assert( - fc.property( - fc.integer({ min: 1, max: 60 }), - (frameRate: number) => { - const result1 = isValidGifFrameRate(frameRate); - const result2 = isValidGifFrameRate(frameRate); - expect(result1).toBe(result2); - } - ), - { numRuns: 100 } - ); - }); - }); + // Property test: Frame rate validation is deterministic + it("should return consistent results for the same input", () => { + fc.assert( + fc.property(fc.integer({ min: 1, max: 60 }), (frameRate: number) => { + const result1 = isValidGifFrameRate(frameRate); + const result2 = isValidGifFrameRate(frameRate); + expect(result1).toBe(result2); + }), + { numRuns: 100 }, + ); + }); + }); }); - diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index 73a323951..cdda5ea78 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -1,134 +1,133 @@ export interface ExportConfig { - width: number; - height: number; - frameRate: number; - bitrate: number; - codec?: string; - encodingMode?: ExportEncodingMode; - backendPreference?: ExportBackendPreference; - experimentalNativeExport?: boolean; - maxEncodeQueue?: number; - maxDecodeQueue?: number; - maxPendingFrames?: number; - maxInFlightNativeWrites?: number; + width: number; + height: number; + frameRate: number; + bitrate: number; + codec?: string; + encodingMode?: ExportEncodingMode; + backendPreference?: ExportBackendPreference; + experimentalNativeExport?: boolean; + maxEncodeQueue?: number; + maxDecodeQueue?: number; + maxPendingFrames?: number; + maxInFlightNativeWrites?: number; } -export type ExportRenderBackend = 'webgpu' | 'webgl'; -export type ExportEncodeBackend = 'ffmpeg' | 'webcodecs'; -export type ExportBackendPreference = 'auto' | 'webcodecs' | 'breeze'; -export type ExportPipelineModel = 'modern' | 'legacy'; +export type ExportRenderBackend = "webgpu" | "webgl"; +export type ExportEncodeBackend = "ffmpeg" | "webcodecs"; +export type ExportBackendPreference = "auto" | "webcodecs" | "breeze"; +export type ExportPipelineModel = "modern" | "legacy"; export interface ExportProgress { - currentFrame: number; - totalFrames: number; - percentage: number; - estimatedTimeRemaining: number; // in seconds - renderFps?: number; - renderBackend?: ExportRenderBackend; - encodeBackend?: ExportEncodeBackend; - encoderName?: string; - phase?: 'extracting' | 'finalizing' | 'saving'; // Phase of export - renderProgress?: number; // 0-100, progress of GIF rendering phase - audioProgress?: number; // 0-1, progress of real-time audio rendering (speed/audio regions) + currentFrame: number; + totalFrames: number; + percentage: number; + estimatedTimeRemaining: number; // in seconds + renderFps?: number; + renderBackend?: ExportRenderBackend; + encodeBackend?: ExportEncodeBackend; + encoderName?: string; + phase?: "extracting" | "finalizing" | "saving"; // Phase of export + renderProgress?: number; // 0-100, progress of GIF rendering phase + audioProgress?: number; // 0-1, progress of real-time audio rendering (speed/audio regions) } export interface ExportMetrics { - totalElapsedMs: number; - metadataLoadMs?: number; - rendererInitMs?: number; - nativeSessionStartMs?: number; - decodeLoopMs?: number; - frameCallbackMs?: number; - renderFrameMs?: number; - encodeWaitMs?: number; - encodeWaitEvents?: number; - peakEncodeQueueSize?: number; - peakNativeWriteInFlight?: number; - nativeCaptureMs?: number; - nativeWriteMs?: number; - finalizationMs?: number; - frameCount?: number; - renderBackend?: ExportRenderBackend; - encodeBackend?: ExportEncodeBackend; - encoderName?: string; - backpressureProfile?: string; - averageFrameCallbackMs?: number; - averageRenderFrameMs?: number; - averageEncodeWaitMs?: number; - averageNativeCaptureMs?: number; - averageNativeWriteMs?: number; + totalElapsedMs: number; + metadataLoadMs?: number; + rendererInitMs?: number; + nativeSessionStartMs?: number; + decodeLoopMs?: number; + frameCallbackMs?: number; + renderFrameMs?: number; + encodeWaitMs?: number; + encodeWaitEvents?: number; + peakEncodeQueueSize?: number; + peakNativeWriteInFlight?: number; + nativeCaptureMs?: number; + nativeWriteMs?: number; + finalizationMs?: number; + frameCount?: number; + renderBackend?: ExportRenderBackend; + encodeBackend?: ExportEncodeBackend; + encoderName?: string; + backpressureProfile?: string; + averageFrameCallbackMs?: number; + averageRenderFrameMs?: number; + averageEncodeWaitMs?: number; + averageNativeCaptureMs?: number; + averageNativeWriteMs?: number; } export interface ExportResult { - success: boolean; - blob?: Blob; - filePath?: string; - error?: string; - metrics?: ExportMetrics; + success: boolean; + blob?: Blob; + filePath?: string; + error?: string; + metrics?: ExportMetrics; } export interface VideoFrameData { - frame: VideoFrame; - timestamp: number; // in microseconds - duration: number; // in microseconds + frame: VideoFrame; + timestamp: number; // in microseconds + duration: number; // in microseconds } -export type ExportEncodingMode = 'fast' | 'balanced' | 'quality'; +export type ExportEncodingMode = "fast" | "balanced" | "quality"; -export type ExportQuality = 'medium' | 'good' | 'high' | 'source'; +export type ExportQuality = "medium" | "good" | "high" | "source"; export type ExportMp4FrameRate = 24 | 30 | 60; // GIF Export Types -export type ExportFormat = 'mp4' | 'gif'; +export type ExportFormat = "mp4" | "gif"; export type GifFrameRate = 15 | 20 | 25 | 30; -export type GifSizePreset = 'medium' | 'large' | 'original'; +export type GifSizePreset = "medium" | "large" | "original"; export interface GifExportConfig { - frameRate: GifFrameRate; - loop: boolean; - sizePreset: GifSizePreset; - width: number; - height: number; + frameRate: GifFrameRate; + loop: boolean; + sizePreset: GifSizePreset; + width: number; + height: number; } export interface ExportSettings { - format: ExportFormat; - // MP4 settings - quality?: ExportQuality; - encodingMode?: ExportEncodingMode; - mp4FrameRate?: ExportMp4FrameRate; - backendPreference?: ExportBackendPreference; - pipelineModel?: ExportPipelineModel; - // GIF settings - gifConfig?: GifExportConfig; + format: ExportFormat; + // MP4 settings + quality?: ExportQuality; + encodingMode?: ExportEncodingMode; + mp4FrameRate?: ExportMp4FrameRate; + backendPreference?: ExportBackendPreference; + pipelineModel?: ExportPipelineModel; + // GIF settings + gifConfig?: GifExportConfig; } export const MP4_FRAME_RATES: readonly ExportMp4FrameRate[] = [24, 30, 60] as const; export function isValidMp4FrameRate(rate: number): rate is ExportMp4FrameRate { - return MP4_FRAME_RATES.includes(rate as ExportMp4FrameRate); + return MP4_FRAME_RATES.includes(rate as ExportMp4FrameRate); } export const GIF_SIZE_PRESETS: Record = { - medium: { maxHeight: 720, label: 'Medium (720p)' }, - large: { maxHeight: 1080, label: 'Large (1080p)' }, - original: { maxHeight: Infinity, label: 'Original' }, + medium: { maxHeight: 720, label: "Medium (720p)" }, + large: { maxHeight: 1080, label: "Large (1080p)" }, + original: { maxHeight: Infinity, label: "Original" }, }; export const GIF_FRAME_RATES: { value: GifFrameRate; label: string }[] = [ - { value: 15, label: '15 FPS - Balanced' }, - { value: 20, label: '20 FPS - Smooth' }, - { value: 25, label: '25 FPS - Very smooth' }, - { value: 30, label: '30 FPS - Maximum' }, + { value: 15, label: "15 FPS - Balanced" }, + { value: 20, label: "20 FPS - Smooth" }, + { value: 25, label: "25 FPS - Very smooth" }, + { value: 30, label: "30 FPS - Maximum" }, ]; // Valid frame rates for validation export const VALID_GIF_FRAME_RATES: readonly GifFrameRate[] = [15, 20, 25, 30] as const; export function isValidGifFrameRate(rate: number): rate is GifFrameRate { - return VALID_GIF_FRAME_RATES.includes(rate as GifFrameRate); + return VALID_GIF_FRAME_RATES.includes(rate as GifFrameRate); } - diff --git a/src/lib/exporter/videoDecoder.ts b/src/lib/exporter/videoDecoder.ts index 4c1e14d5c..4ed1157ac 100644 --- a/src/lib/exporter/videoDecoder.ts +++ b/src/lib/exporter/videoDecoder.ts @@ -1,58 +1,57 @@ export interface DecodedVideoInfo { - width: number; - height: number; - duration: number; // in seconds - frameRate: number; - codec: string; + width: number; + height: number; + duration: number; // in seconds + frameRate: number; + codec: string; } export class VideoFileDecoder { - private info: DecodedVideoInfo | null = null; - private videoElement: HTMLVideoElement | null = null; - - async loadVideo(videoUrl: string): Promise { - this.videoElement = document.createElement('video'); - this.videoElement.src = videoUrl; - this.videoElement.preload = 'metadata'; - - return new Promise((resolve, reject) => { - this.videoElement!.addEventListener('loadedmetadata', () => { - const video = this.videoElement!; - - this.info = { - width: video.videoWidth, - height: video.videoHeight, - duration: video.duration, - frameRate: 60, - codec: 'avc1.640033', - }; - - resolve(this.info); - }); - - this.videoElement!.addEventListener('error', (e) => { - reject(new Error(`Failed to load video: ${e}`)); - }); - }); - } - - /** - * Get video element for seeking - */ - getVideoElement(): HTMLVideoElement | null { - return this.videoElement; - } - - getInfo(): DecodedVideoInfo | null { - return this.info; - } - - destroy(): void { - if (this.videoElement) { - this.videoElement.pause(); - this.videoElement.src = ''; - this.videoElement = null; - } - } + private info: DecodedVideoInfo | null = null; + private videoElement: HTMLVideoElement | null = null; + + async loadVideo(videoUrl: string): Promise { + this.videoElement = document.createElement("video"); + this.videoElement.src = videoUrl; + this.videoElement.preload = "metadata"; + + return new Promise((resolve, reject) => { + this.videoElement!.addEventListener("loadedmetadata", () => { + const video = this.videoElement!; + + this.info = { + width: video.videoWidth, + height: video.videoHeight, + duration: video.duration, + frameRate: 60, + codec: "avc1.640033", + }; + + resolve(this.info); + }); + + this.videoElement!.addEventListener("error", (e) => { + reject(new Error(`Failed to load video: ${e}`)); + }); + }); + } + + /** + * Get video element for seeking + */ + getVideoElement(): HTMLVideoElement | null { + return this.videoElement; + } + + getInfo(): DecodedVideoInfo | null { + return this.info; + } + + destroy(): void { + if (this.videoElement) { + this.videoElement.pause(); + this.videoElement.src = ""; + this.videoElement = null; + } + } } - diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index a346b7fac..a6360a3fb 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -1,7 +1,7 @@ import type { - AutoCaptionSettings, AnnotationRegion, AudioRegion, + AutoCaptionSettings, CaptionCue, CropRegion, CursorStyle, @@ -9,20 +9,18 @@ import type { SpeedRegion, TrimRegion, WebcamOverlaySettings, - ZoomTransitionEasing, ZoomRegion, + ZoomTransitionEasing, } from "@/components/video-editor/types"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; import { FrameRenderer } from "./frameRenderer"; import type { SupportedMp4EncoderPath } from "./mp4Support"; -import { captureCanvasFrameForNativeExport } from "./nativeFrameCapture"; import { VideoMuxer } from "./muxer"; import { type DecodedVideoInfo, StreamingVideoDecoder } from "./streamingDecoder"; import type { ExportConfig, ExportProgress, ExportResult } from "./types"; const DEFAULT_MAX_ENCODE_QUEUE = 240; const PROGRESS_SAMPLE_WINDOW_MS = 1_000; -let nativeExportDisabledWarningShown = false; interface VideoExporterConfig extends ExportConfig { videoUrl: string; @@ -103,6 +101,12 @@ export class VideoExporter { private progressSampleStartFrame = 0; private encoderError: Error | null = null; private nativeExportSessionId: string | null = null; + private nativeH264Encoder: VideoEncoder | null = null; + private nativePendingWrite: Promise = Promise.resolve(); + private nativeWritePromises = new Set>(); + private nativeWriteError: Error | null = null; + private maxNativeWriteInFlight = 1; + private nativeEncoderError: Error | null = null; constructor(config: VideoExporterConfig) { this.config = config; @@ -113,6 +117,14 @@ export class VideoExporter { this.cleanup(); this.cancelled = false; this.encoderError = null; + this.nativeEncoderError = null; + this.nativePendingWrite = Promise.resolve(); + this.nativeWritePromises = new Set(); + this.nativeWriteError = null; + this.maxNativeWriteInFlight = Math.max( + 1, + Math.floor(this.config.maxInFlightNativeWrites ?? 1), + ); this.exportStartTimeMs = this.getNowMs(); this.progressSampleStartTimeMs = this.exportStartTimeMs; this.progressSampleStartFrame = 0; @@ -142,7 +154,7 @@ export class VideoExporter { this.renderer = new FrameRenderer({ width: this.config.width, height: this.config.height, - preferredRenderBackend: useNativeEncoder ? "webgl" : undefined, + preferredRenderBackend: undefined, wallpaper: this.config.wallpaper, zoomRegions: this.config.zoomRegions, showShadow: this.config.showShadow, @@ -226,11 +238,15 @@ export class VideoExporter { const timestamp = frameIndex * frameDuration; const sourceTimestampUs = sourceTimestampMs * 1000; const cursorTimestampUs = cursorTimestampMs * 1000; - await this.renderer!.renderFrame(videoFrame, sourceTimestampUs, cursorTimestampUs); + await this.renderer!.renderFrame( + videoFrame, + sourceTimestampUs, + cursorTimestampUs, + ); videoFrame.close(); if (useNativeEncoder) { - await this.encodeRenderedFrameNative(timestamp); + await this.encodeRenderedFrameNative(timestamp, frameDuration, frameIndex); } else { await this.encodeRenderedFrame(timestamp, frameDuration, frameIndex); } @@ -251,6 +267,15 @@ export class VideoExporter { this.reportFinalizingProgress(totalFrames, 96); if (useNativeEncoder && nativeAudioPlan) { + if (this.nativeH264Encoder) { + await this.nativeH264Encoder.flush(); + await this.awaitPendingNativeWrites(); + if (this.nativeEncoderError) { + throw this.nativeEncoderError; + } + this.nativeH264Encoder.close(); + this.nativeH264Encoder = null; + } this.reportFinalizingProgress(totalFrames, 99, 0); return await this.finishNativeVideoExport(nativeAudioPlan, totalFrames); } @@ -263,7 +288,10 @@ export class VideoExporter { // Wait for queued muxing operations to complete this.reportFinalizingProgress(totalFrames, 98); - await this.awaitWithFinalizationTimeout(this.pendingMuxing, "muxing queued video chunks"); + await this.awaitWithFinalizationTimeout( + this.pendingMuxing, + "muxing queued video chunks", + ); if (hasAudio && !shouldUseFfmpegAudioFallback && !this.cancelled) { const demuxer = this.streamingDecoder.getDemuxer(); @@ -291,7 +319,10 @@ export class VideoExporter { // Finalize muxer and get output blob this.reportFinalizingProgress(totalFrames, 99); - const blob = await this.awaitWithFinalizationTimeout(this.muxer!.finalize(), "muxer finalization"); + const blob = await this.awaitWithFinalizationTimeout( + this.muxer!.finalize(), + "muxer finalization", + ); if (shouldUseFfmpegAudioFallback) { console.warn( @@ -310,7 +341,8 @@ export class VideoExporter { console.error("Export error:", error); return { success: false, - error: resolvedError instanceof Error ? resolvedError.message : String(resolvedError), + error: + resolvedError instanceof Error ? resolvedError.message : String(resolvedError), }; } finally { this.cleanup(); @@ -318,18 +350,15 @@ export class VideoExporter { } private shouldUseExperimentalNativeExport(): boolean { - if (this.config.experimentalNativeExport === true) { - return true; - } - - if (typeof window !== "undefined" && !nativeExportDisabledWarningShown) { - nativeExportDisabledWarningShown = true; - console.info( - "[VideoExporter] Native ffmpeg export is disabled by default until direct GPU readback is restored.", - ); - } - - return false; + return ( + typeof window !== "undefined" && + typeof VideoEncoder !== "undefined" && + typeof VideoEncoder.isConfigSupported === "function" && + typeof window.electronAPI?.nativeVideoExportStart === "function" && + typeof window.electronAPI?.nativeVideoExportWriteFrame === "function" && + typeof window.electronAPI?.nativeVideoExportFinish === "function" && + typeof window.electronAPI?.nativeVideoExportCancel === "function" + ); } private async awaitWithFinalizationTimeout(promise: Promise, stage: string): Promise { @@ -389,7 +418,9 @@ export class VideoExporter { } private buildNativeTrimSegments(durationMs: number): Array<{ startMs: number; endMs: number }> { - const trimRegions = [...(this.config.trimRegions ?? [])].sort((a, b) => a.startMs - b.startMs); + const trimRegions = [...(this.config.trimRegions ?? [])].sort( + (a, b) => a.startMs - b.startMs, + ); if (trimRegions.length === 0) { return [{ startMs: 0, endMs: Math.max(0, durationMs) }]; } @@ -421,13 +452,23 @@ export class VideoExporter { ); const localVideoSourcePath = this.getNativeVideoSourcePath(); const primaryAudioSourcePath = - (videoInfo.hasAudio ? localVideoSourcePath : null) ?? sourceAudioFallbackPaths[0] ?? null; + (videoInfo.hasAudio ? localVideoSourcePath : null) ?? + sourceAudioFallbackPaths[0] ?? + null; - if (!videoInfo.hasAudio && sourceAudioFallbackPaths.length === 0 && audioRegions.length === 0) { + if ( + !videoInfo.hasAudio && + sourceAudioFallbackPaths.length === 0 && + audioRegions.length === 0 + ) { return { audioMode: "none" }; } - if (speedRegions.length > 0 || audioRegions.length > 0 || sourceAudioFallbackPaths.length > 1) { + if ( + speedRegions.length > 0 || + audioRegions.length > 0 || + sourceAudioFallbackPaths.length > 1 + ) { return { audioMode: "edited-track" }; } @@ -459,7 +500,7 @@ export class VideoExporter { } private async tryStartNativeVideoExport(): Promise { - if (typeof window === "undefined" || !window.electronAPI?.nativeVideoExportStart) { + if (!this.shouldUseExperimentalNativeExport()) { return false; } @@ -470,12 +511,36 @@ export class VideoExporter { return false; } + const encoderConfig: VideoEncoderConfig = { + codec: "avc1.640034", + width: this.config.width, + height: this.config.height, + bitrate: this.config.bitrate, + framerate: this.config.frameRate, + hardwareAcceleration: "prefer-hardware", + avc: { format: "annexb" }, + }; + + try { + const support = await VideoEncoder.isConfigSupported(encoderConfig); + if (!support.supported) { + console.warn( + `[VideoExporter] Native H.264 Annex B encoding is unsupported at ${this.config.width}x${this.config.height}`, + ); + return false; + } + } catch (error) { + console.warn("[VideoExporter] Native encoder support check failed:", error); + return false; + } + const result = await window.electronAPI.nativeVideoExportStart({ width: this.config.width, height: this.config.height, frameRate: this.config.frameRate, bitrate: this.config.bitrate, encodingMode: this.config.encodingMode ?? "balanced", + inputMode: "h264-stream", }); if (!result.success || !result.sessionId) { @@ -484,40 +549,124 @@ export class VideoExporter { } this.nativeExportSessionId = result.sessionId; - return true; - } + this.nativePendingWrite = Promise.resolve(); + this.nativeWritePromises = new Set(); + this.nativeWriteError = null; + this.maxNativeWriteInFlight = Math.max( + 1, + Math.floor(this.config.maxInFlightNativeWrites ?? 1), + ); - private async encodeRenderedFrameNative(timestamp: number): Promise { - const sessionId = this.nativeExportSessionId; - if (!sessionId) { - if (this.cancelled) { - return; + // Initialize the browser-side H.264 encoder (hardware-accelerated where available). + // Encoded Annex B chunks are sent over IPC and FFmpeg stream-copies them into MP4. + const sessionId = result.sessionId; + const encoder = new VideoEncoder({ + output: (chunk) => { + if (this.cancelled || !this.nativeExportSessionId) return; + const buffer = new ArrayBuffer(chunk.byteLength); + chunk.copyTo(buffer); + const writePromise = this.nativePendingWrite + .then(async () => { + const writeResult = await window.electronAPI.nativeVideoExportWriteFrame( + sessionId, + new Uint8Array(buffer), + ); + if (!writeResult.success && !this.cancelled) { + throw new Error( + writeResult.error || "Failed to write H.264 chunk to native encoder", + ); + } + }) + .catch((error) => { + if (!this.cancelled && !this.nativeEncoderError) { + this.nativeEncoderError = + error instanceof Error ? error : new Error(String(error)); + } + if (!this.cancelled && !this.nativeWriteError) { + this.nativeWriteError = + error instanceof Error ? error : new Error(String(error)); + } + }); + this.nativePendingWrite = writePromise; + this.trackNativeWritePromise(writePromise); + }, + error: (e) => { + this.nativeEncoderError = e; + }, + }); + + try { + encoder.configure(encoderConfig); + } catch (error) { + this.nativeEncoderError = + error instanceof Error ? error : new Error(String(error)); + try { + encoder.close(); + } catch (closeError) { + console.debug( + "[VideoExporter] Ignoring error closing native encoder after startup failure:", + closeError, + ); } + this.nativeExportSessionId = null; + await window.electronAPI.nativeVideoExportCancel(sessionId); + return false; + } + + this.nativeH264Encoder = encoder; + return true; + } + private async encodeRenderedFrameNative( + timestamp: number, + frameDuration: number, + frameIndex: number, + ): Promise { + if (!this.nativeH264Encoder || !this.nativeExportSessionId) { + if (this.cancelled) return; throw new Error("Native export session is not active"); } - const frameData = await captureCanvasFrameForNativeExport( - this.renderer!.getCanvas(), - timestamp, - true, - ); + if (this.nativeEncoderError) throw this.nativeEncoderError; + if (this.nativeWriteError) throw this.nativeWriteError; - if (this.cancelled) { - return; + while (this.nativeWritePromises.size >= this.maxNativeWriteInFlight && !this.cancelled) { + await this.awaitOldestNativeWrite(); + if (this.nativeEncoderError) throw this.nativeEncoderError; + if (this.nativeWriteError) throw this.nativeWriteError; } - const result = await window.electronAPI.nativeVideoExportWriteFrame(sessionId, frameData); - if (!result.success) { - if (this.cancelled || result.error === "Native video export session was cancelled") { - return; - } - - throw new Error(result.error || "Failed to write frame to native encoder"); + // Apply backpressure: don't queue too far ahead of FFmpeg's stdin pipe + while ( + this.nativeH264Encoder.encodeQueueSize >= + Math.max(1, Math.floor(this.config.maxEncodeQueue ?? DEFAULT_MAX_ENCODE_QUEUE)) + ) { + await new Promise((r) => setTimeout(r, 2)); + if (this.cancelled) return; + if (this.nativeEncoderError) throw this.nativeEncoderError; + if (this.nativeWriteError) throw this.nativeWriteError; } + + const canvas = this.renderer!.getCanvas(); + // @ts-expect-error - colorSpace not in TypeScript definitions but works at runtime + const frame = new VideoFrame(canvas, { + timestamp, + duration: frameDuration, + colorSpace: { + primaries: "bt709", + transfer: "iec61966-2-1", + matrix: "rgb", + fullRange: true, + }, + }); + this.nativeH264Encoder.encode(frame, { keyFrame: frameIndex % 300 === 0 }); + frame.close(); } - private async finishNativeVideoExport(audioPlan: NativeAudioPlan, totalFrames: number): Promise { + private async finishNativeVideoExport( + audioPlan: NativeAudioPlan, + totalFrames: number, + ): Promise { if (!this.nativeExportSessionId) { return { success: false, error: "Native export session is not active" }; } @@ -554,7 +703,8 @@ export class VideoExporter { audioPlan.audioMode === "copy-source" || audioPlan.audioMode === "trim-source" ? audioPlan.audioSourcePath : null, - trimSegments: audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, + trimSegments: + audioPlan.audioMode === "trim-source" ? audioPlan.trimSegments : undefined, editedAudioData: editedAudioBuffer, editedAudioMimeType, }), @@ -641,7 +791,11 @@ export class VideoExporter { }; } - private async encodeRenderedFrame(timestamp: number, frameDuration: number, frameIndex: number) { + private async encodeRenderedFrame( + timestamp: number, + frameDuration: number, + frameIndex: number, + ) { const canvas = this.renderer!.getCanvas(); // @ts-expect-error - colorSpace not in TypeScript definitions but works at runtime @@ -675,7 +829,42 @@ export class VideoExporter { exportFrame.close(); } - private reportFinalizingProgress(totalFrames: number, renderProgress: number, audioProgress?: number) { + private trackNativeWritePromise(writePromise: Promise): void { + this.nativeWritePromises.add(writePromise); + + void writePromise.finally(() => { + this.nativeWritePromises.delete(writePromise); + }); + } + + private async awaitOldestNativeWrite(): Promise { + const oldestWritePromise = this.nativeWritePromises.values().next().value; + if (!oldestWritePromise) { + return; + } + + await this.awaitWithFinalizationTimeout(oldestWritePromise, "native frame write"); + + if (this.nativeWriteError) { + throw this.nativeWriteError; + } + } + + private async awaitPendingNativeWrites(): Promise { + while (this.nativeWritePromises.size > 0) { + await this.awaitOldestNativeWrite(); + } + + if (this.nativeWriteError) { + throw this.nativeWriteError; + } + } + + private reportFinalizingProgress( + totalFrames: number, + renderProgress: number, + audioProgress?: number, + ) { this.reportProgress(totalFrames, totalFrames, "finalizing", renderProgress, audioProgress); } @@ -696,12 +885,10 @@ export class VideoExporter { const estimatedTimeRemaining = averageRenderFps > 0 ? remainingFrames / averageRenderFps : 0; const safeRenderProgress = - phase === "finalizing" - ? Math.max(0, Math.min(renderProgress ?? 99, 99)) - : undefined; + phase === "finalizing" ? Math.max(0, Math.min(renderProgress ?? 99, 99)) : undefined; const percentage = phase === "finalizing" - ? safeRenderProgress ?? 99 + ? (safeRenderProgress ?? 99) : totalFrames > 0 ? (currentFrame / totalFrames) * 100 : 100; @@ -720,7 +907,10 @@ export class VideoExporter { renderFps, phase, renderProgress: safeRenderProgress, - audioProgress: typeof audioProgress === "number" ? Math.max(0, Math.min(audioProgress, 1)) : undefined, + audioProgress: + typeof audioProgress === "number" + ? Math.max(0, Math.min(audioProgress, 1)) + : undefined, }); } } @@ -821,7 +1011,9 @@ export class VideoExporter { const hwSupport = await VideoEncoder.isConfigSupported(hwConfig); if (hwSupport.supported) { resolvedCodec = candidateCodec; - console.log(`[VideoExporter] Using hardware acceleration with codec ${candidateCodec}`); + console.log( + `[VideoExporter] Using hardware acceleration with codec ${candidateCodec}`, + ); this.encoder.configure(hwConfig); return; } @@ -864,6 +1056,17 @@ export class VideoExporter { } private cleanup(): void { + if (this.nativeH264Encoder) { + try { + if (this.nativeH264Encoder.state === "configured") { + this.nativeH264Encoder.close(); + } + } catch (e) { + console.warn("Error closing native H264 encoder:", e); + } + this.nativeH264Encoder = null; + } + if (this.nativeExportSessionId) { if (typeof window !== "undefined") { void window.electronAPI?.nativeVideoExportCancel?.(this.nativeExportSessionId); @@ -912,6 +1115,7 @@ export class VideoExporter { this.audioProcessor = null; this.encodeQueue = 0; this.pendingMuxing = Promise.resolve(); + this.nativePendingWrite = Promise.resolve(); this.chunkCount = 0; this.encoderError = null; this.videoDescription = undefined;