diff --git a/docs/guides/rendering.mdx b/docs/guides/rendering.mdx index 9faf6c2685..bbf0d64182 100644 --- a/docs/guides/rendering.mdx +++ b/docs/guides/rendering.mdx @@ -122,6 +122,7 @@ Render your Hyperframes [compositions](/concepts/compositions) to MP4, MOV, WebM | `--quality` | draft, standard, high | standard | Encoding quality preset | | `--crf` | 0–51 | — | Override CRF (lower = higher quality). Cannot combine with `--video-bitrate` | | `--video-bitrate` | e.g. `10M`, `5000k` | — | Target bitrate encoding. Cannot combine with `--crf` | +| `--video-frame-format` | auto, jpg, png | auto | Source video frame extraction format. Use `png` for UI recordings, screen captures, and color-sensitive source videos | | `--workers` | 1-8 or `auto` | auto | Parallel render workers (see [Workers](#workers) below) | | `--max-concurrent-renders` | 1-10 | 2 | Max simultaneous renders via the producer server (see [Concurrent Renders](#concurrent-renders) below) | | `--batch` | path | — | JSON array of variable rows (or `{ "rows": [...] }`), rendering one output per row | @@ -168,6 +169,8 @@ GIF output uses a two-pass FFmpeg palette encode (`palettegen` with diff statist GIF does not carry audio and only has 1-bit transparency. For transparent overlays, use `--format webm`, `--format mov`, or `--format png-sequence` instead. +For UI recordings, screen captures, or other source videos where saturated interface colors matter, pass `--video-frame-format png` to extract source video layers as PNG before browser capture. The default `auto` mode preserves the historical behavior: alpha-capable sources use PNG, opaque sources use JPG. + ## GPU Acceleration Hyperframes has two separate GPU acceleration surfaces: diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index a725e09deb..b2a439be80 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -664,6 +664,7 @@ This is suppressed in CI environments, non-TTY shells, and when `HYPERFRAMES_NO_ | `--quality` | draft, standard, high | standard | Encoding quality preset (drives CRF/bitrate) | | `--crf` | 0-51 | — | Override encoder CRF (lower = higher quality). Mutually exclusive with `--video-bitrate` | | `--video-bitrate` | e.g. `10M`, `5000k` | — | Target video bitrate. Mutually exclusive with `--crf` | + | `--video-frame-format` | auto, jpg, png | auto | Source video frame extraction format. Use `png` for UI recordings, screen captures, and color-sensitive source videos | | `--resolution` | landscape, portrait, landscape-4k, portrait-4k, square, square-4k (aliases: `1080p`, `4k`, `uhd`, `1080p-square`, `square-1080p`, `4k-square`) | — | Output resolution preset. Supersamples a smaller composition via Chrome `deviceScaleFactor` so the screenshot lands at the requested dimensions. Aspect ratio must match the composition; the scale must be an integer multiple. Not supported with `--hdr`. See [4K Rendering](/guides/4k-rendering) | | `--hdr` | — | off | Force HDR output even if no HDR sources are detected. MP4 only. See [HDR Rendering](/guides/hdr) | | `--sdr` | — | off | Force SDR output even if HDR sources are detected | @@ -678,7 +679,7 @@ This is suppressed in CI environments, non-TTY shells, and when `HYPERFRAMES_NO_ | `--strict-variables` | — | off | Fail render if any `--variables` key is undeclared or has a wrong type vs the composition's `data-composition-variables`. Without this flag, mismatches print as warnings and the render continues. | | `--browser-timeout` | seconds (0.001–86400) | 60 | Puppeteer page-navigation timeout for the entry HTML. Increase when heavy compositions (many videos, fonts, or asset requests) cannot reach `domcontentloaded` within the default 60 s. The flag takes **seconds**; the env fallback `PRODUCER_PAGE_NAVIGATION_TIMEOUT_MS` takes **milliseconds**. This controls `page.goto` only — very heavy compositions may also need `PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS` and/or `PRODUCER_PLAYER_READY_TIMEOUT_MS` bumped (post-navigation `window.__hf` readiness has its own 45 s budget). | - CRF and target bitrate default to the `--quality` preset. Use `--crf` or `--video-bitrate` for fine-grained overrides; `RenderConfig.crf` and `RenderConfig.videoBitrate` accept the same overrides programmatically. + CRF and target bitrate default to the `--quality` preset. Use `--crf` or `--video-bitrate` for fine-grained overrides; `RenderConfig.crf` and `RenderConfig.videoBitrate` accept the same overrides programmatically. Use `--video-frame-format png` when source videos are UI recordings, screen captures, or other color-sensitive clips that should avoid JPEG frame extraction. #### Parametrized renders diff --git a/packages/aws-lambda/src/sdk/validateConfig.test.ts b/packages/aws-lambda/src/sdk/validateConfig.test.ts index 76fdb96022..daf2250f7d 100644 --- a/packages/aws-lambda/src/sdk/validateConfig.test.ts +++ b/packages/aws-lambda/src/sdk/validateConfig.test.ts @@ -30,6 +30,7 @@ describe("validateDistributedRenderConfig", () => { maxParallelChunks: 16, runtimeCap: "lambda", hdrMode: "force-sdr", + videoFrameFormat: "png", }; expect(validateDistributedRenderConfig(cfg)).toBe(cfg); }); @@ -92,6 +93,14 @@ describe("validateDistributedRenderConfig", () => { { ...VALID, bitrate: "fast" } satisfies SerializableDistributedRenderConfig, "config.bitrate", ], + [ + "unsupported videoFrameFormat", + { + ...VALID, + videoFrameFormat: "webp", + } as unknown as SerializableDistributedRenderConfig, + "config.videoFrameFormat", + ], [ "non-positive chunkSize", { ...VALID, chunkSize: 0 } satisfies SerializableDistributedRenderConfig, diff --git a/packages/cli/src/commands/render.test.ts b/packages/cli/src/commands/render.test.ts index 2e71e6383c..d4228055d9 100644 --- a/packages/cli/src/commands/render.test.ts +++ b/packages/cli/src/commands/render.test.ts @@ -237,6 +237,21 @@ describe("renderLocal browser GPU config", () => { expect(producerState.createdJobs[0]?.gifLoop).toBe(3); }); + it("forwards videoFrameFormat to createRenderJob", async () => { + await renderLocal("/tmp/project", "/tmp/out.mp4", { + fps: { num: 30, den: 1 }, + quality: "standard", + format: "mp4", + gpu: false, + browserGpuMode: "software", + hdrMode: "auto", + quiet: true, + videoFrameFormat: "png", + }); + + expect(producerState.createdJobs[0]?.videoFrameFormat).toBe("png"); + }); + it("omits variables from createRenderJob when not provided", async () => { await renderLocal("/tmp/project", "/tmp/out.mp4", { fps: { num: 30, den: 1 }, diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index c99d3ed980..35dcdb1c20 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -71,6 +71,7 @@ import { buildDockerRunArgs, resolveDockerPlatform } from "../utils/dockerRunArg import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { runEnvironmentChecks } from "../browser/preflight.js"; import type { ProducerLogger, RenderJob } from "@hyperframes/producer"; +import { isVideoFrameFormat, type VideoFrameFormat } from "@hyperframes/engine"; import { normalizeResolutionFlag, parseFps, @@ -181,6 +182,14 @@ export default defineCommand({ type: "string", description: "GIF loop count, 0 = infinite. Range: 0-65535. Only used with --format gif.", }, + "video-frame-format": { + type: "string", + description: + "Source video frame extraction format: auto, jpg, png (default: auto). " + + "Use png for UI recordings, screen captures, and color-sensitive source videos; " + + "alpha-capable sources always extract as PNG.", + default: "auto", + }, workers: { type: "string", alias: "w", @@ -375,6 +384,16 @@ export default defineCommand({ } const gifLoop = gifLoopParse.value ?? (format === "gif" ? 0 : undefined); + const videoFrameFormatRaw = args["video-frame-format"] ?? "auto"; + if (!isVideoFrameFormat(videoFrameFormatRaw)) { + errorBox( + "Invalid video-frame-format", + `Got "${videoFrameFormatRaw}". Must be auto, jpg, or png.`, + ); + process.exit(1); + } + const videoFrameFormat = videoFrameFormatRaw; + // ── Validate resolution ──────────────────────────────────────────────── let outputResolution: CanvasResolution | undefined; if (args.resolution !== undefined) { @@ -768,6 +787,7 @@ export default defineCommand({ hdrMode: args.sdr ? "force-sdr" : args.hdr ? "force-hdr" : "auto", crf, videoBitrate, + videoFrameFormat, quiet, variables, entryFile, @@ -790,6 +810,7 @@ export default defineCommand({ hdrMode: args.sdr ? "force-sdr" : args.hdr ? "force-hdr" : "auto", crf, videoBitrate, + videoFrameFormat, quiet, browserPath, variables, @@ -825,6 +846,7 @@ interface RenderOptions { hdrMode: "auto" | "force-hdr" | "force-sdr"; crf?: number; videoBitrate?: string; + videoFrameFormat?: VideoFrameFormat; quiet: boolean; browserPath?: string; variables?: Record; @@ -1067,6 +1089,7 @@ async function renderDocker( hdrMode: options.hdrMode, crf: options.crf, videoBitrate: options.videoBitrate, + videoFrameFormat: options.videoFrameFormat, quiet: options.quiet, variables: options.variables, entryFile: options.entryFile, @@ -1176,6 +1199,7 @@ export async function renderLocal( hdrMode: options.hdrMode, crf: options.crf, videoBitrate: options.videoBitrate, + videoFrameFormat: options.videoFrameFormat, variables: options.variables, entryFile: options.entryFile, outputResolution: options.outputResolution, diff --git a/packages/cli/src/docs/rendering.md b/packages/cli/src/docs/rendering.md index 188ef42066..253eb33a41 100644 --- a/packages/cli/src/docs/rendering.md +++ b/packages/cli/src/docs/rendering.md @@ -19,6 +19,7 @@ Requires: Docker installed and running. - `-w, --workers` — Parallel workers 1-8 (default: auto) - `--crf` — Override encoder CRF (mutually exclusive with `--video-bitrate`) - `--video-bitrate` — Target video bitrate such as `10M` (mutually exclusive with `--crf`) +- `--video-frame-format` — Source video frame extraction format: `auto`, `jpg`, or `png` (default: `auto`). Use `png` for UI recordings, screen captures, and color-sensitive source videos. - `--gpu` — Use GPU encoding (NVENC, VideoToolbox, AMF, VAAPI, QSV) - `--browser-gpu` / `--no-browser-gpu` — Force host GPU or software (SwiftShader) for Chrome/WebGL capture. Default for local renders is `auto` — probe WebGL availability on first launch and fall back to software if no GPU is reachable. Docker mode always uses software. - `-o, --output` — Custom output path @@ -28,5 +29,6 @@ Requires: Docker installed and running. - Use `draft` quality for fast previews during development - Local renders auto-detect GPU on first launch; use `--browser-gpu` to force hardware (errors if no GPU) or `--no-browser-gpu` to force SwiftShader - Use `--gpu` when a local render also benefits from hardware FFmpeg encoding +- Use `--video-frame-format png` when source videos contain saturated UI colors that should avoid JPEG extraction - Use `npx hyperframes benchmark` to find optimal settings - 4 workers is usually the sweet spot for most compositions diff --git a/packages/cli/src/utils/dockerRunArgs.test.ts b/packages/cli/src/utils/dockerRunArgs.test.ts index a48dc967bd..a48c28ff4e 100644 --- a/packages/cli/src/utils/dockerRunArgs.test.ts +++ b/packages/cli/src/utils/dockerRunArgs.test.ts @@ -168,6 +168,7 @@ describe("buildDockerRunArgs", () => { hdrMode: "force-hdr", crf: 16, videoBitrate: undefined, + videoFrameFormat: "png", quiet: true, entryFile: "compositions/intro.html", }, @@ -181,6 +182,8 @@ describe("buildDockerRunArgs", () => { expect(args).toContain("8"); expect(args).toContain("--crf"); expect(args).toContain("16"); + expect(args).toContain("--video-frame-format"); + expect(args).toContain("png"); expect(args).toContain("--quiet"); expect(args).toContain("--gpu"); expect(args).toContain("--no-browser-gpu"); @@ -224,6 +227,27 @@ describe("buildDockerRunArgs", () => { expect(args).not.toContain("--crf"); }); + it("forwards --video-frame-format to the container when set to png", () => { + const args = buildDockerRunArgs({ + ...FIXED_INPUT, + options: { ...BASE, videoFrameFormat: "png" }, + }); + expect(args).toContain("--video-frame-format"); + expect(args).toContain("png"); + }); + + it("omits --video-frame-format when it is auto or unset", () => { + expect(buildDockerRunArgs({ ...FIXED_INPUT, options: BASE })).not.toContain( + "--video-frame-format", + ); + expect( + buildDockerRunArgs({ + ...FIXED_INPUT, + options: { ...BASE, videoFrameFormat: "auto" }, + }), + ).not.toContain("--video-frame-format"); + }); + it("forwards --variables JSON to the container when set", () => { const args = buildDockerRunArgs({ ...FIXED_INPUT, diff --git a/packages/cli/src/utils/dockerRunArgs.ts b/packages/cli/src/utils/dockerRunArgs.ts index 1a89f85032..df4b502dd2 100644 --- a/packages/cli/src/utils/dockerRunArgs.ts +++ b/packages/cli/src/utils/dockerRunArgs.ts @@ -47,6 +47,7 @@ export interface DockerRenderOptions { hdrMode: "auto" | "force-hdr" | "force-sdr"; crf?: number; videoBitrate?: string; + videoFrameFormat?: "auto" | "jpg" | "png"; quiet: boolean; variables?: Record; entryFile?: string; @@ -121,6 +122,9 @@ export function buildDockerRunArgs(input: DockerRunArgsInput): string[] { ...(options.workers != null ? ["--workers", String(options.workers)] : []), ...(options.crf != null ? ["--crf", String(options.crf)] : []), ...(options.videoBitrate ? ["--video-bitrate", options.videoBitrate] : []), + ...(options.videoFrameFormat && options.videoFrameFormat !== "auto" + ? ["--video-frame-format", options.videoFrameFormat] + : []), ...(options.quiet ? ["--quiet"] : []), ...(options.gpu ? ["--gpu"] : []), ...(options.browserGpu ? [] : ["--no-browser-gpu"]), diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 3c9034b6fd..a569e6699e 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -135,6 +135,9 @@ export { type ExtractionOptions, type ExtractionResult, type ExtractionPhaseBreakdown, + type VideoFrameFormat, + VIDEO_FRAME_FORMATS, + isVideoFrameFormat, } from "./services/videoFrameExtractor.js"; export { createVideoFrameInjector } from "./services/videoFrameInjector.js"; diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 49d92103c2..ebb439594a 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -18,13 +18,14 @@ import { extractAllVideoFrames, createFrameLookupTable, resolveProjectRelativeSrc, + resolveFrameFormat, codecMayHaveAlpha, decoderForCodec, getFrameAtTime, type VideoElement, type ExtractedFrames, } from "./videoFrameExtractor.js"; -import { extractVideoMetadata } from "../utils/ffprobe.js"; +import { extractVideoMetadata, type VideoMetadata } from "../utils/ffprobe.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; // ffmpeg is not preinstalled on GitHub's ubuntu-24.04 runners. The producer @@ -71,6 +72,45 @@ describe("codec alpha capability", () => { }); }); +describe("resolveFrameFormat", () => { + function metadata(overrides: Partial = {}): VideoMetadata { + return { + durationSeconds: 1, + width: 320, + height: 180, + fps: 30, + hasAudio: false, + videoCodec: "h264", + colorSpace: { + colorTransfer: "bt709", + colorPrimaries: "bt709", + colorSpace: "bt709", + }, + isVFR: false, + hasAlpha: false, + ...overrides, + }; + } + + it("keeps opaque non-alpha sources on jpg by default", () => { + expect(resolveFrameFormat(metadata(), undefined)).toBe("jpg"); + expect(resolveFrameFormat(metadata(), "auto")).toBe("jpg"); + }); + + it("honors explicit png for opaque videos", () => { + expect(resolveFrameFormat(metadata(), "png")).toBe("png"); + }); + + it("honors explicit jpg for opaque videos", () => { + expect(resolveFrameFormat(metadata(), "jpg")).toBe("jpg"); + }); + + it("forces png when alpha is present or the codec can carry alpha", () => { + expect(resolveFrameFormat(metadata({ hasAlpha: true }), "jpg")).toBe("png"); + expect(resolveFrameFormat(metadata({ videoCodec: "vp9" }), "jpg")).toBe("png"); + }); +}); + // Regression: a long-standing footgun where `