feat(web): add sprite sheet scrub previews for video progress bar

apps/media-server/src/lib/ffmpeg-video.ts

@@ -985,6 +985,156 @@ export async function generateThumbnail(
 	}
 }
 
+const SPRITE_TIMEOUT_MS = 120_000;
+const MAX_SPRITE_FRAMES = 300;
+
+export interface SpriteSheetOptions {
+	frameInterval?: number;
+	frameWidth?: number;
+	frameHeight?: number;
+	columns?: number;
+	quality?: number;
+}
+
+export interface SpriteSheetResult {
+	imageData: Uint8Array;
+	vttContent: string;
+	frameCount: number;
+}
+
+export async function generateSpriteSheet(
+	inputPath: string,
+	duration: number,
+	options: SpriteSheetOptions = {},
+): Promise<SpriteSheetResult> {
+	const frameWidth = options.frameWidth ?? 160;
+	const frameHeight = options.frameHeight ?? 90;
+	const columns = options.columns ?? 10;
+	const quality = options.quality ?? 5;
+
+	let frameInterval = options.frameInterval ?? 2;
+	let frameCount = Math.max(1, Math.floor(duration / frameInterval));
+	if (frameCount > MAX_SPRITE_FRAMES) {
+		frameInterval = duration / MAX_SPRITE_FRAMES;
+		frameCount = MAX_SPRITE_FRAMES;
+	}
+
+	const rows = Math.ceil(frameCount / columns);
+
+	const ffmpegArgs = [
+		"ffmpeg",
+		"-i",
+		inputPath,
+		"-vf",
+		`fps=1/${frameInterval},scale=${frameWidth}:${frameHeight},tile=${columns}x${rows}`,
+		"-q:v",
+		quality.toString(),
+		"-frames:v",
+		"1",
+		"-f",
+		"image2",
+		"pipe:1",
+	];
+
+	const proc = registerSubprocess(
+		spawn({
+			cmd: ffmpegArgs,
+			stdout: "pipe",
+			stderr: "pipe",
+		}),
+	);
+
+	try {
+		const imageData = await withTimeout(
+			(async () => {
+				const stderrPromise = readStreamWithLimit(
+					proc.stderr as ReadableStream<Uint8Array>,
+					MAX_STDERR_BYTES,
+				);
+
+				const chunks: Uint8Array[] = [];
+				let totalBytes = 0;
+				const reader = (proc.stdout as ReadableStream<Uint8Array>).getReader();
+
+				try {
+					while (true) {
+						const { done, value } = await reader.read();
+						if (done) break;
+						chunks.push(value);
+						totalBytes += value.length;
+					}
+				} finally {
+					reader.releaseLock();
+				}
+
+				const [, exitCode] = await Promise.all([stderrPromise, proc.exited]);
+
+				if (exitCode !== 0) {
+					throw new Error(`FFmpeg sprite sheet exited with code ${exitCode}`);
+				}
+
+				if (totalBytes === 0) {
+					throw new Error("FFmpeg produced empty sprite sheet");
+				}
+
+				const output = new Uint8Array(totalBytes);
+				let offset = 0;
+				for (const chunk of chunks) {
+					output.set(chunk, offset);
+					offset += chunk.length;
+				}
+
+				return output;
+			})(),
+			SPRITE_TIMEOUT_MS,
+			() => terminateProcess(proc),
+		);
+
+		const vttLines = ["WEBVTT", ""];
+		for (let i = 0; i < frameCount; i++) {
+			const startTime = i * frameInterval;
+			const endTime = Math.min((i + 1) * frameInterval, duration);
+			const col = i % columns;
+			const row = Math.floor(i / columns);
+			const x = col * frameWidth;
+			const y = row * frameHeight;
+
+			vttLines.push(
+				`${formatVttTimestamp(startTime)} --> ${formatVttTimestamp(endTime)}`,
+			);
+			vttLines.push(
+				`__SPRITE_URL__#xywh=${x},${y},${frameWidth},${frameHeight}`,
+			);
+			vttLines.push("");
+		}
+
+		return {
+			imageData,
+			vttContent: vttLines.join("\n"),
+			frameCount,
+		};
+	} finally {
+		await terminateProcess(proc);
+	}
+}
+
+function formatVttTimestamp(seconds: number): string {
+	const hrs = Math.floor(seconds / 3600);
+	const mins = Math.floor((seconds % 3600) / 60);
+	const secs = seconds % 60;
+	const wholeSecs = Math.floor(secs);
+	const ms = Math.round((secs - wholeSecs) * 1000);
+	return (
+		String(hrs).padStart(2, "0") +
+		":" +
+		String(mins).padStart(2, "0") +
+		":" +
+		String(wholeSecs).padStart(2, "0") +
+		"." +
+		String(ms).padStart(3, "0")
+	);
+}
+
 export async function uploadToS3(
 	data: Uint8Array | Blob,
 	presignedUrl: string,

apps/media-server/src/lib/job-manager.ts

@@ -9,6 +9,7 @@ export type JobPhase =
 	| "processing"
 	| "uploading"
 	| "generating_thumbnail"
+	| "generating_sprites"
 	| "complete"
 	| "error"
 	| "cancelled";

apps/media-server/src/routes/video.ts

@@ -4,6 +4,7 @@ import { z } from "zod";
 import type { ResilientInputFlags } from "../lib/ffmpeg-video";
 import {
 	downloadVideoToTemp,
+	generateSpriteSheet,
 	generateThumbnail,
 	processVideo,
 	repairContainer,
@@ -73,6 +74,8 @@ const processSchema = z.object({
 	videoUrl: z.string().url(),
 	outputPresignedUrl: z.string().url(),
 	thumbnailPresignedUrl: z.string().url().optional(),
+	spriteSheetPresignedUrl: z.string().url().optional(),
+	spriteVttPresignedUrl: z.string().url().optional(),
 	webhookUrl: z.string().url().optional(),
 	webhookSecret: z.string().optional(),
 	inputExtension: z.string().optional(),
@@ -434,6 +437,8 @@ video.post("/process", async (c) => {
 		videoUrl,
 		outputPresignedUrl,
 		thumbnailPresignedUrl,
+		spriteSheetPresignedUrl,
+		spriteVttPresignedUrl,
 		webhookUrl,
 		webhookSecret,
 	} = result.data;
@@ -446,6 +451,8 @@ video.post("/process", async (c) => {
 		videoUrl,
 		outputPresignedUrl,
 		thumbnailPresignedUrl,
+		spriteSheetPresignedUrl,
+		spriteVttPresignedUrl,
 		result.data,
 	).catch((err) => {
 		console.error(
@@ -667,6 +674,8 @@ async function processVideoAsync(
 	videoUrl: string,
 	outputPresignedUrl: string,
 	thumbnailPresignedUrl: string | undefined,
+	spriteSheetPresignedUrl: string | undefined,
+	spriteVttPresignedUrl: string | undefined,
 	options: z.infer<typeof processSchema>,
 ): Promise<void> {
 	const job = getJob(jobId);
@@ -762,6 +771,40 @@ async function processVideoAsync(
 			await uploadToS3(thumbnailData, thumbnailPresignedUrl, "image/jpeg");
 		}
 
+		if (spriteSheetPresignedUrl && spriteVttPresignedUrl) {
+			try {
+				updateJob(jobId, {
+					phase: "generating_sprites",
+					progress: 93,
+					message: "Generating preview sprites...",
+				});
+				await sendWebhook(job);
+
+				const spriteResult = await generateSpriteSheet(
+					outputTempFile.path,
+					metadata.duration,
+				);
+				await uploadToS3(
+					spriteResult.imageData,
+					spriteSheetPresignedUrl,
+					"image/jpeg",
+				);
+				const vttBlob = new Blob([spriteResult.vttContent], {
+					type: "text/vtt",
+				});
+				await uploadToS3(
+					new Uint8Array(await vttBlob.arrayBuffer()),
+					spriteVttPresignedUrl,
+					"text/vtt",
+				);
+			} catch (spriteErr) {
+				console.error(
+					`[video/process] Sprite generation failed for job ${jobId} (non-fatal):`,
+					spriteErr,
+				);
+			}
+		}
+
 		updateJob(jobId, {
 			phase: "complete",
 			progress: 100,
@@ -956,6 +999,8 @@ const muxSegmentsSchema = z.object({
 	userId: z.string(),
 	outputPresignedUrl: z.string().url(),
 	thumbnailPresignedUrl: z.string().url().optional(),
+	spriteSheetPresignedUrl: z.string().url().optional(),
+	spriteVttPresignedUrl: z.string().url().optional(),
 	webhookUrl: z.string().url().optional(),
 	webhookSecret: z.string().optional(),
 	videoInitUrl: z.string().url(),
@@ -982,6 +1027,8 @@ video.post("/mux-segments", async (c) => {
 		userId,
 		outputPresignedUrl,
 		thumbnailPresignedUrl,
+		spriteSheetPresignedUrl,
+		spriteVttPresignedUrl,
 		webhookUrl,
 		webhookSecret,
 	} = body.data;
@@ -1011,6 +1058,8 @@ video.post("/mux-segments", async (c) => {
 		videoId,
 		outputPresignedUrl,
 		thumbnailPresignedUrl,
+		spriteSheetPresignedUrl,
+		spriteVttPresignedUrl,
 		videoInitUrl,
 		videoSegUrls,
 		audioInitUrl ?? null,
@@ -1152,6 +1201,8 @@ async function muxSegmentsAsync(
 	videoId: string,
 	outputPresignedUrl: string,
 	thumbnailPresignedUrl: string | undefined,
+	spriteSheetPresignedUrl: string | undefined,
+	spriteVttPresignedUrl: string | undefined,
 	videoInitUrl: string,
 	videoSegmentUrls: string[],
 	audioInitUrl: string | null,
@@ -1333,6 +1384,38 @@ async function muxSegmentsAsync(
 			}
 		}
 
+		if (spriteSheetPresignedUrl && spriteVttPresignedUrl) {
+			try {
+				updateJob(jobId, {
+					phase: "generating_sprites",
+					progress: 93,
+					message: "Generating preview sprites...",
+				});
+				sendWebhook(getJob(jobId)!);
+
+				const duration = metadata?.duration ?? 0;
+				const spriteResult = await generateSpriteSheet(resultPath, duration);
+				await uploadToS3(
+					spriteResult.imageData,
+					spriteSheetPresignedUrl,
+					"image/jpeg",
+				);
+				const vttBlob = new Blob([spriteResult.vttContent], {
+					type: "text/vtt",
+				});
+				await uploadToS3(
+					new Uint8Array(await vttBlob.arrayBuffer()),
+					spriteVttPresignedUrl,
+					"text/vtt",
+				);
+			} catch (spriteErr) {
+				console.warn(
+					`[mux-segments] Sprite generation failed for ${videoId} (non-fatal):`,
+					spriteErr,
+				);
+			}
+		}
+
 		updateJob(jobId, {
 			phase: "complete",
 			progress: 100,

apps/web/app/api/playlist/route.ts

@@ -265,6 +265,34 @@ const getPlaylistResponse = (
 			});
 		}
 
+		if (
+			Option.isSome(urlParams.fileType) &&
+			urlParams.fileType.value === "thumbnails-vtt"
+		) {
+			const vttKey = `${video.ownerId}/${video.id}/sprites/thumbnails.vtt`;
+			const spriteKey = `${video.ownerId}/${video.id}/sprites/sprite.jpg`;
+			return yield* Effect.gen(function* () {
+				const vttContent = yield* s3.getObject(vttKey);
+				if (Option.isNone(vttContent)) {
+					return yield* new HttpApiError.NotFound();
+				}
+				const spriteUrl = yield* s3.getSignedObjectUrl(spriteKey);
+				const resolvedVtt = vttContent.value.replaceAll(
+					"__SPRITE_URL__",
+					spriteUrl,
+				);
+				return HttpServerResponse.text(resolvedVtt).pipe(
+					HttpServerResponse.setHeaders({
+						...CACHE_CONTROL_HEADERS,
+						"Content-Type": "text/vtt",
+					}),
+				);
+			}).pipe(
+				Effect.catchTag("S3Error", () => new HttpApiError.NotFound()),
+				Effect.withSpan("fetchThumbnailsVtt"),
+			);
+		}
+
 		if (Option.isNone(customBucket)) {
 			let redirect = `${video.ownerId}/${video.id}/combined-source/stream.m3u8`;