diff --git a/apps/media-server/src/lib/ffmpeg-video.ts b/apps/media-server/src/lib/ffmpeg-video.ts index e9c98f09c9..018d8a29ad 100644 --- a/apps/media-server/src/lib/ffmpeg-video.ts +++ b/apps/media-server/src/lib/ffmpeg-video.ts @@ -985,6 +985,156 @@ export async function generateThumbnail( } } +const SPRITE_TIMEOUT_MS = 120_000; +const MAX_SPRITE_FRAMES = 300; + +export interface SpriteSheetOptions { + frameInterval?: number; + frameWidth?: number; + frameHeight?: number; + columns?: number; + quality?: number; +} + +export interface SpriteSheetResult { + imageData: Uint8Array; + vttContent: string; + frameCount: number; +} + +export async function generateSpriteSheet( + inputPath: string, + duration: number, + options: SpriteSheetOptions = {}, +): Promise { + const frameWidth = options.frameWidth ?? 160; + const frameHeight = options.frameHeight ?? 90; + const columns = options.columns ?? 10; + const quality = options.quality ?? 5; + + let frameInterval = options.frameInterval ?? 2; + let frameCount = Math.max(1, Math.floor(duration / frameInterval)); + if (frameCount > MAX_SPRITE_FRAMES) { + frameInterval = duration / MAX_SPRITE_FRAMES; + frameCount = MAX_SPRITE_FRAMES; + } + + const rows = Math.ceil(frameCount / columns); + + const ffmpegArgs = [ + "ffmpeg", + "-i", + inputPath, + "-vf", + `fps=1/${frameInterval},scale=${frameWidth}:${frameHeight},tile=${columns}x${rows}`, + "-q:v", + quality.toString(), + "-frames:v", + "1", + "-f", + "image2", + "pipe:1", + ]; + + const proc = registerSubprocess( + spawn({ + cmd: ffmpegArgs, + stdout: "pipe", + stderr: "pipe", + }), + ); + + try { + const imageData = await withTimeout( + (async () => { + const stderrPromise = readStreamWithLimit( + proc.stderr as ReadableStream, + MAX_STDERR_BYTES, + ); + + const chunks: Uint8Array[] = []; + let totalBytes = 0; + const reader = (proc.stdout as ReadableStream).getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + totalBytes += value.length; + } + } finally { + reader.releaseLock(); + } + + const [, exitCode] = await Promise.all([stderrPromise, proc.exited]); + + if (exitCode !== 0) { + throw new Error(`FFmpeg sprite sheet exited with code ${exitCode}`); + } + + if (totalBytes === 0) { + throw new Error("FFmpeg produced empty sprite sheet"); + } + + const output = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.length; + } + + return output; + })(), + SPRITE_TIMEOUT_MS, + () => terminateProcess(proc), + ); + + const vttLines = ["WEBVTT", ""]; + for (let i = 0; i < frameCount; i++) { + const startTime = i * frameInterval; + const endTime = Math.min((i + 1) * frameInterval, duration); + const col = i % columns; + const row = Math.floor(i / columns); + const x = col * frameWidth; + const y = row * frameHeight; + + vttLines.push( + `${formatVttTimestamp(startTime)} --> ${formatVttTimestamp(endTime)}`, + ); + vttLines.push( + `__SPRITE_URL__#xywh=${x},${y},${frameWidth},${frameHeight}`, + ); + vttLines.push(""); + } + + return { + imageData, + vttContent: vttLines.join("\n"), + frameCount, + }; + } finally { + await terminateProcess(proc); + } +} + +function formatVttTimestamp(seconds: number): string { + const hrs = Math.floor(seconds / 3600); + const mins = Math.floor((seconds % 3600) / 60); + const secs = seconds % 60; + const wholeSecs = Math.floor(secs); + const ms = Math.round((secs - wholeSecs) * 1000); + return ( + String(hrs).padStart(2, "0") + + ":" + + String(mins).padStart(2, "0") + + ":" + + String(wholeSecs).padStart(2, "0") + + "." + + String(ms).padStart(3, "0") + ); +} + export async function uploadToS3( data: Uint8Array | Blob, presignedUrl: string, diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index eddc8e1317..325d66271b 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -9,6 +9,7 @@ export type JobPhase = | "processing" | "uploading" | "generating_thumbnail" + | "generating_sprites" | "complete" | "error" | "cancelled"; diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index 91becb8569..f2c2326c7c 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import type { ResilientInputFlags } from "../lib/ffmpeg-video"; import { downloadVideoToTemp, + generateSpriteSheet, generateThumbnail, processVideo, repairContainer, @@ -73,6 +74,8 @@ const processSchema = z.object({ videoUrl: z.string().url(), outputPresignedUrl: z.string().url(), thumbnailPresignedUrl: z.string().url().optional(), + spriteSheetPresignedUrl: z.string().url().optional(), + spriteVttPresignedUrl: z.string().url().optional(), webhookUrl: z.string().url().optional(), webhookSecret: z.string().optional(), inputExtension: z.string().optional(), @@ -434,6 +437,8 @@ video.post("/process", async (c) => { videoUrl, outputPresignedUrl, thumbnailPresignedUrl, + spriteSheetPresignedUrl, + spriteVttPresignedUrl, webhookUrl, webhookSecret, } = result.data; @@ -446,6 +451,8 @@ video.post("/process", async (c) => { videoUrl, outputPresignedUrl, thumbnailPresignedUrl, + spriteSheetPresignedUrl, + spriteVttPresignedUrl, result.data, ).catch((err) => { console.error( @@ -667,6 +674,8 @@ async function processVideoAsync( videoUrl: string, outputPresignedUrl: string, thumbnailPresignedUrl: string | undefined, + spriteSheetPresignedUrl: string | undefined, + spriteVttPresignedUrl: string | undefined, options: z.infer, ): Promise { const job = getJob(jobId); @@ -762,6 +771,40 @@ async function processVideoAsync( await uploadToS3(thumbnailData, thumbnailPresignedUrl, "image/jpeg"); } + if (spriteSheetPresignedUrl && spriteVttPresignedUrl) { + try { + updateJob(jobId, { + phase: "generating_sprites", + progress: 93, + message: "Generating preview sprites...", + }); + await sendWebhook(job); + + const spriteResult = await generateSpriteSheet( + outputTempFile.path, + metadata.duration, + ); + await uploadToS3( + spriteResult.imageData, + spriteSheetPresignedUrl, + "image/jpeg", + ); + const vttBlob = new Blob([spriteResult.vttContent], { + type: "text/vtt", + }); + await uploadToS3( + new Uint8Array(await vttBlob.arrayBuffer()), + spriteVttPresignedUrl, + "text/vtt", + ); + } catch (spriteErr) { + console.error( + `[video/process] Sprite generation failed for job ${jobId} (non-fatal):`, + spriteErr, + ); + } + } + updateJob(jobId, { phase: "complete", progress: 100, @@ -956,6 +999,8 @@ const muxSegmentsSchema = z.object({ userId: z.string(), outputPresignedUrl: z.string().url(), thumbnailPresignedUrl: z.string().url().optional(), + spriteSheetPresignedUrl: z.string().url().optional(), + spriteVttPresignedUrl: z.string().url().optional(), webhookUrl: z.string().url().optional(), webhookSecret: z.string().optional(), videoInitUrl: z.string().url(), @@ -982,6 +1027,8 @@ video.post("/mux-segments", async (c) => { userId, outputPresignedUrl, thumbnailPresignedUrl, + spriteSheetPresignedUrl, + spriteVttPresignedUrl, webhookUrl, webhookSecret, } = body.data; @@ -1011,6 +1058,8 @@ video.post("/mux-segments", async (c) => { videoId, outputPresignedUrl, thumbnailPresignedUrl, + spriteSheetPresignedUrl, + spriteVttPresignedUrl, videoInitUrl, videoSegUrls, audioInitUrl ?? null, @@ -1152,6 +1201,8 @@ async function muxSegmentsAsync( videoId: string, outputPresignedUrl: string, thumbnailPresignedUrl: string | undefined, + spriteSheetPresignedUrl: string | undefined, + spriteVttPresignedUrl: string | undefined, videoInitUrl: string, videoSegmentUrls: string[], audioInitUrl: string | null, @@ -1333,6 +1384,38 @@ async function muxSegmentsAsync( } } + if (spriteSheetPresignedUrl && spriteVttPresignedUrl) { + try { + updateJob(jobId, { + phase: "generating_sprites", + progress: 93, + message: "Generating preview sprites...", + }); + sendWebhook(getJob(jobId)!); + + const duration = metadata?.duration ?? 0; + const spriteResult = await generateSpriteSheet(resultPath, duration); + await uploadToS3( + spriteResult.imageData, + spriteSheetPresignedUrl, + "image/jpeg", + ); + const vttBlob = new Blob([spriteResult.vttContent], { + type: "text/vtt", + }); + await uploadToS3( + new Uint8Array(await vttBlob.arrayBuffer()), + spriteVttPresignedUrl, + "text/vtt", + ); + } catch (spriteErr) { + console.warn( + `[mux-segments] Sprite generation failed for ${videoId} (non-fatal):`, + spriteErr, + ); + } + } + updateJob(jobId, { phase: "complete", progress: 100, diff --git a/apps/web/app/api/playlist/route.ts b/apps/web/app/api/playlist/route.ts index 19cd1161f2..663c15c283 100644 --- a/apps/web/app/api/playlist/route.ts +++ b/apps/web/app/api/playlist/route.ts @@ -265,6 +265,34 @@ const getPlaylistResponse = ( }); } + if ( + Option.isSome(urlParams.fileType) && + urlParams.fileType.value === "thumbnails-vtt" + ) { + const vttKey = `${video.ownerId}/${video.id}/sprites/thumbnails.vtt`; + const spriteKey = `${video.ownerId}/${video.id}/sprites/sprite.jpg`; + return yield* Effect.gen(function* () { + const vttContent = yield* s3.getObject(vttKey); + if (Option.isNone(vttContent)) { + return yield* new HttpApiError.NotFound(); + } + const spriteUrl = yield* s3.getSignedObjectUrl(spriteKey); + const resolvedVtt = vttContent.value.replaceAll( + "__SPRITE_URL__", + spriteUrl, + ); + return HttpServerResponse.text(resolvedVtt).pipe( + HttpServerResponse.setHeaders({ + ...CACHE_CONTROL_HEADERS, + "Content-Type": "text/vtt", + }), + ); + }).pipe( + Effect.catchTag("S3Error", () => new HttpApiError.NotFound()), + Effect.withSpan("fetchThumbnailsVtt"), + ); + } + if (Option.isNone(customBucket)) { let redirect = `${video.ownerId}/${video.id}/combined-source/stream.m3u8`; diff --git a/apps/web/app/api/upload/[...route]/recording-complete.ts b/apps/web/app/api/upload/[...route]/recording-complete.ts index ce5515b9a6..cec86bd3ad 100644 --- a/apps/web/app/api/upload/[...route]/recording-complete.ts +++ b/apps/web/app/api/upload/[...route]/recording-complete.ts @@ -141,6 +141,8 @@ export const app = new Hono().post( const outputKey = `${user.id}/${videoIdRaw}/result.mp4`; const thumbnailKey = `${user.id}/${videoIdRaw}/screenshot/screen-capture.jpg`; + const spriteSheetKey = `${user.id}/${videoIdRaw}/sprites/sprite.jpg`; + const spriteVttKey = `${user.id}/${videoIdRaw}/sprites/thumbnails.vtt`; const outputPresignedUrl = yield* bucket.getInternalPresignedPutUrl( outputKey, @@ -154,10 +156,22 @@ export const app = new Hono().post( ContentType: "image/jpeg", }, ); + const spriteSheetPresignedUrl = + yield* bucket.getInternalPresignedPutUrl(spriteSheetKey, { + ContentType: "image/jpeg", + }); + const spriteVttPresignedUrl = yield* bucket.getInternalPresignedPutUrl( + spriteVttKey, + { + ContentType: "text/vtt", + }, + ); return { outputPresignedUrl, thumbnailPresignedUrl, + spriteSheetPresignedUrl, + spriteVttPresignedUrl, videoInitUrl, videoSegmentUrls, audioInitUrl, @@ -215,6 +229,8 @@ export const app = new Hono().post( userId: user.id, outputPresignedUrl: muxPayload.outputPresignedUrl, thumbnailPresignedUrl: muxPayload.thumbnailPresignedUrl, + spriteSheetPresignedUrl: muxPayload.spriteSheetPresignedUrl, + spriteVttPresignedUrl: muxPayload.spriteVttPresignedUrl, videoInitUrl: muxPayload.videoInitUrl, videoSegmentUrls: muxPayload.videoSegmentUrls, audioInitUrl: muxPayload.audioInitUrl, diff --git a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx index 32afbddfc9..787609641e 100644 --- a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx +++ b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx @@ -45,7 +45,6 @@ import { MediaPlayerVolumeIndicator, } from "./video/media-player"; import { Tooltip, TooltipContent, TooltipTrigger } from "./video/tooltip"; -import { captureVideoFrameDataUrl } from "./video-frame-thumbnail"; const { circumference } = getProgressCircleConfig(); @@ -402,11 +401,7 @@ export function CapVideoPlayer({ videoRef.current, ]); - const generateVideoFrameThumbnail = useCallback( - (_time: number): string | undefined => - captureVideoFrameDataUrl({ video: videoRef.current }), - [], - ); + const thumbnailsVttUrl = `/api/playlist?videoId=${videoId}&videoType=mp4&fileType=thumbnails-vtt`; const isUploadFailed = uploadProgress?.status === "failed"; const isUploadError = uploadProgress?.status === "error"; @@ -595,6 +590,12 @@ export function CapVideoPlayer({ src={captionsSrc} /> )} + )} @@ -734,14 +735,7 @@ export function CapVideoPlayer({ isUploadingOrFailed={blockPlaybackControls} > - +
diff --git a/apps/web/workflows/process-video.ts b/apps/web/workflows/process-video.ts index 08f1526d8a..612b4fc9c8 100644 --- a/apps/web/workflows/process-video.ts +++ b/apps/web/workflows/process-video.ts @@ -135,6 +135,8 @@ async function startMediaServerProcessJob( videoUrl: string; outputPresignedUrl: string; thumbnailPresignedUrl: string; + spriteSheetPresignedUrl: string; + spriteVttPresignedUrl: string; webhookUrl: string; webhookSecret?: string; inputExtension: string; @@ -233,6 +235,8 @@ async function processVideoOnMediaServer( const outputKey = `${userId}/${videoId}/result.mp4`; const thumbnailKey = `${userId}/${videoId}/screenshot/screen-capture.jpg`; + const spriteSheetKey = `${userId}/${videoId}/sprites/sprite.jpg`; + const spriteVttKey = `${userId}/${videoId}/sprites/thumbnails.vtt`; const outputPresignedUrl = await bucket .getInternalPresignedPutUrl(outputKey, { @@ -246,6 +250,18 @@ async function processVideoOnMediaServer( }) .pipe(runPromise); + const spriteSheetPresignedUrl = await bucket + .getInternalPresignedPutUrl(spriteSheetKey, { + ContentType: "image/jpeg", + }) + .pipe(runPromise); + + const spriteVttPresignedUrl = await bucket + .getInternalPresignedPutUrl(spriteVttKey, { + ContentType: "text/vtt", + }) + .pipe(runPromise); + const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress`; const webhookSecret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET; @@ -255,6 +271,8 @@ async function processVideoOnMediaServer( videoUrl: rawVideoUrl, outputPresignedUrl, thumbnailPresignedUrl, + spriteSheetPresignedUrl, + spriteVttPresignedUrl, webhookUrl, webhookSecret: webhookSecret || undefined, inputExtension: getInputExtension(rawFileKey),