From 5f598fbcb2377265f89f66b6339bb0974cb5a5eb Mon Sep 17 00:00:00 2001 From: Sergiy Dybskiy Date: Wed, 11 Feb 2026 13:07:38 -0500 Subject: [PATCH] fix: parallelize R2 uploads in md-exports child-listings phase The child-listings phase uploaded ~1651 section index files to R2 sequentially (one await per file inside a for...of loop). At ~300ms per request, this added ~8 minutes to every other production build, causing an alternating SLOW/FAST pattern (19 min vs 12 min). Collect uploads during the loop and batch them with pLimit(50), reducing the upload phase from ~8 min to ~10-20 seconds. Co-Authored-By: Claude --- scripts/generate-md-exports.mjs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 44b8f602255af3..547709bf8e2190 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -301,6 +301,7 @@ ${ // Append child listings to parent index files let updatedCount = 0; + const r2Uploads = []; for (const [parentPath, children] of pathsByParent) { const parentFile = path.join(OUTPUT_DIR, parentPath); let existingContent; @@ -348,17 +349,28 @@ ${ await writeFile(parentFile, updatedContent, {encoding: 'utf8'}); updatedCount++; - // Upload modified parent file to R2 if configured + // Collect R2 uploads needed (will be uploaded in parallel below) if (accessKeyId && secretAccessKey && existingFilesOnR2) { const fileHash = md5(updatedContent); const existingHash = existingFilesOnR2.get(parentPath); if (existingHash !== fileHash) { - const s3Client = getS3Client(); - await uploadToCFR2(s3Client, parentPath, updatedContent); + r2Uploads.push({parentPath, updatedContent}); } } } } + + // Upload all modified section index files to R2 in parallel + if (r2Uploads.length > 0) { + const limit = pLimit(50); + const s3Client = getS3Client(); + await Promise.all( + r2Uploads.map(({parentPath, updatedContent}) => + limit(() => uploadToCFR2(s3Client, parentPath, updatedContent)) + ) + ); + console.log(`📤 Uploaded ${r2Uploads.length} section index files to R2`); + } console.log(`📑 Added child page listings to ${updatedCount} section index files`); // Upload index.md to R2 if configured