From a88011acc6677706cd8329c67710beab0f66ec3a Mon Sep 17 00:00:00 2001 From: Chase Sillevis Date: Mon, 22 Sep 2025 20:01:33 +0200 Subject: [PATCH 1/2] feat(prerenderer): add prom + 503 overload protection --- prerender-server/package.json | 3 +- prerender-server/src/cluster.js | 18 +++++- prerender-server/src/server.js | 101 +++++++++++++++++++++++++++----- 3 files changed, 104 insertions(+), 18 deletions(-) diff --git a/prerender-server/package.json b/prerender-server/package.json index a6d6b707..92f1f310 100644 --- a/prerender-server/package.json +++ b/prerender-server/package.json @@ -6,7 +6,8 @@ "dependencies": { "body-parser": "^1.19.0", "cookie-parser": "^1.4.5", - "superagent": "^6.1.0" + "superagent": "^6.1.0", + "prom-client": "^14.0.0" }, "scripts": { "start": "./start.sh", diff --git a/prerender-server/src/cluster.js b/prerender-server/src/cluster.js index 3c083447..006815e8 100644 --- a/prerender-server/src/cluster.js +++ b/prerender-server/src/cluster.js @@ -1,13 +1,29 @@ const cluster = require('cluster') - , numCPUs = require('os').cpus().length + , numCPUs = require('os').cpus().length if (cluster.isMaster) { console.log(`Master ${process.pid} is running`); + let activeRenders = 0 + const MAX_TOTAL_RENDERS = parseInt(process.env.MAX_TOTAL_RENDERS || '1') + for (let i = 0; i < numCPUs; i++) { cluster.fork() } + cluster.on('message', (worker, message) => { + if (message.type === 'startRender') { + if (activeRenders < MAX_TOTAL_RENDERS) { + activeRenders++ + worker.send({ type: 'renderAllowed', requestId: message.requestId }) + } else { + worker.send({ type: 'renderDenied', requestId: message.requestId }) + } + } else if (message.type === 'endRender') { + activeRenders = Math.max(0, activeRenders - 1) + } + }) + cluster.on('exit', (worker, code, signal) => { console.log(`worker ${worker.process.pid} died`, { worker, code, signal }); }); diff --git a/prerender-server/src/server.js b/prerender-server/src/server.js index 41fbfad7..de7a94d6 100644 --- a/prerender-server/src/server.js +++ b/prerender-server/src/server.js @@ -3,6 +3,7 @@ import pug from 'pug' import path from 'path' import express from 'express' import request from 'superagent' +import promClient from 'prom-client' import l10n from '../client/l10n' import render from '../client/run-server' @@ -17,9 +18,37 @@ const rpath = p => path.join(__dirname, p) const indexView = rpath('../../client/index.pug') +const register = new promClient.Registry() + +const activeRenders = new promClient.Gauge({ + name: 'prerender_active_renders', + help: 'Number of active renders' +}) + +const totalRenders = new promClient.Counter({ + name: 'prerender_total_renders', + help: 'Total number of renders completed' +}) + +const renderDuration = new promClient.Histogram({ + name: 'prerender_render_duration_seconds', + help: 'Duration of renders in seconds' +}) + +register.registerMetric(activeRenders) +register.registerMetric(totalRenders) +register.registerMetric(renderDuration) + +let requestCounter = 0 + const app = express() app.engine('pug', pug.__express) +app.get('/metrics', async (req, res) => { + res.set('Content-Type', register.contentType) + res.end(await register.metrics()) +}) + if (app.settings.env == 'development') app.use(require('morgan')('dev')) @@ -37,25 +66,65 @@ app.use((req, res, next) => { if (!langs.includes(lang)) lang = 'en' if (req.query.lang && req.cookies.lang !== lang) res.cookie('lang', lang) - render(req._parsedUrl.pathname, req._parsedUrl.query || '', req.body, { theme, lang, isHead: req.method === 'HEAD' }, (err, resp) => { - if (err) return next(err) - if (resp.redirect) return res.redirect(301, baseHref + resp.redirect.substr(1)) - if (resp.errorCode) { - console.error(`Failed with code ${resp.errorCode}:`, resp) - return res.sendStatus(resp.errorCode) + if (typeof process.send === 'function') { + const requestId = ++requestCounter + process.send({ type: 'startRender', requestId }) + let responded = false + const handler = (msg) => { + if (msg.requestId === requestId && !responded) { + responded = true + clearTimeout(timeout) + process.removeListener('message', handler) + if (msg.type === 'renderAllowed') { + doRender() + } else if (msg.type === 'renderDenied') { + res.status(503).send('Server overloaded') + } + } } + process.on('message', handler) + const timeout = setTimeout(() => { + if (!responded) { + responded = true + process.removeListener('message', handler) + console.error('IPC timeout for request', requestId) + res.status(500).send('Internal server error') + } + }, 5000) // 5 second timeout + } else { + doRender() + } - res.status(resp.status || 200) - res.render(indexView, { - prerender_title: resp.title - , prerender_html: resp.html - , canon_url: canonBase ? canonBase + req.url : null - , noscript: true - , theme - , t: l10n[lang] - }) - }) + function doRender() { + activeRenders.inc() + const end = renderDuration.startTimer() + let metricsUpdated = false + render(req._parsedUrl.pathname, req._parsedUrl.query || '', req.body, { theme, lang, isHead: req.method === 'HEAD' }, (err, resp) => { + if (!metricsUpdated) { + metricsUpdated = true + if (typeof process.send === 'function') process.send({ type: 'endRender' }) + activeRenders.dec() + end() + totalRenders.inc() + } + if (err) return next(err) + if (resp.redirect) return res.redirect(301, baseHref + resp.redirect.substr(1)) + if (resp.errorCode) { + console.error(`Failed with code ${resp.errorCode}:`, resp) + return res.sendStatus(resp.errorCode) + } + res.status(resp.status || 200) + res.render(indexView, { + prerender_title: resp.title + , prerender_html: resp.html + , canon_url: canonBase ? canonBase + req.url : null + , noscript: true + , theme + , t: l10n[lang] + }) + }) + } }) // Cleanup socket file from previous executions From 950ebf367086223a22faf51f65a892813f85f063 Mon Sep 17 00:00:00 2001 From: Chase Date: Tue, 23 Sep 2025 18:37:42 +0200 Subject: [PATCH 2/2] fix(prerenderer): configure trust proxy to fix http -> https redirects (#596) --- prerender-server/src/server.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prerender-server/src/server.js b/prerender-server/src/server.js index de7a94d6..f3a8fa87 100644 --- a/prerender-server/src/server.js +++ b/prerender-server/src/server.js @@ -42,6 +42,7 @@ register.registerMetric(renderDuration) let requestCounter = 0 const app = express() +app.set('trust proxy', true) app.engine('pug', pug.__express) app.get('/metrics', async (req, res) => { @@ -108,7 +109,7 @@ app.use((req, res, next) => { totalRenders.inc() } if (err) return next(err) - if (resp.redirect) return res.redirect(301, baseHref + resp.redirect.substr(1)) + if (resp.redirect) return res.redirect(301, baseHref + resp.redirect) if (resp.errorCode) { console.error(`Failed with code ${resp.errorCode}:`, resp) return res.sendStatus(resp.errorCode)