Load whitelist only on server startup

app.js

@@ -45,6 +45,8 @@ app.use(cacheMiddleware);
 
 import apiViews from './modules/api/views.js';
 import debugViews from './modules/debug/views.js';
+import { initializeWhitelistLoader } from './lib/whitelist.js';
+initializeWhitelistLoader();
 apiViews(app);
 debugViews(app);
 
@@ -174,4 +176,4 @@ app.get('/', function(req, res) {
   res.end();
 });
 
-process.title = "iframely";
+process.title = "iframely";

audit.log

@@ -41,5 +41,19 @@
 ├─────────────────────┼────────────────────────────────────────────────────────┤
 │ More info           │ https://github.com/advisories/GHSA-73rr-hh4g-fpgx      │
 └─────────────────────┴────────────────────────────────────────────────────────┘
-3 vulnerabilities found
-Severity: 1 low | 1 moderate | 1 high
+┌─────────────────────┬────────────────────────────────────────────────────────┐
+│ low                 │ qs's arrayLimit bypass in comma parsing allows denial  │
+│                     │ of service                                             │
+├─────────────────────┼────────────────────────────────────────────────────────┤
+│ Package             │ qs                                                     │
+├─────────────────────┼────────────────────────────────────────────────────────┤
+│ Vulnerable versions │ >=6.7.0 <=6.14.1                                       │
+├─────────────────────┼────────────────────────────────────────────────────────┤
+│ Patched versions    │ >=6.14.2                                               │
+├─────────────────────┼────────────────────────────────────────────────────────┤
+│ Paths               │ .>mock-http-server>body-parser>qs                      │
+├─────────────────────┼────────────────────────────────────────────────────────┤
+│ More info           │ https://github.com/advisories/GHSA-w7fw-mjwx-w883      │
+└─────────────────────┴────────────────────────────────────────────────────────┘
+4 vulnerabilities found
+Severity: 2 low | 1 moderate | 1 high

config.js

@@ -20,6 +20,7 @@
         use_http2: true,
         DEBUG: false,
 
+        WHITELIST_AUTO_LOAD: true,  // Whitelist is always loaded by lib init. Otherwise use `initializeWhitelistLoader();`.
         WHITELIST_URL: 'https://iframely.com/qa/domains.json',
         WHITELIST_URL_RELOAD_PERIOD: 60 * 60 * 1000,  // will reload WL every hour, if no local files are found in /whitelist folder
 
@@ -440,4 +441,4 @@
         config.HTTP2_RETRY_CODES[item] = 1;
     });
 
-    export default config;
+    export default config;

lib/core.js

@@ -1612,6 +1612,51 @@
             return value;
         };
 
+        options.getDomainOptions = function(path, defaultValue) {
+            /*
+            p1. providerOptions:    {_: {prerender: true}} -> from api get params
+            p2. link url:           "https://domain.com/path?_prerender=true" -> urlOptions={prerender: true}
+            p3. domainOptions:      {meta: {prerender: true}}
+            */
+
+            const bits = path.split('.');
+
+            const alternatePath = path.replace(/^[^.]+\./, '_');    // "meta.prerender" -> "_.prerender"
+
+            // p1.
+            var value = options.getProviderOptions(alternatePath);
+            if (typeof value === 'undefined') {
+                if (!options.getProviderOptions('app.disable_url_options', false)) {
+                    const alternatePathForUrlOptions = alternatePath.replace(/^_/, '');
+                    // p2.
+                    value = urlOptions && urlOptions[alternatePathForUrlOptions];
+                }
+                if (typeof value === 'undefined') {
+                    // p3.
+                    value = searchParamInObj(0, bits, options.domainOptions);
+                }
+            }
+
+            if (
+                 typeof defaultValue !== "undefined"
+                 && typeof value !== typeof defaultValue
+            ) {
+                // Type mismatch fallback.
+                if (typeof defaultValue === "string") {
+                    return "" + value;
+                } else {
+                    return defaultValue;
+                }
+            }
+
+            // Read from CONFIG.
+            if (typeof value === 'undefined' && CONFIG.domainOptions) {
+                value = searchParamInObj(0, bits, CONFIG.domainOptions);
+            }
+
+            return typeof value !== 'undefined' ? value : defaultValue;
+        };
+
         options.digitize = function(params) {
             return Object.keys(params).reduce((acc, k) => {
                 acc[k] = typeof params[k] === 'boolean' ? (params[k] ? 1 : 0) : params[k];
@@ -1969,6 +2014,10 @@
             }
         }
 
+        if (options.getDomainOptionsByUrl) {
+            options.domainOptions = options.getDomainOptionsByUrl(uri);
+        }
+
         asyncMethodCb('initial');
     };
 

lib/utils.js

@@ -91,17 +91,28 @@ export function prepareRequestOptions(request_options, options) {
 
     var disable_language = false;
 
+    function setPrerender(prerender_option) {
+        if (options?.allowPrerender && CONFIG.PRERENDER_URL) {
+            request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
+
+            // Use string `proxy.prerender` as additional prerender param.
+            if (typeof prerender_option === 'string') {
+                request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`;
+            }
+        }
+    }
+
+    var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender');
+    if (enable_domain_prerender) {
+        setPrerender(enable_domain_prerender);
+    }
+
     if (CONFIG.PROXY || (options && options.proxy)) {
 
         var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options);
         if (proxy) {
-            if (proxy.prerender && CONFIG.PRERENDER_URL && options?.allowPrerender) {
-                request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
-
-                // Use string `proxy.prerender` as additional prerender param.
-                if (typeof proxy.prerender === 'string') {
-                    request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${proxy.prerender}`;
-                }
+            if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) {
+                setPrerender(proxy.prerender);
 
             } else if (proxy.proxy && CONFIG.PROXY_URL) {
                 request_options.uri = /{url}/.test(CONFIG.PROXY_URL) 

lib/whitelist.js

@@ -313,6 +313,8 @@
 
     }
 
+    var loadFileTimeout;
+
     function startScanWhitelist() {
 
         var watcher = chokidar.watch(WHITELIST_DIR, {
@@ -326,7 +328,8 @@
             // Check if newer file added.
             if (p.match(/iframely-.*\.json/)) {
                 // Wait sometime to be sure write finished.
-                setTimeout(function() {
+                clearTimeout(loadFileTimeout);
+                loadFileTimeout = setTimeout(function() {
                     loadLastWhitelist();
                 }, 5000);
             }
@@ -337,7 +340,8 @@
             // Reload last whitelist.
             if (p == currentWhitelistFilename) {
                 // Wait sometime to be sure write finished.
-                setTimeout(function() {
+                clearTimeout(loadFileTimeout);
+                loadFileTimeout = setTimeout(function() {
                     readWhitelist(p);
                 }, 5000);
             }
@@ -346,9 +350,6 @@
         loadLastWhitelist();
     }
 
-    startScanWhitelist();
-    loadWhitelistUrl();
-
     function loadWhitelistUrl() {
 
         if (!currentWhitelistFilename && CONFIG.WHITELIST_URL && CONFIG.WHITELIST_URL_RELOAD_PERIOD) {
@@ -394,3 +395,18 @@
                 });
         }
     }
+
+    function privateInitializeWhitelistLoader() {
+        startScanWhitelist();
+        loadWhitelistUrl();
+    }
+
+    export function initializeWhitelistLoader() {
+        if (!CONFIG.WHITELIST_AUTO_LOAD) {
+            privateInitializeWhitelistLoader();
+        }
+    }
+
+    if (CONFIG.WHITELIST_AUTO_LOAD) {
+        privateInitializeWhitelistLoader();
+    }

logging.js

@@ -1,4 +1,5 @@
 import moment from 'moment';
+import CONFIG from './config.loader.js';
 
 export default function log() {
     var args = Array.prototype.slice.apply(arguments);
@@ -20,4 +21,4 @@ export default function log() {
     }
 
     console.log.apply(console, args);
-};
+};

plugins/links/prerender/prerender.js

@@ -8,7 +8,8 @@ export default {
 
     getData: function(url, __allowJSRender, iframelyRun, options, meta, cb) {
 
-        if (CONFIG.PRERENDER && CONFIG.PRERENDER_URL && options.user_agent === CONFIG.FB_USER_AGENT) {
+        if (CONFIG.PRERENDER && CONFIG.PRERENDER_URL && options.user_agent === CONFIG.FB_USER_AGENT
+            && !url.startsWith(CONFIG.PRERENDER_URL)) {
 
             var prerenderUrl = CONFIG.PRERENDER_URL + encodeURIComponent(url);
             var options2 = {...options, ...{

plugins/links/prerender/react-app-fb-fallback.js

@@ -8,7 +8,8 @@ export default {
 
     getData: function(url, __allowJSRender, iframelyRun, options, cb) {
 
-        if (options.user_agent === CONFIG.FB_USER_AGENT) {
+        if (options.user_agent === CONFIG.FB_USER_AGENT 
+            || CONFIG.PRERENDER_URL && url.startsWith(CONFIG.PRERENDER_URL)) {
             return cb();
         }