From 959aa3688d023b815862a0fc11dd826e71a40f65 Mon Sep 17 00:00:00 2001 From: Jacek Date: Mon, 16 Mar 2026 20:22:04 -0500 Subject: [PATCH 1/4] feat(e2e): add staging instance settings validation script Compares FAPI /v1/environment responses between production and staging instance pairs to detect configuration drift (auth strategies, MFA, org settings, user requirements, etc.). Runs as a non-blocking warning step in the e2e-staging workflow before integration tests. Also runnable locally via: node scripts/validate-staging-instances.mjs --- .github/workflows/e2e-staging.yml | 17 ++ scripts/validate-staging-instances.mjs | 249 +++++++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 scripts/validate-staging-instances.mjs diff --git a/.github/workflows/e2e-staging.yml b/.github/workflows/e2e-staging.yml index c40b5a102dc..9c1d23ece74 100644 --- a/.github/workflows/e2e-staging.yml +++ b/.github/workflows/e2e-staging.yml @@ -37,6 +37,23 @@ concurrency: cancel-in-progress: true jobs: + validate-instances: + name: Validate Staging Instances + runs-on: 'blacksmith-8vcpu-ubuntu-2204' + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.ref || github.event.client_payload.ref || 'main' }} + sparse-checkout: scripts/validate-staging-instances.mjs + fetch-depth: 1 + + - name: Validate staging instance settings + run: node scripts/validate-staging-instances.mjs + env: + INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }} + INTEGRATION_STAGING_INSTANCE_KEYS: ${{ secrets.INTEGRATION_STAGING_INSTANCE_KEYS }} + integration-tests: name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}) runs-on: 'blacksmith-8vcpu-ubuntu-2204' diff --git a/scripts/validate-staging-instances.mjs b/scripts/validate-staging-instances.mjs new file mode 100644 index 00000000000..3de85275706 --- /dev/null +++ b/scripts/validate-staging-instances.mjs @@ -0,0 +1,249 @@ +#!/usr/bin/env node + +/** + * Validates that staging Clerk instances have the same settings as their + * production counterparts by comparing FAPI /v1/environment responses. + * + * Usage: + * node scripts/validate-staging-instances.mjs + * + * Reads keys from INTEGRATION_INSTANCE_KEYS / INTEGRATION_STAGING_INSTANCE_KEYS + * env vars, or from integration/.keys.json / integration/.keys.staging.json. + */ + +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const STAGING_KEY_PREFIX = 'clerkstage-'; + +/** + * Paths to ignore during comparison — these are expected to differ between + * production and staging environments. + */ +const IGNORED_PATHS = [ + // Resource IDs are always different per instance + /\.id$/, + /^auth_config\.id$/, + // Logo/image URLs use different CDN domains (img.clerk.com vs img.clerkstage.dev) + /\.logo_url$/, + // Captcha settings may intentionally differ between environments + /\.captcha_enabled$/, + /\.captcha_widget_type$/, + // HIBP (breach detection) enforcement may differ + /\.enforce_hibp_on_sign_in$/, + /\.disable_hibp$/, +]; + +function isIgnored(path) { + return IGNORED_PATHS.some(pattern => pattern.test(path)); +} + +// ── Key loading ────────────────────────────────────────────────────────────── + +function loadKeys(envVar, filePath) { + if (process.env[envVar]) { + return JSON.parse(process.env[envVar]); + } + try { + return JSON.parse(readFileSync(resolve(filePath), 'utf-8')); + } catch { + return null; + } +} + +// ── PK parsing ─────────────────────────────────────────────────────────────── + +function parseFapiDomain(pk) { + // pk_test_$ or pk_live_$ + const parts = pk.split('_'); + const encoded = parts.slice(2).join('_'); + const decoded = Buffer.from(encoded, 'base64').toString('utf-8'); + // Remove trailing '$' + return decoded.replace(/\$$/, ''); +} + +// ── Environment fetching ───────────────────────────────────────────────────── + +async function fetchEnvironment(fapiDomain) { + const url = `https://${fapiDomain}/v1/environment`; + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`); + } + return res.json(); +} + +// ── Comparison ─────────────────────────────────────────────────────────────── + +const COMPARED_SECTIONS = ['user_settings', 'organization_settings', 'auth_config']; + +const COMPARED_USER_SETTINGS_FIELDS = [ + 'attributes', + 'social', + 'sign_in', + 'sign_up', + 'password_settings', +]; + +/** + * Recursively compare two values and collect paths where they differ. + */ +function diffObjects(a, b, path = '') { + const mismatches = []; + + if (a === b) return mismatches; + if (a == null || b == null || typeof a !== typeof b) { + mismatches.push({ path, prod: a, staging: b }); + return mismatches; + } + if (typeof a !== 'object') { + if (a !== b) { + mismatches.push({ path, prod: a, staging: b }); + } + return mismatches; + } + if (Array.isArray(a) && Array.isArray(b)) { + const sortedA = [...a].sort(); + const sortedB = [...b].sort(); + if (JSON.stringify(sortedA) !== JSON.stringify(sortedB)) { + mismatches.push({ path, prod: a, staging: b }); + } + return mismatches; + } + + const allKeys = new Set([...Object.keys(a), ...Object.keys(b)]); + for (const key of allKeys) { + const childPath = path ? `${path}.${key}` : key; + mismatches.push(...diffObjects(a[key], b[key], childPath)); + } + return mismatches; +} + +function compareEnvironments(prodEnv, stagingEnv) { + const mismatches = []; + + // auth_config + mismatches.push(...diffObjects(prodEnv.auth_config, stagingEnv.auth_config, 'auth_config')); + + // organization_settings + const orgFields = ['enabled', 'force_organization_selection']; + for (const field of orgFields) { + mismatches.push( + ...diffObjects( + prodEnv.organization_settings?.[field], + stagingEnv.organization_settings?.[field], + `organization_settings.${field}`, + ), + ); + } + + // user_settings — selected fields only + for (const field of COMPARED_USER_SETTINGS_FIELDS) { + if (field === 'social') { + // Only compare social providers that are enabled in at least one environment + const prodSocial = prodEnv.user_settings?.social ?? {}; + const stagingSocial = stagingEnv.user_settings?.social ?? {}; + const allProviders = new Set([...Object.keys(prodSocial), ...Object.keys(stagingSocial)]); + for (const provider of allProviders) { + const prodProvider = prodSocial[provider]; + const stagingProvider = stagingSocial[provider]; + if (!prodProvider?.enabled && !stagingProvider?.enabled) continue; + mismatches.push( + ...diffObjects(prodProvider, stagingProvider, `user_settings.social.${provider}`), + ); + } + } else { + mismatches.push( + ...diffObjects(prodEnv.user_settings?.[field], stagingEnv.user_settings?.[field], `user_settings.${field}`), + ); + } + } + + return mismatches; +} + +// ── Output formatting ──────────────────────────────────────────────────────── + +function formatValue(val) { + if (val === undefined) return 'undefined'; + if (val === null) return 'null'; + if (typeof val === 'object') return JSON.stringify(val); + return String(val); +} + +// ── Main ───────────────────────────────────────────────────────────────────── + +async function main() { + const prodKeys = loadKeys('INTEGRATION_INSTANCE_KEYS', 'integration/.keys.json'); + if (!prodKeys) { + console.error('No production instance keys found.'); + process.exit(0); + } + + const stagingKeys = loadKeys('INTEGRATION_STAGING_INSTANCE_KEYS', 'integration/.keys.staging.json'); + if (!stagingKeys) { + console.error('No staging instance keys found. Skipping validation.'); + process.exit(0); + } + + // Find pairs + const pairs = []; + for (const [name, keys] of Object.entries(prodKeys)) { + const stagingName = STAGING_KEY_PREFIX + name; + if (stagingKeys[stagingName]) { + pairs.push({ name, prod: keys, staging: stagingKeys[stagingName] }); + } + } + + if (pairs.length === 0) { + console.log('No production/staging key pairs found. Skipping validation.'); + process.exit(0); + } + + console.log(`Validating ${pairs.length} staging instance pair(s)...\n`); + + let mismatchCount = 0; + + for (const pair of pairs) { + const prodDomain = parseFapiDomain(pair.prod.pk); + const stagingDomain = parseFapiDomain(pair.staging.pk); + + let prodEnv, stagingEnv; + try { + [prodEnv, stagingEnv] = await Promise.all([fetchEnvironment(prodDomain), fetchEnvironment(stagingDomain)]); + } catch (err) { + console.log(`⚠️ ${pair.name}: failed to fetch environment`); + console.log(` ${err.message}\n`); + continue; + } + + const mismatches = compareEnvironments(prodEnv, stagingEnv).filter(m => !isIgnored(m.path)); + + if (mismatches.length === 0) { + console.log(`✅ ${pair.name}: matched`); + } else { + mismatchCount++; + console.log(`❌ ${pair.name} (${mismatches.length} mismatch${mismatches.length === 1 ? '' : 'es'}):`); + for (const m of mismatches) { + const prodVal = formatValue(m.prod); + const stagingVal = formatValue(m.staging); + console.log(` ${m.path}`); + console.log(` prod: ${prodVal}`); + console.log(` staging: ${stagingVal}`); + } + } + console.log(); + } + + // Summary + if (mismatchCount > 0) { + console.log(`Summary: ${mismatchCount} of ${pairs.length} instance pair(s) have mismatches`); + } else { + console.log(`Summary: all ${pairs.length} instance pair(s) matched`); + } +} + +main().catch(err => { + console.error('Unexpected error:', err); + process.exit(0); // Don't fail the CI run +}); From f6389559737a8fd24d3f4b9a09eae66191cef1d2 Mon Sep 17 00:00:00 2001 From: Jacek Date: Mon, 16 Mar 2026 21:25:45 -0500 Subject: [PATCH 2/4] fix(e2e): improve staging validation output formatting Group mismatches by section with aligned columns, collapse child fields when a parent attribute is disabled, show array diffs as missing/extra items instead of raw JSON, and collapse wholly missing social providers. --- scripts/validate-staging-instances.mjs | 184 +++++++++++++++++++------ 1 file changed, 142 insertions(+), 42 deletions(-) diff --git a/scripts/validate-staging-instances.mjs b/scripts/validate-staging-instances.mjs index 3de85275706..f3ac094fb3c 100644 --- a/scripts/validate-staging-instances.mjs +++ b/scripts/validate-staging-instances.mjs @@ -21,15 +21,11 @@ const STAGING_KEY_PREFIX = 'clerkstage-'; * production and staging environments. */ const IGNORED_PATHS = [ - // Resource IDs are always different per instance /\.id$/, /^auth_config\.id$/, - // Logo/image URLs use different CDN domains (img.clerk.com vs img.clerkstage.dev) /\.logo_url$/, - // Captcha settings may intentionally differ between environments /\.captcha_enabled$/, /\.captcha_widget_type$/, - // HIBP (breach detection) enforcement may differ /\.enforce_hibp_on_sign_in$/, /\.disable_hibp$/, ]; @@ -54,11 +50,9 @@ function loadKeys(envVar, filePath) { // ── PK parsing ─────────────────────────────────────────────────────────────── function parseFapiDomain(pk) { - // pk_test_$ or pk_live_$ const parts = pk.split('_'); const encoded = parts.slice(2).join('_'); const decoded = Buffer.from(encoded, 'base64').toString('utf-8'); - // Remove trailing '$' return decoded.replace(/\$$/, ''); } @@ -75,18 +69,11 @@ async function fetchEnvironment(fapiDomain) { // ── Comparison ─────────────────────────────────────────────────────────────── -const COMPARED_SECTIONS = ['user_settings', 'organization_settings', 'auth_config']; - -const COMPARED_USER_SETTINGS_FIELDS = [ - 'attributes', - 'social', - 'sign_in', - 'sign_up', - 'password_settings', -]; +const COMPARED_USER_SETTINGS_FIELDS = ['attributes', 'social', 'sign_in', 'sign_up', 'password_settings']; /** * Recursively compare two values and collect paths where they differ. + * For arrays of primitives (like strategy lists), stores structured diff info. */ function diffObjects(a, b, path = '') { const mismatches = []; @@ -103,10 +90,21 @@ function diffObjects(a, b, path = '') { return mismatches; } if (Array.isArray(a) && Array.isArray(b)) { - const sortedA = [...a].sort(); - const sortedB = [...b].sort(); - if (JSON.stringify(sortedA) !== JSON.stringify(sortedB)) { - mismatches.push({ path, prod: a, staging: b }); + const sortedA = JSON.stringify([...a].sort()); + const sortedB = JSON.stringify([...b].sort()); + if (sortedA !== sortedB) { + // For arrays of primitives, compute added/removed + const flatA = a.flat(Infinity); + const flatB = b.flat(Infinity); + if (flatA.every(v => typeof v !== 'object') && flatB.every(v => typeof v !== 'object')) { + const setA = new Set(flatA); + const setB = new Set(flatB); + const missingOnStaging = [...new Set(flatA.filter(v => !setB.has(v)))]; + const extraOnStaging = [...new Set(flatB.filter(v => !setA.has(v)))]; + mismatches.push({ path, prod: a, staging: b, missingOnStaging, extraOnStaging }); + } else { + mismatches.push({ path, prod: a, staging: b }); + } } return mismatches; } @@ -140,7 +138,6 @@ function compareEnvironments(prodEnv, stagingEnv) { // user_settings — selected fields only for (const field of COMPARED_USER_SETTINGS_FIELDS) { if (field === 'social') { - // Only compare social providers that are enabled in at least one environment const prodSocial = prodEnv.user_settings?.social ?? {}; const stagingSocial = stagingEnv.user_settings?.social ?? {}; const allProviders = new Set([...Object.keys(prodSocial), ...Object.keys(stagingSocial)]); @@ -148,9 +145,7 @@ function compareEnvironments(prodEnv, stagingEnv) { const prodProvider = prodSocial[provider]; const stagingProvider = stagingSocial[provider]; if (!prodProvider?.enabled && !stagingProvider?.enabled) continue; - mismatches.push( - ...diffObjects(prodProvider, stagingProvider, `user_settings.social.${provider}`), - ); + mismatches.push(...diffObjects(prodProvider, stagingProvider, `user_settings.social.${provider}`)); } } else { mismatches.push( @@ -164,13 +159,130 @@ function compareEnvironments(prodEnv, stagingEnv) { // ── Output formatting ──────────────────────────────────────────────────────── -function formatValue(val) { +/** + * Section display names and the path prefixes they cover. + */ +const SECTIONS = [ + { label: 'Auth Config', prefix: 'auth_config.' }, + { label: 'Organization Settings', prefix: 'organization_settings.' }, + { label: 'Attributes', prefix: 'user_settings.attributes.' }, + { label: 'Social Providers', prefix: 'user_settings.social.' }, + { label: 'Sign In', prefix: 'user_settings.sign_in.' }, + { label: 'Sign Up', prefix: 'user_settings.sign_up.' }, + { label: 'Password Settings', prefix: 'user_settings.password_settings.' }, +]; + +const COL_FIELD = 40; +const COL_VAL = 14; + +function pad(str, len) { + return str.length >= len ? str : str + ' '.repeat(len - str.length); +} + +function formatScalar(val) { if (val === undefined) return 'undefined'; if (val === null) return 'null'; if (typeof val === 'object') return JSON.stringify(val); return String(val); } +/** + * Collapse attribute mismatches: if .enabled differs, skip the child + * fields (first_factors, second_factors, verifications, etc.) since the root + * cause is the enabled flag. + */ +function collapseAttributeMismatches(mismatches) { + const disabledAttrs = new Set(); + for (const m of mismatches) { + if (m.path.startsWith('user_settings.attributes.') && m.path.endsWith('.enabled')) { + disabledAttrs.add(m.path.replace('.enabled', '')); + } + } + return mismatches.filter(m => { + if (!m.path.startsWith('user_settings.attributes.')) return true; + // Keep the .enabled entry itself + if (m.path.endsWith('.enabled')) return true; + // Drop children of disabled attributes + const parentAttr = m.path.replace(/\.[^.]+$/, ''); + return !disabledAttrs.has(parentAttr); + }); +} + +/** + * For social providers that are entirely present/missing, collapse to one line. + */ +function collapseSocialMismatches(mismatches) { + const wholeMissing = new Set(); + for (const m of mismatches) { + if (m.path.startsWith('user_settings.social.') && !m.path.includes('.', 'user_settings.social.x'.length)) { + if ((m.prod && !m.staging) || (!m.prod && m.staging)) { + wholeMissing.add(m.path); + } + } + } + return mismatches.filter(m => { + if (!m.path.startsWith('user_settings.social.')) return true; + // Keep the top-level entry + const parts = m.path.split('.'); + if (parts.length <= 3) return true; + // Drop children of wholly missing providers + const parentPath = parts.slice(0, 3).join('.'); + return !wholeMissing.has(parentPath); + }); +} + +function formatMismatch(m, prefix) { + const field = m.path.slice(prefix.length); + + // Array diff with missing/extra items + if (m.missingOnStaging || m.extraOnStaging) { + const parts = []; + if (m.missingOnStaging?.length) { + parts.push(`missing on staging: ${m.missingOnStaging.join(', ')}`); + } + if (m.extraOnStaging?.length) { + parts.push(`extra on staging: ${m.extraOnStaging.join(', ')}`); + } + return ` ${pad(field, COL_FIELD)} ${parts.join('; ')}`; + } + + // Social provider entirely present/missing + if (prefix === 'user_settings.social.' && !field.includes('.')) { + if (m.prod && !m.staging) { + return ` ${pad(field, COL_FIELD)} ${pad('present', COL_VAL)} missing`; + } + if (!m.prod && m.staging) { + return ` ${pad(field, COL_FIELD)} ${pad('missing', COL_VAL)} present`; + } + } + + const prodVal = formatScalar(m.prod); + const stagingVal = formatScalar(m.staging); + return ` ${pad(field, COL_FIELD)} ${pad(prodVal, COL_VAL)} ${stagingVal}`; +} + +function printReport(name, mismatches) { + if (mismatches.length === 0) { + console.log(`✅ ${name}: matched\n`); + return; + } + + console.log(`❌ ${name} (${mismatches.length} mismatch${mismatches.length === 1 ? '' : 'es'})\n`); + + for (const section of SECTIONS) { + const sectionMismatches = mismatches.filter(m => m.path.startsWith(section.prefix)); + if (sectionMismatches.length === 0) continue; + + console.log(` ${section.label}`); + console.log(` ${pad('', COL_FIELD)} ${pad('prod', COL_VAL)} staging`); + + for (const m of sectionMismatches) { + console.log(formatMismatch(m, section.prefix)); + } + console.log(); + } +} + // ── Main ───────────────────────────────────────────────────────────────────── async function main() { @@ -186,7 +298,6 @@ async function main() { process.exit(0); } - // Find pairs const pairs = []; for (const [name, keys] of Object.entries(prodKeys)) { const stagingName = STAGING_KEY_PREFIX + name; @@ -217,25 +328,14 @@ async function main() { continue; } - const mismatches = compareEnvironments(prodEnv, stagingEnv).filter(m => !isIgnored(m.path)); + let mismatches = compareEnvironments(prodEnv, stagingEnv).filter(m => !isIgnored(m.path)); + mismatches = collapseAttributeMismatches(mismatches); + mismatches = collapseSocialMismatches(mismatches); - if (mismatches.length === 0) { - console.log(`✅ ${pair.name}: matched`); - } else { - mismatchCount++; - console.log(`❌ ${pair.name} (${mismatches.length} mismatch${mismatches.length === 1 ? '' : 'es'}):`); - for (const m of mismatches) { - const prodVal = formatValue(m.prod); - const stagingVal = formatValue(m.staging); - console.log(` ${m.path}`); - console.log(` prod: ${prodVal}`); - console.log(` staging: ${stagingVal}`); - } - } - console.log(); + if (mismatches.length > 0) mismatchCount++; + printReport(pair.name, mismatches); } - // Summary if (mismatchCount > 0) { console.log(`Summary: ${mismatchCount} of ${pairs.length} instance pair(s) have mismatches`); } else { @@ -245,5 +345,5 @@ async function main() { main().catch(err => { console.error('Unexpected error:', err); - process.exit(0); // Don't fail the CI run + process.exit(0); }); From a933a2a241717d08b83e2d30159a8fd9e664c40e Mon Sep 17 00:00:00 2001 From: Jacek Date: Mon, 16 Mar 2026 21:50:22 -0500 Subject: [PATCH 3/4] fix(e2e): add fetch timeout and track fetch failures in summary Add 10s fetch timeout via AbortSignal. Track fetch failures separately so the summary never falsely reports 'all matched' when fetches failed. --- scripts/validate-staging-instances.mjs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/validate-staging-instances.mjs b/scripts/validate-staging-instances.mjs index f3ac094fb3c..a97f8f04903 100644 --- a/scripts/validate-staging-instances.mjs +++ b/scripts/validate-staging-instances.mjs @@ -60,7 +60,7 @@ function parseFapiDomain(pk) { async function fetchEnvironment(fapiDomain) { const url = `https://${fapiDomain}/v1/environment`; - const res = await fetch(url); + const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); if (!res.ok) { throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`); } @@ -314,6 +314,7 @@ async function main() { console.log(`Validating ${pairs.length} staging instance pair(s)...\n`); let mismatchCount = 0; + let fetchFailCount = 0; for (const pair of pairs) { const prodDomain = parseFapiDomain(pair.prod.pk); @@ -323,6 +324,7 @@ async function main() { try { [prodEnv, stagingEnv] = await Promise.all([fetchEnvironment(prodDomain), fetchEnvironment(stagingDomain)]); } catch (err) { + fetchFailCount++; console.log(`⚠️ ${pair.name}: failed to fetch environment`); console.log(` ${err.message}\n`); continue; @@ -336,11 +338,12 @@ async function main() { printReport(pair.name, mismatches); } - if (mismatchCount > 0) { - console.log(`Summary: ${mismatchCount} of ${pairs.length} instance pair(s) have mismatches`); - } else { - console.log(`Summary: all ${pairs.length} instance pair(s) matched`); - } + const parts = []; + if (mismatchCount > 0) parts.push(`${mismatchCount} mismatched`); + if (fetchFailCount > 0) parts.push(`${fetchFailCount} failed to fetch`); + const matchedCount = pairs.length - mismatchCount - fetchFailCount; + if (matchedCount > 0) parts.push(`${matchedCount} matched`); + console.log(`Summary: ${parts.join(', ')} (${pairs.length} total)`); } main().catch(err => { From ba1e1b046ba09a2dcb8106abba07f56d3b293a1a Mon Sep 17 00:00:00 2001 From: Jacek Date: Mon, 16 Mar 2026 21:56:37 -0500 Subject: [PATCH 4/4] chore: add empty changeset --- .changeset/validate-staging-instances.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changeset/validate-staging-instances.md diff --git a/.changeset/validate-staging-instances.md b/.changeset/validate-staging-instances.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/validate-staging-instances.md @@ -0,0 +1,2 @@ +--- +---