improvement(memory): replace unbounded server caches with lru-cache to fix heap growth (#4652)

* fix(memory): prune toolSchemaCache and semaphores to prevent heap growth

toolSchemaCache (lib/copilot/chat/payload.ts): module-level Map keyed by
userId:workspaceId never deleted expired entries, only checked TTL on read.
With 100K+ unique user/workspace pairs each holding 50-200KB of tool schemas,
this was the primary driver of the 24MB -> 25GB heap growth observed in
CloudWatch. Add a setInterval sweep every 30s (matching the TTL) with .unref()
so it does not prevent graceful shutdown.

semaphores (lib/core/async-jobs/backends/database.ts): acquireSlot created
Semaphore entries that releaseSlot never deleted. With per-execution UUID keys
(e.g. scheduleJobId), each scheduled workflow run would add a permanent entry.
Store the concurrency limit on the Semaphore struct and delete the entry from
the Map when all slots are free and no waiters remain.

validatorCache (lib/copilot/tools/server/generated-schema.ts): validated as
bounded (93 tools x 2 schema kinds = 186 max entries, ~2-9MB). No fix needed.

isolated-vm nativeContexts: validated as deferred GC, self-healed by worker
rotation at MAX_EXECUTIONS_PER_WORKER=200. externalMB spikes trace to
concurrent isolate heaps at peak load (128MB limit x active isolates), not a
reference leak. No fix needed.

* fix(memory): prune effectiveEnvCache and instrument cache sizes in telemetry

effectiveEnvCache (lib/environment/utils.ts): same unbounded accumulation
pattern as toolSchemaCache — module-level Map keyed by userId:workspaceId
with a 15s TTL that is only checked on read, never proactively evicted.
Adds a periodic sweep matching the TTL interval with .unref().

cache-registry (lib/monitoring/cache-registry.ts): lightweight registry
so modules can expose their cache sizes to telemetry without coupling.
toolSchemaCache and effectiveEnvCache both register on module load.

memory-telemetry: emits cacheSizes in every Memory snapshot log so
CloudWatch can confirm the caches stay bounded post-deploy.

* improvement(memory): replace manual TTL Maps with lru-cache for toolSchemaCache and effectiveEnvCache

Replaces the homegrown Map + setInterval sweep pattern with LRUCache from
the lru-cache npm package, which is the standard Node.js solution for
bounded in-process caching with TTL.

Changes per cache:
- Removes manual ToolSchemaCacheEntry / EffectiveEnvCacheEntry types
- Removes setInterval sweep timers (and the .unref() boilerplate)
- Removes the two-phase promise->value entry update inside the IIFE
- Stores Promise<T> directly — in-flight and resolved states share one type
- max: 200 (toolSchemaCache) / max: 500 (effectiveEnvCache) as hard ceilings
- TTL behaviour and concurrent-request deduplication are preserved exactly
- cache-registry .size reporting works unchanged via lru-cache's .size prop

* fix(memory): remove redundant waiters guard in releaseSlot

Author: waleedlatif1

apps/sim/lib/copilot/chat/payload.ts

@@ -1,10 +1,12 @@
 import { createLogger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
+import { LRUCache } from 'lru-cache'
 import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
 import { isPaid } from '@/lib/billing/plan-helpers'
 import { getToolEntry } from '@/lib/copilot/tool-executor/router'
 import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions'
 import { isHosted } from '@/lib/core/config/feature-flags'
+import { registerCache } from '@/lib/monitoring/cache-registry'
 import { buildMothershipToolsForRequest } from '@/lib/mothership/settings/runtime'
 import { trackChatUpload } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
 import { tools } from '@/tools/registry'
@@ -13,13 +15,12 @@ import { getLatestVersionTools, stripVersionSuffix } from '@/tools/utils'
 const logger = createLogger('CopilotChatPayload')
 const TOOL_SCHEMA_CACHE_TTL_MS = 30_000
 
-type ToolSchemaCacheEntry = {
-  expiresAt: number
-  value?: ToolSchema[]
-  promise?: Promise<ToolSchema[]>
-}
+const toolSchemaCache = new LRUCache<string, Promise<ToolSchema[]>>({
+  max: 200,
+  ttl: TOOL_SCHEMA_CACHE_TTL_MS,
+})
 
-const toolSchemaCache = new Map<string, ToolSchemaCacheEntry>()
+registerCache('toolSchemaCache', () => toolSchemaCache.size)
 
 interface BuildPayloadParams {
   message: string
@@ -74,13 +75,10 @@ export async function buildIntegrationToolSchemas(
   workspaceId?: string
 ): Promise<ToolSchema[]> {
   const cacheKey = `${userId}:${workspaceId ?? ''}:${options.schemaSurface ?? 'copilot'}`
-  const now = Date.now()
+
   const cached = toolSchemaCache.get(cacheKey)
-  if (cached?.value && cached.expiresAt > now) {
-    return cached.value.map((tool) => ({ ...tool, input_schema: { ...tool.input_schema } }))
-  }
-  if (cached?.promise) {
-    const tools = await cached.promise
+  if (cached) {
+    const tools = await cached
     return tools.map((tool) => ({ ...tool, input_schema: { ...tool.input_schema } }))
   }
 
@@ -187,18 +185,10 @@ export async function buildIntegrationToolSchemas(
       )
     }
 
-    toolSchemaCache.set(cacheKey, {
-      value: integrationTools,
-      expiresAt: Date.now() + TOOL_SCHEMA_CACHE_TTL_MS,
-    })
-
     return integrationTools
   })()
 
-  toolSchemaCache.set(cacheKey, {
-    expiresAt: now + TOOL_SCHEMA_CACHE_TTL_MS,
-    promise,
-  })
+  toolSchemaCache.set(cacheKey, promise)
 
   const integrationTools = await promise
   return integrationTools.map((tool) => ({ ...tool, input_schema: { ...tool.input_schema } }))

apps/sim/lib/core/async-jobs/backends/database.ts

@@ -38,6 +38,7 @@ function rowToJob(row: AsyncJobRow): Job {
 const inlineAbortControllers = new Map<string, AbortController>()
 
 interface Semaphore {
+  limit: number
   available: number
   waiters: Array<() => void>
 }
@@ -46,7 +47,7 @@ const semaphores = new Map<string, Semaphore>()
 async function acquireSlot(key: string, limit: number): Promise<void> {
   let s = semaphores.get(key)
   if (!s) {
-    s = { available: limit, waiters: [] }
+    s = { limit, available: limit, waiters: [] }
     semaphores.set(key, s)
   }
   if (s.available > 0) {
@@ -65,6 +66,9 @@ function releaseSlot(key: string): void {
     return
   }
   s.available += 1
+  if (s.available === s.limit) {
+    semaphores.delete(key)
+  }
 }
 
 export class DatabaseJobQueue implements JobQueueBackend {

apps/sim/lib/environment/utils.ts

@@ -4,24 +4,25 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { eq, inArray } from 'drizzle-orm'
+import { LRUCache } from 'lru-cache'
 import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption'
 import {
   createWorkspaceEnvCredentials,
   getAccessibleEnvCredentials,
   syncPersonalEnvCredentialsForUser,
 } from '@/lib/credentials/environment'
+import { registerCache } from '@/lib/monitoring/cache-registry'
 import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils'
 
 const logger = createLogger('EnvironmentUtils')
 const EFFECTIVE_ENV_CACHE_TTL_MS = 15_000
 
-type EffectiveEnvCacheEntry = {
-  expiresAt: number
-  value?: Record<string, string>
-  promise?: Promise<Record<string, string>>
-}
+const effectiveEnvCache = new LRUCache<string, Promise<Record<string, string>>>({
+  max: 500,
+  ttl: EFFECTIVE_ENV_CACHE_TTL_MS,
+})
 
-const effectiveEnvCache = new Map<string, EffectiveEnvCacheEntry>()
+registerCache('effectiveEnvCache', () => effectiveEnvCache.size)
 
 function getEffectiveEnvCacheKey(userId: string, workspaceId?: string) {
   return `${userId}:${workspaceId ?? ''}`
@@ -325,37 +326,24 @@ export async function getEffectiveDecryptedEnv(
   workspaceId?: string
 ): Promise<Record<string, string>> {
   const cacheKey = getEffectiveEnvCacheKey(userId, workspaceId)
-  const now = Date.now()
-  const cached = effectiveEnvCache.get(cacheKey)
-
-  if (cached?.value && cached.expiresAt > now) {
-    return { ...cached.value }
-  }
 
-  if (cached?.promise) {
-    const value = await cached.promise
+  const cached = effectiveEnvCache.get(cacheKey)
+  if (cached) {
+    const value = await cached
     return { ...value }
   }
 
   const promise = getPersonalAndWorkspaceEnv(userId, workspaceId)
-    .then(({ personalDecrypted, workspaceDecrypted }) => {
-      const value = { ...personalDecrypted, ...workspaceDecrypted }
-      effectiveEnvCache.set(cacheKey, {
-        value,
-        expiresAt: Date.now() + EFFECTIVE_ENV_CACHE_TTL_MS,
-      })
-      return value
-    })
+    .then(({ personalDecrypted, workspaceDecrypted }) => ({
+      ...personalDecrypted,
+      ...workspaceDecrypted,
+    }))
     .catch((error) => {
       effectiveEnvCache.delete(cacheKey)
       throw error
     })
 
-  effectiveEnvCache.set(cacheKey, {
-    expiresAt: now + EFFECTIVE_ENV_CACHE_TTL_MS,
-    promise,
-  })
-
+  effectiveEnvCache.set(cacheKey, promise)
   const value = await promise
   return { ...value }
 }

apps/sim/lib/monitoring/cache-registry.ts

@@ -0,0 +1,13 @@
+const registry = new Map<string, () => number>()
+
+export function registerCache(name: string, getSize: () => number): void {
+  registry.set(name, getSize)
+}
+
+export function getCacheSizes(): Record<string, number> {
+  const sizes: Record<string, number> = {}
+  for (const [name, getSize] of registry) {
+    sizes[name] = getSize()
+  }
+  return sizes
+}

apps/sim/lib/monitoring/memory-telemetry.ts

@@ -5,6 +5,7 @@
 
 import v8 from 'node:v8'
 import { createLogger } from '@sim/logger'
+import { getCacheSizes } from '@/lib/monitoring/cache-registry'
 
 const logger = createLogger('MemoryTelemetry', { logLevel: 'INFO' })
 
@@ -33,6 +34,7 @@ export function startMemoryTelemetry(intervalMs = 60_000) {
           ? process.getActiveResourcesInfo().length
           : -1,
       uptimeMin: Math.round(process.uptime() / 60),
+      cacheSizes: getCacheSizes(),
     })
   }, intervalMs)
   timer.unref()

apps/sim/package.json

@@ -148,6 +148,7 @@
     "json5": "2.2.3",
     "jszip": "3.10.1",
     "jwt-decode": "^4.0.0",
+    "lru-cache": "11.3.6",
     "lucide-react": "^0.479.0",
     "mammoth": "^1.9.0",
     "mermaid": "11.15.0",