fix(envvars): resolution standardized (#2957)

* fix(envvars): resolution standardized

* remove comments

* address bugbot

* fix highlighting for env vars

* remove comments

* address greptile

* address bugbot

Author: icecrasher321

apps/sim/app/api/copilot/execute-tool/route.ts

@@ -104,17 +104,11 @@ export async function POST(req: NextRequest) {
     })
 
     // Build execution params starting with LLM-provided arguments
-    // Resolve all {{ENV_VAR}} references in the arguments
+    // Resolve all {{ENV_VAR}} references in the arguments (deep for nested objects)
     const executionParams: Record<string, any> = resolveEnvVarReferences(
       toolArgs,
       decryptedEnvVars,
-      {
-        resolveExactMatch: true,
-        allowEmbedded: true,
-        trimKeys: true,
-        onMissing: 'keep',
-        deep: true,
-      }
+      { deep: true }
     ) as Record<string, any>
 
     logger.info(`[${tracker.requestId}] Resolved env var references in arguments`, {

apps/sim/app/api/mcp/servers/test-connection/route.ts

@@ -1,11 +1,10 @@
 import { createLogger } from '@sim/logger'
 import type { NextRequest } from 'next/server'
-import { getEffectiveDecryptedEnv } from '@/lib/environment/utils'
 import { McpClient } from '@/lib/mcp/client'
 import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
-import type { McpServerConfig, McpTransport } from '@/lib/mcp/types'
+import { resolveMcpConfigEnvVars } from '@/lib/mcp/resolve-config'
+import type { McpTransport } from '@/lib/mcp/types'
 import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
-import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
 
 const logger = createLogger('McpServerTestAPI')
 
@@ -19,30 +18,6 @@ function isUrlBasedTransport(transport: McpTransport): boolean {
   return transport === 'streamable-http'
 }
 
-/**
- * Resolve environment variables in strings
- */
-function resolveEnvVars(value: string, envVars: Record<string, string>): string {
-  const missingVars: string[] = []
-  const resolvedValue = resolveEnvVarReferences(value, envVars, {
-    allowEmbedded: true,
-    resolveExactMatch: true,
-    trimKeys: true,
-    onMissing: 'keep',
-    deep: false,
-    missingKeys: missingVars,
-  }) as string
-
-  if (missingVars.length > 0) {
-    const uniqueMissing = Array.from(new Set(missingVars))
-    uniqueMissing.forEach((envKey) => {
-      logger.warn(`Environment variable "${envKey}" not found in MCP server test`)
-    })
-  }
-
-  return resolvedValue
-}
-
 interface TestConnectionRequest {
   name: string
   transport: McpTransport
@@ -96,39 +71,30 @@ export const POST = withMcpAuth('write')(
         )
       }
 
-      let resolvedUrl = body.url
-      let resolvedHeaders = body.headers || {}
-
-      try {
-        const envVars = await getEffectiveDecryptedEnv(userId, workspaceId)
-
-        if (resolvedUrl) {
-          resolvedUrl = resolveEnvVars(resolvedUrl, envVars)
-        }
-
-        const resolvedHeadersObj: Record<string, string> = {}
-        for (const [key, value] of Object.entries(resolvedHeaders)) {
-          resolvedHeadersObj[key] = resolveEnvVars(value, envVars)
-        }
-        resolvedHeaders = resolvedHeadersObj
-      } catch (envError) {
-        logger.warn(
-          `[${requestId}] Failed to resolve environment variables, using raw values:`,
-          envError
-        )
-      }
-
-      const testConfig: McpServerConfig = {
+      // Build initial config for resolution
+      const initialConfig = {
         id: `test-${requestId}`,
         name: body.name,
         transport: body.transport,
-        url: resolvedUrl,
-        headers: resolvedHeaders,
+        url: body.url,
+        headers: body.headers || {},
         timeout: body.timeout || 10000,
         retries: 1, // Only one retry for tests
         enabled: true,
       }
 
+      // Resolve env vars using shared utility (non-strict mode for testing)
+      const { config: testConfig, missingVars } = await resolveMcpConfigEnvVars(
+        initialConfig,
+        userId,
+        workspaceId,
+        { strict: false }
+      )
+
+      if (missingVars.length > 0) {
+        logger.warn(`[${requestId}] Some environment variables not found:`, { missingVars })
+      }
+
       const testSecurityPolicy = {
         requireConsent: false,
         auditLevel: 'none' as const,

apps/sim/app/api/webhooks/[id]/route.ts

@@ -7,6 +7,7 @@ import { getSession } from '@/lib/auth'
 import { validateInteger } from '@/lib/core/security/input-validation'
 import { PlatformEvents } from '@/lib/core/telemetry'
 import { generateRequestId } from '@/lib/core/utils/request'
+import { resolveEnvVarsInObject } from '@/lib/webhooks/env-resolver'
 import {
   cleanupExternalWebhook,
   createExternalWebhookSubscription,
@@ -112,9 +113,9 @@ export async function PATCH(request: NextRequest, { params }: { params: Promise<
       }
     }
 
+    const originalProviderConfig = providerConfig
     let resolvedProviderConfig = providerConfig
     if (providerConfig) {
-      const { resolveEnvVarsInObject } = await import('@/lib/webhooks/env-resolver')
       const webhookDataForResolve = await db
         .select({
           workspaceId: workflow.workspaceId,
@@ -230,19 +231,23 @@ export async function PATCH(request: NextRequest, { params }: { params: Promise<
       hasFailedCountUpdate: failedCount !== undefined,
     })
 
-    // Merge providerConfig to preserve credential-related fields
     let finalProviderConfig = webhooks[0].webhook.providerConfig
-    if (providerConfig !== undefined) {
+    if (providerConfig !== undefined && originalProviderConfig) {
       const existingConfig = existingProviderConfig
       finalProviderConfig = {
-        ...nextProviderConfig,
+        ...originalProviderConfig,
         credentialId: existingConfig.credentialId,
         credentialSetId: existingConfig.credentialSetId,
         userId: existingConfig.userId,
         historyId: existingConfig.historyId,
         lastCheckedTimestamp: existingConfig.lastCheckedTimestamp,
         setupCompleted: existingConfig.setupCompleted,
-        externalId: nextProviderConfig.externalId ?? existingConfig.externalId,
+        externalId: existingConfig.externalId,
+      }
+      for (const [key, value] of Object.entries(nextProviderConfig)) {
+        if (!(key in originalProviderConfig)) {
+          ;(finalProviderConfig as Record<string, unknown>)[key] = value
+        }
       }
     }
 

apps/sim/app/api/webhooks/route.ts

@@ -7,6 +7,7 @@ import { type NextRequest, NextResponse } from 'next/server'
 import { getSession } from '@/lib/auth'
 import { PlatformEvents } from '@/lib/core/telemetry'
 import { generateRequestId } from '@/lib/core/utils/request'
+import { resolveEnvVarsInObject } from '@/lib/webhooks/env-resolver'
 import { createExternalWebhookSubscription } from '@/lib/webhooks/provider-subscriptions'
 import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
 
@@ -298,14 +299,10 @@ export async function POST(request: NextRequest) {
       }
     }
 
-    let savedWebhook: any = null // Variable to hold the result of save/update
-
-    // Use the original provider config - Gmail/Outlook configuration functions will inject userId automatically
-    const finalProviderConfig = providerConfig || {}
-
-    const { resolveEnvVarsInObject } = await import('@/lib/webhooks/env-resolver')
+    let savedWebhook: any = null
+    const originalProviderConfig = providerConfig || {}
     let resolvedProviderConfig = await resolveEnvVarsInObject(
-      finalProviderConfig,
+      originalProviderConfig,
       userId,
       workflowRecord.workspaceId || undefined
     )
@@ -469,6 +466,8 @@ export async function POST(request: NextRequest) {
       providerConfig: providerConfigOverride,
     })
 
+    const configToSave = { ...originalProviderConfig }
+
     try {
       const result = await createExternalWebhookSubscription(
         request,
@@ -477,7 +476,13 @@ export async function POST(request: NextRequest) {
         userId,
         requestId
       )
-      resolvedProviderConfig = result.updatedProviderConfig as Record<string, unknown>
+      const updatedConfig = result.updatedProviderConfig as Record<string, unknown>
+      for (const [key, value] of Object.entries(updatedConfig)) {
+        if (!(key in originalProviderConfig)) {
+          configToSave[key] = value
+        }
+      }
+      resolvedProviderConfig = updatedConfig
       externalSubscriptionCreated = result.externalSubscriptionCreated
     } catch (err) {
       logger.error(`[${requestId}] Error creating external webhook subscription`, err)
@@ -490,25 +495,22 @@ export async function POST(request: NextRequest) {
       )
     }
 
-    // Now save to database (only if subscription succeeded or provider doesn't need external subscription)
     try {
       if (targetWebhookId) {
         logger.info(`[${requestId}] Updating existing webhook for path: ${finalPath}`, {
           webhookId: targetWebhookId,
           provider,
-          hasCredentialId: !!(resolvedProviderConfig as any)?.credentialId,
-          credentialId: (resolvedProviderConfig as any)?.credentialId,
+          hasCredentialId: !!(configToSave as any)?.credentialId,
+          credentialId: (configToSave as any)?.credentialId,
         })
         const updatedResult = await db
           .update(webhook)
           .set({
             blockId,
             provider,
-            providerConfig: resolvedProviderConfig,
+            providerConfig: configToSave,
             credentialSetId:
-              ((resolvedProviderConfig as Record<string, unknown>)?.credentialSetId as
-                | string
-                | null) || null,
+              ((configToSave as Record<string, unknown>)?.credentialSetId as string | null) || null,
             isActive: true,
             updatedAt: new Date(),
           })
@@ -531,11 +533,9 @@ export async function POST(request: NextRequest) {
             blockId,
             path: finalPath,
             provider,
-            providerConfig: resolvedProviderConfig,
+            providerConfig: configToSave,
             credentialSetId:
-              ((resolvedProviderConfig as Record<string, unknown>)?.credentialSetId as
-                | string
-                | null) || null,
+              ((configToSave as Record<string, unknown>)?.credentialSetId as string | null) || null,
             isActive: true,
             createdAt: new Date(),
             updatedAt: new Date(),
@@ -549,7 +549,7 @@ export async function POST(request: NextRequest) {
         try {
           const { cleanupExternalWebhook } = await import('@/lib/webhooks/provider-subscriptions')
           await cleanupExternalWebhook(
-            createTempWebhookData(resolvedProviderConfig),
+            createTempWebhookData(configToSave),
             workflowRecord,
             requestId
           )

apps/sim/app/api/workflows/[id]/execute/route.ts

@@ -116,7 +116,6 @@ type AsyncExecutionParams = {
   userId: string
   input: any
   triggerType: CoreTriggerType
-  preflighted?: boolean
 }
 
 /**
@@ -139,7 +138,6 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise<NextR
     userId,
     input,
     triggerType,
-    preflighted: params.preflighted,
   }
 
   try {
@@ -276,7 +274,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
       requestId
     )
 
-    const shouldPreflightEnvVars = isAsyncMode && isTriggerDevEnabled
     const preprocessResult = await preprocessExecution({
       workflowId,
       userId,
@@ -285,9 +282,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
       requestId,
       checkDeployment: !shouldUseDraftState,
       loggingSession,
-      preflightEnvVars: shouldPreflightEnvVars,
       useDraftState: shouldUseDraftState,
-      envUserId: isClientSession ? userId : undefined,
     })
 
     if (!preprocessResult.success) {
@@ -319,7 +314,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
         userId: actorUserId,
         input,
         triggerType: loggingTriggerType,
-        preflighted: shouldPreflightEnvVars,
       })
     }
 

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/code/code.tsx

@@ -38,6 +38,7 @@ import type { GenerationType } from '@/blocks/types'
 import { normalizeName } from '@/executor/constants'
 import { createEnvVarPattern, createReferencePattern } from '@/executor/utils/reference-validation'
 import { useTagSelection } from '@/hooks/kb/use-tag-selection'
+import { createShouldHighlightEnvVar, useAvailableEnvVarKeys } from '@/hooks/use-available-env-vars'
 
 const logger = createLogger('Code')
 
@@ -88,21 +89,27 @@ interface CodePlaceholder {
 /**
  * Creates a syntax highlighter function with custom reference and environment variable highlighting.
  * @param effectiveLanguage - The language to use for syntax highlighting
- * @param shouldHighlightReference - Function to determine if a reference should be highlighted
+ * @param shouldHighlightReference - Function to determine if a block reference should be highlighted
+ * @param shouldHighlightEnvVar - Function to determine if an env var should be highlighted
  * @returns A function that highlights code with syntax and custom highlights
  */
 const createHighlightFunction = (
   effectiveLanguage: 'javascript' | 'python' | 'json',
-  shouldHighlightReference: (part: string) => boolean
+  shouldHighlightReference: (part: string) => boolean,
+  shouldHighlightEnvVar: (varName: string) => boolean
 ) => {
   return (codeToHighlight: string): string => {
     const placeholders: CodePlaceholder[] = []
     let processedCode = codeToHighlight
 
     processedCode = processedCode.replace(createEnvVarPattern(), (match) => {
-      const placeholder = `__ENV_VAR_${placeholders.length}__`
-      placeholders.push({ placeholder, original: match, type: 'env' })
-      return placeholder
+      const varName = match.slice(2, -2).trim()
+      if (shouldHighlightEnvVar(varName)) {
+        const placeholder = `__ENV_VAR_${placeholders.length}__`
+        placeholders.push({ placeholder, original: match, type: 'env' })
+        return placeholder
+      }
+      return match
     })
 
     processedCode = processedCode.replace(createReferencePattern(), (match) => {
@@ -212,6 +219,7 @@ export const Code = memo(function Code({
   const accessiblePrefixes = useAccessibleReferencePrefixes(blockId)
   const emitTagSelection = useTagSelection(blockId, subBlockId)
   const [languageValue] = useSubBlockValue<string>(blockId, 'language')
+  const availableEnvVars = useAvailableEnvVarKeys(workspaceId)
 
   const effectiveLanguage = (languageValue as 'javascript' | 'python' | 'json') || language
 
@@ -603,9 +611,15 @@ export const Code = memo(function Code({
     [generateCodeStream, isPromptVisible, isAiStreaming]
   )
 
+  const shouldHighlightEnvVar = useMemo(
+    () => createShouldHighlightEnvVar(availableEnvVars),
+    [availableEnvVars]
+  )
+
   const highlightCode = useMemo(
-    () => createHighlightFunction(effectiveLanguage, shouldHighlightReference),
-    [effectiveLanguage, shouldHighlightReference]
+    () =>
+      createHighlightFunction(effectiveLanguage, shouldHighlightReference, shouldHighlightEnvVar),
+    [effectiveLanguage, shouldHighlightReference, shouldHighlightEnvVar]
   )
 
   const handleValueChange = useCallback(