improvement(serializer): canonical subblock, serialization cleanups, schedules/webhooks are deployment version friendly

apps/docs/content/docs/en/execution/meta.json

@@ -1,3 +1,3 @@
 {
-  "pages": ["index", "basics", "api", "form", "logging", "costs"]
+  "pages": ["index", "basics", "api", "logging", "costs"]
 }

apps/sim/app/api/copilot/execute-tool/route.ts

@@ -14,8 +14,7 @@ import {
 import { generateRequestId } from '@/lib/core/utils/request'
 import { getEffectiveDecryptedEnv } from '@/lib/environment/utils'
 import { refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils'
-import { REFERENCE } from '@/executor/constants'
-import { createEnvVarPattern } from '@/executor/utils/reference-validation'
+import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
 import { executeTool } from '@/tools'
 import { getTool, resolveToolId } from '@/tools/utils'
 
@@ -28,45 +27,6 @@ const ExecuteToolSchema = z.object({
   workflowId: z.string().optional(),
 })
 
-/**
- * Resolves all {{ENV_VAR}} references in a value recursively
- * Works with strings, arrays, and objects
- */
-function resolveEnvVarReferences(value: any, envVars: Record<string, string>): any {
-  if (typeof value === 'string') {
-    // Check for exact match: entire string is "{{VAR_NAME}}"
-    const exactMatchPattern = new RegExp(
-      `^\\${REFERENCE.ENV_VAR_START}([^}]+)\\${REFERENCE.ENV_VAR_END}$`
-    )
-    const exactMatch = exactMatchPattern.exec(value)
-    if (exactMatch) {
-      const envVarName = exactMatch[1].trim()
-      return envVars[envVarName] ?? value
-    }
-
-    // Check for embedded references: "prefix {{VAR}} suffix"
-    const envVarPattern = createEnvVarPattern()
-    return value.replace(envVarPattern, (match, varName) => {
-      const trimmedName = varName.trim()
-      return envVars[trimmedName] ?? match
-    })
-  }
-
-  if (Array.isArray(value)) {
-    return value.map((item) => resolveEnvVarReferences(item, envVars))
-  }
-
-  if (value !== null && typeof value === 'object') {
-    const resolved: Record<string, any> = {}
-    for (const [key, val] of Object.entries(value)) {
-      resolved[key] = resolveEnvVarReferences(val, envVars)
-    }
-    return resolved
-  }
-
-  return value
-}
-
 export async function POST(req: NextRequest) {
   const tracker = createRequestTracker()
 
@@ -145,7 +105,17 @@ export async function POST(req: NextRequest) {
 
     // Build execution params starting with LLM-provided arguments
     // Resolve all {{ENV_VAR}} references in the arguments
-    const executionParams: Record<string, any> = resolveEnvVarReferences(toolArgs, decryptedEnvVars)
+    const executionParams: Record<string, any> = resolveEnvVarReferences(
+      toolArgs,
+      decryptedEnvVars,
+      {
+        resolveExactMatch: true,
+        allowEmbedded: true,
+        trimKeys: true,
+        onMissing: 'keep',
+        deep: true,
+      }
+    ) as Record<string, any>
 
     logger.info(`[${tracker.requestId}] Resolved env var references in arguments`, {
       toolName,

apps/sim/app/api/function/execute/route.ts

@@ -9,6 +9,7 @@ import { escapeRegExp, normalizeName, REFERENCE } from '@/executor/constants'
 import {
   createEnvVarPattern,
   createWorkflowVariablePattern,
+  resolveEnvVarReferences,
 } from '@/executor/utils/reference-validation'
 export const dynamic = 'force-dynamic'
 export const runtime = 'nodejs'
@@ -479,9 +480,29 @@ function resolveEnvironmentVariables(
   const replacements: Array<{ match: string; index: number; varName: string; varValue: string }> =
     []
 
+  const resolverVars: Record<string, string> = {}
+  Object.entries(params).forEach(([key, value]) => {
+    if (value) {
+      resolverVars[key] = String(value)
+    }
+  })
+  Object.entries(envVars).forEach(([key, value]) => {
+    if (value) {
+      resolverVars[key] = value
+    }
+  })
+
   while ((match = regex.exec(code)) !== null) {
     const varName = match[1].trim()
-    const varValue = envVars[varName] || params[varName] || ''
+    const resolved = resolveEnvVarReferences(match[0], resolverVars, {
+      allowEmbedded: true,
+      resolveExactMatch: true,
+      trimKeys: true,
+      onMissing: 'empty',
+      deep: false,
+    })
+    const varValue =
+      typeof resolved === 'string' ? resolved : resolved == null ? '' : String(resolved)
     replacements.push({
       match: match[0],
       index: match.index,

apps/sim/app/api/mcp/servers/test-connection/route.ts

@@ -5,8 +5,7 @@ import { McpClient } from '@/lib/mcp/client'
 import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
 import type { McpServerConfig, McpTransport } from '@/lib/mcp/types'
 import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
-import { REFERENCE } from '@/executor/constants'
-import { createEnvVarPattern } from '@/executor/utils/reference-validation'
+import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
 
 const logger = createLogger('McpServerTestAPI')
 
@@ -24,22 +23,23 @@ function isUrlBasedTransport(transport: McpTransport): boolean {
  * Resolve environment variables in strings
  */
 function resolveEnvVars(value: string, envVars: Record<string, string>): string {
-  const envVarPattern = createEnvVarPattern()
-  const envMatches = value.match(envVarPattern)
-  if (!envMatches) return value
-
-  let resolvedValue = value
-  for (const match of envMatches) {
-    const envKey = match.slice(REFERENCE.ENV_VAR_START.length, -REFERENCE.ENV_VAR_END.length).trim()
-    const envValue = envVars[envKey]
-
-    if (envValue === undefined) {
+  const missingVars: string[] = []
+  const resolvedValue = resolveEnvVarReferences(value, envVars, {
+    allowEmbedded: true,
+    resolveExactMatch: true,
+    trimKeys: true,
+    onMissing: 'keep',
+    deep: false,
+    missingKeys: missingVars,
+  }) as string
+
+  if (missingVars.length > 0) {
+    const uniqueMissing = Array.from(new Set(missingVars))
+    uniqueMissing.forEach((envKey) => {
       logger.warn(`Environment variable "${envKey}" not found in MCP server test`)
-      continue
-    }
-
-    resolvedValue = resolvedValue.replace(match, envValue)
+    })
   }
+
   return resolvedValue
 }
 

apps/sim/app/api/schedules/execute/route.test.ts

@@ -6,6 +6,11 @@
 import type { NextRequest } from 'next/server'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
+// Mock the preflight module before any imports to avoid cascade of db/schema imports
+vi.mock('@/lib/workflows/executor/preflight', () => ({
+  preflightWorkflowEnvVars: vi.fn().mockResolvedValue(undefined),
+}))
+
 function createMockRequest(): NextRequest {
   const mockHeaders = new Map([
     ['authorization', 'Bearer test-cron-secret'],
@@ -93,6 +98,11 @@ describe('Scheduled Workflow Execution API Route', () => {
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
         },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
+        },
       }
     })
 
@@ -170,6 +180,11 @@ describe('Scheduled Workflow Execution API Route', () => {
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
         },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
+        },
       }
     })
 
@@ -229,6 +244,11 @@ describe('Scheduled Workflow Execution API Route', () => {
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
         },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
+        },
       }
     })
 
@@ -311,6 +331,11 @@ describe('Scheduled Workflow Execution API Route', () => {
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
         },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
+        },
       }
     })
 

apps/sim/app/api/schedules/execute/route.ts

@@ -1,11 +1,12 @@
-import { db, workflowSchedule } from '@sim/db'
+import { db, workflow, workflowSchedule } from '@sim/db'
 import { createLogger } from '@sim/logger'
 import { tasks } from '@trigger.dev/sdk'
 import { and, eq, isNull, lt, lte, not, or } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { verifyCronAuth } from '@/lib/auth/internal'
 import { isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
 import { generateRequestId } from '@/lib/core/utils/request'
+import { preflightWorkflowEnvVars } from '@/lib/workflows/executor/preflight'
 import { executeScheduleJob } from '@/background/schedule-execution'
 
 export const dynamic = 'force-dynamic'
@@ -68,6 +69,39 @@ export async function GET(request: NextRequest) {
             failedCount: schedule.failedCount || 0,
             now: queueTime.toISOString(),
             scheduledFor: schedule.nextRunAt?.toISOString(),
+            preflighted: true,
+          }
+
+          const [workflowRecord] = await db
+            .select({ userId: workflow.userId, workspaceId: workflow.workspaceId })
+            .from(workflow)
+            .where(eq(workflow.id, schedule.workflowId))
+            .limit(1)
+
+          if (!workflowRecord?.userId || !workflowRecord.workspaceId) {
+            logger.warn(
+              `[${requestId}] Missing workflow metadata for preflight. Skipping Trigger.dev enqueue.`,
+              { workflowId: schedule.workflowId }
+            )
+            await executeScheduleJob({ ...payload, preflighted: false })
+            return null
+          }
+
+          try {
+            await preflightWorkflowEnvVars({
+              workflowId: schedule.workflowId,
+              workspaceId: workflowRecord.workspaceId,
+              envUserId: workflowRecord.userId,
+              requestId,
+              useDraftState: false,
+            })
+          } catch (error) {
+            logger.warn(
+              `[${requestId}] Env preflight failed. Skipping Trigger.dev enqueue for workflow ${schedule.workflowId}`,
+              { error: error instanceof Error ? error.message : String(error) }
+            )
+            await executeScheduleJob({ ...payload, preflighted: false })
+            return null
           }
 
           const handle = await tasks.trigger('schedule-execution', payload)

apps/sim/app/api/workflows/[id]/execute/route.ts

@@ -110,6 +110,7 @@ type AsyncExecutionParams = {
   userId: string
   input: any
   triggerType: CoreTriggerType
+  preflighted?: boolean
 }
 
 /**
@@ -132,6 +133,7 @@ async function handleAsyncExecution(params: AsyncExecutionParams): Promise<NextR
     userId,
     input,
     triggerType,
+    preflighted: params.preflighted,
   }
 
   try {
@@ -264,6 +266,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
       requestId
     )
 
+    const shouldPreflightEnvVars = isAsyncMode && isTriggerDevEnabled
     const preprocessResult = await preprocessExecution({
       workflowId,
       userId,
@@ -272,6 +275,9 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
       requestId,
       checkDeployment: !shouldUseDraftState,
       loggingSession,
+      preflightEnvVars: shouldPreflightEnvVars,
+      useDraftState: shouldUseDraftState,
+      envUserId: isClientSession ? userId : undefined,
     })
 
     if (!preprocessResult.success) {
@@ -303,6 +309,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
         userId: actorUserId,
         input,
         triggerType: loggingTriggerType,
+        preflighted: shouldPreflightEnvVars,
       })
     }
 

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/combobox/combobox.tsx

@@ -2,16 +2,19 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { useReactFlow } from 'reactflow'
 import { Combobox, type ComboboxOption } from '@/components/emcn/components'
 import { cn } from '@/lib/core/utils/cn'
+import { buildCanonicalIndex, resolveDependencyValue } from '@/lib/workflows/subblocks/visibility'
 import { formatDisplayText } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/formatted-text'
 import { SubBlockInputController } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/sub-block-input-controller'
 import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value'
 import { useAccessibleReferencePrefixes } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-accessible-reference-prefixes'
+import { getBlock } from '@/blocks/registry'
 import type { SubBlockConfig } from '@/blocks/types'
 import { getDependsOnFields } from '@/blocks/utils'
 import { usePermissionConfig } from '@/hooks/use-permission-config'
 import { getProviderFromModel } from '@/providers/utils'
 import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
 import { useSubBlockStore } from '@/stores/workflows/subblock/store'
+import { useWorkflowStore } from '@/stores/workflows/workflow/store'
 
 /**
  * Constants for ComboBox component behavior
@@ -91,15 +94,24 @@ export function ComboBox({
   // Dependency tracking for fetchOptions
   const dependsOnFields = useMemo(() => getDependsOnFields(dependsOn), [dependsOn])
   const activeWorkflowId = useWorkflowRegistry((s) => s.activeWorkflowId)
+  const blockState = useWorkflowStore((state) => state.blocks[blockId])
+  const blockConfig = blockState?.type ? getBlock(blockState.type) : null
+  const canonicalIndex = useMemo(
+    () => buildCanonicalIndex(blockConfig?.subBlocks || []),
+    [blockConfig?.subBlocks]
+  )
+  const canonicalModeOverrides = blockState?.data?.canonicalModes
   const dependencyValues = useSubBlockStore(
     useCallback(
       (state) => {
         if (dependsOnFields.length === 0 || !activeWorkflowId) return []
         const workflowValues = state.workflowValues[activeWorkflowId] || {}
         const blockValues = workflowValues[blockId] || {}
-        return dependsOnFields.map((depKey) => blockValues[depKey] ?? null)
+        return dependsOnFields.map((depKey) =>
+          resolveDependencyValue(depKey, blockValues, canonicalIndex, canonicalModeOverrides)
+        )
       },
-      [dependsOnFields, activeWorkflowId, blockId]
+      [dependsOnFields, activeWorkflowId, blockId, canonicalIndex, canonicalModeOverrides]
     )
   )