added cost tracking for router & evaluator blocks, consolidated model information into a single file, hosted keys for evaluator & router, parallelized unit tests (#516)

Author: waleedlatif1

apps/sim/app/w/[id]/components/workflow-block/components/sub-block/components/tool-input/tool-input.tsx

@@ -14,8 +14,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip
 import type { OAuthProvider } from '@/lib/oauth/oauth'
 import { cn } from '@/lib/utils'
 import { getAllBlocks } from '@/blocks'
-import { supportsToolUsageControl } from '@/providers/model-capabilities'
-import { getProviderFromModel } from '@/providers/utils'
+import { getProviderFromModel, supportsToolUsageControl } from '@/providers/utils'
 import { useCustomToolsStore } from '@/stores/custom-tools/store'
 import { useGeneralStore } from '@/stores/settings/general/store'
 import { useSubBlockStore } from '@/stores/workflows/subblock/store'

apps/sim/app/w/[id]/hooks/use-workflow-execution.ts

@@ -92,9 +92,13 @@ export function useWorkflowExecution() {
           const streamingBlockId = (result.metadata as any)?.streamingBlockId || null
 
           for (const log of enrichedResult.logs) {
-            // Only update the specific agent block that was streamed
+            // Only update the specific LLM block (agent/router) that was streamed
             const isStreamingBlock = streamingBlockId && log.blockId === streamingBlockId
-            if (isStreamingBlock && log.blockType === 'agent' && log.output?.response) {
+            if (
+              isStreamingBlock &&
+              (log.blockType === 'agent' || log.blockType === 'router') &&
+              log.output?.response
+            ) {
               log.output.response.content = streamContent
             }
           }

apps/sim/app/w/logs/components/sidebar/sidebar.tsx

@@ -556,7 +556,8 @@ export function Sidebar({
                     {isWorkflowWithCost && (
                       <div className='border-t bg-muted p-3 text-muted-foreground text-xs'>
                         <p>
-                          This is the total cost for all agent blocks in this workflow execution.
+                          This is the total cost for all LLM-based blocks in this workflow
+                          execution.
                         </p>
                       </div>
                     )}

apps/sim/blocks/blocks/agent.ts

@@ -1,8 +1,14 @@
 import { AgentIcon } from '@/components/icons'
 import { isHosted } from '@/lib/environment'
 import { createLogger } from '@/lib/logs/console-logger'
-import { MODELS_TEMP_RANGE_0_1, MODELS_TEMP_RANGE_0_2 } from '@/providers/model-capabilities'
-import { getAllModelProviders, getBaseModelProviders } from '@/providers/utils'
+import {
+  getAllModelProviders,
+  getBaseModelProviders,
+  getHostedModels,
+  MODELS_TEMP_RANGE_0_1,
+  MODELS_TEMP_RANGE_0_2,
+  providers,
+} from '@/providers/utils'
 import { useOllamaStore } from '@/stores/ollama/store'
 import type { ToolResponse } from '@/tools/types'
 import type { BlockConfig } from '../types'
@@ -121,29 +127,11 @@ export const AgentBlock: BlockConfig<AgentResponse> = {
       placeholder: 'Enter your API key',
       password: true,
       connectionDroppable: false,
-      // Hide API key for all OpenAI and Claude models when running on hosted version
+      // Hide API key for all hosted models when running on hosted version
       condition: isHosted
         ? {
             field: 'model',
-            // Include all OpenAI models and Claude models for which we don't show the API key field
-            value: [
-              // OpenAI models
-              'gpt-4o',
-              'o1',
-              'o1-mini',
-              'o1-preview',
-              'o3',
-              'o3-preview',
-              'o4-mini',
-              'gpt-4.1',
-              'gpt-4.1-nano',
-              'gpt-4.1-mini',
-              // Claude models
-              'claude-sonnet-4-0',
-              'claude-opus-4-0',
-              'claude-3-7-sonnet-latest',
-              'claude-3-5-sonnet-latest',
-            ],
+            value: getHostedModels(),
             not: true, // Show for all models EXCEPT those listed
           }
         : undefined, // Show for all models in non-hosted environments
@@ -158,7 +146,7 @@ export const AgentBlock: BlockConfig<AgentResponse> = {
       connectionDroppable: false,
       condition: {
         field: 'model',
-        value: ['azure/gpt-4o', 'azure/o3', 'azure/o4-mini', 'azure/gpt-4.1', 'azure/model-router'],
+        value: providers['azure-openai'].models,
       },
     },
     {
@@ -170,7 +158,7 @@ export const AgentBlock: BlockConfig<AgentResponse> = {
       connectionDroppable: false,
       condition: {
         field: 'model',
-        value: ['azure/gpt-4o', 'azure/o3', 'azure/o4-mini', 'azure/gpt-4.1', 'azure/model-router'],
+        value: providers['azure-openai'].models,
       },
     },
     {

apps/sim/blocks/blocks/evaluator.ts

@@ -1,7 +1,8 @@
 import { ChartBarIcon } from '@/components/icons'
+import { isHosted } from '@/lib/environment'
 import { createLogger } from '@/lib/logs/console-logger'
 import type { ProviderId } from '@/providers/types'
-import { getAllModelProviders, getBaseModelProviders } from '@/providers/utils'
+import { getAllModelProviders, getBaseModelProviders, getHostedModels } from '@/providers/utils'
 import { useOllamaStore } from '@/stores/ollama/store'
 import type { ToolResponse } from '@/tools/types'
 import type { BlockConfig, ParamType } from '../types'
@@ -26,6 +27,11 @@ interface EvaluatorResponse extends ToolResponse {
       completion?: number
       total?: number
     }
+    cost?: {
+      input: number
+      output: number
+      total: number
+    }
     [metricName: string]: any // Allow dynamic metric fields
   }
 }
@@ -181,6 +187,13 @@ export const EvaluatorBlock: BlockConfig<EvaluatorResponse> = {
       placeholder: 'Enter your API key',
       password: true,
       connectionDroppable: false,
+      condition: isHosted
+        ? {
+            field: 'model',
+            value: getHostedModels(),
+            not: true,
+          }
+        : undefined,
     },
     {
       id: 'systemPrompt',
@@ -299,6 +312,7 @@ export const EvaluatorBlock: BlockConfig<EvaluatorResponse> = {
         content: 'string',
         model: 'string',
         tokens: 'any',
+        cost: 'any',
       },
       dependsOn: {
         subBlockId: 'metrics',
@@ -307,6 +321,7 @@ export const EvaluatorBlock: BlockConfig<EvaluatorResponse> = {
             content: 'string',
             model: 'string',
             tokens: 'any',
+            cost: 'any',
           },
           whenFilled: 'json',
         },

apps/sim/blocks/blocks/router.ts

@@ -1,6 +1,7 @@
 import { ConnectIcon } from '@/components/icons'
+import { isHosted } from '@/lib/environment'
 import type { ProviderId } from '@/providers/types'
-import { getAllModelProviders, getBaseModelProviders } from '@/providers/utils'
+import { getAllModelProviders, getBaseModelProviders, getHostedModels } from '@/providers/utils'
 import { useOllamaStore } from '@/stores/ollama/store'
 import type { ToolResponse } from '@/tools/types'
 import type { BlockConfig } from '../types'
@@ -14,6 +15,11 @@ interface RouterResponse extends ToolResponse {
       completion?: number
       total?: number
     }
+    cost?: {
+      input: number
+      output: number
+      total: number
+    }
     selectedPath: {
       blockId: string
       blockType: string
@@ -125,6 +131,14 @@ export const RouterBlock: BlockConfig<RouterResponse> = {
       placeholder: 'Enter your API key',
       password: true,
       connectionDroppable: false,
+      // Hide API key for all hosted models when running on hosted version
+      condition: isHosted
+        ? {
+            field: 'model',
+            value: getHostedModels(),
+            not: true, // Show for all models EXCEPT those listed
+          }
+        : undefined, // Show for all models in non-hosted environments
     },
     {
       id: 'systemPrompt',
@@ -171,6 +185,7 @@ export const RouterBlock: BlockConfig<RouterResponse> = {
         content: 'string',
         model: 'string',
         tokens: 'any',
+        cost: 'any',
         selectedPath: 'json',
       },
     },

apps/sim/executor/handlers/evaluator/evaluator-handler.test.ts

@@ -91,6 +91,11 @@ describe('EvaluatorBlockHandler', () => {
         content: 'This is the content to evaluate.',
         model: 'mock-model',
         tokens: { prompt: 50, completion: 10, total: 60 },
+        cost: {
+          input: 0,
+          output: 0,
+          total: 0,
+        },
         score1: 5,
         score2: 8,
       },

apps/sim/executor/handlers/evaluator/evaluator-handler.ts

@@ -1,7 +1,7 @@
 import { env } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console-logger'
 import type { BlockOutput } from '@/blocks/types'
-import { getProviderFromModel } from '@/providers/utils'
+import { calculateCost, getProviderFromModel } from '@/providers/utils'
 import type { SerializedBlock } from '@/serializer/types'
 import type { BlockHandler, ExecutionContext } from '../../types'
 
@@ -241,6 +241,14 @@ export class EvaluatorBlockHandler implements BlockHandler {
         logger.error('Error extracting metric scores:', e)
       }
 
+      // Calculate cost based on token usage, similar to how providers do it
+      const costCalculation = calculateCost(
+        result.model,
+        result.tokens?.prompt || 0,
+        result.tokens?.completion || 0,
+        false // Evaluator blocks don't typically use cached input
+      )
+
       // Create result with metrics as direct fields for easy access
       const outputResult = {
         response: {
@@ -251,6 +259,11 @@ export class EvaluatorBlockHandler implements BlockHandler {
             completion: result.tokens?.completion || 0,
             total: result.tokens?.total || 0,
           },
+          cost: {
+            input: costCalculation.input,
+            output: costCalculation.output,
+            total: costCalculation.total,
+          },
           ...metricScores,
         },
       }

apps/sim/executor/handlers/router/router-handler.test.ts

@@ -152,6 +152,11 @@ describe('RouterBlockHandler', () => {
         content: 'Choose the best option.',
         model: 'mock-model',
         tokens: { prompt: 100, completion: 5, total: 105 },
+        cost: {
+          input: 0,
+          output: 0,
+          total: 0,
+        },
         selectedPath: {
           blockId: 'target-block-1',
           blockType: 'target',

apps/sim/executor/handlers/router/router-handler.ts

@@ -2,7 +2,7 @@ import { env } from '@/lib/env'
 import { createLogger } from '@/lib/logs/console-logger'
 import { generateRouterPrompt } from '@/blocks/blocks/router'
 import type { BlockOutput } from '@/blocks/types'
-import { getProviderFromModel } from '@/providers/utils'
+import { calculateCost, getProviderFromModel } from '@/providers/utils'
 import type { SerializedBlock } from '@/serializer/types'
 import type { PathTracker } from '../../path'
 import type { BlockHandler, ExecutionContext } from '../../types'
@@ -92,6 +92,14 @@ export class RouterBlockHandler implements BlockHandler {
 
       const tokens = result.tokens || { prompt: 0, completion: 0, total: 0 }
 
+      // Calculate cost based on token usage, similar to how providers do it
+      const cost = calculateCost(
+        result.model,
+        tokens.prompt || 0,
+        tokens.completion || 0,
+        false // Router blocks don't typically use cached input
+      )
+
       return {
         response: {
           content: inputs.prompt,
@@ -101,6 +109,11 @@ export class RouterBlockHandler implements BlockHandler {
             completion: tokens.completion || 0,
             total: tokens.total || 0,
           },
+          cost: {
+            input: cost.input,
+            output: cost.output,
+            total: cost.total,
+          },
           selectedPath: {
             blockId: chosenBlock.id,
             blockType: chosenBlock.type || 'unknown',