Renamed llm_usage_v1 to llm_metrics_v1, added some additional fields to track as well

Author: ericallam

.server-changes/llm-cost-tracking.md

@@ -3,4 +3,4 @@ area: webapp
 type: feature
 ---
 
-Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_usage_v1` ClickHouse table.
+Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_metrics_v1` ClickHouse table that captures usage, cost, performance (TTFC, tokens/sec), and behavioral (finish reason, operation type) metrics.

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx

@@ -126,25 +126,25 @@ LIMIT 100`,
   SUM(total_cost) AS total_cost,
   SUM(input_tokens) AS input_tokens,
   SUM(output_tokens) AS output_tokens
-FROM llm_usage
+FROM llm_metrics
 WHERE start_time > now() - INTERVAL 7 DAY
 GROUP BY response_model
 ORDER BY total_cost DESC`,
     scope: "environment",
-    table: "llm_usage",
+    table: "llm_metrics",
   },
   {
     title: "LLM cost over time",
     description: "Total LLM cost bucketed over time. The bucket size adjusts automatically.",
     query: `SELECT
   timeBucket(),
   SUM(total_cost) AS total_cost
-FROM llm_usage
+FROM llm_metrics
 GROUP BY timeBucket
 ORDER BY timeBucket
 LIMIT 1000`,
     scope: "environment",
-    table: "llm_usage",
+    table: "llm_metrics",
   },
   {
     title: "Most expensive runs by LLM cost (top 50)",
@@ -155,12 +155,12 @@ LIMIT 1000`,
   SUM(total_cost) AS llm_cost,
   SUM(input_tokens) AS input_tokens,
   SUM(output_tokens) AS output_tokens
-FROM llm_usage
+FROM llm_metrics
 GROUP BY run_id, task_identifier
 ORDER BY llm_cost DESC
 LIMIT 50`,
     scope: "environment",
-    table: "llm_usage",
+    table: "llm_metrics",
   },
   {
     title: "LLM calls by provider",
@@ -169,11 +169,11 @@ LIMIT 50`,
   gen_ai_system,
   count() AS call_count,
   SUM(total_cost) AS total_cost
-FROM llm_usage
+FROM llm_metrics
 GROUP BY gen_ai_system
 ORDER BY total_cost DESC`,
     scope: "environment",
-    table: "llm_usage",
+    table: "llm_metrics",
   },
   {
     title: "LLM cost by user",
@@ -184,13 +184,13 @@ ORDER BY total_cost DESC`,
   SUM(total_cost) AS total_cost,
   SUM(total_tokens) AS total_tokens,
   count() AS call_count
-FROM llm_usage
+FROM llm_metrics
 WHERE metadata.userId != ''
 GROUP BY metadata.userId
 ORDER BY total_cost DESC
 LIMIT 50`,
     scope: "environment",
-    table: "llm_usage",
+    table: "llm_metrics",
   },
   {
     title: "LLM cost by metadata key",
@@ -202,11 +202,11 @@ LIMIT 50`,
   total_cost,
   total_tokens,
   run_id
-FROM llm_usage
+FROM llm_metrics
 ORDER BY start_time DESC
 LIMIT 20`,
     scope: "environment",
-    table: "llm_usage",
+    table: "llm_metrics",
   },
 ];
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx

@@ -120,6 +120,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
     return typedjson({ ...result, regions: regionsResult.regions });
   } catch (error) {
+    logger.error("Failed to load test page", {
+      taskParam,
+      error: error instanceof Error ? error.message : error,
+      stack: error instanceof Error ? error.stack : undefined,
+    });
+
     return redirectWithErrorMessage(
       v3TestPath({ slug: organizationSlug }, { slug: projectParam }, environment),
       request,

apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts

@@ -1,6 +1,6 @@
 import type {
   ClickHouse,
-  LlmUsageV1Input,
+  LlmMetricsV1Input,
   TaskEventDetailedSummaryV1Result,
   TaskEventDetailsV1Result,
   TaskEventSummaryV1Result,
@@ -96,7 +96,7 @@ export class ClickhouseEventRepository implements IEventRepository {
   private _clickhouse: ClickHouse;
   private _config: ClickhouseEventRepositoryConfig;
   private readonly _flushScheduler: DynamicFlushScheduler<TaskEventV1Input | TaskEventV2Input>;
-  private readonly _llmUsageFlushScheduler: DynamicFlushScheduler<LlmUsageV1Input>;
+  private readonly _llmMetricsFlushScheduler: DynamicFlushScheduler<LlmMetricsV1Input>;
   private _tracer: Tracer;
   private _version: "v1" | "v2";
 
@@ -122,10 +122,10 @@ export class ClickhouseEventRepository implements IEventRepository {
       },
     });
 
-    this._llmUsageFlushScheduler = new DynamicFlushScheduler({
+    this._llmMetricsFlushScheduler = new DynamicFlushScheduler({
       batchSize: 5000,
       flushInterval: 2000,
-      callback: this.#flushLlmUsageBatch.bind(this),
+      callback: this.#flushLlmMetricsBatch.bind(this),
       minConcurrency: 1,
       maxConcurrency: 2,
       maxBatchSize: 10000,
@@ -230,9 +230,9 @@ export class ClickhouseEventRepository implements IEventRepository {
     });
   }
 
-  async #flushLlmUsageBatch(flushId: string, rows: LlmUsageV1Input[]) {
+  async #flushLlmMetricsBatch(flushId: string, rows: LlmMetricsV1Input[]) {
 
-    const [insertError] = await this._clickhouse.llmUsage.insert(rows, {
+    const [insertError] = await this._clickhouse.llmMetrics.insert(rows, {
       params: {
         clickhouse_settings: this.#getClickhouseInsertSettings(),
       },
@@ -242,13 +242,13 @@ export class ClickhouseEventRepository implements IEventRepository {
       throw insertError;
     }
 
-    logger.info("ClickhouseEventRepository.flushLlmUsageBatch Inserted LLM usage batch", {
+    logger.info("ClickhouseEventRepository.flushLlmMetricsBatch Inserted LLM metrics batch", {
       rows: rows.length,
     });
   }
 
-  #createLlmUsageInput(event: CreateEventInput): LlmUsageV1Input {
-    const llmUsage = event._llmUsage!;
+  #createLlmMetricsInput(event: CreateEventInput): LlmMetricsV1Input {
+    const llmMetrics = event._llmMetrics!;
 
     return {
       organization_id: event.organizationId,
@@ -258,22 +258,27 @@ export class ClickhouseEventRepository implements IEventRepository {
       task_identifier: event.taskSlug,
       trace_id: event.traceId,
       span_id: event.spanId,
-      gen_ai_system: llmUsage.genAiSystem,
-      request_model: llmUsage.requestModel,
-      response_model: llmUsage.responseModel,
-      matched_model_id: llmUsage.matchedModelId,
-      operation_name: llmUsage.operationName,
-      pricing_tier_id: llmUsage.pricingTierId,
-      pricing_tier_name: llmUsage.pricingTierName,
-      input_tokens: llmUsage.inputTokens,
-      output_tokens: llmUsage.outputTokens,
-      total_tokens: llmUsage.totalTokens,
-      usage_details: llmUsage.usageDetails,
-      input_cost: llmUsage.inputCost,
-      output_cost: llmUsage.outputCost,
-      total_cost: llmUsage.totalCost,
-      cost_details: llmUsage.costDetails,
-      metadata: llmUsage.metadata,
+      gen_ai_system: llmMetrics.genAiSystem,
+      request_model: llmMetrics.requestModel,
+      response_model: llmMetrics.responseModel,
+      matched_model_id: llmMetrics.matchedModelId,
+      operation_id: llmMetrics.operationId,
+      finish_reason: llmMetrics.finishReason,
+      cost_source: llmMetrics.costSource,
+      pricing_tier_id: llmMetrics.pricingTierId,
+      pricing_tier_name: llmMetrics.pricingTierName,
+      input_tokens: llmMetrics.inputTokens,
+      output_tokens: llmMetrics.outputTokens,
+      total_tokens: llmMetrics.totalTokens,
+      usage_details: llmMetrics.usageDetails,
+      input_cost: llmMetrics.inputCost,
+      output_cost: llmMetrics.outputCost,
+      total_cost: llmMetrics.totalCost,
+      cost_details: llmMetrics.costDetails,
+      provider_cost: llmMetrics.providerCost,
+      ms_to_first_chunk: llmMetrics.msToFirstChunk,
+      tokens_per_second: llmMetrics.tokensPerSecond,
+      metadata: llmMetrics.metadata,
       start_time: this.#clampAndFormatStartTime(event.startTime.toString()),
       duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0),
     };
@@ -300,13 +305,13 @@ export class ClickhouseEventRepository implements IEventRepository {
   async insertMany(events: CreateEventInput[]): Promise<void> {
     this.addToBatch(events.flatMap((event) => this.createEventToTaskEventV1Input(event)));
 
-    // Dual-write LLM usage records for spans with cost enrichment
-    const llmUsageRows = events
-      .filter((e) => e._llmUsage != null)
-      .map((e) => this.#createLlmUsageInput(e));
+    // Dual-write LLM metrics records for spans with cost enrichment
+    const llmMetricsRows = events
+      .filter((e) => e._llmMetrics != null)
+      .map((e) => this.#createLlmMetricsInput(e));
 
-    if (llmUsageRows.length > 0) {
-      this._llmUsageFlushScheduler.addToBatch(llmUsageRows);
+    if (llmMetricsRows.length > 0) {
+      this._llmMetricsFlushScheduler.addToBatch(llmMetricsRows);
     }
   }
 

apps/webapp/app/v3/eventRepository/eventRepository.types.ts

@@ -21,12 +21,14 @@ export type { ExceptionEventProperties };
 // Event Creation Types
 // ============================================================================
 
-export type LlmUsageData = {
+export type LlmMetricsData = {
   genAiSystem: string;
   requestModel: string;
   responseModel: string;
   matchedModelId: string;
-  operationName: string;
+  operationId: string;
+  finishReason: string;
+  costSource: string;
   pricingTierId: string;
   pricingTierName: string;
   inputTokens: number;
@@ -37,6 +39,9 @@ export type LlmUsageData = {
   outputCost: number;
   totalCost: number;
   costDetails: Record<string, number>;
+  providerCost: number;
+  msToFirstChunk: number;
+  tokensPerSecond: number;
   metadata: Record<string, string>;
 };
 
@@ -78,7 +83,7 @@ export type CreateEventInput = Omit<
   machineId?: string;
   runTags?: string[];
   /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */
-  _llmUsage?: LlmUsageData;
+  _llmMetrics?: LlmMetricsData;
 };
 
 export type CreatableEventKind = TaskEventKind;

apps/webapp/app/v3/querySchemas.ts

@@ -600,12 +600,12 @@ export const metricsSchema: TableSchema = {
  * All available schemas for the query editor
  */
 /**
- * Schema definition for the llm_usage table (trigger_dev.llm_usage_v1)
+ * Schema definition for the llm_metrics table (trigger_dev.llm_metrics_v1)
  */
-export const llmUsageSchema: TableSchema = {
-  name: "llm_usage",
-  clickhouseName: "trigger_dev.llm_usage_v1",
-  description: "LLM token usage and cost data from GenAI spans",
+export const llmMetricsSchema: TableSchema = {
+  name: "llm_metrics",
+  clickhouseName: "trigger_dev.llm_metrics_v1",
+  description: "LLM metrics: token usage, cost, performance, and behavior from GenAI spans",
   timeConstraint: "start_time",
   tenantColumns: {
     organizationId: "organization_id",
@@ -669,11 +669,26 @@ export const llmUsageSchema: TableSchema = {
         coreColumn: true,
       }),
     },
-    operation_name: {
-      name: "operation_name",
+    operation_id: {
+      name: "operation_id",
       ...column("LowCardinality(String)", {
-        description: "Operation type (e.g. chat, completion)",
-        example: "chat",
+        description: "Operation type (e.g. ai.streamText.doStream, ai.generateText.doGenerate)",
+        example: "ai.streamText.doStream",
+      }),
+    },
+    finish_reason: {
+      name: "finish_reason",
+      ...column("LowCardinality(String)", {
+        description: "Why the LLM stopped generating (e.g. stop, tool-calls, length)",
+        example: "stop",
+        coreColumn: true,
+      }),
+    },
+    cost_source: {
+      name: "cost_source",
+      ...column("LowCardinality(String)", {
+        description: "Where cost data came from (registry, gateway, openrouter)",
+        example: "registry",
       }),
     },
     input_tokens: {
@@ -700,14 +715,14 @@ export const llmUsageSchema: TableSchema = {
     input_cost: {
       name: "input_cost",
       ...column("Decimal64(12)", {
-        description: "Input cost in USD",
+        description: "Input cost in USD (from pricing registry)",
         customRenderType: "costInDollars",
       }),
     },
     output_cost: {
       name: "output_cost",
       ...column("Decimal64(12)", {
-        description: "Output cost in USD",
+        description: "Output cost in USD (from pricing registry)",
         customRenderType: "costInDollars",
       }),
     },
@@ -719,6 +734,28 @@ export const llmUsageSchema: TableSchema = {
         coreColumn: true,
       }),
     },
+    provider_cost: {
+      name: "provider_cost",
+      ...column("Decimal64(12)", {
+        description: "Provider-reported cost in USD (from gateway or openrouter)",
+        customRenderType: "costInDollars",
+      }),
+    },
+    ms_to_first_chunk: {
+      name: "ms_to_first_chunk",
+      ...column("Float64", {
+        description: "Time to first chunk in milliseconds (TTFC)",
+        example: "245.3",
+        coreColumn: true,
+      }),
+    },
+    tokens_per_second: {
+      name: "tokens_per_second",
+      ...column("Float64", {
+        description: "Average output tokens per second",
+        example: "72.5",
+      }),
+    },
     pricing_tier_name: {
       name: "pricing_tier_name",
       ...column("LowCardinality(String)", {
@@ -751,7 +788,7 @@ export const llmUsageSchema: TableSchema = {
   },
 };
 
-export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmUsageSchema];
+export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmMetricsSchema];
 
 /**
  * Default query for the query editor