feat: add metadata map to llm_usage_v1 for cost attribution by user/tenant

Author: ericallam

.changeset/llm-metadata-run-tags.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Propagate run tags to span attributes so they can be extracted server-side for LLM cost attribution metadata.

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx

@@ -175,6 +175,39 @@ ORDER BY total_cost DESC`,
     scope: "environment",
     table: "llm_usage",
   },
+  {
+    title: "LLM cost by user",
+    description:
+      "Total LLM cost per user from run tags or AI SDK telemetry metadata. Uses metadata.userId which comes from experimental_telemetry metadata or run tags like user:123.",
+    query: `SELECT
+  metadata.userId AS user_id,
+  SUM(total_cost) AS total_cost,
+  SUM(total_tokens) AS total_tokens,
+  count() AS call_count
+FROM llm_usage
+WHERE metadata.userId != ''
+GROUP BY metadata.userId
+ORDER BY total_cost DESC
+LIMIT 50`,
+    scope: "environment",
+    table: "llm_usage",
+  },
+  {
+    title: "LLM cost by metadata key",
+    description:
+      "Browse all metadata keys and their LLM cost. Metadata comes from run tags (key:value) and AI SDK telemetry metadata.",
+    query: `SELECT
+  metadata,
+  response_model,
+  total_cost,
+  total_tokens,
+  run_id
+FROM llm_usage
+ORDER BY start_time DESC
+LIMIT 20`,
+    scope: "environment",
+    table: "llm_usage",
+  },
 ];
 
 const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name }));

apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts

@@ -279,6 +279,7 @@ export class ClickhouseEventRepository implements IEventRepository {
       output_cost: llmUsage.outputCost,
       total_cost: llmUsage.totalCost,
       cost_details: llmUsage.costDetails,
+      metadata: llmUsage.metadata,
       start_time: this.#clampAndFormatStartTime(event.startTime.toString()),
       duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0),
     };
@@ -311,7 +312,7 @@ export class ClickhouseEventRepository implements IEventRepository {
       .map((e) => this.#createLlmUsageInput(e));
 
     if (llmUsageRows.length > 0) {
-      crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id }); // @crumbs
+      crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id, metadataKeys: llmUsageRows.map((r) => Object.keys(r.metadata)) }); // @crumbs
       this._llmUsageFlushScheduler.addToBatch(llmUsageRows);
     }
   }

apps/webapp/app/v3/eventRepository/eventRepository.types.ts

@@ -37,6 +37,7 @@ export type LlmUsageData = {
   outputCost: number;
   totalCost: number;
   costDetails: Record<string, number>;
+  metadata: Record<string, string>;
 };
 
 export type CreateEventInput = Omit<
@@ -75,6 +76,7 @@ export type CreateEventInput = Omit<
   metadata: Attributes | undefined;
   style: Attributes | undefined;
   machineId?: string;
+  runTags?: string[];
   /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */
   _llmUsage?: LlmUsageData;
 };

apps/webapp/app/v3/otlpExporter.server.ts

@@ -38,6 +38,8 @@ import type {
 import { startSpan } from "./tracing.server";
 import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server";
 import "./llmPricingRegistry.server"; // Initialize LLM pricing registry on startup
+import { trail } from "agentcrumbs"; // @crumbs
+const crumbOtlp = trail("webapp:otlp-exporter"); // @crumbs
 import { env } from "~/env.server";
 import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings";
 import { singleton } from "~/utils/singleton";
@@ -392,6 +394,9 @@ function convertSpansToCreateableEvents(
           SemanticInternalAttributes.METADATA
         );
 
+        const runTags = extractArrayAttribute(span.attributes ?? [], SemanticInternalAttributes.RUN_TAGS);
+        if (runTags && runTags.length > 0) { crumbOtlp("extracted runTags from span", { runTags, spanId: binaryToHex(span.spanId) }); } // @crumbs
+
         const properties =
           truncateAttributes(
             convertKeyValueItemsToMap(span.attributes ?? [], [], undefined, [
@@ -440,6 +445,7 @@ function convertSpansToCreateableEvents(
           runId: spanProperties.runId ?? resourceProperties.runId ?? "unknown",
           taskSlug: spanProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown",
           machineId: spanProperties.machineId ?? resourceProperties.machineId,
+          runTags,
           attemptNumber:
             extractNumberAttribute(
               span.attributes ?? [],
@@ -1001,6 +1007,21 @@ function extractBooleanAttribute(
   return isBoolValue(attribute?.value) ? attribute.value.boolValue : fallback;
 }
 
+function extractArrayAttribute(
+  attributes: KeyValue[],
+  name: string | Array<string | undefined>
+): string[] | undefined {
+  const key = Array.isArray(name) ? name.filter(Boolean).join(".") : name;
+
+  const attribute = attributes.find((attribute) => attribute.key === key);
+
+  if (!attribute?.value?.arrayValue?.values) return undefined;
+
+  return attribute.value.arrayValue.values
+    .filter((v): v is { stringValue: string } => isStringValue(v))
+    .map((v) => v.stringValue);
+}
+
 function isPartialSpan(span: Span): boolean {
   if (!span.attributes) return false;
 

apps/webapp/app/v3/querySchemas.ts

@@ -740,6 +740,15 @@ export const llmUsageSchema: TableSchema = {
         customRenderType: "durationNs",
       }),
     },
+    metadata: {
+      name: "metadata",
+      ...column("Map(LowCardinality(String), String)", {
+        description:
+          "Key-value metadata from run tags (key:value format) and AI SDK telemetry metadata. Access keys with dot notation (metadata.userId) or bracket syntax (metadata['userId']).",
+        example: "{'userId':'user_123','org':'acme'}",
+        coreColumn: true,
+      }),
+    },
   },
 };
 

apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts

@@ -48,7 +48,24 @@ function enrichLlmCost(event: CreateEventInput): void {
   const props = event.properties;
   if (!props) return;
 
-  crumb("enrichLlmCost called", { kind: event.kind, isPartial: event.isPartial, spanId: event.spanId, message: event.message, props }); // @crumbs
+  // #region @crumbs
+  // Log all spans (not just gen_ai) that have conversation/chat/session/user context
+  if (!event.isPartial) {
+    const contextKeys = Object.entries(props).filter(([k]) =>
+      k.startsWith("ai.telemetry.") || k.startsWith("gen_ai.conversation") ||
+      k.startsWith("chat.") || k.includes("session") || k.includes("user")
+    );
+    if (contextKeys.length > 0) {
+      crumb("span with context", {
+        spanId: event.spanId,
+        parentId: event.parentId,
+        runId: event.runId,
+        message: event.message,
+        contextAttrs: Object.fromEntries(contextKeys),
+      });
+    }
+  }
+  // #endregion @crumbs
 
   // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED)
   const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]);
@@ -130,6 +147,39 @@ function enrichLlmCost(event: CreateEventInput): void {
     },
   };
 
+  // Build metadata map from run tags and ai.telemetry.metadata.*
+  const metadata: Record<string, string> = {};
+
+  if (event.runTags) {
+    for (const tag of event.runTags) {
+      const colonIdx = tag.indexOf(":");
+      if (colonIdx > 0) {
+        metadata[tag.substring(0, colonIdx)] = tag.substring(colonIdx + 1);
+      }
+    }
+  }
+
+  for (const [key, value] of Object.entries(props)) {
+    if (key.startsWith("ai.telemetry.metadata.") && typeof value === "string") {
+      metadata[key.slice("ai.telemetry.metadata.".length)] = value;
+    }
+  }
+
+  // #region @crumbs
+  const metadataKeyCount = Object.keys(metadata).length;
+  if (metadataKeyCount > 0) {
+    crumb("llm metadata built", {
+      spanId: event.spanId,
+      runId: event.runId,
+      responseModel,
+      metadataKeyCount,
+      metadataKeys: Object.keys(metadata),
+      fromRunTags: event.runTags?.length ?? 0,
+      fromTelemetry: metadataKeyCount - (event.runTags?.filter((t) => t.includes(":")).length ?? 0),
+    });
+  }
+  // #endregion @crumbs
+
   // Set _llmUsage side-channel for dual-write to llm_usage_v1
   const llmUsage: LlmUsageData = {
     genAiSystem: (props["gen_ai.system"] as string) ?? "unknown",
@@ -147,6 +197,7 @@ function enrichLlmCost(event: CreateEventInput): void {
     outputCost: cost.outputCost,
     totalCost: cost.totalCost,
     costDetails: cost.costDetails,
+    metadata,
   };
 
   event._llmUsage = llmUsage;

internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql

@@ -27,13 +27,16 @@ CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1
   total_cost        Decimal64(12) DEFAULT 0,
   cost_details      Map(LowCardinality(String), Decimal64(12)),
 
+  metadata          Map(LowCardinality(String), String),
+
   start_time        DateTime64(9) CODEC(Delta(8), ZSTD(1)),
   duration          UInt64 DEFAULT 0 CODEC(ZSTD(1)),
   inserted_at       DateTime64(3) DEFAULT now64(3),
 
   INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1,
   INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1,
-  INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1
+  INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1,
+  INDEX idx_metadata_keys mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
 )
 ENGINE = MergeTree
 PARTITION BY toDate(inserted_at)

internal-packages/clickhouse/src/llmUsage.ts

@@ -28,6 +28,8 @@ export const LlmUsageV1Input = z.object({
   total_cost: z.number(),
   cost_details: z.record(z.string(), z.number()),
 
+  metadata: z.record(z.string(), z.string()),
+
   start_time: z.string(),
   duration: z.string(),
 });

internal-packages/tsql/src/query/printer.ts

@@ -2370,6 +2370,21 @@ export class ClickHousePrinter {
     // Try to resolve column names through table context
     const resolvedChain = this.resolveFieldChain(chainWithPrefix);
 
+    // For Map columns, convert dot-notation to bracket syntax:
+    // metadata.user -> metadata['user']
+    if (resolvedChain.length > 1) {
+      const rootColumnSchema = this.resolveFieldToColumnSchema([node.chain[0]]);
+      if (rootColumnSchema?.type.startsWith("Map(")) {
+        const rootCol = this.printIdentifierOrIndex(resolvedChain[0]);
+        const mapKeys = resolvedChain.slice(1);
+        let result = rootCol;
+        for (const key of mapKeys) {
+          result = `${result}[${this.context.addValue(String(key))}]`;
+        }
+        return result;
+      }
+    }
+
     // Print each chain element
     let result = resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join(".");