From 1b7b7e32c578edf72b00d11c3feef9d9693d82e4 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 11 Mar 2026 10:09:31 +0000 Subject: [PATCH 01/15] feat: add automatic LLM cost tracking for GenAI spans Calculates costs from gen_ai.* span attributes using an in-memory pricing registry backed by Postgres, with model prices synced from Langfuse (145 models). Costs are dual-written to span attributes (trigger.llm.*) and a new llm_usage_v1 ClickHouse table for efficient aggregation. - New @internal/llm-pricing package with ModelPricingRegistry - Prisma schema for llm_models, llm_pricing_tiers, llm_prices - ClickHouse llm_usage_v1 table with DynamicFlushScheduler batching - Cost enrichment in enrichCreatableEvents() with gen_ai.usage.* extraction - TRQL llm_usage table schema for querying - Admin API endpoints for model CRUD, seed, and registry reload - Pill-style accessories on spans showing model, tokens, and cost - Anthropic logo icon for RunIcon - Style merge fix for partial/completed span deduplication - Env vars: LLM_COST_TRACKING_ENABLED, LLM_PRICING_RELOAD_INTERVAL_MS refs TRI-7773 --- .server-changes/llm-cost-tracking.md | 6 + .../app/assets/icons/AnthropicLogoIcon.tsx | 12 + .../webapp/app/components/runs/v3/RunIcon.tsx | 3 + .../app/components/runs/v3/SpanTitle.tsx | 26 + apps/webapp/app/env.server.ts | 4 + .../admin.api.v1.llm-models.$modelId.ts | 149 + .../routes/admin.api.v1.llm-models.reload.ts | 24 + .../routes/admin.api.v1.llm-models.seed.ts | 30 + .../app/routes/admin.api.v1.llm-models.ts | 132 + .../clickhouseEventRepository.server.ts | 86 +- .../eventRepository/eventRepository.types.ts | 20 + .../app/v3/llmPricingRegistry.server.ts | 47 + apps/webapp/app/v3/otlpExporter.server.ts | 1 + apps/webapp/app/v3/querySchemas.ts | 146 +- .../v3/utils/enrichCreatableEvents.server.ts | 183 +- apps/webapp/package.json | 1 + apps/webapp/test/otlpExporter.test.ts | 240 +- .../schema/024_create_llm_usage_v1.sql | 45 + internal-packages/clickhouse/src/index.ts | 8 + internal-packages/clickhouse/src/llmUsage.ts | 42 + .../migration.sql | 63 + .../database/prisma/schema.prisma | 56 + internal-packages/llm-pricing/package.json | 17 + .../llm-pricing/scripts/sync-model-prices.sh | 77 + .../llm-pricing/src/default-model-prices.json | 3838 +++++++++++++++++ .../llm-pricing/src/defaultPrices.ts | 2984 +++++++++++++ internal-packages/llm-pricing/src/index.ts | 11 + .../llm-pricing/src/registry.test.ts | 233 + internal-packages/llm-pricing/src/registry.ts | 209 + internal-packages/llm-pricing/src/seed.ts | 59 + internal-packages/llm-pricing/src/types.ts | 53 + internal-packages/llm-pricing/tsconfig.json | 19 + packages/core/src/v3/schemas/style.ts | 3 +- 33 files changed, 8821 insertions(+), 6 deletions(-) create mode 100644 .server-changes/llm-cost-tracking.md create mode 100644 apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.ts create mode 100644 apps/webapp/app/v3/llmPricingRegistry.server.ts create mode 100644 internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql create mode 100644 internal-packages/clickhouse/src/llmUsage.ts create mode 100644 internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql create mode 100644 internal-packages/llm-pricing/package.json create mode 100755 internal-packages/llm-pricing/scripts/sync-model-prices.sh create mode 100644 internal-packages/llm-pricing/src/default-model-prices.json create mode 100644 internal-packages/llm-pricing/src/defaultPrices.ts create mode 100644 internal-packages/llm-pricing/src/index.ts create mode 100644 internal-packages/llm-pricing/src/registry.test.ts create mode 100644 internal-packages/llm-pricing/src/registry.ts create mode 100644 internal-packages/llm-pricing/src/seed.ts create mode 100644 internal-packages/llm-pricing/src/types.ts create mode 100644 internal-packages/llm-pricing/tsconfig.json diff --git a/.server-changes/llm-cost-tracking.md b/.server-changes/llm-cost-tracking.md new file mode 100644 index 00000000000..b68302731a7 --- /dev/null +++ b/.server-changes/llm-cost-tracking.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_usage_v1` ClickHouse table. diff --git a/apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx b/apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx new file mode 100644 index 00000000000..3e647284cce --- /dev/null +++ b/apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx @@ -0,0 +1,12 @@ +export function AnthropicLogoIcon({ className }: { className?: string }) { + return ( + + + + ); +} diff --git a/apps/webapp/app/components/runs/v3/RunIcon.tsx b/apps/webapp/app/components/runs/v3/RunIcon.tsx index 846d7cae0a4..73963ea09b9 100644 --- a/apps/webapp/app/components/runs/v3/RunIcon.tsx +++ b/apps/webapp/app/components/runs/v3/RunIcon.tsx @@ -7,6 +7,7 @@ import { TableCellsIcon, TagIcon, } from "@heroicons/react/20/solid"; +import { AnthropicLogoIcon } from "~/assets/icons/AnthropicLogoIcon"; import { AttemptIcon } from "~/assets/icons/AttemptIcon"; import { TaskIcon } from "~/assets/icons/TaskIcon"; import { cn } from "~/utils/cn"; @@ -112,6 +113,8 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) { return ; case "streams": return ; + case "tabler-brand-anthropic": + return ; } return ; diff --git a/apps/webapp/app/components/runs/v3/SpanTitle.tsx b/apps/webapp/app/components/runs/v3/SpanTitle.tsx index fb0105c45db..fe85fd70c9e 100644 --- a/apps/webapp/app/components/runs/v3/SpanTitle.tsx +++ b/apps/webapp/app/components/runs/v3/SpanTitle.tsx @@ -3,6 +3,8 @@ import { TaskEventStyle } from "@trigger.dev/core/v3"; import type { TaskEventLevel } from "@trigger.dev/database"; import { Fragment } from "react"; import { cn } from "~/utils/cn"; +import { tablerIcons } from "~/utils/tablerIcons"; +import tablerSpritePath from "~/components/primitives/tabler-sprite.svg"; type SpanTitleProps = { message: string; @@ -45,6 +47,15 @@ function SpanAccessory({ /> ); } + case "pills": { + return ( +
+ {accessory.items.map((item, index) => ( + + ))} +
+ ); + } default: { return (
@@ -59,6 +70,21 @@ function SpanAccessory({ } } +function SpanPill({ text, icon }: { text: string; icon?: string }) { + const hasIcon = icon && tablerIcons.has(icon); + + return ( + + {hasIcon && ( + + + + )} + {text} + + ); +} + export function SpanCodePathAccessory({ accessory, className, diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index bdfdbea6b3e..8877c982ae0 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1277,6 +1277,10 @@ const EnvironmentSchema = z EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(5_000), EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING: z.coerce.number().int().default(2000), + // LLM cost tracking + LLM_COST_TRACKING_ENABLED: BoolEnv.default(true), + LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes + // Bootstrap TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: z.string().optional(), diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts new file mode 100644 index 00000000000..8cf132daab9 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts @@ -0,0 +1,149 @@ +import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; + +async function requireAdmin(request: Request) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + throw json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + throw json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + return user; +} + +export async function loader({ request, params }: LoaderFunctionArgs) { + await requireAdmin(request); + + const model = await prisma.llmModel.findUnique({ + where: { id: params.modelId }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + }); + + if (!model) { + return json({ error: "Model not found" }, { status: 404 }); + } + + return json({ model }); +} + +const UpdateModelSchema = z.object({ + modelName: z.string().min(1).optional(), + matchPattern: z.string().min(1).optional(), + startDate: z.string().nullable().optional(), + pricingTiers: z + .array( + z.object({ + name: z.string().min(1), + isDefault: z.boolean().default(true), + priority: z.number().int().default(0), + conditions: z + .array( + z.object({ + usageDetailPattern: z.string(), + operator: z.enum(["gt", "gte", "lt", "lte", "eq", "neq"]), + value: z.number(), + }) + ) + .default([]), + prices: z.record(z.string(), z.number()), + }) + ) + .optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + await requireAdmin(request); + + const modelId = params.modelId!; + + if (request.method === "DELETE") { + const existing = await prisma.llmModel.findUnique({ where: { id: modelId } }); + if (!existing) { + return json({ error: "Model not found" }, { status: 404 }); + } + + await prisma.llmModel.delete({ where: { id: modelId } }); + return json({ success: true }); + } + + if (request.method !== "PUT") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const body = await request.json(); + const parsed = UpdateModelSchema.safeParse(body); + + if (!parsed.success) { + return json({ error: "Invalid request body", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, startDate, pricingTiers } = parsed.data; + + // Validate regex if provided + if (matchPattern) { + try { + new RegExp(matchPattern); + } catch { + return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + } + + // Update model fields + const model = await prisma.llmModel.update({ + where: { id: modelId }, + data: { + ...(modelName !== undefined && { modelName }), + ...(matchPattern !== undefined && { matchPattern }), + ...(startDate !== undefined && { startDate: startDate ? new Date(startDate) : null }), + }, + }); + + // If pricing tiers provided, replace them entirely + if (pricingTiers) { + // Delete existing tiers (cascades to prices) + await prisma.llmPricingTier.deleteMany({ where: { modelId } }); + + // Create new tiers + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId, + usageType, + price, + })), + }, + }, + }); + } + } + + const updated = await prisma.llmModel.findUnique({ + where: { id: modelId }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + }); + + return json({ model: updated }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts new file mode 100644 index 00000000000..747722b352a --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts @@ -0,0 +1,24 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export async function action({ request }: ActionFunctionArgs) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + return json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + if (!llmPricingRegistry) { + return json({ error: "LLM cost tracking is disabled" }, { status: 400 }); + } + + await llmPricingRegistry.reload(); + + return json({ success: true, message: "LLM pricing registry reloaded" }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts new file mode 100644 index 00000000000..805f97ad233 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts @@ -0,0 +1,30 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { seedLlmPricing } from "@internal/llm-pricing"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export async function action({ request }: ActionFunctionArgs) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + return json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + const result = await seedLlmPricing(prisma); + + // Reload the in-memory registry after seeding (if enabled) + if (llmPricingRegistry) { + await llmPricingRegistry.reload(); + } + + return json({ + success: true, + ...result, + message: `Seeded ${result.modelsCreated} models, skipped ${result.modelsSkipped} existing`, + }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts new file mode 100644 index 00000000000..7b1b2226ca5 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts @@ -0,0 +1,132 @@ +import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; + +async function requireAdmin(request: Request) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + throw json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + throw json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + return user; +} + +export async function loader({ request }: LoaderFunctionArgs) { + await requireAdmin(request); + + const url = new URL(request.url); + const page = parseInt(url.searchParams.get("page") ?? "1"); + const pageSize = parseInt(url.searchParams.get("pageSize") ?? "50"); + + const [models, total] = await Promise.all([ + prisma.llmModel.findMany({ + where: { projectId: null }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + orderBy: { modelName: "asc" }, + skip: (page - 1) * pageSize, + take: pageSize, + }), + prisma.llmModel.count({ where: { projectId: null } }), + ]); + + return json({ models, total, page, pageSize }); +} + +const CreateModelSchema = z.object({ + modelName: z.string().min(1), + matchPattern: z.string().min(1), + startDate: z.string().optional(), + source: z.enum(["default", "admin"]).optional().default("admin"), + pricingTiers: z.array( + z.object({ + name: z.string().min(1), + isDefault: z.boolean().default(true), + priority: z.number().int().default(0), + conditions: z + .array( + z.object({ + usageDetailPattern: z.string(), + operator: z.enum(["gt", "gte", "lt", "lte", "eq", "neq"]), + value: z.number(), + }) + ) + .default([]), + prices: z.record(z.string(), z.number()), + }) + ), +}); + +export async function action({ request }: ActionFunctionArgs) { + await requireAdmin(request); + + if (request.method !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const body = await request.json(); + const parsed = CreateModelSchema.safeParse(body); + + if (!parsed.success) { + return json({ error: "Invalid request body", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, startDate, source, pricingTiers } = parsed.data; + + // Validate regex pattern + try { + new RegExp(matchPattern); + } catch { + return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + + // Create model first, then tiers with explicit model connection + const model = await prisma.llmModel.create({ + data: { + modelName, + matchPattern, + startDate: startDate ? new Date(startDate) : null, + source, + }, + }); + + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + const created = await prisma.llmModel.findUnique({ + where: { id: model.id }, + include: { + pricingTiers: { + include: { prices: true }, + }, + }, + }); + + return json({ model: created }, { status: 201 }); +} diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 9100ad84fec..780b74d85e1 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -1,5 +1,6 @@ import type { ClickHouse, + LlmUsageV1Input, TaskEventDetailedSummaryV1Result, TaskEventDetailsV1Result, TaskEventSummaryV1Result, @@ -7,6 +8,9 @@ import type { TaskEventV2Input, } from "@internal/clickhouse"; import { Attributes, startSpan, trace, Tracer } from "@internal/tracing"; +import { trail } from "agentcrumbs"; // @crumbs + +const crumb = trail("webapp:llm-dual-write"); // @crumbs import { createJsonErrorObject } from "@trigger.dev/core/v3/errors"; import { serializeTraceparent } from "@trigger.dev/core/v3/isomorphic"; import { @@ -94,6 +98,7 @@ export class ClickhouseEventRepository implements IEventRepository { private _clickhouse: ClickHouse; private _config: ClickhouseEventRepositoryConfig; private readonly _flushScheduler: DynamicFlushScheduler; + private readonly _llmUsageFlushScheduler: DynamicFlushScheduler; private _tracer: Tracer; private _version: "v1" | "v2"; @@ -118,6 +123,17 @@ export class ClickhouseEventRepository implements IEventRepository { return event.kind === "DEBUG_EVENT"; }, }); + + this._llmUsageFlushScheduler = new DynamicFlushScheduler({ + batchSize: 5000, + flushInterval: 2000, + callback: this.#flushLlmUsageBatch.bind(this), + minConcurrency: 1, + maxConcurrency: 2, + maxBatchSize: 10000, + memoryPressureThreshold: 10000, + loadSheddingEnabled: false, + }); } get version() { @@ -216,6 +232,58 @@ export class ClickhouseEventRepository implements IEventRepository { }); } + async #flushLlmUsageBatch(flushId: string, rows: LlmUsageV1Input[]) { + crumb("flushing llm usage batch", { flushId, rows: rows.length }); // @crumbs + + const [insertError] = await this._clickhouse.llmUsage.insert(rows, { + params: { + clickhouse_settings: this.#getClickhouseInsertSettings(), + }, + }); + + if (insertError) { + crumb("llm usage batch insert failed", { flushId, error: String(insertError) }); // @crumbs + throw insertError; + } + + crumb("llm usage batch inserted", { flushId, rows: rows.length }); // @crumbs + + logger.info("ClickhouseEventRepository.flushLlmUsageBatch Inserted LLM usage batch", { + rows: rows.length, + }); + } + + #createLlmUsageInput(event: CreateEventInput): LlmUsageV1Input { + const llmUsage = event._llmUsage!; + + return { + organization_id: event.organizationId, + project_id: event.projectId, + environment_id: event.environmentId, + run_id: event.runId, + task_identifier: event.taskSlug, + trace_id: event.traceId, + span_id: event.spanId, + gen_ai_system: llmUsage.genAiSystem, + request_model: llmUsage.requestModel, + response_model: llmUsage.responseModel, + matched_model_id: llmUsage.matchedModelId, + operation_name: llmUsage.operationName, + pricing_tier_id: llmUsage.pricingTierId, + pricing_tier_name: llmUsage.pricingTierName, + input_tokens: llmUsage.inputTokens, + output_tokens: llmUsage.outputTokens, + total_tokens: llmUsage.totalTokens, + usage_details: llmUsage.usageDetails, + input_cost: llmUsage.inputCost, + output_cost: llmUsage.outputCost, + total_cost: llmUsage.totalCost, + cost_details: llmUsage.costDetails, + start_time: this.#clampAndFormatStartTime(event.startTime.toString()), + duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0), + }; + } + #getClickhouseInsertSettings() { if (this._config.insertStrategy === "insert") { return {}; @@ -236,6 +304,16 @@ export class ClickhouseEventRepository implements IEventRepository { async insertMany(events: CreateEventInput[]): Promise { this.addToBatch(events.flatMap((event) => this.createEventToTaskEventV1Input(event))); + + // Dual-write LLM usage records for spans with cost enrichment + const llmUsageRows = events + .filter((e) => e._llmUsage != null) + .map((e) => this.#createLlmUsageInput(e)); + + if (llmUsageRows.length > 0) { + crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id }); // @crumbs + this._llmUsageFlushScheduler.addToBatch(llmUsageRows); + } } async insertManyImmediate(events: CreateEventInput[]): Promise { @@ -1525,7 +1603,13 @@ export class ClickhouseEventRepository implements IEventRepository { } if (parsedMetadata && "style" in parsedMetadata && parsedMetadata.style) { - span.data.style = parsedMetadata.style as TaskEventStyle; + const newStyle = parsedMetadata.style as TaskEventStyle; + // Merge styles: prefer the most complete value for each field + span.data.style = { + icon: newStyle.icon ?? span.data.style.icon, + variant: newStyle.variant ?? span.data.style.variant, + accessory: newStyle.accessory ?? span.data.style.accessory, + }; } if (record.kind === "SPAN") { diff --git a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts index 75664ad0525..b750786a651 100644 --- a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts +++ b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts @@ -21,6 +21,24 @@ export type { ExceptionEventProperties }; // Event Creation Types // ============================================================================ +export type LlmUsageData = { + genAiSystem: string; + requestModel: string; + responseModel: string; + matchedModelId: string; + operationName: string; + pricingTierId: string; + pricingTierName: string; + inputTokens: number; + outputTokens: number; + totalTokens: number; + usageDetails: Record; + inputCost: number; + outputCost: number; + totalCost: number; + costDetails: Record; +}; + export type CreateEventInput = Omit< Prisma.TaskEventCreateInput, | "id" @@ -57,6 +75,8 @@ export type CreateEventInput = Omit< metadata: Attributes | undefined; style: Attributes | undefined; machineId?: string; + /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */ + _llmUsage?: LlmUsageData; }; export type CreatableEventKind = TaskEventKind; diff --git a/apps/webapp/app/v3/llmPricingRegistry.server.ts b/apps/webapp/app/v3/llmPricingRegistry.server.ts new file mode 100644 index 00000000000..4a2c67e247b --- /dev/null +++ b/apps/webapp/app/v3/llmPricingRegistry.server.ts @@ -0,0 +1,47 @@ +import { ModelPricingRegistry } from "@internal/llm-pricing"; +import { trail } from "agentcrumbs"; // @crumbs +import { $replica } from "~/db.server"; +import { env } from "~/env.server"; +import { singleton } from "~/utils/singleton"; +import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server"; + +const crumb = trail("webapp:llm-registry"); // @crumbs + +export const llmPricingRegistry = singleton("llmPricingRegistry", () => { + if (!env.LLM_COST_TRACKING_ENABLED) { + crumb("llm cost tracking disabled via env"); // @crumbs + return null; + } + + crumb("initializing registry singleton"); // @crumbs + const registry = new ModelPricingRegistry($replica); + + // Wire up the registry so enrichCreatableEvents can use it + setLlmPricingRegistry(registry); + + registry + .loadFromDatabase() + .then(() => { + crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs + }) + .catch((err) => { + crumb("registry load failed", { error: String(err) }); // @crumbs + console.error("Failed to load LLM pricing registry", err); + }); + + // Periodic reload + const reloadInterval = env.LLM_PRICING_RELOAD_INTERVAL_MS; + setInterval(() => { + registry + .reload() + .then(() => { + crumb("registry reloaded"); // @crumbs + }) + .catch((err) => { + crumb("registry reload failed", { error: String(err) }); // @crumbs + console.error("Failed to reload LLM pricing registry", err); + }); + }, reloadInterval); + + return registry; +}); diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 837071b7de7..0079cb5f0be 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -37,6 +37,7 @@ import type { } from "./eventRepository/eventRepository.types"; import { startSpan } from "./tracing.server"; import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server"; +import "./llmPricingRegistry.server"; // Initialize LLM pricing registry on startup import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 53c3be60fa2..323310ebf84 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -599,7 +599,151 @@ export const metricsSchema: TableSchema = { /** * All available schemas for the query editor */ -export const querySchemas: TableSchema[] = [runsSchema, metricsSchema]; +/** + * Schema definition for the llm_usage table (trigger_dev.llm_usage_v1) + */ +export const llmUsageSchema: TableSchema = { + name: "llm_usage", + clickhouseName: "trigger_dev.llm_usage_v1", + description: "LLM token usage and cost data from GenAI spans", + timeConstraint: "start_time", + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + columns: { + environment: { + name: "environment", + clickhouseName: "environment_id", + ...column("String", { description: "The environment slug", example: "prod" }), + fieldMapping: "environment", + customRenderType: "environment", + }, + project: { + name: "project", + clickhouseName: "project_id", + ...column("String", { + description: "The project reference, they always start with `proj_`.", + example: "proj_howcnaxbfxdmwmxazktx", + }), + fieldMapping: "project", + customRenderType: "project", + }, + run_id: { + name: "run_id", + ...column("String", { + description: "The run ID", + customRenderType: "runId", + coreColumn: true, + }), + }, + task_identifier: { + name: "task_identifier", + ...column("LowCardinality(String)", { + description: "The task identifier", + example: "my-task", + coreColumn: true, + }), + }, + gen_ai_system: { + name: "gen_ai_system", + ...column("LowCardinality(String)", { + description: "AI provider (e.g. openai, anthropic)", + example: "openai", + coreColumn: true, + }), + }, + request_model: { + name: "request_model", + ...column("String", { + description: "The model name requested", + example: "gpt-4o", + }), + }, + response_model: { + name: "response_model", + ...column("String", { + description: "The model name returned by the provider", + example: "gpt-4o-2024-08-06", + coreColumn: true, + }), + }, + operation_name: { + name: "operation_name", + ...column("LowCardinality(String)", { + description: "Operation type (e.g. chat, completion)", + example: "chat", + }), + }, + input_tokens: { + name: "input_tokens", + ...column("UInt64", { + description: "Number of input tokens", + example: "702", + }), + }, + output_tokens: { + name: "output_tokens", + ...column("UInt64", { + description: "Number of output tokens", + example: "22", + }), + }, + total_tokens: { + name: "total_tokens", + ...column("UInt64", { + description: "Total token count", + example: "724", + }), + }, + input_cost: { + name: "input_cost", + ...column("Decimal64(12)", { + description: "Input cost in USD", + customRenderType: "costInDollars", + }), + }, + output_cost: { + name: "output_cost", + ...column("Decimal64(12)", { + description: "Output cost in USD", + customRenderType: "costInDollars", + }), + }, + total_cost: { + name: "total_cost", + ...column("Decimal64(12)", { + description: "Total cost in USD", + customRenderType: "costInDollars", + coreColumn: true, + }), + }, + pricing_tier_name: { + name: "pricing_tier_name", + ...column("LowCardinality(String)", { + description: "The matched pricing tier name", + example: "Standard", + }), + }, + start_time: { + name: "start_time", + ...column("DateTime64(9)", { + description: "When the LLM call started", + coreColumn: true, + }), + }, + duration: { + name: "duration", + ...column("UInt64", { + description: "Span duration in nanoseconds", + customRenderType: "durationNs", + }), + }, + }, +}; + +export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmUsageSchema]; /** * Default query for the query editor diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index f718c13d2dd..9dfe091aa10 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -1,4 +1,31 @@ -import type { CreateEventInput } from "../eventRepository/eventRepository.types"; +import { trail } from "agentcrumbs"; // @crumbs +import type { CreateEventInput, LlmUsageData } from "../eventRepository/eventRepository.types"; + +const crumb = trail("webapp:llm-enrich"); // @crumbs + +// Registry interface — matches ModelPricingRegistry from @internal/llm-pricing +type CostRegistry = { + isLoaded: boolean; + calculateCost( + responseModel: string, + usageDetails: Record + ): { + matchedModelId: string; + matchedModelName: string; + pricingTierId: string; + pricingTierName: string; + inputCost: number; + outputCost: number; + totalCost: number; + costDetails: Record; + } | null; +}; + +let _registry: CostRegistry | undefined; + +export function setLlmPricingRegistry(registry: CostRegistry): void { + _registry = registry; +} export function enrichCreatableEvents(events: CreateEventInput[]) { return events.map((event) => { @@ -12,9 +39,151 @@ function enrichCreatableEvent(event: CreateEventInput): CreateEventInput { event.message = message; event.style = enrichStyle(event); + enrichLlmCost(event); + return event; } +function enrichLlmCost(event: CreateEventInput): void { + const props = event.properties; + if (!props) return; + + crumb("enrichLlmCost called", { kind: event.kind, isPartial: event.isPartial, spanId: event.spanId, message: event.message, props }); // @crumbs + + // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED) + const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); + if (!enrichableKinds.has(event.kind as string)) return; + + // Skip partial spans (they don't have final token counts) + if (event.isPartial) return; + + // Only use gen_ai.* attributes for model resolution to avoid double-counting. + // The Vercel AI SDK emits both a parent span (ai.streamText with ai.usage.*) + // and a child span (ai.streamText.doStream with gen_ai.*). We only enrich the + // child span that has the canonical gen_ai.response.model attribute. + const responseModel = + typeof props["gen_ai.response.model"] === "string" + ? props["gen_ai.response.model"] + : typeof props["gen_ai.request.model"] === "string" + ? props["gen_ai.request.model"] + : null; + + if (!responseModel) { + // #region @crumbs + const genAiModel = props["gen_ai.response.model"]; + const aiModel = props["ai.model.id"]; + if (genAiModel || aiModel) { + crumb("responseModel null despite gen_ai attrs", { genAiModel, genAiModelType: typeof genAiModel, aiModel, spanId: event.spanId }); + } + // #endregion @crumbs + return; + } + + crumb("llm span detected", { responseModel, kind: event.kind, spanId: event.spanId }); // @crumbs + + // Extract usage details, normalizing attribute names + const usageDetails = extractUsageDetails(props); + + // Need at least some token usage + const hasTokens = Object.values(usageDetails).some((v) => v > 0); + if (!hasTokens) { + crumb("llm span skipped: no tokens", { responseModel, spanId: event.spanId }); // @crumbs + return; + } + + if (!_registry?.isLoaded) { + crumb("llm span skipped: registry not loaded", { responseModel }); // @crumbs + return; + } + + const cost = _registry.calculateCost(responseModel, usageDetails); + if (!cost) return; + + crumb("llm cost enriched", { responseModel, totalCost: cost.totalCost, matchedModel: cost.matchedModelName, spanId: event.spanId }); // @crumbs + + // Add trigger.llm.* attributes to the span + event.properties = { + ...props, + "trigger.llm.input_cost": cost.inputCost, + "trigger.llm.output_cost": cost.outputCost, + "trigger.llm.total_cost": cost.totalCost, + "trigger.llm.matched_model": cost.matchedModelName, + "trigger.llm.matched_model_id": cost.matchedModelId, + "trigger.llm.pricing_tier": cost.pricingTierName, + "trigger.llm.pricing_tier_id": cost.pricingTierId, + }; + + // Add style accessories for model, tokens, and cost + const inputTokens = usageDetails["input"] ?? 0; + const outputTokens = usageDetails["output"] ?? 0; + const totalTokens = inputTokens + outputTokens; + + event.style = { + ...event.style, + accessory: { + style: "pills", + items: [ + { text: responseModel, icon: "tabler-cube" }, + { text: formatTokenCount(totalTokens), icon: "tabler-hash" }, + { text: formatCost(cost.totalCost), icon: "tabler-currency-dollar" }, + ], + }, + }; + + // Set _llmUsage side-channel for dual-write to llm_usage_v1 + const llmUsage: LlmUsageData = { + genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", + requestModel: (props["gen_ai.request.model"] as string) ?? responseModel, + responseModel, + matchedModelId: cost.matchedModelId, + operationName: (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? "", + pricingTierId: cost.pricingTierId, + pricingTierName: cost.pricingTierName, + inputTokens: usageDetails["input"] ?? 0, + outputTokens: usageDetails["output"] ?? 0, + totalTokens: Object.values(usageDetails).reduce((sum, v) => sum + v, 0), + usageDetails, + inputCost: cost.inputCost, + outputCost: cost.outputCost, + totalCost: cost.totalCost, + costDetails: cost.costDetails, + }; + + event._llmUsage = llmUsage; +} + +function extractUsageDetails(props: Record): Record { + const details: Record = {}; + + // Only map gen_ai.usage.* attributes — NOT ai.usage.* from parent spans. + // This prevents double-counting when both parent (ai.streamText) and child + // (ai.streamText.doStream) spans carry token counts. + const mappings: Record = { + "gen_ai.usage.input_tokens": "input", + "gen_ai.usage.output_tokens": "output", + "gen_ai.usage.prompt_tokens": "input", + "gen_ai.usage.completion_tokens": "output", + "gen_ai.usage.total_tokens": "total", + "gen_ai.usage.cache_read_input_tokens": "input_cached_tokens", + "gen_ai.usage.input_tokens_cache_read": "input_cached_tokens", + "gen_ai.usage.cache_creation_input_tokens": "cache_creation_input_tokens", + "gen_ai.usage.input_tokens_cache_write": "cache_creation_input_tokens", + "gen_ai.usage.reasoning_tokens": "reasoning_tokens", + }; + + for (const [attrKey, usageKey] of Object.entries(mappings)) { + const value = props[attrKey]; + if (typeof value === "number" && value > 0) { + // Don't overwrite if already set (first mapping wins) + if (details[usageKey] === undefined) { + details[usageKey] = value; + } + } + } + + return details; +} + function enrichStyle(event: CreateEventInput) { const baseStyle = event.style ?? {}; const props = event.properties; @@ -49,6 +218,18 @@ function enrichStyle(event: CreateEventInput) { return baseStyle; } +function formatTokenCount(tokens: number): string { + if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}k`; + return tokens.toString(); +} + +function formatCost(cost: number): string { + if (cost >= 1) return `$${cost.toFixed(2)}`; + if (cost >= 0.01) return `$${cost.toFixed(4)}`; + return `$${cost.toFixed(6)}`; +} + function repr(value: any): string { if (typeof value === "string") { return `'${value}'`; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 139a0ce2d0d..fce7f2769c1 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -56,6 +56,7 @@ "@heroicons/react": "^2.0.12", "@jsonhero/schema-infer": "^0.1.5", "@internal/cache": "workspace:*", + "@internal/llm-pricing": "workspace:*", "@internal/redis": "workspace:*", "@internal/run-engine": "workspace:*", "@internal/schedule-engine": "workspace:*", diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index 98380f2a596..a1d1baa3b1c 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -1,5 +1,8 @@ -import { describe, it, expect } from "vitest"; -import { enrichCreatableEvents } from "../app/v3/utils/enrichCreatableEvents.server.js"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + enrichCreatableEvents, + setLlmPricingRegistry, +} from "../app/v3/utils/enrichCreatableEvents.server.js"; import { RuntimeEnvironmentType, TaskEventKind, @@ -394,4 +397,237 @@ describe("OTLPExporter", () => { }); }); }); + + describe("LLM cost enrichment", () => { + const mockRegistry = { + isLoaded: true, + calculateCost: (responseModel: string, usageDetails: Record) => { + if (responseModel.startsWith("gpt-4o")) { + const inputCost = (usageDetails["input"] ?? 0) * 0.0000025; + const outputCost = (usageDetails["output"] ?? 0) * 0.00001; + return { + matchedModelId: "model-gpt4o", + matchedModelName: "gpt-4o", + pricingTierId: "tier-standard", + pricingTierName: "Standard", + inputCost, + outputCost, + totalCost: inputCost + outputCost, + costDetails: { input: inputCost, output: outputCost }, + }; + } + return null; + }, + }; + + beforeEach(() => { + setLlmPricingRegistry(mockRegistry); + }); + + afterEach(() => { + setLlmPricingRegistry(undefined as any); + }); + + function makeGenAiEvent(overrides: Record = {}) { + return { + message: "ai.streamText.doStream", + traceId: "test-trace", + spanId: "test-span", + parentId: "test-parent", + isPartial: false, + isError: false, + kind: TaskEventKind.INTERNAL, + level: TaskEventLevel.TRACE, + status: TaskEventStatus.UNSET, + startTime: BigInt(1), + duration: 5000000000, + style: {}, + serviceName: "test", + environmentId: "env-1", + environmentType: RuntimeEnvironmentType.DEVELOPMENT, + organizationId: "org-1", + projectId: "proj-1", + projectRef: "proj_test", + runId: "run_test", + runIsTest: false, + taskSlug: "my-task", + metadata: undefined, + properties: { + "gen_ai.system": "openai", + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.model": "gpt-4o-2024-08-06", + "gen_ai.usage.input_tokens": 702, + "gen_ai.usage.output_tokens": 22, + "operation.name": "ai.streamText.doStream", + ...overrides, + }, + }; + } + + it("should enrich spans with cost attributes and accessories", () => { + const events = [makeGenAiEvent()]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + const event = $events[0]; + + // Cost attributes + expect(event.properties["trigger.llm.total_cost"]).toBeCloseTo(0.001975); + expect(event.properties["trigger.llm.input_cost"]).toBeCloseTo(0.001755); + expect(event.properties["trigger.llm.output_cost"]).toBeCloseTo(0.00022); + expect(event.properties["trigger.llm.matched_model"]).toBe("gpt-4o"); + expect(event.properties["trigger.llm.pricing_tier"]).toBe("Standard"); + + // Accessories (pills style) + expect(event.style.accessory).toBeDefined(); + expect(event.style.accessory.style).toBe("pills"); + expect(event.style.accessory.items).toHaveLength(3); + expect(event.style.accessory.items[0]).toEqual({ + text: "gpt-4o-2024-08-06", + icon: "tabler-cube", + }); + expect(event.style.accessory.items[1]).toEqual({ + text: "724", + icon: "tabler-hash", + }); + expect(event.style.accessory.items[2]).toEqual({ + text: "$0.001975", + icon: "tabler-currency-dollar", + }); + }); + + it("should set _llmUsage side-channel for dual-write", () => { + const events = [makeGenAiEvent()]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + const event = $events[0]; + + expect(event._llmUsage).toBeDefined(); + expect(event._llmUsage.genAiSystem).toBe("openai"); + expect(event._llmUsage.responseModel).toBe("gpt-4o-2024-08-06"); + expect(event._llmUsage.inputTokens).toBe(702); + expect(event._llmUsage.outputTokens).toBe(22); + expect(event._llmUsage.totalCost).toBeCloseTo(0.001975); + expect(event._llmUsage.operationName).toBe("ai.streamText.doStream"); + }); + + it("should skip partial spans", () => { + const events = [makeGenAiEvent()]; + events[0].isPartial = true; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + expect($events[0]._llmUsage).toBeUndefined(); + }); + + it("should skip spans without gen_ai.response.model or gen_ai.request.model", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.response.model": undefined, + "gen_ai.request.model": undefined, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should fall back to gen_ai.request.model when response.model is missing", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.response.model": undefined, + "gen_ai.request.model": "gpt-4o", + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.matched_model"]).toBe("gpt-4o"); + }); + + it("should skip spans with no token usage", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.usage.input_tokens": 0, + "gen_ai.usage.output_tokens": 0, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should skip spans with unknown models", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.response.model": "unknown-model-xyz", + "gen_ai.request.model": "unknown-model-xyz", + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should not enrich non-span kinds like SPAN_EVENT or LOG", () => { + const events = [makeGenAiEvent()]; + events[0].kind = "SPAN_EVENT" as any; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should enrich SERVER kind events", () => { + const events = [makeGenAiEvent()]; + events[0].kind = TaskEventKind.SERVER; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeCloseTo(0.001975); + }); + + it("should not enrich when registry is not loaded", () => { + setLlmPricingRegistry({ isLoaded: false, calculateCost: () => null }); + const events = [makeGenAiEvent()]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should format token counts with k/M suffixes in accessories", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.usage.input_tokens": 150000, + "gen_ai.usage.output_tokens": 2000, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].style.accessory.items[1].text).toBe("152.0k"); + }); + + it("should normalize alternate token attribute names", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.usage.input_tokens": undefined, + "gen_ai.usage.output_tokens": undefined, + "gen_ai.usage.prompt_tokens": 500, + "gen_ai.usage.completion_tokens": 100, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0]._llmUsage.inputTokens).toBe(500); + expect($events[0]._llmUsage.outputTokens).toBe(100); + }); + }); }); diff --git a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql new file mode 100644 index 00000000000..65bd9c68ebd --- /dev/null +++ b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql @@ -0,0 +1,45 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 +( + organization_id LowCardinality(String), + project_id LowCardinality(String), + environment_id String CODEC(ZSTD(1)), + run_id String CODEC(ZSTD(1)), + task_identifier LowCardinality(String), + trace_id String CODEC(ZSTD(1)), + span_id String CODEC(ZSTD(1)), + + gen_ai_system LowCardinality(String), + request_model String CODEC(ZSTD(1)), + response_model String CODEC(ZSTD(1)), + matched_model_id String CODEC(ZSTD(1)), + operation_name LowCardinality(String), + pricing_tier_id String CODEC(ZSTD(1)), + pricing_tier_name LowCardinality(String), + + input_tokens UInt64 DEFAULT 0, + output_tokens UInt64 DEFAULT 0, + total_tokens UInt64 DEFAULT 0, + usage_details Map(LowCardinality(String), UInt64), + + input_cost Decimal64(12) DEFAULT 0, + output_cost Decimal64(12) DEFAULT 0, + total_cost Decimal64(12) DEFAULT 0, + cost_details Map(LowCardinality(String), Decimal64(12)), + + start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)), + duration UInt64 DEFAULT 0 CODEC(ZSTD(1)), + inserted_at DateTime64(3) DEFAULT now64(3), + + INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(inserted_at) +ORDER BY (organization_id, project_id, environment_id, toDate(inserted_at), run_id) +TTL toDateTime(inserted_at) + INTERVAL 365 DAY +SETTINGS ttl_only_drop_parts = 1; + +-- +goose Down +DROP TABLE IF EXISTS trigger_dev.llm_usage_v1; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index b6fbd92177b..336abf3761f 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -27,6 +27,7 @@ import { getLogsSearchListQueryBuilder, } from "./taskEvents.js"; import { insertMetrics } from "./metrics.js"; +import { insertLlmUsage } from "./llmUsage.js"; import { getErrorGroups, getErrorInstances, @@ -44,6 +45,7 @@ import type { Agent as HttpsAgent } from "https"; export type * from "./taskRuns.js"; export type * from "./taskEvents.js"; export type * from "./metrics.js"; +export type * from "./llmUsage.js"; export type * from "./errors.js"; export type * from "./client/queryBuilder.js"; @@ -225,6 +227,12 @@ export class ClickHouse { }; } + get llmUsage() { + return { + insert: insertLlmUsage(this.writer), + }; + } + get taskEventsV2() { return { insert: insertTaskEventsV2(this.writer), diff --git a/internal-packages/clickhouse/src/llmUsage.ts b/internal-packages/clickhouse/src/llmUsage.ts new file mode 100644 index 00000000000..fd9deccb562 --- /dev/null +++ b/internal-packages/clickhouse/src/llmUsage.ts @@ -0,0 +1,42 @@ +import { z } from "zod"; +import { ClickhouseWriter } from "./client/types.js"; + +export const LlmUsageV1Input = z.object({ + organization_id: z.string(), + project_id: z.string(), + environment_id: z.string(), + run_id: z.string(), + task_identifier: z.string(), + trace_id: z.string(), + span_id: z.string(), + + gen_ai_system: z.string(), + request_model: z.string(), + response_model: z.string(), + matched_model_id: z.string(), + operation_name: z.string(), + pricing_tier_id: z.string(), + pricing_tier_name: z.string(), + + input_tokens: z.number(), + output_tokens: z.number(), + total_tokens: z.number(), + usage_details: z.record(z.string(), z.number()), + + input_cost: z.number(), + output_cost: z.number(), + total_cost: z.number(), + cost_details: z.record(z.string(), z.number()), + + start_time: z.string(), + duration: z.string(), +}); + +export type LlmUsageV1Input = z.input; + +export function insertLlmUsage(ch: ClickhouseWriter) { + return ch.insertUnsafe({ + name: "insertLlmUsage", + table: "trigger_dev.llm_usage_v1", + }); +} diff --git a/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql new file mode 100644 index 00000000000..44cc581616f --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql @@ -0,0 +1,63 @@ +-- CreateTable +CREATE TABLE "public"."llm_models" ( + "id" TEXT NOT NULL, + "project_id" TEXT, + "model_name" TEXT NOT NULL, + "match_pattern" TEXT NOT NULL, + "start_date" TIMESTAMP(3), + "source" TEXT NOT NULL DEFAULT 'default', + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "llm_models_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."llm_pricing_tiers" ( + "id" TEXT NOT NULL, + "model_id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "is_default" BOOLEAN NOT NULL DEFAULT true, + "priority" INTEGER NOT NULL DEFAULT 0, + "conditions" JSONB NOT NULL DEFAULT '[]', + + CONSTRAINT "llm_pricing_tiers_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."llm_prices" ( + "id" TEXT NOT NULL, + "model_id" TEXT NOT NULL, + "pricing_tier_id" TEXT NOT NULL, + "usage_type" TEXT NOT NULL, + "price" DECIMAL(20,12) NOT NULL, + + CONSTRAINT "llm_prices_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "llm_models_project_id_idx" ON "public"."llm_models"("project_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_models_project_id_model_name_start_date_key" ON "public"."llm_models"("project_id", "model_name", "start_date"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_pricing_tiers_model_id_priority_key" ON "public"."llm_pricing_tiers"("model_id", "priority"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_pricing_tiers_model_id_name_key" ON "public"."llm_pricing_tiers"("model_id", "name"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_prices_model_id_usage_type_pricing_tier_id_key" ON "public"."llm_prices"("model_id", "usage_type", "pricing_tier_id"); + +-- AddForeignKey +ALTER TABLE "public"."llm_models" ADD CONSTRAINT "llm_models_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "public"."Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."llm_pricing_tiers" ADD CONSTRAINT "llm_pricing_tiers_model_id_fkey" FOREIGN KEY ("model_id") REFERENCES "public"."llm_models"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."llm_prices" ADD CONSTRAINT "llm_prices_model_id_fkey" FOREIGN KEY ("model_id") REFERENCES "public"."llm_models"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."llm_prices" ADD CONSTRAINT "llm_prices_pricing_tier_id_fkey" FOREIGN KEY ("pricing_tier_id") REFERENCES "public"."llm_pricing_tiers"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index f6986be42c0..65878a8af7c 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -417,6 +417,7 @@ model Project { onboardingData Json? taskScheduleInstances TaskScheduleInstance[] metricsDashboards MetricsDashboard[] + llmModels LlmModel[] } enum ProjectVersion { @@ -2577,3 +2578,58 @@ model MetricsDashboard { /// Fast lookup for the list @@index([projectId, createdAt(sort: Desc)]) } + +// ==================================================== +// LLM Pricing Models +// ==================================================== + +/// A known LLM model or model pattern for cost tracking +model LlmModel { + id String @id @default(cuid()) + projectId String? @map("project_id") + project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade) + modelName String @map("model_name") + matchPattern String @map("match_pattern") + startDate DateTime? @map("start_date") + source String @default("default") // "default", "admin", "project" + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + pricingTiers LlmPricingTier[] + prices LlmPrice[] + + @@unique([projectId, modelName, startDate]) + @@index([projectId]) + @@map("llm_models") +} + +/// A pricing tier for a model (supports volume-based or conditional pricing) +model LlmPricingTier { + id String @id @default(cuid()) + modelId String @map("model_id") + model LlmModel @relation(fields: [modelId], references: [id], onDelete: Cascade) + name String + isDefault Boolean @default(true) @map("is_default") + priority Int @default(0) + conditions Json @default("[]") @db.JsonB + + prices LlmPrice[] + + @@unique([modelId, priority]) + @@unique([modelId, name]) + @@map("llm_pricing_tiers") +} + +/// A price point for a usage type within a pricing tier +model LlmPrice { + id String @id @default(cuid()) + modelId String @map("model_id") + model LlmModel @relation(fields: [modelId], references: [id], onDelete: Cascade) + pricingTierId String @map("pricing_tier_id") + pricingTier LlmPricingTier @relation(fields: [pricingTierId], references: [id], onDelete: Cascade) + usageType String @map("usage_type") + price Decimal @db.Decimal(20, 12) + + @@unique([modelId, usageType, pricingTierId]) + @@map("llm_prices") +} diff --git a/internal-packages/llm-pricing/package.json b/internal-packages/llm-pricing/package.json new file mode 100644 index 00000000000..423dae77eb4 --- /dev/null +++ b/internal-packages/llm-pricing/package.json @@ -0,0 +1,17 @@ +{ + "name": "@internal/llm-pricing", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "types": "./src/index.ts", + "type": "module", + "dependencies": { + "@trigger.dev/database": "workspace:*" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "generate": "./scripts/sync-model-prices.sh", + "sync-prices": "./scripts/sync-model-prices.sh", + "sync-prices:check": "./scripts/sync-model-prices.sh --check" + } +} diff --git a/internal-packages/llm-pricing/scripts/sync-model-prices.sh b/internal-packages/llm-pricing/scripts/sync-model-prices.sh new file mode 100755 index 00000000000..9bb6af11cbb --- /dev/null +++ b/internal-packages/llm-pricing/scripts/sync-model-prices.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Sync default model prices from Langfuse's repository and generate the TS module. +# Usage: ./scripts/sync-model-prices.sh [--check] +# --check: Exit 1 if prices are outdated (for CI) + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PACKAGE_DIR="$(dirname "$SCRIPT_DIR")" +JSON_TARGET="$PACKAGE_DIR/src/default-model-prices.json" +TS_TARGET="$PACKAGE_DIR/src/defaultPrices.ts" +SOURCE_URL="https://raw.githubusercontent.com/langfuse/langfuse/main/worker/src/constants/default-model-prices.json" + +CHECK_MODE=false +if [[ "${1:-}" == "--check" ]]; then + CHECK_MODE=true +fi + +echo "Fetching latest model prices from Langfuse..." +TMPFILE=$(mktemp) +trap 'rm -f "$TMPFILE"' EXIT + +if ! curl -fsSL "$SOURCE_URL" -o "$TMPFILE"; then + echo "ERROR: Failed to fetch from $SOURCE_URL" + exit 1 +fi + +# Validate it's valid JSON with at least some models +MODEL_COUNT=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$TMPFILE','utf-8')).length)" 2>/dev/null || echo "0") +if [[ "$MODEL_COUNT" -lt 10 ]]; then + echo "ERROR: Downloaded file has only $MODEL_COUNT models (expected 100+). Aborting." + exit 1 +fi + +if $CHECK_MODE; then + if diff -q "$JSON_TARGET" "$TMPFILE" > /dev/null 2>&1; then + echo "Model prices are up to date ($MODEL_COUNT models)" + exit 0 + else + echo "Model prices are OUTDATED. Run 'pnpm run sync-prices' in @internal/llm-pricing to update." + exit 1 + fi +fi + +cp "$TMPFILE" "$JSON_TARGET" +echo "Updated default-model-prices.json ($MODEL_COUNT models)" + +# Generate the TypeScript module from the JSON +echo "Generating defaultPrices.ts..." +node -e " +const data = JSON.parse(require('fs').readFileSync('$JSON_TARGET', 'utf-8')); +const stripped = data.map(e => ({ + modelName: e.modelName, + matchPattern: e.matchPattern, + startDate: e.createdAt, + pricingTiers: e.pricingTiers.map(t => ({ + name: t.name, + isDefault: t.isDefault, + priority: t.priority, + conditions: t.conditions.map(c => ({ + usageDetailPattern: c.usageDetailPattern, + operator: c.operator, + value: c.value, + })), + prices: t.prices, + })), +})); + +let out = 'import type { DefaultModelDefinition } from \"./types.js\";\n\n'; +out += '// Auto-generated from Langfuse default-model-prices.json — do not edit manually.\n'; +out += '// Run \`pnpm run sync-prices\` to update from upstream.\n'; +out += '// Source: https://github.com/langfuse/langfuse\n\n'; +out += 'export const defaultModelPrices: DefaultModelDefinition[] = '; +out += JSON.stringify(stripped, null, 2) + ';\n'; +require('fs').writeFileSync('$TS_TARGET', out); +console.log('Generated defaultPrices.ts with ' + stripped.length + ' models'); +" diff --git a/internal-packages/llm-pricing/src/default-model-prices.json b/internal-packages/llm-pricing/src/default-model-prices.json new file mode 100644 index 00000000000..486c6c512b5 --- /dev/null +++ b/internal-packages/llm-pricing/src/default-model-prices.json @@ -0,0 +1,3838 @@ +[ + { + "id": "b9854a5c92dc496b997d99d20", + "modelName": "gpt-4o", + "matchPattern": "(?i)^(openai\/)?(gpt-4o)$", + "createdAt": "2024-05-13T23:15:07.670Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "b9854a5c92dc496b997d99d20_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "id": "b9854a5c92dc496b997d99d21", + "modelName": "gpt-4o-2024-05-13", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-2024-05-13)$", + "createdAt": "2024-05-13T23:15:07.670Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o-2024-05-13", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "b9854a5c92dc496b997d99d21_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "id": "clrkvq6iq000008ju6c16gynt", + "modelName": "gpt-4-1106-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-1106-preview)$", + "createdAt": "2024-04-23T10:37:17.092Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-1106-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkvq6iq000008ju6c16gynt_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clrkvx5gp000108juaogs54ea", + "modelName": "gpt-4-turbo-vision", + "matchPattern": "(?i)^(openai\/)?(gpt-4(-\\d{4})?-vision-preview)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-vision-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkvx5gp000108juaogs54ea_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clrkvyzgw000308jue4hse4j9", + "modelName": "gpt-4-32k", + "matchPattern": "(?i)^(openai\/)?(gpt-4-32k)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-32k", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkvyzgw000308jue4hse4j9_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "id": "clrkwk4cb000108l5hwwh3zdi", + "modelName": "gpt-4-32k-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-4-32k-0613)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-32k-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cb000108l5hwwh3zdi_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "id": "clrkwk4cb000208l59yvb9yq8", + "modelName": "gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-1106)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-1106", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cb000208l59yvb9yq8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrkwk4cc000808l51xmk4uic", + "modelName": "gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-0613)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cc000808l51xmk4uic_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrkwk4cc000908l537kl0rx3", + "modelName": "gpt-4-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-4-0613)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cc000908l537kl0rx3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "id": "clrkwk4cc000a08l562uc3s9g", + "modelName": "gpt-3.5-turbo-instruct", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-instruct)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cc000a08l562uc3s9g_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrntjt89000108jwcou1af71", + "modelName": "text-ada-001", + "matchPattern": "(?i)^(text-ada-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-ada-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000108jwcou1af71_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.000004 + } + } + ] + }, + { + "id": "clrntjt89000208jwawjr894q", + "modelName": "text-babbage-001", + "matchPattern": "(?i)^(text-babbage-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-babbage-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000208jwawjr894q_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 5e-7 + } + } + ] + }, + { + "id": "clrntjt89000308jw0jtfa4rs", + "modelName": "text-curie-001", + "matchPattern": "(?i)^(text-curie-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-curie-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000308jw0jtfa4rs_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000408jwc2c93h6i", + "modelName": "text-davinci-001", + "matchPattern": "(?i)^(text-davinci-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-davinci-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000408jwc2c93h6i_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000508jw192m64qi", + "modelName": "text-davinci-002", + "matchPattern": "(?i)^(text-davinci-002)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-davinci-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000508jw192m64qi_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000608jw4m3x5s55", + "modelName": "text-davinci-003", + "matchPattern": "(?i)^(text-davinci-003)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-davinci-003" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000608jw4m3x5s55_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000908jwhvkz5crg", + "modelName": "text-embedding-ada-002-v2", + "matchPattern": "(?i)^(text-embedding-ada-002-v2)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000908jwhvkz5crg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "clrntjt89000908jwhvkz5crm", + "modelName": "text-embedding-ada-002", + "matchPattern": "(?i)^(text-embedding-ada-002)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000908jwhvkz5crm_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "clrntjt89000a08jw0gcdbd5a", + "modelName": "gpt-3.5-turbo-16k-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-16k-0613)$", + "createdAt": "2024-02-03T17:29:57.350Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-16k-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000a08jw0gcdbd5a_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000004 + } + } + ] + }, + { + "id": "clrntkjgy000a08jx4e062mr0", + "modelName": "gpt-3.5-turbo-0301", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-0301)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": -1, + "tokenizerModel": "gpt-3.5-turbo-0301", + "tokensPerMessage": 4 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000a08jx4e062mr0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrntkjgy000d08jx0p4y9h4l", + "modelName": "gpt-4-32k-0314", + "matchPattern": "(?i)^(openai\/)?(gpt-4-32k-0314)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-32k-0314", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000d08jx0p4y9h4l_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "id": "clrntkjgy000e08jx4x6uawoo", + "modelName": "gpt-4-0314", + "matchPattern": "(?i)^(openai\/)?(gpt-4-0314)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-0314", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000e08jx4x6uawoo_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "id": "clrntkjgy000f08jx79v9g1xj", + "modelName": "gpt-4", + "matchPattern": "(?i)^(openai\/)?(gpt-4)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000f08jx79v9g1xj_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "id": "clrnwb41q000308jsfrac9uh6", + "modelName": "claude-instant-1.2", + "matchPattern": "(?i)^(anthropic\/)?(claude-instant-1.2)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwb41q000308jsfrac9uh6_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "id": "clrnwb836000408jsallr6u11", + "modelName": "claude-2.0", + "matchPattern": "(?i)^(anthropic\/)?(claude-2.0)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwb836000408jsallr6u11_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbd1m000508js4hxu6o7n", + "modelName": "claude-2.1", + "matchPattern": "(?i)^(anthropic\/)?(claude-2.1)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbd1m000508js4hxu6o7n_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbg2b000608jse2pp4q2d", + "modelName": "claude-1.3", + "matchPattern": "(?i)^(anthropic\/)?(claude-1.3)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbg2b000608jse2pp4q2d_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbi9d000708jseiy44k26", + "modelName": "claude-1.2", + "matchPattern": "(?i)^(anthropic\/)?(claude-1.2)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbi9d000708jseiy44k26_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwblo0000808jsc1385hdp", + "modelName": "claude-1.1", + "matchPattern": "(?i)^(anthropic\/)?(claude-1.1)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwblo0000808jsc1385hdp_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbota000908jsgg9mb1ml", + "modelName": "claude-instant-1", + "matchPattern": "(?i)^(anthropic\/)?(claude-instant-1)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbota000908jsgg9mb1ml_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "id": "clrs2dnql000108l46vo0gp2t", + "modelName": "babbage-002", + "matchPattern": "(?i)^(babbage-002)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "babbage-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrs2dnql000108l46vo0gp2t_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "output": 0.0000016 + } + } + ] + }, + { + "id": "clrs2ds35000208l4g4b0hi3u", + "modelName": "davinci-002", + "matchPattern": "(?i)^(davinci-002)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "davinci-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrs2ds35000208l4g4b0hi3u_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000006, + "output": 0.000012 + } + } + ] + }, + { + "id": "clruwn3pc00010al7bl611c8o", + "modelName": "text-embedding-3-small", + "matchPattern": "(?i)^(text-embedding-3-small)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwn3pc00010al7bl611c8o_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 2e-8 + } + } + ] + }, + { + "id": "clruwn76700020al7gp8e4g4l", + "modelName": "text-embedding-3-large", + "matchPattern": "(?i)^(text-embedding-3-large)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwn76700020al7gp8e4g4l_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1.3e-7 + } + } + ] + }, + { + "id": "clruwnahl00030al7ab9rark7", + "modelName": "gpt-3.5-turbo-0125", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-0125)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwnahl00030al7ab9rark7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "id": "clruwnahl00040al78f1lb0at", + "modelName": "gpt-3.5-turbo", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo)$", + "createdAt": "2024-02-13T12:00:37.424Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwnahl00040al78f1lb0at_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "id": "clruwnahl00050al796ck3p44", + "modelName": "gpt-4-0125-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-0125-preview)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwnahl00050al796ck3p44_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "cls08r8sq000308jq14ae96f0", + "modelName": "ft:gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-1106", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08r8sq000308jq14ae96f0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000006 + } + } + ] + }, + { + "id": "cls08rp99000408jqepxoakjv", + "modelName": "ft:gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-0613:)(.+)(:)(.*)(:)(.+)$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08rp99000408jqepxoakjv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000016 + } + } + ] + }, + { + "id": "cls08rv9g000508jq5p4z4nlr", + "modelName": "ft:davinci-002", + "matchPattern": "(?i)^(ft:)(davinci-002:)(.+)(:)(.*)(:)(.+)$$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokenizerModel": "davinci-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08rv9g000508jq5p4z4nlr_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000012 + } + } + ] + }, + { + "id": "cls08s2bw000608jq57wj4un2", + "modelName": "ft:babbage-002", + "matchPattern": "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokenizerModel": "babbage-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08s2bw000608jq57wj4un2_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000016, + "output": 0.0000016 + } + } + ] + }, + { + "id": "cls0iv12d000108l251gf3038", + "modelName": "chat-bison", + "matchPattern": "(?i)^(chat-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0iv12d000108l251gf3038_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0j33v1000008joagkc4lql", + "modelName": "codechat-bison-32k", + "matchPattern": "(?i)^(codechat-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0j33v1000008joagkc4lql_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jmc9v000008l8ee6r3gsd", + "modelName": "codechat-bison", + "matchPattern": "(?i)^(codechat-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jmc9v000008l8ee6r3gsd_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jmjt3000108l83ix86w0d", + "modelName": "text-bison-32k", + "matchPattern": "(?i)^(text-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jmjt3000108l83ix86w0d_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jni4t000008jk3kyy803r", + "modelName": "chat-bison-32k", + "matchPattern": "(?i)^(chat-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jni4t000008jk3kyy803r_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jungb000208jk12gm4gk1", + "modelName": "text-unicorn", + "matchPattern": "(?i)^(text-unicorn)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jungb000208jk12gm4gk1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "id": "cls0juygp000308jk2a6x9my2", + "modelName": "text-bison", + "matchPattern": "(?i)^(text-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0juygp000308jk2a6x9my2_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls1nyj5q000208l33ne901d8", + "modelName": "textembedding-gecko", + "matchPattern": "(?i)^(textembedding-gecko)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nyj5q000208l33ne901d8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "cls1nyyjp000308l31gxy1bih", + "modelName": "textembedding-gecko-multilingual", + "matchPattern": "(?i)^(textembedding-gecko-multilingual)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nyyjp000308l31gxy1bih_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "cls1nzjt3000508l3dnwad3g0", + "modelName": "code-gecko", + "matchPattern": "(?i)^(code-gecko)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nzjt3000508l3dnwad3g0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls1nzwx4000608l38va7e4tv", + "modelName": "code-bison", + "matchPattern": "(?i)^(code-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nzwx4000608l38va7e4tv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls1o053j000708l39f8g4bgs", + "modelName": "code-bison-32k", + "matchPattern": "(?i)^(code-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1o053j000708l39f8g4bgs_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "clsk9lntu000008jwfc51bbqv", + "modelName": "gpt-3.5-turbo-16k", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-16k)$", + "createdAt": "2024-02-13T12:00:37.424Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-16k", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clsk9lntu000008jwfc51bbqv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "id": "clsnq07bn000008l4e46v1ll8", + "modelName": "gpt-4-turbo-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-turbo-preview)$", + "createdAt": "2024-02-15T21:21:50.947Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clsnq07bn000008l4e46v1ll8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "cltgy0iuw000008le3vod1hhy", + "modelName": "claude-3-opus-20240229", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-opus-20240229|anthropic\\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$", + "createdAt": "2024-03-07T17:55:38.139Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cltgy0iuw000008le3vod1hhy_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.000075 + } + } + ] + }, + { + "id": "cltgy0pp6000108le56se7bl3", + "modelName": "claude-3-sonnet-20240229", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-sonnet-20240229|anthropic\\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$", + "createdAt": "2024-03-07T17:55:38.139Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cltgy0pp6000108le56se7bl3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cltr0w45b000008k1407o9qv1", + "modelName": "claude-3-haiku-20240307", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-haiku-20240307|anthropic\\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$", + "createdAt": "2024-03-14T09:41:18.736Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cltr0w45b000008k1407o9qv1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 0.00000125 + } + } + ] + }, + { + "id": "cluv2sjeo000008ih0fv23hi0", + "modelName": "gemini-1.0-pro-latest", + "matchPattern": "(?i)^(google\/)?(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2sjeo000008ih0fv23hi0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cluv2subq000108ih2mlrga6a", + "modelName": "gemini-1.0-pro", + "matchPattern": "(?i)^(google\/)?(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2subq000108ih2mlrga6a_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "id": "cluv2sx04000208ihbek75lsz", + "modelName": "gemini-1.0-pro-001", + "matchPattern": "(?i)^(google\/)?(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2sx04000208ihbek75lsz_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "id": "cluv2szw0000308ihch3n79x7", + "modelName": "gemini-pro", + "matchPattern": "(?i)^(google\/)?(gemini-pro)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2szw0000308ihch3n79x7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "id": "cluv2t2x0000408ihfytl45l1", + "modelName": "gemini-1.5-pro-latest", + "matchPattern": "(?i)^(google\/)?(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2t2x0000408ihfytl45l1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "id": "cluv2t5k3000508ih5kve9zag", + "modelName": "gpt-4-turbo-2024-04-09", + "matchPattern": "(?i)^(openai\/)?(gpt-4-turbo-2024-04-09)$", + "createdAt": "2024-04-23T10:37:17.092Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-turbo-2024-04-09", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cluv2t5k3000508ih5kve9zag_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "cluvpl4ls000008l6h2gx3i07", + "modelName": "gpt-4-turbo", + "matchPattern": "(?i)^(openai\/)?(gpt-4-turbo)$", + "createdAt": "2024-04-11T21:13:44.989Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-1106-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cluvpl4ls000008l6h2gx3i07_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clv2o2x0p000008jsf9afceau", + "modelName": " gpt-4-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-preview)$", + "createdAt": "2024-04-23T10:37:17.092Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-turbo-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clv2o2x0p000008jsf9afceau_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clxt0n0m60000pumz1j5b7zsf", + "modelName": "claude-3-5-sonnet-20240620", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-sonnet-20240620|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$", + "createdAt": "2024-06-25T11:47:24.475Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clxt0n0m60000pumz1j5b7zsf_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "clyrjp56f0000t0mzapoocd7u", + "modelName": "gpt-4o-mini", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-mini)$", + "createdAt": "2024-07-18T17:56:09.591Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clyrjp56f0000t0mzapoocd7u_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "output": 6e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8 + } + } + ] + }, + { + "id": "clyrjpbe20000t0mzcbwc42rg", + "modelName": "gpt-4o-mini-2024-07-18", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-mini-2024-07-18)$", + "createdAt": "2024-07-18T17:56:09.591Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clyrjpbe20000t0mzcbwc42rg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8, + "output": 6e-7 + } + } + ] + }, + { + "id": "clzjr85f70000ymmzg7hqffra", + "modelName": "gpt-4o-2024-08-06", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-2024-08-06)$", + "createdAt": "2024-08-07T11:54:31.298Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clzjr85f70000ymmzg7hqffra_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "id": "cm10ivcdp0000gix7lelmbw80", + "modelName": "o1-preview", + "matchPattern": "(?i)^(openai\/)?(o1-preview)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10ivcdp0000gix7lelmbw80_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm10ivo130000n8x7qopcjjcg", + "modelName": "o1-preview-2024-09-12", + "matchPattern": "(?i)^(openai\/)?(o1-preview-2024-09-12)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10ivo130000n8x7qopcjjcg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm10ivwo40000r1x7gg3syjq0", + "modelName": "o1-mini", + "matchPattern": "(?i)^(openai\/)?(o1-mini)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10ivwo40000r1x7gg3syjq0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm10iw6p20000wgx7it1hlb22", + "modelName": "o1-mini-2024-09-12", + "matchPattern": "(?i)^(openai\/)?(o1-mini-2024-09-12)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10iw6p20000wgx7it1hlb22_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm2krz1uf000208jjg5653iud", + "modelName": "claude-3.5-sonnet-20241022", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-sonnet-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20241022-v2:0|claude-3-5-sonnet-V2@20241022)$", + "createdAt": "2024-10-22T18:48:01.676Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm2krz1uf000208jjg5653iud_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm2ks2vzn000308jjh4ze1w7q", + "modelName": "claude-3.5-sonnet-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-sonnet-latest)$", + "createdAt": "2024-10-22T18:48:01.676Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm2ks2vzn000308jjh4ze1w7q_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm34aq60d000207ml0j1h31ar", + "modelName": "claude-3-5-haiku-20241022", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-haiku-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$", + "createdAt": "2024-11-05T10:30:50.566Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm34aq60d000207ml0j1h31ar_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "id": "cm34aqb9h000307ml6nypd618", + "modelName": "claude-3.5-haiku-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-haiku-latest)$", + "createdAt": "2024-11-05T10:30:50.566Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm34aqb9h000307ml6nypd618_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "id": "cm3x0p8ev000008kyd96800c8", + "modelName": "chatgpt-4o-latest", + "matchPattern": "(?i)^(chatgpt-4o-latest)$", + "createdAt": "2024-11-25T12:47:17.504Z", + "updatedAt": "2024-11-25T12:47:17.504Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm3x0p8ev000008kyd96800c8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "id": "cm48akqgo000008ldbia24qg0", + "modelName": "gpt-4o-2024-11-20", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-2024-11-20)$", + "createdAt": "2024-12-03T10:06:12.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm48akqgo000008ldbia24qg0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "id": "cm48b2ksh000008l0hn3u0hl3", + "modelName": "gpt-4o-audio-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-audio-preview)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48b2ksh000008l0hn3u0hl3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48bbm0k000008l69nsdakwf", + "modelName": "gpt-4o-audio-preview-2024-10-01", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-audio-preview-2024-10-01)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48bbm0k000008l69nsdakwf_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48c2qh4000008mhgy4mg2qc", + "modelName": "gpt-4o-realtime-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-realtime-preview)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48c2qh4000008mhgy4mg2qc_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48cjxtc000008jrcsso3avv", + "modelName": "gpt-4o-realtime-preview-2024-10-01", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-realtime-preview-2024-10-01)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48cjxtc000008jrcsso3avv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48cjxtc000108jrcsso3avv", + "modelName": "o1", + "matchPattern": "(?i)^(openai\/)?(o1)$", + "createdAt": "2025-01-17T00:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48cjxtc000108jrcsso3avv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm48cjxtc000208jrcsso3avv", + "modelName": "o1-2024-12-17", + "matchPattern": "(?i)^(openai\/)?(o1-2024-12-17)$", + "createdAt": "2025-01-17T00:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48cjxtc000208jrcsso3avv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm6l8j7vs0000tymz9vk7ew8t", + "modelName": "o3-mini", + "matchPattern": "(?i)^(openai\/)?(o3-mini)$", + "createdAt": "2025-01-31T20:41:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8j7vs0000tymz9vk7ew8t_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm6l8jan90000tymz52sh0ql8", + "modelName": "o3-mini-2025-01-31", + "matchPattern": "(?i)^(openai\/)?(o3-mini-2025-01-31)$", + "createdAt": "2025-01-31T20:41:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8jan90000tymz52sh0ql8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm6l8jdef0000tymz52sh0ql0", + "modelName": "gemini-2.0-flash-001", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash-001)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-02-06T11:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8jdef0000tymz52sh0ql0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm6l8jfgh0000tymz52sh0ql1", + "modelName": "gemini-2.0-flash-lite-preview-02-05", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash-lite-preview-02-05)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-02-06T11:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8jfgh0000tymz52sh0ql1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "id": "cm7ka7561000108js3t9tb3at", + "modelName": "claude-3.7-sonnet-20250219", + "matchPattern": "(?i)^(anthropic\/)?(claude-3.7-sonnet-20250219|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3.7-sonnet-20250219-v1:0|claude-3-7-sonnet-V1@20250219)$", + "createdAt": "2025-02-25T09:35:39.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm7ka7561000108js3t9tb3at_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm7ka7zob000208jsfs9h5ajj", + "modelName": "claude-3.7-sonnet-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-7-sonnet-latest)$", + "createdAt": "2025-02-25T09:35:39.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm7ka7zob000208jsfs9h5ajj_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm7nusjvk0000tvmz71o85jwg", + "modelName": "gpt-4.5-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4.5-preview)$", + "createdAt": "2025-02-27T21:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7nusjvk0000tvmz71o85jwg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "id": "cm7nusn640000tvmzf10z2x65", + "modelName": "gpt-4.5-preview-2025-02-27", + "matchPattern": "(?i)^(openai\/)?(gpt-4.5-preview-2025-02-27)$", + "createdAt": "2025-02-27T21:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7nusn640000tvmzf10z2x65_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "id": "cm7nusn643377tvmzh27m33kl", + "modelName": "gpt-4.1", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7nusn643377tvmzh27m33kl_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "id": "cm7qahw732891bpmzy45r3x70", + "modelName": "gpt-4.1-2025-04-14", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-2025-04-14)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7qahw732891bpmzy45r3x70_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "id": "cm7sglt825463kxnza72p6v81", + "modelName": "gpt-4.1-mini-2025-04-14", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-mini-2025-04-14)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7sglt825463kxnza72p6v81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "id": "cm7vxpz967124dhjtb95w8f92", + "modelName": "gpt-4.1-nano-2025-04-14", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-nano-2025-04-14)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7vxpz967124dhjtb95w8f92_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm7wmny967124dhjtb95w8f81", + "modelName": "o3", + "matchPattern": "(?i)^(openai\/)?(o3)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7wmny967124dhjtb95w8f81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "id": "cm7wopq3327124dhjtb95w8f81", + "modelName": "o3-2025-04-16", + "matchPattern": "(?i)^(openai\/)?(o3-2025-04-16)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7wopq3327124dhjtb95w8f81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "id": "cm7wqrs1327124dhjtb95w8f81", + "modelName": "o4-mini", + "matchPattern": "(?i)^(o4-mini)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "id": "cm7wqrs1327124dhjtb95w8f81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm7zqrs1327124dhjtb95w8f82", + "modelName": "o4-mini-2025-04-16", + "matchPattern": "(?i)^(o4-mini-2025-04-16)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "id": "cm7zqrs1327124dhjtb95w8f82_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm7zsrs1327124dhjtb95w8f74", + "modelName": "gemini-2.0-flash", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7zsrs1327124dhjtb95w8f74_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm7ztrs1327124dhjtb95w8f19", + "modelName": "gemini-2.0-flash-lite-preview", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash-lite-preview)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7ztrs1327124dhjtb95w8f19_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "id": "cm7zxrs1327124dhjtb95w8f45", + "modelName": "gpt-4.1-nano", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-nano)$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7zxrs1327124dhjtb95w8f45_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm7zzrs1327124dhjtb95w8p96", + "modelName": "gpt-4.1-mini", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-mini)$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7zzrs1327124dhjtb95w8p96_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "id": "c5qmrqolku82tra3vgdixmys", + "modelName": "claude-sonnet-4-5-20250929", + "matchPattern": "(?i)^(anthropic\/)?(claude-sonnet-4-5(-20250929)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-5(-20250929)?-v1(:0)?|claude-sonnet-4-5-V1(@20250929)?|claude-sonnet-4-5(@20250929)?)$", + "createdAt": "2025-09-29T00:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "c5qmrqolku82tra3vgdixmys_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "id": "00b65240-047b-4722-9590-808edbc2067f", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "id": "cmazmkzlm00000djp1e1qe4k4", + "modelName": "claude-sonnet-4-20250514", + "matchPattern": "(?i)^(anthropic\/)?(claude-sonnet-4(-20250514)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4(-20250514)?-v1(:0)?|claude-sonnet-4-V1(@20250514)?|claude-sonnet-4(@20250514)?)$", + "createdAt": "2025-05-22T17:09:02.131Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmazmkzlm00000djp1e1qe4k4_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cmazmlbnv00010djpazed91va", + "modelName": "claude-sonnet-4-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-sonnet-4-latest)$", + "createdAt": "2025-05-22T17:09:02.131Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmazmlbnv00010djpazed91va_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cmazmlm2p00020djpa9s64jw5", + "modelName": "claude-opus-4-20250514", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4(-20250514)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4(-20250514)?-v1(:0)?|claude-opus-4(@20250514)?)$", + "createdAt": "2025-05-22T17:09:02.131Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmazmlm2p00020djpa9s64jw5_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "id": "cmz9x72kq55721pqrs83y4n2bx", + "modelName": "o3-pro", + "matchPattern": "(?i)^(openai\/)?(o3-pro)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmz9x72kq55721pqrs83y4n2bx_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "id": "cmz9x72kq55721pqrs83y4n2by", + "modelName": "o3-pro-2025-06-10", + "matchPattern": "(?i)^(openai\/)?(o3-pro-2025-06-10)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmz9x72kq55721pqrs83y4n2by_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "id": "cmbrold5b000107lbftb9fdoo", + "modelName": "o1-pro", + "matchPattern": "(?i)^(openai\/)?(o1-pro)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmbrold5b000107lbftb9fdoo_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "id": "cmbrolpax000207lb3xkedysz", + "modelName": "o1-pro-2025-03-19", + "matchPattern": "(?i)^(openai\/)?(o1-pro-2025-03-19)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmbrolpax000207lb3xkedysz_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "id": "cmcnjkfwn000107l43bf5e8ax", + "modelName": "gemini-2.5-flash", + "matchPattern": "(?i)^(google\/)?(gemini-2.5-flash)$", + "createdAt": "2025-07-03T13:44:06.964Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "pricingTiers": [ + { + "id": "cmcnjkfwn000107l43bf5e8ax_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 3e-7, + "input_text": 3e-7, + "input_modality_1": 3e-7, + "prompt_token_count": 3e-7, + "promptTokenCount": 3e-7, + "input_cached_tokens": 3e-8, + "cached_content_token_count": 3e-8, + "output": 0.0000025, + "output_modality_1": 0.0000025, + "candidates_token_count": 0.0000025, + "candidatesTokenCount": 0.0000025, + "thoughtsTokenCount": 0.0000025, + "thoughts_token_count": 0.0000025, + "output_reasoning": 0.0000025, + "input_audio_tokens": 0.000001 + } + } + ] + }, + { + "id": "cmcnjkrfa000207l4fpnh5mnv", + "modelName": "gemini-2.5-flash-lite", + "matchPattern": "(?i)^(google\/)?(gemini-2.5-flash-lite)$", + "createdAt": "2025-07-03T13:44:06.964Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "pricingTiers": [ + { + "id": "cmcnjkrfa000207l4fpnh5mnv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_text": 1e-7, + "input_modality_1": 1e-7, + "prompt_token_count": 1e-7, + "promptTokenCount": 1e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 4e-7, + "output_modality_1": 4e-7, + "candidates_token_count": 4e-7, + "candidatesTokenCount": 4e-7, + "thoughtsTokenCount": 4e-7, + "thoughts_token_count": 4e-7, + "output_reasoning": 4e-7, + "input_audio_tokens": 5e-7 + } + } + ] + }, + { + "id": "cmdysde5w0000rkmzbc1g5au3", + "modelName": "claude-opus-4-1-20250805", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4-1(-20250805)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4-1(-20250805)?-v1(:0)?|claude-opus-4-1(@20250805)?)$", + "createdAt": "2025-08-05T15:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmdysde5w0000rkmzbc1g5au3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "id": "38c3822a-09a3-457b-b200-2c6f17f7cf2f", + "modelName": "gpt-5", + "matchPattern": "(?i)^(openai\/)?(gpt-5)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "38c3822a-09a3-457b-b200-2c6f17f7cf2f_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "12543803-2d5f-4189-addc-821ad71c8b55", + "modelName": "gpt-5-2025-08-07", + "matchPattern": "(?i)^(openai\/)?(gpt-5-2025-08-07)$", + "createdAt": "2025-08-11T08:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "12543803-2d5f-4189-addc-821ad71c8b55_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "3d6a975a-a42d-4ea2-a3ec-4ae567d5a364", + "modelName": "gpt-5-mini", + "matchPattern": "(?i)^(openai\/)?(gpt-5-mini)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "3d6a975a-a42d-4ea2-a3ec-4ae567d5a364_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "id": "03b83894-7172-4e1e-8e8b-37d792484efd", + "modelName": "gpt-5-mini-2025-08-07", + "matchPattern": "(?i)^(openai\/)?(gpt-5-mini-2025-08-07)$", + "createdAt": "2025-08-11T08:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "03b83894-7172-4e1e-8e8b-37d792484efd_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "id": "f0b40234-b694-4c40-9494-7b0efd860fb9", + "modelName": "gpt-5-nano", + "matchPattern": "(?i)^(openai\/)?(gpt-5-nano)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "f0b40234-b694-4c40-9494-7b0efd860fb9_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "id": "4489fde4-a594-4011-948b-526989300cd3", + "modelName": "gpt-5-nano-2025-08-07", + "matchPattern": "(?i)^(openai\/)?(gpt-5-nano-2025-08-07)$", + "createdAt": "2025-08-11T08:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "4489fde4-a594-4011-948b-526989300cd3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "id": "8ba72ee3-ebe8-4110-a614-bf81094447e5", + "modelName": "gpt-5-chat-latest", + "matchPattern": "(?i)^(openai\/)?(gpt-5-chat-latest)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "8ba72ee3-ebe8-4110-a614-bf81094447e5_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "cmgg9zco3000004l258um9xk8", + "modelName": "gpt-5-pro", + "matchPattern": "(?i)^(openai\/)?(gpt-5-pro)$", + "createdAt": "2025-10-07T08:03:54.727Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmgg9zco3000004l258um9xk8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "id": "cmgga0vh9000104l22qe4fes4", + "modelName": "gpt-5-pro-2025-10-06", + "matchPattern": "(?i)^(openai\/)?(gpt-5-pro-2025-10-06)$", + "createdAt": "2025-10-07T08:03:54.727Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmgga0vh9000104l22qe4fes4_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "id": "cmgt5gnkv000104jx171tbq4e", + "modelName": "claude-haiku-4-5-20251001", + "matchPattern": "(?i)^(anthropic\/)?(claude-haiku-4-5-20251001|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-haiku-4-5-20251001-v1:0|claude-4-5-haiku@20251001)$", + "createdAt": "2025-10-16T08:20:44.558Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmgt5gnkv000104jx171tbq4e_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "input_tokens": 0.000001, + "output": 0.000005, + "output_tokens": 0.000005, + "cache_creation_input_tokens": 0.00000125, + "input_cache_creation": 0.00000125, + "input_cache_creation_5m": 0.00000125, + "input_cache_creation_1h": 0.000002, + "cache_read_input_tokens": 1e-7, + "input_cache_read": 1e-7 + } + } + ] + }, + { + "id": "cmhymgpym000d04ih34rndvhr", + "modelName": "gpt-5.1", + "matchPattern": "(?i)^(openai\/)?(gpt-5.1)$", + "createdAt": "2025-11-14T08:57:23.481Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmhymgpym000d04ih34rndvhr_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "cmhymgxiw000e04ihh9pw12ef", + "modelName": "gpt-5.1-2025-11-13", + "matchPattern": "(?i)^(openai\/)?(gpt-5.1-2025-11-13)$", + "createdAt": "2025-11-14T08:57:23.481Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmhymgxiw000e04ihh9pw12ef_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "cmieupdva000004l541kwae70", + "modelName": "claude-opus-4-5-20251101", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4-5(-20251101)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-5(-20251101)?-v1(:0)?|claude-opus-4-5(@20251101)?)$", + "createdAt": "2025-11-24T20:53:27.571Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerConfig": null, + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmieupdva000004l541kwae70_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-6, + "input_tokens": 5e-6, + "output": 25e-6, + "output_tokens": 25e-6, + "cache_creation_input_tokens": 6.25e-6, + "input_cache_creation": 6.25e-6, + "input_cache_creation_5m": 6.25e-6, + "input_cache_creation_1h": 10e-6, + "cache_read_input_tokens": 0.5e-6, + "input_cache_read": 0.5e-6 + } + } + ] + }, + { + "id": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelName": "claude-sonnet-4-6", + "matchPattern": "(?i)^(anthropic\\/)?(claude-sonnet-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-6(-v1(:0)?)?|claude-sonnet-4-6)$", + "createdAt": "2026-02-18T00:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerConfig": null, + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 3e-6, + "input_tokens": 3e-6, + "output": 15e-6, + "output_tokens": 15e-6, + "cache_creation_input_tokens": 3.75e-6, + "input_cache_creation": 3.75e-6, + "input_cache_creation_5m": 3.75e-6, + "input_cache_creation_1h": 6e-6, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "id": "7830bfc2-c464-4ffe-b9a2-6e741f6c5486", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 6e-6, + "input_tokens": 6e-6, + "output": 22.5e-6, + "output_tokens": 22.5e-6, + "cache_creation_input_tokens": 7.5e-6, + "input_cache_creation": 7.5e-6, + "input_cache_creation_5m": 7.5e-6, + "input_cache_creation_1h": 12e-6, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "id": "13458bc0-1c20-44c2-8753-172f54b67647", + "modelName": "claude-opus-4-6", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-6-v1(:0)?|claude-opus-4-6)$", + "createdAt": "2026-02-09T00:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerConfig": null, + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "13458bc0-1c20-44c2-8753-172f54b67647_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-6, + "input_tokens": 5e-6, + "output": 25e-6, + "output_tokens": 25e-6, + "cache_creation_input_tokens": 6.25e-6, + "input_cache_creation": 6.25e-6, + "input_cache_creation_5m": 6.25e-6, + "input_cache_creation_1h": 10e-6, + "cache_read_input_tokens": 0.5e-6, + "input_cache_read": 0.5e-6 + } + } + ] + }, + { + "id": "cmig1hb7i000104l72qrzgc6h", + "modelName": "gemini-2.5-pro", + "matchPattern": "(?i)^(google\/)?(gemini-2.5-pro)$", + "createdAt": "2025-11-26T13:27:53.545Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "pricingTiers": [ + { + "id": "cmig1hb7i000104l72qrzgc6h_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-6, + "input_text": 1.25e-6, + "input_modality_1": 1.25e-6, + "prompt_token_count": 1.25e-6, + "promptTokenCount": 1.25e-6, + "input_cached_tokens": 0.125e-6, + "cached_content_token_count": 0.125e-6, + "output": 10e-6, + "output_modality_1": 10e-6, + "candidates_token_count": 10e-6, + "candidatesTokenCount": 10e-6, + "thoughtsTokenCount": 10e-6, + "thoughts_token_count": 10e-6, + "output_reasoning": 10e-6 + } + }, + { + "id": "bcf39e8f-9969-455f-be9a-541a00256092", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 2.5e-6, + "input_text": 2.5e-6, + "input_modality_1": 2.5e-6, + "prompt_token_count": 2.5e-6, + "promptTokenCount": 2.5e-6, + "input_cached_tokens": 0.25e-6, + "cached_content_token_count": 0.25e-6, + "output": 15e-6, + "output_modality_1": 15e-6, + "candidates_token_count": 15e-6, + "candidatesTokenCount": 15e-6, + "thoughtsTokenCount": 15e-6, + "thoughts_token_count": 15e-6, + "output_reasoning": 15e-6 + } + } + ] + }, + { + "id": "cmig1wmep000404l7fh6q5uog", + "modelName": "gemini-3-pro-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3-pro-preview)$", + "createdAt": "2025-11-26T13:27:53.545Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmig1wmep000404l7fh6q5uog_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2e-6, + "input_modality_1": 2e-6, + "prompt_token_count": 2e-6, + "promptTokenCount": 2e-6, + "input_cached_tokens": 0.2e-6, + "cached_content_token_count": 0.2e-6, + "output": 12e-6, + "output_modality_1": 12e-6, + "candidates_token_count": 12e-6, + "candidatesTokenCount": 12e-6, + "thoughtsTokenCount": 12e-6, + "thoughts_token_count": 12e-6, + "output_reasoning": 12e-6 + } + }, + { + "id": "4da930c8-7146-4e27-b66c-b62f2c2ec357", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 4e-6, + "input_modality_1": 4e-6, + "prompt_token_count": 4e-6, + "promptTokenCount": 4e-6, + "input_cached_tokens": 0.4e-6, + "cached_content_token_count": 0.4e-6, + "output": 18e-6, + "output_modality_1": 18e-6, + "candidates_token_count": 18e-6, + "candidatesTokenCount": 18e-6, + "thoughtsTokenCount": 18e-6, + "thoughts_token_count": 18e-6, + "output_reasoning": 18e-6 + } + } + ] + }, + { + "id": "55106bba-a5dd-441b-bc0d-5652582b349d", + "modelName": "gemini-3.1-pro-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3.1-pro-preview(-customtools)?)$", + "createdAt": "2026-02-19T00:00:00.000Z", + "updatedAt": "2026-02-19T00:00:00.000Z", + "pricingTiers": [ + { + "id": "55106bba-a5dd-441b-bc0d-5652582b349d_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2e-6, + "input_modality_1": 2e-6, + "prompt_token_count": 2e-6, + "promptTokenCount": 2e-6, + "input_cached_tokens": 0.2e-6, + "cached_content_token_count": 0.2e-6, + "output": 12e-6, + "output_modality_1": 12e-6, + "candidates_token_count": 12e-6, + "candidatesTokenCount": 12e-6, + "thoughtsTokenCount": 12e-6, + "thoughts_token_count": 12e-6, + "output_reasoning": 12e-6 + } + }, + { + "id": "ada11e9f-fe0d-465a-92af-ce334d0eedeb", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 4e-6, + "input_modality_1": 4e-6, + "prompt_token_count": 4e-6, + "promptTokenCount": 4e-6, + "input_cached_tokens": 0.4e-6, + "cached_content_token_count": 0.4e-6, + "output": 18e-6, + "output_modality_1": 18e-6, + "candidates_token_count": 18e-6, + "candidatesTokenCount": 18e-6, + "thoughtsTokenCount": 18e-6, + "thoughts_token_count": 18e-6, + "output_reasoning": 18e-6 + } + } + ] + }, + { + "id": "cmj2n4f2a000304kz49g4c43u", + "modelName": "gpt-5.2", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2n4f2a000304kz49g4c43u_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.75e-6, + "input_cached_tokens": 0.175e-6, + "input_cache_read": 0.175e-6, + "output": 14e-6, + "output_reasoning_tokens": 14e-6, + "output_reasoning": 14e-6 + } + } + ] + }, + { + "id": "cmj2muxg6000104kzd2tc8953", + "modelName": "gpt-5.2-2025-12-11", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2-2025-12-11)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2muxg6000104kzd2tc8953_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.75e-6, + "input_cached_tokens": 0.175e-6, + "input_cache_read": 0.175e-6, + "output": 14e-6, + "output_reasoning_tokens": 14e-6, + "output_reasoning": 14e-6 + } + } + ] + }, + { + "id": "cmj2n6pkq000404kz2s0b6if7", + "modelName": "gpt-5.2-pro", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2-pro)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2n6pkq000404kz2s0b6if7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 21e-6, + "output": 168e-6, + "output_reasoning_tokens": 168e-6, + "output_reasoning": 168e-6 + } + } + ] + }, + { + "id": "cmj2n70oe000504kz21b76mes", + "modelName": "gpt-5.2-pro-2025-12-11", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2-pro-2025-12-11)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2n70oe000504kz21b76mes_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 21e-6, + "output": 168e-6, + "output_reasoning_tokens": 168e-6, + "output_reasoning": 168e-6 + } + } + ] + }, + { + "id": "bee3c111-fe6f-4641-8775-73ea33b29fca", + "modelName": "gpt-5.4", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "bee3c111-fe6f-4641-8775-73ea33b29fca_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-6, + "input_cached_tokens": 0.25e-6, + "input_cache_read": 0.25e-6, + "output": 15e-6, + "output_reasoning_tokens": 15e-6, + "output_reasoning": 15e-6 + } + } + ] + }, + { + "id": "68d32054-8748-4d25-9f64-d78d483601bd", + "modelName": "gpt-5.4-pro", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4-pro)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "68d32054-8748-4d25-9f64-d78d483601bd_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 30e-6, + "output": 180e-6, + "output_reasoning_tokens": 180e-6, + "output_reasoning": 180e-6 + } + } + ] + }, + { + "id": "22dfc7e1-1fe1-4286-b1af-928635e7ecb9", + "modelName": "gpt-5.4-2026-03-05", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4-2026-03-05)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "22dfc7e1-1fe1-4286-b1af-928635e7ecb9_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-6, + "input_cached_tokens": 0.25e-6, + "input_cache_read": 0.25e-6, + "output": 15e-6, + "output_reasoning_tokens": 15e-6, + "output_reasoning": 15e-6 + } + } + ] + }, + { + "id": "d8873413-05ab-4374-8223-e8c9005c4a0e", + "modelName": "gpt-5.4-pro-2026-03-05", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4-pro-2026-03-05)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "d8873413-05ab-4374-8223-e8c9005c4a0e_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 30e-6, + "output": 180e-6, + "output_reasoning_tokens": 180e-6, + "output_reasoning": 180e-6 + } + } + ] + }, + { + "id": "cmjfoeykl000004l8ffzra8c7", + "modelName": "gemini-3-flash-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3-flash-preview)$", + "createdAt": "2025-12-21T12:01:42.282Z", + "updatedAt": "2025-12-21T12:01:42.282Z", + "pricingTiers": [ + { + "id": "cmjfoeykl000004l8ffzra8c7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.5e-6, + "input_modality_1": 0.5e-6, + "prompt_token_count": 0.5e-6, + "promptTokenCount": 0.5e-6, + "input_cached_tokens": 0.05e-6, + "cached_content_token_count": 0.05e-6, + "output": 3e-6, + "output_modality_1": 3e-6, + "candidates_token_count": 3e-6, + "candidatesTokenCount": 3e-6, + "thoughtsTokenCount": 3e-6, + "thoughts_token_count": 3e-6, + "output_reasoning": 3e-6 + } + } + ] + }, + { + "id": "0707bef8-10c8-46e2-a871-0436f05f0b92", + "modelName": "gemini-3.1-flash-lite-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3.1-flash-lite-preview)$", + "createdAt": "2026-03-03T00:00:00.000Z", + "updatedAt": "2026-03-03T00:00:00.000Z", + "pricingTiers": [ + { + "id": "0707bef8-10c8-46e2-a871-0436f05f0b92_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.25e-6, + "input_modality_1": 0.25e-6, + "prompt_token_count": 0.25e-6, + "promptTokenCount": 0.25e-6, + "input_cached_tokens": 0.025e-6, + "cached_content_token_count": 0.025e-6, + "output": 1.5e-6, + "output_modality_1": 1.5e-6, + "candidates_token_count": 1.5e-6, + "candidatesTokenCount": 1.5e-6, + "thoughtsTokenCount": 1.5e-6, + "thoughts_token_count": 1.5e-6, + "output_reasoning": 1.5e-6, + "input_audio_tokens": 0.5e-6 + } + } + ] + } +] diff --git a/internal-packages/llm-pricing/src/defaultPrices.ts b/internal-packages/llm-pricing/src/defaultPrices.ts new file mode 100644 index 00000000000..91b8a6f89f6 --- /dev/null +++ b/internal-packages/llm-pricing/src/defaultPrices.ts @@ -0,0 +1,2984 @@ +import type { DefaultModelDefinition } from "./types.js"; + +// Auto-generated from Langfuse default-model-prices.json — do not edit manually. +// Run `pnpm run sync-prices` to update from upstream. +// Source: https://github.com/langfuse/langfuse + +export const defaultModelPrices: DefaultModelDefinition[] = [ + { + "modelName": "gpt-4o", + "matchPattern": "(?i)^(openai/)?(gpt-4o)$", + "startDate": "2024-05-13T23:15:07.670Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-05-13", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-05-13)$", + "startDate": "2024-05-13T23:15:07.670Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-4-1106-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-1106-preview)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-vision", + "matchPattern": "(?i)^(openai/)?(gpt-4(-\\d{4})?-vision-preview)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-32k", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-4-32k-0613", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-1106)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-4-0613", + "matchPattern": "(?i)^(openai/)?(gpt-4-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-instruct", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-instruct)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "text-ada-001", + "matchPattern": "(?i)^(text-ada-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.000004 + } + } + ] + }, + { + "modelName": "text-babbage-001", + "matchPattern": "(?i)^(text-babbage-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 5e-7 + } + } + ] + }, + { + "modelName": "text-curie-001", + "matchPattern": "(?i)^(text-curie-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-001", + "matchPattern": "(?i)^(text-davinci-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-002", + "matchPattern": "(?i)^(text-davinci-002)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-003", + "matchPattern": "(?i)^(text-davinci-003)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-embedding-ada-002-v2", + "matchPattern": "(?i)^(text-embedding-ada-002-v2)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "text-embedding-ada-002", + "matchPattern": "(?i)^(text-embedding-ada-002)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-16k-0613", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k-0613)$", + "startDate": "2024-02-03T17:29:57.350Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000004 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0301", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0301)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-4-32k-0314", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k-0314)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-4-0314", + "matchPattern": "(?i)^(openai/)?(gpt-4-0314)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "gpt-4", + "matchPattern": "(?i)^(openai/)?(gpt-4)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "claude-instant-1.2", + "matchPattern": "(?i)^(anthropic/)?(claude-instant-1.2)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "modelName": "claude-2.0", + "matchPattern": "(?i)^(anthropic/)?(claude-2.0)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-2.1", + "matchPattern": "(?i)^(anthropic/)?(claude-2.1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.3", + "matchPattern": "(?i)^(anthropic/)?(claude-1.3)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.2", + "matchPattern": "(?i)^(anthropic/)?(claude-1.2)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.1", + "matchPattern": "(?i)^(anthropic/)?(claude-1.1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-instant-1", + "matchPattern": "(?i)^(anthropic/)?(claude-instant-1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "modelName": "babbage-002", + "matchPattern": "(?i)^(babbage-002)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "davinci-002", + "matchPattern": "(?i)^(davinci-002)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000006, + "output": 0.000012 + } + } + ] + }, + { + "modelName": "text-embedding-3-small", + "matchPattern": "(?i)^(text-embedding-3-small)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 2e-8 + } + } + ] + }, + { + "modelName": "text-embedding-3-large", + "matchPattern": "(?i)^(text-embedding-3-large)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1.3e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0125", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0125)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo)$", + "startDate": "2024-02-13T12:00:37.424Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-4-0125-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-0125-preview)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "ft:gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000006 + } + } + ] + }, + { + "modelName": "ft:gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-0613:)(.+)(:)(.*)(:)(.+)$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000016 + } + } + ] + }, + { + "modelName": "ft:davinci-002", + "matchPattern": "(?i)^(ft:)(davinci-002:)(.+)(:)(.*)(:)(.+)$$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000012 + } + } + ] + }, + { + "modelName": "ft:babbage-002", + "matchPattern": "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000016, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "chat-bison", + "matchPattern": "(?i)^(chat-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "codechat-bison-32k", + "matchPattern": "(?i)^(codechat-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "codechat-bison", + "matchPattern": "(?i)^(codechat-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "text-bison-32k", + "matchPattern": "(?i)^(text-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "chat-bison-32k", + "matchPattern": "(?i)^(chat-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "text-unicorn", + "matchPattern": "(?i)^(text-unicorn)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "modelName": "text-bison", + "matchPattern": "(?i)^(text-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "textembedding-gecko", + "matchPattern": "(?i)^(textembedding-gecko)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "textembedding-gecko-multilingual", + "matchPattern": "(?i)^(textembedding-gecko-multilingual)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "code-gecko", + "matchPattern": "(?i)^(code-gecko)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "code-bison", + "matchPattern": "(?i)^(code-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "code-bison-32k", + "matchPattern": "(?i)^(code-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-16k", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k)$", + "startDate": "2024-02-13T12:00:37.424Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo-preview)$", + "startDate": "2024-02-15T21:21:50.947Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "claude-3-opus-20240229", + "matchPattern": "(?i)^(anthropic/)?(claude-3-opus-20240229|anthropic\\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$", + "startDate": "2024-03-07T17:55:38.139Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.000075 + } + } + ] + }, + { + "modelName": "claude-3-sonnet-20240229", + "matchPattern": "(?i)^(anthropic/)?(claude-3-sonnet-20240229|anthropic\\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$", + "startDate": "2024-03-07T17:55:38.139Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3-haiku-20240307", + "matchPattern": "(?i)^(anthropic/)?(claude-3-haiku-20240307|anthropic\\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$", + "startDate": "2024-03-14T09:41:18.736Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 0.00000125 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro-latest", + "matchPattern": "(?i)^(google/)?(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro", + "matchPattern": "(?i)^(google/)?(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro-001", + "matchPattern": "(?i)^(google/)?(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-pro", + "matchPattern": "(?i)^(google/)?(gemini-pro)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-1.5-pro-latest", + "matchPattern": "(?i)^(google/)?(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-2024-04-09", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo-2024-04-09)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-turbo", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo)$", + "startDate": "2024-04-11T21:13:44.989Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": " gpt-4-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-preview)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "claude-3-5-sonnet-20240620", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-20240620|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$", + "startDate": "2024-06-25T11:47:24.475Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4o-mini", + "matchPattern": "(?i)^(openai/)?(gpt-4o-mini)$", + "startDate": "2024-07-18T17:56:09.591Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "output": 6e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8 + } + } + ] + }, + { + "modelName": "gpt-4o-mini-2024-07-18", + "matchPattern": "(?i)^(openai/)?(gpt-4o-mini-2024-07-18)$", + "startDate": "2024-07-18T17:56:09.591Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8, + "output": 6e-7 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-08-06", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-08-06)$", + "startDate": "2024-08-07T11:54:31.298Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "o1-preview", + "matchPattern": "(?i)^(openai/)?(o1-preview)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-preview-2024-09-12", + "matchPattern": "(?i)^(openai/)?(o1-preview-2024-09-12)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-mini", + "matchPattern": "(?i)^(openai/)?(o1-mini)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o1-mini-2024-09-12", + "matchPattern": "(?i)^(openai/)?(o1-mini-2024-09-12)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "claude-3.5-sonnet-20241022", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20241022-v2:0|claude-3-5-sonnet-V2@20241022)$", + "startDate": "2024-10-22T18:48:01.676Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.5-sonnet-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-latest)$", + "startDate": "2024-10-22T18:48:01.676Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3-5-haiku-20241022", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-haiku-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$", + "startDate": "2024-11-05T10:30:50.566Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "modelName": "claude-3.5-haiku-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-haiku-latest)$", + "startDate": "2024-11-05T10:30:50.566Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "modelName": "chatgpt-4o-latest", + "matchPattern": "(?i)^(chatgpt-4o-latest)$", + "startDate": "2024-11-25T12:47:17.504Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-11-20", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-11-20)$", + "startDate": "2024-12-03T10:06:12.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-4o-audio-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4o-audio-preview)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-audio-preview-2024-10-01", + "matchPattern": "(?i)^(openai/)?(gpt-4o-audio-preview-2024-10-01)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-realtime-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4o-realtime-preview)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-realtime-preview-2024-10-01", + "matchPattern": "(?i)^(openai/)?(gpt-4o-realtime-preview-2024-10-01)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "o1", + "matchPattern": "(?i)^(openai/)?(o1)$", + "startDate": "2025-01-17T00:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-2024-12-17", + "matchPattern": "(?i)^(openai/)?(o1-2024-12-17)$", + "startDate": "2025-01-17T00:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o3-mini", + "matchPattern": "(?i)^(openai/)?(o3-mini)$", + "startDate": "2025-01-31T20:41:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o3-mini-2025-01-31", + "matchPattern": "(?i)^(openai/)?(o3-mini-2025-01-31)$", + "startDate": "2025-01-31T20:41:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-001", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash-001)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-02-06T11:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-lite-preview-02-05", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash-lite-preview-02-05)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-02-06T11:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.7-sonnet-20250219", + "matchPattern": "(?i)^(anthropic/)?(claude-3.7-sonnet-20250219|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3.7-sonnet-20250219-v1:0|claude-3-7-sonnet-V1@20250219)$", + "startDate": "2025-02-25T09:35:39.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.7-sonnet-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-7-sonnet-latest)$", + "startDate": "2025-02-25T09:35:39.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4.5-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4.5-preview)$", + "startDate": "2025-02-27T21:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "modelName": "gpt-4.5-preview-2025-02-27", + "matchPattern": "(?i)^(openai/)?(gpt-4.5-preview-2025-02-27)$", + "startDate": "2025-02-27T21:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "modelName": "gpt-4.1", + "matchPattern": "(?i)^(openai/)?(gpt-4.1)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "modelName": "gpt-4.1-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "modelName": "gpt-4.1-mini-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-mini-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "gpt-4.1-nano-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-nano-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "o3", + "matchPattern": "(?i)^(openai/)?(o3)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "modelName": "o3-2025-04-16", + "matchPattern": "(?i)^(openai/)?(o3-2025-04-16)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "modelName": "o4-mini", + "matchPattern": "(?i)^(o4-mini)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o4-mini-2025-04-16", + "matchPattern": "(?i)^(o4-mini-2025-04-16)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-lite-preview", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash-lite-preview)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4.1-nano", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-nano)$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-4.1-mini", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-mini)$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-5-20250929", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4-5(-20250929)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-5(-20250929)?-v1(:0)?|claude-sonnet-4-5-V1(@20250929)?|claude-sonnet-4-5(@20250929)?)$", + "startDate": "2025-09-29T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-20250514", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4(-20250514)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4(-20250514)?-v1(:0)?|claude-sonnet-4-V1(@20250514)?|claude-sonnet-4(@20250514)?)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4-latest)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-20250514", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4(-20250514)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4(-20250514)?-v1(:0)?|claude-opus-4(@20250514)?)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "modelName": "o3-pro", + "matchPattern": "(?i)^(openai/)?(o3-pro)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "modelName": "o3-pro-2025-06-10", + "matchPattern": "(?i)^(openai/)?(o3-pro-2025-06-10)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "modelName": "o1-pro", + "matchPattern": "(?i)^(openai/)?(o1-pro)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "modelName": "o1-pro-2025-03-19", + "matchPattern": "(?i)^(openai/)?(o1-pro-2025-03-19)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "modelName": "gemini-2.5-flash", + "matchPattern": "(?i)^(google/)?(gemini-2.5-flash)$", + "startDate": "2025-07-03T13:44:06.964Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 3e-7, + "input_text": 3e-7, + "input_modality_1": 3e-7, + "prompt_token_count": 3e-7, + "promptTokenCount": 3e-7, + "input_cached_tokens": 3e-8, + "cached_content_token_count": 3e-8, + "output": 0.0000025, + "output_modality_1": 0.0000025, + "candidates_token_count": 0.0000025, + "candidatesTokenCount": 0.0000025, + "thoughtsTokenCount": 0.0000025, + "thoughts_token_count": 0.0000025, + "output_reasoning": 0.0000025, + "input_audio_tokens": 0.000001 + } + } + ] + }, + { + "modelName": "gemini-2.5-flash-lite", + "matchPattern": "(?i)^(google/)?(gemini-2.5-flash-lite)$", + "startDate": "2025-07-03T13:44:06.964Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_text": 1e-7, + "input_modality_1": 1e-7, + "prompt_token_count": 1e-7, + "promptTokenCount": 1e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 4e-7, + "output_modality_1": 4e-7, + "candidates_token_count": 4e-7, + "candidatesTokenCount": 4e-7, + "thoughtsTokenCount": 4e-7, + "thoughts_token_count": 4e-7, + "output_reasoning": 4e-7, + "input_audio_tokens": 5e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-1-20250805", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-1(-20250805)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4-1(-20250805)?-v1(:0)?|claude-opus-4-1(@20250805)?)$", + "startDate": "2025-08-05T15:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-5", + "matchPattern": "(?i)^(openai/)?(gpt-5)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-mini", + "matchPattern": "(?i)^(openai/)?(gpt-5-mini)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-5-mini-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-mini-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-5-nano", + "matchPattern": "(?i)^(openai/)?(gpt-5-nano)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-5-nano-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-nano-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-5-chat-latest", + "matchPattern": "(?i)^(openai/)?(gpt-5-chat-latest)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5-pro)$", + "startDate": "2025-10-07T08:03:54.727Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-5-pro-2025-10-06", + "matchPattern": "(?i)^(openai/)?(gpt-5-pro-2025-10-06)$", + "startDate": "2025-10-07T08:03:54.727Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "modelName": "claude-haiku-4-5-20251001", + "matchPattern": "(?i)^(anthropic/)?(claude-haiku-4-5-20251001|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-haiku-4-5-20251001-v1:0|claude-4-5-haiku@20251001)$", + "startDate": "2025-10-16T08:20:44.558Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "input_tokens": 0.000001, + "output": 0.000005, + "output_tokens": 0.000005, + "cache_creation_input_tokens": 0.00000125, + "input_cache_creation": 0.00000125, + "input_cache_creation_5m": 0.00000125, + "input_cache_creation_1h": 0.000002, + "cache_read_input_tokens": 1e-7, + "input_cache_read": 1e-7 + } + } + ] + }, + { + "modelName": "gpt-5.1", + "matchPattern": "(?i)^(openai/)?(gpt-5.1)$", + "startDate": "2025-11-14T08:57:23.481Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5.1-2025-11-13", + "matchPattern": "(?i)^(openai/)?(gpt-5.1-2025-11-13)$", + "startDate": "2025-11-14T08:57:23.481Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "claude-opus-4-5-20251101", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-5(-20251101)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-5(-20251101)?-v1(:0)?|claude-opus-4-5(@20251101)?)$", + "startDate": "2025-11-24T20:53:27.571Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "input_tokens": 0.000005, + "output": 0.000025, + "output_tokens": 0.000025, + "cache_creation_input_tokens": 0.00000625, + "input_cache_creation": 0.00000625, + "input_cache_creation_5m": 0.00000625, + "input_cache_creation_1h": 0.00001, + "cache_read_input_tokens": 5e-7, + "input_cache_read": 5e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-6", + "matchPattern": "(?i)^(anthropic\\/)?(claude-sonnet-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-6(-v1(:0)?)?|claude-sonnet-4-6)$", + "startDate": "2026-02-18T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-6", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-6-v1(:0)?|claude-opus-4-6)$", + "startDate": "2026-02-09T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "input_tokens": 0.000005, + "output": 0.000025, + "output_tokens": 0.000025, + "cache_creation_input_tokens": 0.00000625, + "input_cache_creation": 0.00000625, + "input_cache_creation_5m": 0.00000625, + "input_cache_creation_1h": 0.00001, + "cache_read_input_tokens": 5e-7, + "input_cache_read": 5e-7 + } + } + ] + }, + { + "modelName": "gemini-2.5-pro", + "matchPattern": "(?i)^(google/)?(gemini-2.5-pro)$", + "startDate": "2025-11-26T13:27:53.545Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_text": 0.00000125, + "input_modality_1": 0.00000125, + "prompt_token_count": 0.00000125, + "promptTokenCount": 0.00000125, + "input_cached_tokens": 1.25e-7, + "cached_content_token_count": 1.25e-7, + "output": 0.00001, + "output_modality_1": 0.00001, + "candidates_token_count": 0.00001, + "candidatesTokenCount": 0.00001, + "thoughtsTokenCount": 0.00001, + "thoughts_token_count": 0.00001, + "output_reasoning": 0.00001 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.0000025, + "input_text": 0.0000025, + "input_modality_1": 0.0000025, + "prompt_token_count": 0.0000025, + "promptTokenCount": 0.0000025, + "input_cached_tokens": 2.5e-7, + "cached_content_token_count": 2.5e-7, + "output": 0.000015, + "output_modality_1": 0.000015, + "candidates_token_count": 0.000015, + "candidatesTokenCount": 0.000015, + "thoughtsTokenCount": 0.000015, + "thoughts_token_count": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gemini-3-pro-preview", + "matchPattern": "(?i)^(google/)?(gemini-3-pro-preview)$", + "startDate": "2025-11-26T13:27:53.545Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_modality_1": 0.000002, + "prompt_token_count": 0.000002, + "promptTokenCount": 0.000002, + "input_cached_tokens": 2e-7, + "cached_content_token_count": 2e-7, + "output": 0.000012, + "output_modality_1": 0.000012, + "candidates_token_count": 0.000012, + "candidatesTokenCount": 0.000012, + "thoughtsTokenCount": 0.000012, + "thoughts_token_count": 0.000012, + "output_reasoning": 0.000012 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000004, + "input_modality_1": 0.000004, + "prompt_token_count": 0.000004, + "promptTokenCount": 0.000004, + "input_cached_tokens": 4e-7, + "cached_content_token_count": 4e-7, + "output": 0.000018, + "output_modality_1": 0.000018, + "candidates_token_count": 0.000018, + "candidatesTokenCount": 0.000018, + "thoughtsTokenCount": 0.000018, + "thoughts_token_count": 0.000018, + "output_reasoning": 0.000018 + } + } + ] + }, + { + "modelName": "gemini-3.1-pro-preview", + "matchPattern": "(?i)^(google/)?(gemini-3.1-pro-preview(-customtools)?)$", + "startDate": "2026-02-19T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_modality_1": 0.000002, + "prompt_token_count": 0.000002, + "promptTokenCount": 0.000002, + "input_cached_tokens": 2e-7, + "cached_content_token_count": 2e-7, + "output": 0.000012, + "output_modality_1": 0.000012, + "candidates_token_count": 0.000012, + "candidatesTokenCount": 0.000012, + "thoughtsTokenCount": 0.000012, + "thoughts_token_count": 0.000012, + "output_reasoning": 0.000012 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000004, + "input_modality_1": 0.000004, + "prompt_token_count": 0.000004, + "promptTokenCount": 0.000004, + "input_cached_tokens": 4e-7, + "cached_content_token_count": 4e-7, + "output": 0.000018, + "output_modality_1": 0.000018, + "candidates_token_count": 0.000018, + "candidatesTokenCount": 0.000018, + "thoughtsTokenCount": 0.000018, + "thoughts_token_count": 0.000018, + "output_reasoning": 0.000018 + } + } + ] + }, + { + "modelName": "gpt-5.2", + "matchPattern": "(?i)^(openai/)?(gpt-5.2)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000175, + "input_cached_tokens": 1.75e-7, + "input_cache_read": 1.75e-7, + "output": 0.000014, + "output_reasoning_tokens": 0.000014, + "output_reasoning": 0.000014 + } + } + ] + }, + { + "modelName": "gpt-5.2-2025-12-11", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-2025-12-11)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000175, + "input_cached_tokens": 1.75e-7, + "input_cache_read": 1.75e-7, + "output": 0.000014, + "output_reasoning_tokens": 0.000014, + "output_reasoning": 0.000014 + } + } + ] + }, + { + "modelName": "gpt-5.2-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-pro)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000021, + "output": 0.000168, + "output_reasoning_tokens": 0.000168, + "output_reasoning": 0.000168 + } + } + ] + }, + { + "modelName": "gpt-5.2-pro-2025-12-11", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-pro-2025-12-11)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000021, + "output": 0.000168, + "output_reasoning_tokens": 0.000168, + "output_reasoning": 0.000168 + } + } + ] + }, + { + "modelName": "gpt-5.4", + "matchPattern": "(?i)^(openai/)?(gpt-5.4)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 2.5e-7, + "input_cache_read": 2.5e-7, + "output": 0.000015, + "output_reasoning_tokens": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-5.4-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-pro)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00018, + "output_reasoning_tokens": 0.00018, + "output_reasoning": 0.00018 + } + } + ] + }, + { + "modelName": "gpt-5.4-2026-03-05", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-2026-03-05)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 2.5e-7, + "input_cache_read": 2.5e-7, + "output": 0.000015, + "output_reasoning_tokens": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-5.4-pro-2026-03-05", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-pro-2026-03-05)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00018, + "output_reasoning_tokens": 0.00018, + "output_reasoning": 0.00018 + } + } + ] + }, + { + "modelName": "gemini-3-flash-preview", + "matchPattern": "(?i)^(google/)?(gemini-3-flash-preview)$", + "startDate": "2025-12-21T12:01:42.282Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "input_modality_1": 5e-7, + "prompt_token_count": 5e-7, + "promptTokenCount": 5e-7, + "input_cached_tokens": 5e-8, + "cached_content_token_count": 5e-8, + "output": 0.000003, + "output_modality_1": 0.000003, + "candidates_token_count": 0.000003, + "candidatesTokenCount": 0.000003, + "thoughtsTokenCount": 0.000003, + "thoughts_token_count": 0.000003, + "output_reasoning": 0.000003 + } + } + ] + }, + { + "modelName": "gemini-3.1-flash-lite-preview", + "matchPattern": "(?i)^(google/)?(gemini-3.1-flash-lite-preview)$", + "startDate": "2026-03-03T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_modality_1": 2.5e-7, + "prompt_token_count": 2.5e-7, + "promptTokenCount": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 0.0000015, + "output_modality_1": 0.0000015, + "candidates_token_count": 0.0000015, + "candidatesTokenCount": 0.0000015, + "thoughtsTokenCount": 0.0000015, + "thoughts_token_count": 0.0000015, + "output_reasoning": 0.0000015, + "input_audio_tokens": 5e-7 + } + } + ] + } +]; diff --git a/internal-packages/llm-pricing/src/index.ts b/internal-packages/llm-pricing/src/index.ts new file mode 100644 index 00000000000..3632434c137 --- /dev/null +++ b/internal-packages/llm-pricing/src/index.ts @@ -0,0 +1,11 @@ +export { ModelPricingRegistry } from "./registry.js"; +export { seedLlmPricing } from "./seed.js"; +export { defaultModelPrices } from "./defaultPrices.js"; +export type { + LlmModelWithPricing, + LlmCostResult, + LlmPricingTierWithPrices, + LlmPriceEntry, + PricingCondition, + DefaultModelDefinition, +} from "./types.js"; diff --git a/internal-packages/llm-pricing/src/registry.test.ts b/internal-packages/llm-pricing/src/registry.test.ts new file mode 100644 index 00000000000..7732c596493 --- /dev/null +++ b/internal-packages/llm-pricing/src/registry.test.ts @@ -0,0 +1,233 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { ModelPricingRegistry } from "./registry.js"; +import { defaultModelPrices } from "./defaultPrices.js"; +import type { LlmModelWithPricing } from "./types.js"; + +// Convert POSIX-style (?i) inline flag to JS RegExp 'i' flag +function compilePattern(pattern: string): RegExp { + if (pattern.startsWith("(?i)")) { + return new RegExp(pattern.slice(4), "i"); + } + return new RegExp(pattern); +} + +// Create a mock registry that we can load with test data without Prisma +class TestableRegistry extends ModelPricingRegistry { + loadPatterns(models: LlmModelWithPricing[]) { + // Access private fields via any cast for testing + const self = this as any; + self._patterns = models.map((model) => ({ + regex: compilePattern(model.matchPattern), + model, + })); + self._exactMatchCache = new Map(); + self._loaded = true; + } +} + +const gpt4o: LlmModelWithPricing = { + id: "model-gpt4o", + modelName: "gpt-4o", + matchPattern: "^gpt-4o(-\\d{4}-\\d{2}-\\d{2})?$", + startDate: null, + pricingTiers: [ + { + id: "tier-gpt4o-standard", + name: "Standard", + isDefault: true, + priority: 0, + conditions: [], + prices: [ + { usageType: "input", price: 0.0000025 }, + { usageType: "output", price: 0.00001 }, + { usageType: "input_cached_tokens", price: 0.00000125 }, + ], + }, + ], +}; + +const claudeSonnet: LlmModelWithPricing = { + id: "model-claude-sonnet", + modelName: "claude-sonnet-4-0", + matchPattern: "^claude-sonnet-4-0(-\\d{8})?$", + startDate: null, + pricingTiers: [ + { + id: "tier-claude-sonnet-standard", + name: "Standard", + isDefault: true, + priority: 0, + conditions: [], + prices: [ + { usageType: "input", price: 0.000003 }, + { usageType: "output", price: 0.000015 }, + { usageType: "input_cached_tokens", price: 0.0000015 }, + ], + }, + ], +}; + +describe("ModelPricingRegistry", () => { + let registry: TestableRegistry; + + beforeEach(() => { + registry = new TestableRegistry(null as any); + registry.loadPatterns([gpt4o, claudeSonnet]); + }); + + describe("match", () => { + it("should match exact model name", () => { + const result = registry.match("gpt-4o"); + expect(result).not.toBeNull(); + expect(result!.modelName).toBe("gpt-4o"); + }); + + it("should match model with date suffix", () => { + const result = registry.match("gpt-4o-2024-08-06"); + expect(result).not.toBeNull(); + expect(result!.modelName).toBe("gpt-4o"); + }); + + it("should match claude model", () => { + const result = registry.match("claude-sonnet-4-0-20250514"); + expect(result).not.toBeNull(); + expect(result!.modelName).toBe("claude-sonnet-4-0"); + }); + + it("should return null for unknown model", () => { + const result = registry.match("unknown-model-xyz"); + expect(result).toBeNull(); + }); + + it("should cache exact matches", () => { + registry.match("gpt-4o"); + registry.match("gpt-4o"); + // Second call should use cache - no way to verify without mocking, but it shouldn't error + expect(registry.match("gpt-4o")!.modelName).toBe("gpt-4o"); + }); + + it("should cache misses", () => { + expect(registry.match("unknown")).toBeNull(); + expect(registry.match("unknown")).toBeNull(); + }); + }); + + describe("calculateCost", () => { + it("should calculate cost for input and output tokens", () => { + const result = registry.calculateCost("gpt-4o", { + input: 1000, + output: 100, + }); + + expect(result).not.toBeNull(); + expect(result!.matchedModelName).toBe("gpt-4o"); + expect(result!.pricingTierName).toBe("Standard"); + expect(result!.inputCost).toBeCloseTo(0.0025); // 1000 * 0.0000025 + expect(result!.outputCost).toBeCloseTo(0.001); // 100 * 0.00001 + expect(result!.totalCost).toBeCloseTo(0.0035); + }); + + it("should include cached token costs", () => { + const result = registry.calculateCost("gpt-4o", { + input: 500, + output: 50, + input_cached_tokens: 200, + }); + + expect(result).not.toBeNull(); + expect(result!.costDetails["input"]).toBeCloseTo(0.00125); // 500 * 0.0000025 + expect(result!.costDetails["output"]).toBeCloseTo(0.0005); // 50 * 0.00001 + expect(result!.costDetails["input_cached_tokens"]).toBeCloseTo(0.00025); // 200 * 0.00000125 + expect(result!.totalCost).toBeCloseTo(0.002); + }); + + it("should return null for unknown model", () => { + const result = registry.calculateCost("unknown-model", { input: 100, output: 50 }); + expect(result).toBeNull(); + }); + + it("should handle zero tokens", () => { + const result = registry.calculateCost("gpt-4o", { input: 0, output: 0 }); + expect(result).not.toBeNull(); + expect(result!.totalCost).toBe(0); + }); + + it("should handle missing usage types gracefully", () => { + const result = registry.calculateCost("gpt-4o", { input: 100 }); + expect(result).not.toBeNull(); + expect(result!.inputCost).toBeCloseTo(0.00025); + expect(result!.outputCost).toBe(0); // No output tokens + expect(result!.totalCost).toBeCloseTo(0.00025); + }); + }); + + describe("isLoaded", () => { + it("should return false before loading", () => { + const freshRegistry = new TestableRegistry(null as any); + expect(freshRegistry.isLoaded).toBe(false); + }); + + it("should return true after loading", () => { + expect(registry.isLoaded).toBe(true); + }); + }); + + describe("defaultModelPrices (Langfuse JSON)", () => { + it("should load all models from the JSON file", () => { + expect(defaultModelPrices.length).toBeGreaterThan(100); + }); + + it("should compile all match patterns without errors", () => { + const langfuseRegistry = new TestableRegistry(null as any); + const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ + id: `test-${i}`, + modelName: def.modelName, + matchPattern: def.matchPattern, + startDate: def.startDate ? new Date(def.startDate) : null, + pricingTiers: def.pricingTiers.map((tier, j) => ({ + id: `tier-${i}-${j}`, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: Object.entries(tier.prices).map(([usageType, price]) => ({ + usageType, + price, + })), + })), + })); + + // This should not throw — all 141 patterns should compile + expect(() => langfuseRegistry.loadPatterns(models)).not.toThrow(); + expect(langfuseRegistry.isLoaded).toBe(true); + }); + + it("should match real-world model names from Langfuse patterns", () => { + const langfuseRegistry = new TestableRegistry(null as any); + const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ + id: `test-${i}`, + modelName: def.modelName, + matchPattern: def.matchPattern, + startDate: null, + pricingTiers: def.pricingTiers.map((tier, j) => ({ + id: `tier-${i}-${j}`, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: Object.entries(tier.prices).map(([usageType, price]) => ({ + usageType, + price, + })), + })), + })); + langfuseRegistry.loadPatterns(models); + + // Test real model strings that SDKs send + expect(langfuseRegistry.match("gpt-4o")).not.toBeNull(); + expect(langfuseRegistry.match("gpt-4o-mini")).not.toBeNull(); + expect(langfuseRegistry.match("claude-sonnet-4-5-20250929")).not.toBeNull(); + expect(langfuseRegistry.match("claude-sonnet-4-20250514")).not.toBeNull(); + }); + }); +}); diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts new file mode 100644 index 00000000000..85713e5268a --- /dev/null +++ b/internal-packages/llm-pricing/src/registry.ts @@ -0,0 +1,209 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import { trail } from "agentcrumbs"; // @crumbs +import type { + LlmModelWithPricing, + LlmCostResult, + LlmPricingTierWithPrices, + PricingCondition, +} from "./types.js"; + +const crumb = trail("webapp:llm-pricing"); // @crumbs + +type CompiledPattern = { + regex: RegExp; + model: LlmModelWithPricing; +}; + +// Convert POSIX-style (?i) inline flag to JS RegExp 'i' flag +function compilePattern(pattern: string): RegExp { + if (pattern.startsWith("(?i)")) { + return new RegExp(pattern.slice(4), "i"); + } + return new RegExp(pattern); +} + +export class ModelPricingRegistry { + private _prisma: PrismaClient; + private _patterns: CompiledPattern[] = []; + private _exactMatchCache: Map = new Map(); + private _loaded = false; + + constructor(prisma: PrismaClient) { + this._prisma = prisma; + } + + get isLoaded(): boolean { + return this._loaded; + } + + async loadFromDatabase(): Promise { + const models = await this._prisma.llmModel.findMany({ + where: { projectId: null }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + orderBy: [{ startDate: "desc" }], + }); + + crumb("loaded models from db", { count: models.length }); // @crumbs + + const compiled: CompiledPattern[] = []; + let skippedCount = 0; // @crumbs + + for (const model of models) { + try { + const regex = compilePattern(model.matchPattern); + const tiers: LlmPricingTierWithPrices[] = model.pricingTiers.map((tier) => ({ + id: tier.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: (tier.conditions as PricingCondition[]) ?? [], + prices: tier.prices.map((p) => ({ + usageType: p.usageType, + price: Number(p.price), + })), + })); + + compiled.push({ + regex, + model: { + id: model.id, + modelName: model.modelName, + matchPattern: model.matchPattern, + startDate: model.startDate, + pricingTiers: tiers, + }, + }); + } catch { + skippedCount++; // @crumbs + // Skip models with invalid regex patterns + console.warn(`Invalid regex pattern for model ${model.modelName}: ${model.matchPattern}`); + crumb("invalid regex pattern", { modelName: model.modelName, pattern: model.matchPattern }); // @crumbs + } + } + + this._patterns = compiled; + this._exactMatchCache.clear(); + this._loaded = true; + crumb("registry loaded", { patterns: compiled.length, skipped: skippedCount }); // @crumbs + } + + async reload(): Promise { + await this.loadFromDatabase(); + } + + match(responseModel: string): LlmModelWithPricing | null { + if (!this._loaded) return null; + + // Check exact match cache + const cached = this._exactMatchCache.get(responseModel); + if (cached !== undefined) return cached; + + // Iterate compiled regex patterns + for (const { regex, model } of this._patterns) { + if (regex.test(responseModel)) { + this._exactMatchCache.set(responseModel, model); + crumb("model matched", { responseModel, matchedModel: model.modelName }); // @crumbs + return model; + } + } + + // Cache miss + this._exactMatchCache.set(responseModel, null); + crumb("model not matched", { responseModel }); // @crumbs + return null; + } + + calculateCost( + responseModel: string, + usageDetails: Record + ): LlmCostResult | null { + const model = this.match(responseModel); + if (!model) return null; + + const tier = this._matchPricingTier(model.pricingTiers, usageDetails); + if (!tier) return null; + + const costDetails: Record = {}; + let totalCost = 0; + + for (const priceEntry of tier.prices) { + const tokenCount = usageDetails[priceEntry.usageType] ?? 0; + if (tokenCount === 0) continue; + const cost = tokenCount * priceEntry.price; + costDetails[priceEntry.usageType] = cost; + totalCost += cost; + } + + const inputCost = costDetails["input"] ?? 0; + const outputCost = costDetails["output"] ?? 0; + + return { + matchedModelId: model.id, + matchedModelName: model.modelName, + pricingTierId: tier.id, + pricingTierName: tier.name, + inputCost, + outputCost, + totalCost, + costDetails, + }; + } + + private _matchPricingTier( + tiers: LlmPricingTierWithPrices[], + usageDetails: Record + ): LlmPricingTierWithPrices | null { + if (tiers.length === 0) return null; + + // Tiers are sorted by priority ascending (lowest first) + // Evaluate conditions — first tier whose conditions match wins + for (const tier of tiers) { + if (tier.conditions.length === 0) { + // No conditions = default tier + return tier; + } + + if (this._evaluateConditions(tier.conditions, usageDetails)) { + return tier; + } + } + + // Fallback to default tier + const defaultTier = tiers.find((t) => t.isDefault); + return defaultTier ?? tiers[0] ?? null; + } + + private _evaluateConditions( + conditions: PricingCondition[], + usageDetails: Record + ): boolean { + return conditions.every((condition) => { + // Find matching usage detail key + const regex = new RegExp(condition.usageDetailPattern); + const matchingValue = Object.entries(usageDetails).find(([key]) => regex.test(key)); + const value = matchingValue?.[1] ?? 0; + + switch (condition.operator) { + case "gt": + return value > condition.value; + case "gte": + return value >= condition.value; + case "lt": + return value < condition.value; + case "lte": + return value <= condition.value; + case "eq": + return value === condition.value; + case "neq": + return value !== condition.value; + default: + return false; + } + }); + } +} diff --git a/internal-packages/llm-pricing/src/seed.ts b/internal-packages/llm-pricing/src/seed.ts new file mode 100644 index 00000000000..6289d31176b --- /dev/null +++ b/internal-packages/llm-pricing/src/seed.ts @@ -0,0 +1,59 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import { defaultModelPrices } from "./defaultPrices.js"; + +export async function seedLlmPricing(prisma: PrismaClient): Promise<{ + modelsCreated: number; + modelsSkipped: number; +}> { + let modelsCreated = 0; + let modelsSkipped = 0; + + for (const modelDef of defaultModelPrices) { + // Check if this model already exists (don't overwrite admin changes) + const existing = await prisma.llmModel.findFirst({ + where: { + projectId: null, + modelName: modelDef.modelName, + }, + }); + + if (existing) { + modelsSkipped++; + continue; + } + + // Create model first + const model = await prisma.llmModel.create({ + data: { + modelName: modelDef.modelName, + matchPattern: modelDef.matchPattern, + startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, + source: "default", + }, + }); + + // Create tiers and prices with explicit model connection + for (const tier of modelDef.pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + modelsCreated++; + } + + return { modelsCreated, modelsSkipped }; +} diff --git a/internal-packages/llm-pricing/src/types.ts b/internal-packages/llm-pricing/src/types.ts new file mode 100644 index 00000000000..5e2ffdcaec5 --- /dev/null +++ b/internal-packages/llm-pricing/src/types.ts @@ -0,0 +1,53 @@ +import type { Decimal } from "@trigger.dev/database"; + +export type PricingCondition = { + usageDetailPattern: string; + operator: "gt" | "gte" | "lt" | "lte" | "eq" | "neq"; + value: number; +}; + +export type LlmPriceEntry = { + usageType: string; + price: number; +}; + +export type LlmPricingTierWithPrices = { + id: string; + name: string; + isDefault: boolean; + priority: number; + conditions: PricingCondition[]; + prices: LlmPriceEntry[]; +}; + +export type LlmModelWithPricing = { + id: string; + modelName: string; + matchPattern: string; + startDate: Date | null; + pricingTiers: LlmPricingTierWithPrices[]; +}; + +export type LlmCostResult = { + matchedModelId: string; + matchedModelName: string; + pricingTierId: string; + pricingTierName: string; + inputCost: number; + outputCost: number; + totalCost: number; + costDetails: Record; +}; + +export type DefaultModelDefinition = { + modelName: string; + matchPattern: string; + startDate?: string; + pricingTiers: Array<{ + name: string; + isDefault: boolean; + priority: number; + conditions: PricingCondition[]; + prices: Record; + }>; +}; diff --git a/internal-packages/llm-pricing/tsconfig.json b/internal-packages/llm-pricing/tsconfig.json new file mode 100644 index 00000000000..c64cf33133b --- /dev/null +++ b/internal-packages/llm-pricing/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], + "module": "Node16", + "moduleResolution": "Node16", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "resolveJsonModule": true + }, + "exclude": ["node_modules"] +} diff --git a/packages/core/src/v3/schemas/style.ts b/packages/core/src/v3/schemas/style.ts index eab62c5b41b..2f833b800ac 100644 --- a/packages/core/src/v3/schemas/style.ts +++ b/packages/core/src/v3/schemas/style.ts @@ -11,11 +11,12 @@ const AccessoryItem = z.object({ text: z.string(), variant: z.string().optional(), url: z.string().optional(), + icon: z.string().optional(), }); const Accessory = z.object({ items: z.array(AccessoryItem), - style: z.enum(["codepath"]).optional(), + style: z.enum(["codepath", "pills"]).optional(), }); export type Accessory = z.infer; From 2508ee3a784415343767fb1ea6b9c31535cea15b Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 11 Mar 2026 11:04:21 +0000 Subject: [PATCH 02/15] feat: add friendlyId to LlmModel, TRQL llm_usage integration, and seed-on-startup - Add friendly_id column to llm_models (llm_model_xxx format) - Use friendlyId as matchedModelId in all external surfaces - Add durationNs render type to TSQLResultsTable and QueryResultsChart - Add 4 example queries for llm_usage in query editor - Add LLM_PRICING_SEED_ON_STARTUP env var for local bootstrapping - Update admin API and seed to generate friendlyId refs TRI-7773 --- .../app/components/code/QueryResultsChart.tsx | 5 ++ .../app/components/code/TSQLResultsTable.tsx | 20 +++++++ apps/webapp/app/env.server.ts | 1 + .../ExamplesContent.tsx | 57 +++++++++++++++++++ .../app/routes/admin.api.v1.llm-models.ts | 2 + .../app/v3/llmPricingRegistry.server.ts | 28 +++++---- apps/webapp/test/otlpExporter.test.ts | 2 +- .../migration.sql | 4 ++ .../database/prisma/schema.prisma | 1 + internal-packages/llm-pricing/package.json | 1 + .../llm-pricing/src/registry.test.ts | 4 ++ internal-packages/llm-pricing/src/registry.ts | 3 +- internal-packages/llm-pricing/src/seed.ts | 2 + internal-packages/llm-pricing/src/types.ts | 1 + pnpm-lock.yaml | 12 ++++ 15 files changed, 130 insertions(+), 13 deletions(-) diff --git a/apps/webapp/app/components/code/QueryResultsChart.tsx b/apps/webapp/app/components/code/QueryResultsChart.tsx index 190304a1e9c..aa6bc6d8898 100644 --- a/apps/webapp/app/components/code/QueryResultsChart.tsx +++ b/apps/webapp/app/components/code/QueryResultsChart.tsx @@ -1209,6 +1209,11 @@ function createYAxisFormatter( formatDurationMilliseconds(value * 1000, { style: "short" }); } + if (format === "durationNs") { + return (value: number): string => + formatDurationMilliseconds(value / 1_000_000, { style: "short" }); + } + if (format === "costInDollars" || format === "cost") { return (value: number): string => { const dollars = format === "cost" ? value / 100 : value; diff --git a/apps/webapp/app/components/code/TSQLResultsTable.tsx b/apps/webapp/app/components/code/TSQLResultsTable.tsx index 3eb033c1d09..b2caf74dac6 100644 --- a/apps/webapp/app/components/code/TSQLResultsTable.tsx +++ b/apps/webapp/app/components/code/TSQLResultsTable.tsx @@ -81,6 +81,11 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string return formatDurationMilliseconds(value * 1000, { style: "short" }); } break; + case "durationNs": + if (typeof value === "number") { + return formatDurationMilliseconds(value / 1_000_000, { style: "short" }); + } + break; case "cost": if (typeof value === "number") { return formatCurrencyAccurate(value / 100); @@ -282,6 +287,12 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number return formatted.length; } return 10; + case "durationNs": + if (typeof value === "number") { + const formatted = formatDurationMilliseconds(value / 1_000_000, { style: "short" }); + return formatted.length; + } + return 10; case "cost": case "costInDollars": // Currency format: "$1,234.56" @@ -598,6 +609,15 @@ function CellValue({ ); } return {String(value)}; + case "durationNs": + if (typeof value === "number") { + return ( + + {formatDurationMilliseconds(value / 1_000_000, { style: "short" })} + + ); + } + return {String(value)}; case "cost": if (typeof value === "number") { return {formatCurrencyAccurate(value / 100)}; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 8877c982ae0..711c37157e5 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1280,6 +1280,7 @@ const EnvironmentSchema = z // LLM cost tracking LLM_COST_TRACKING_ENABLED: BoolEnv.default(true), LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes + LLM_PRICING_SEED_ON_STARTUP: BoolEnv.default(false), // Bootstrap TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index 4d027223f14..23bac4b97ad 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -118,6 +118,63 @@ LIMIT 100`, scope: "environment", table: "metrics", }, + { + title: "LLM cost by model (past 7d)", + description: "Total cost, input tokens, and output tokens grouped by model over the last 7 days.", + query: `SELECT + response_model, + SUM(total_cost) AS total_cost, + SUM(input_tokens) AS input_tokens, + SUM(output_tokens) AS output_tokens +FROM llm_usage +WHERE start_time > now() - INTERVAL 7 DAY +GROUP BY response_model +ORDER BY total_cost DESC`, + scope: "environment", + table: "llm_usage", + }, + { + title: "LLM cost over time", + description: "Total LLM cost bucketed over time. The bucket size adjusts automatically.", + query: `SELECT + timeBucket(), + SUM(total_cost) AS total_cost +FROM llm_usage +GROUP BY timeBucket +ORDER BY timeBucket +LIMIT 1000`, + scope: "environment", + table: "llm_usage", + }, + { + title: "Most expensive runs by LLM cost (top 50)", + description: "Top 50 runs by total LLM cost with token breakdown.", + query: `SELECT + run_id, + task_identifier, + SUM(total_cost) AS llm_cost, + SUM(input_tokens) AS input_tokens, + SUM(output_tokens) AS output_tokens +FROM llm_usage +GROUP BY run_id, task_identifier +ORDER BY llm_cost DESC +LIMIT 50`, + scope: "environment", + table: "llm_usage", + }, + { + title: "LLM calls by provider", + description: "Count and cost of LLM calls grouped by AI provider.", + query: `SELECT + gen_ai_system, + count() AS call_count, + SUM(total_cost) AS total_cost +FROM llm_usage +GROUP BY gen_ai_system +ORDER BY total_cost DESC`, + scope: "environment", + table: "llm_usage", + }, ]; const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name })); diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts index 7b1b2226ca5..706c73549de 100644 --- a/apps/webapp/app/routes/admin.api.v1.llm-models.ts +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts @@ -2,6 +2,7 @@ import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-r import { z } from "zod"; import { prisma } from "~/db.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { generateFriendlyId } from "~/v3/friendlyIdentifiers"; async function requireAdmin(request: Request) { const authResult = await authenticateApiRequestWithPersonalAccessToken(request); @@ -93,6 +94,7 @@ export async function action({ request }: ActionFunctionArgs) { // Create model first, then tiers with explicit model connection const model = await prisma.llmModel.create({ data: { + friendlyId: generateFriendlyId("llm_model"), modelName, matchPattern, startDate: startDate ? new Date(startDate) : null, diff --git a/apps/webapp/app/v3/llmPricingRegistry.server.ts b/apps/webapp/app/v3/llmPricingRegistry.server.ts index 4a2c67e247b..d05b1afc2c8 100644 --- a/apps/webapp/app/v3/llmPricingRegistry.server.ts +++ b/apps/webapp/app/v3/llmPricingRegistry.server.ts @@ -1,12 +1,23 @@ -import { ModelPricingRegistry } from "@internal/llm-pricing"; +import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-pricing"; import { trail } from "agentcrumbs"; // @crumbs -import { $replica } from "~/db.server"; +import { prisma, $replica } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server"; const crumb = trail("webapp:llm-registry"); // @crumbs +async function initRegistry(registry: ModelPricingRegistry) { + if (env.LLM_PRICING_SEED_ON_STARTUP) { + crumb("seeding llm pricing on startup"); // @crumbs + const result = await seedLlmPricing(prisma); + crumb("seed complete", { modelsCreated: result.modelsCreated, modelsSkipped: result.modelsSkipped }); // @crumbs + } + + await registry.loadFromDatabase(); + crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs +} + export const llmPricingRegistry = singleton("llmPricingRegistry", () => { if (!env.LLM_COST_TRACKING_ENABLED) { crumb("llm cost tracking disabled via env"); // @crumbs @@ -19,15 +30,10 @@ export const llmPricingRegistry = singleton("llmPricingRegistry", () => { // Wire up the registry so enrichCreatableEvents can use it setLlmPricingRegistry(registry); - registry - .loadFromDatabase() - .then(() => { - crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs - }) - .catch((err) => { - crumb("registry load failed", { error: String(err) }); // @crumbs - console.error("Failed to load LLM pricing registry", err); - }); + initRegistry(registry).catch((err) => { + crumb("registry init failed", { error: String(err) }); // @crumbs + console.error("Failed to initialize LLM pricing registry", err); + }); // Periodic reload const reloadInterval = env.LLM_PRICING_RELOAD_INTERVAL_MS; diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index a1d1baa3b1c..2a569b74d62 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -406,7 +406,7 @@ describe("OTLPExporter", () => { const inputCost = (usageDetails["input"] ?? 0) * 0.0000025; const outputCost = (usageDetails["output"] ?? 0) * 0.00001; return { - matchedModelId: "model-gpt4o", + matchedModelId: "llm_model_gpt4o", matchedModelName: "gpt-4o", pricingTierId: "tier-standard", pricingTierName: "Standard", diff --git a/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql index 44cc581616f..286de6eacfb 100644 --- a/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql +++ b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql @@ -1,6 +1,7 @@ -- CreateTable CREATE TABLE "public"."llm_models" ( "id" TEXT NOT NULL, + "friendly_id" TEXT NOT NULL, "project_id" TEXT, "model_name" TEXT NOT NULL, "match_pattern" TEXT NOT NULL, @@ -35,6 +36,9 @@ CREATE TABLE "public"."llm_prices" ( CONSTRAINT "llm_prices_pkey" PRIMARY KEY ("id") ); +-- CreateIndex +CREATE UNIQUE INDEX "llm_models_friendly_id_key" ON "public"."llm_models"("friendly_id"); + -- CreateIndex CREATE INDEX "llm_models_project_id_idx" ON "public"."llm_models"("project_id"); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 65878a8af7c..9e91fc70f14 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2586,6 +2586,7 @@ model MetricsDashboard { /// A known LLM model or model pattern for cost tracking model LlmModel { id String @id @default(cuid()) + friendlyId String @unique @map("friendly_id") projectId String? @map("project_id") project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade) modelName String @map("model_name") diff --git a/internal-packages/llm-pricing/package.json b/internal-packages/llm-pricing/package.json index 423dae77eb4..2ac51427dd6 100644 --- a/internal-packages/llm-pricing/package.json +++ b/internal-packages/llm-pricing/package.json @@ -6,6 +6,7 @@ "types": "./src/index.ts", "type": "module", "dependencies": { + "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*" }, "scripts": { diff --git a/internal-packages/llm-pricing/src/registry.test.ts b/internal-packages/llm-pricing/src/registry.test.ts index 7732c596493..93d6a362cb3 100644 --- a/internal-packages/llm-pricing/src/registry.test.ts +++ b/internal-packages/llm-pricing/src/registry.test.ts @@ -27,6 +27,7 @@ class TestableRegistry extends ModelPricingRegistry { const gpt4o: LlmModelWithPricing = { id: "model-gpt4o", + friendlyId: "llm_model_gpt4o", modelName: "gpt-4o", matchPattern: "^gpt-4o(-\\d{4}-\\d{2}-\\d{2})?$", startDate: null, @@ -48,6 +49,7 @@ const gpt4o: LlmModelWithPricing = { const claudeSonnet: LlmModelWithPricing = { id: "model-claude-sonnet", + friendlyId: "llm_model_claude_sonnet", modelName: "claude-sonnet-4-0", matchPattern: "^claude-sonnet-4-0(-\\d{8})?$", startDate: null, @@ -181,6 +183,7 @@ describe("ModelPricingRegistry", () => { const langfuseRegistry = new TestableRegistry(null as any); const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ id: `test-${i}`, + friendlyId: `llm_model_test${i}`, modelName: def.modelName, matchPattern: def.matchPattern, startDate: def.startDate ? new Date(def.startDate) : null, @@ -206,6 +209,7 @@ describe("ModelPricingRegistry", () => { const langfuseRegistry = new TestableRegistry(null as any); const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ id: `test-${i}`, + friendlyId: `llm_model_test${i}`, modelName: def.modelName, matchPattern: def.matchPattern, startDate: null, diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 85713e5268a..506a9360bd2 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -72,6 +72,7 @@ export class ModelPricingRegistry { regex, model: { id: model.id, + friendlyId: model.friendlyId, modelName: model.modelName, matchPattern: model.matchPattern, startDate: model.startDate, @@ -143,7 +144,7 @@ export class ModelPricingRegistry { const outputCost = costDetails["output"] ?? 0; return { - matchedModelId: model.id, + matchedModelId: model.friendlyId, matchedModelName: model.modelName, pricingTierId: tier.id, pricingTierName: tier.name, diff --git a/internal-packages/llm-pricing/src/seed.ts b/internal-packages/llm-pricing/src/seed.ts index 6289d31176b..b4f95373eff 100644 --- a/internal-packages/llm-pricing/src/seed.ts +++ b/internal-packages/llm-pricing/src/seed.ts @@ -1,4 +1,5 @@ import type { PrismaClient } from "@trigger.dev/database"; +import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import { defaultModelPrices } from "./defaultPrices.js"; export async function seedLlmPricing(prisma: PrismaClient): Promise<{ @@ -25,6 +26,7 @@ export async function seedLlmPricing(prisma: PrismaClient): Promise<{ // Create model first const model = await prisma.llmModel.create({ data: { + friendlyId: generateFriendlyId("llm_model"), modelName: modelDef.modelName, matchPattern: modelDef.matchPattern, startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, diff --git a/internal-packages/llm-pricing/src/types.ts b/internal-packages/llm-pricing/src/types.ts index 5e2ffdcaec5..2deec6246ed 100644 --- a/internal-packages/llm-pricing/src/types.ts +++ b/internal-packages/llm-pricing/src/types.ts @@ -22,6 +22,7 @@ export type LlmPricingTierWithPrices = { export type LlmModelWithPricing = { id: string; + friendlyId: string; modelName: string; matchPattern: string; startDate: Date | null; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 88ac6ad5421..854d4215447 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -302,6 +302,9 @@ importers: '@internal/cache': specifier: workspace:* version: link:../../internal-packages/cache + '@internal/llm-pricing': + specifier: workspace:* + version: link:../../internal-packages/llm-pricing '@internal/redis': specifier: workspace:* version: link:../../internal-packages/redis @@ -1125,6 +1128,15 @@ importers: specifier: 18.2.69 version: 18.2.69 + internal-packages/llm-pricing: + dependencies: + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core + '@trigger.dev/database': + specifier: workspace:* + version: link:../database + internal-packages/otlp-importer: dependencies: long: From 04742072524dc04a3416c6f8165b3656921ed447 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 11 Mar 2026 13:13:04 +0000 Subject: [PATCH 03/15] feat: add metadata map to llm_usage_v1 for cost attribution by user/tenant --- .changeset/llm-metadata-run-tags.md | 5 ++ .../ExamplesContent.tsx | 33 ++++++++++++ .../clickhouseEventRepository.server.ts | 3 +- .../eventRepository/eventRepository.types.ts | 2 + apps/webapp/app/v3/otlpExporter.server.ts | 21 ++++++++ apps/webapp/app/v3/querySchemas.ts | 9 ++++ .../v3/utils/enrichCreatableEvents.server.ts | 53 ++++++++++++++++++- .../schema/024_create_llm_usage_v1.sql | 5 +- internal-packages/clickhouse/src/llmUsage.ts | 2 + internal-packages/tsql/src/query/printer.ts | 15 ++++++ .../core/src/v3/taskContext/otelProcessors.ts | 6 +++ 11 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 .changeset/llm-metadata-run-tags.md diff --git a/.changeset/llm-metadata-run-tags.md b/.changeset/llm-metadata-run-tags.md new file mode 100644 index 00000000000..85f04c363b8 --- /dev/null +++ b/.changeset/llm-metadata-run-tags.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Propagate run tags to span attributes so they can be extracted server-side for LLM cost attribution metadata. diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index 23bac4b97ad..1235b443348 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -175,6 +175,39 @@ ORDER BY total_cost DESC`, scope: "environment", table: "llm_usage", }, + { + title: "LLM cost by user", + description: + "Total LLM cost per user from run tags or AI SDK telemetry metadata. Uses metadata.userId which comes from experimental_telemetry metadata or run tags like user:123.", + query: `SELECT + metadata.userId AS user_id, + SUM(total_cost) AS total_cost, + SUM(total_tokens) AS total_tokens, + count() AS call_count +FROM llm_usage +WHERE metadata.userId != '' +GROUP BY metadata.userId +ORDER BY total_cost DESC +LIMIT 50`, + scope: "environment", + table: "llm_usage", + }, + { + title: "LLM cost by metadata key", + description: + "Browse all metadata keys and their LLM cost. Metadata comes from run tags (key:value) and AI SDK telemetry metadata.", + query: `SELECT + metadata, + response_model, + total_cost, + total_tokens, + run_id +FROM llm_usage +ORDER BY start_time DESC +LIMIT 20`, + scope: "environment", + table: "llm_usage", + }, ]; const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name })); diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 780b74d85e1..7a46b6b3bfb 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -279,6 +279,7 @@ export class ClickhouseEventRepository implements IEventRepository { output_cost: llmUsage.outputCost, total_cost: llmUsage.totalCost, cost_details: llmUsage.costDetails, + metadata: llmUsage.metadata, start_time: this.#clampAndFormatStartTime(event.startTime.toString()), duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0), }; @@ -311,7 +312,7 @@ export class ClickhouseEventRepository implements IEventRepository { .map((e) => this.#createLlmUsageInput(e)); if (llmUsageRows.length > 0) { - crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id }); // @crumbs + crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id, metadataKeys: llmUsageRows.map((r) => Object.keys(r.metadata)) }); // @crumbs this._llmUsageFlushScheduler.addToBatch(llmUsageRows); } } diff --git a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts index b750786a651..69590dc9493 100644 --- a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts +++ b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts @@ -37,6 +37,7 @@ export type LlmUsageData = { outputCost: number; totalCost: number; costDetails: Record; + metadata: Record; }; export type CreateEventInput = Omit< @@ -75,6 +76,7 @@ export type CreateEventInput = Omit< metadata: Attributes | undefined; style: Attributes | undefined; machineId?: string; + runTags?: string[]; /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */ _llmUsage?: LlmUsageData; }; diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 0079cb5f0be..b5b5fee70fc 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -38,6 +38,8 @@ import type { import { startSpan } from "./tracing.server"; import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server"; import "./llmPricingRegistry.server"; // Initialize LLM pricing registry on startup +import { trail } from "agentcrumbs"; // @crumbs +const crumbOtlp = trail("webapp:otlp-exporter"); // @crumbs import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; @@ -392,6 +394,9 @@ function convertSpansToCreateableEvents( SemanticInternalAttributes.METADATA ); + const runTags = extractArrayAttribute(span.attributes ?? [], SemanticInternalAttributes.RUN_TAGS); + if (runTags && runTags.length > 0) { crumbOtlp("extracted runTags from span", { runTags, spanId: binaryToHex(span.spanId) }); } // @crumbs + const properties = truncateAttributes( convertKeyValueItemsToMap(span.attributes ?? [], [], undefined, [ @@ -440,6 +445,7 @@ function convertSpansToCreateableEvents( runId: spanProperties.runId ?? resourceProperties.runId ?? "unknown", taskSlug: spanProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown", machineId: spanProperties.machineId ?? resourceProperties.machineId, + runTags, attemptNumber: extractNumberAttribute( span.attributes ?? [], @@ -1001,6 +1007,21 @@ function extractBooleanAttribute( return isBoolValue(attribute?.value) ? attribute.value.boolValue : fallback; } +function extractArrayAttribute( + attributes: KeyValue[], + name: string | Array +): string[] | undefined { + const key = Array.isArray(name) ? name.filter(Boolean).join(".") : name; + + const attribute = attributes.find((attribute) => attribute.key === key); + + if (!attribute?.value?.arrayValue?.values) return undefined; + + return attribute.value.arrayValue.values + .filter((v): v is { stringValue: string } => isStringValue(v)) + .map((v) => v.stringValue); +} + function isPartialSpan(span: Span): boolean { if (!span.attributes) return false; diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 323310ebf84..c719ac21378 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -740,6 +740,15 @@ export const llmUsageSchema: TableSchema = { customRenderType: "durationNs", }), }, + metadata: { + name: "metadata", + ...column("Map(LowCardinality(String), String)", { + description: + "Key-value metadata from run tags (key:value format) and AI SDK telemetry metadata. Access keys with dot notation (metadata.userId) or bracket syntax (metadata['userId']).", + example: "{'userId':'user_123','org':'acme'}", + coreColumn: true, + }), + }, }, }; diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 9dfe091aa10..d76fdda5879 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -48,7 +48,24 @@ function enrichLlmCost(event: CreateEventInput): void { const props = event.properties; if (!props) return; - crumb("enrichLlmCost called", { kind: event.kind, isPartial: event.isPartial, spanId: event.spanId, message: event.message, props }); // @crumbs + // #region @crumbs + // Log all spans (not just gen_ai) that have conversation/chat/session/user context + if (!event.isPartial) { + const contextKeys = Object.entries(props).filter(([k]) => + k.startsWith("ai.telemetry.") || k.startsWith("gen_ai.conversation") || + k.startsWith("chat.") || k.includes("session") || k.includes("user") + ); + if (contextKeys.length > 0) { + crumb("span with context", { + spanId: event.spanId, + parentId: event.parentId, + runId: event.runId, + message: event.message, + contextAttrs: Object.fromEntries(contextKeys), + }); + } + } + // #endregion @crumbs // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED) const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); @@ -130,6 +147,39 @@ function enrichLlmCost(event: CreateEventInput): void { }, }; + // Build metadata map from run tags and ai.telemetry.metadata.* + const metadata: Record = {}; + + if (event.runTags) { + for (const tag of event.runTags) { + const colonIdx = tag.indexOf(":"); + if (colonIdx > 0) { + metadata[tag.substring(0, colonIdx)] = tag.substring(colonIdx + 1); + } + } + } + + for (const [key, value] of Object.entries(props)) { + if (key.startsWith("ai.telemetry.metadata.") && typeof value === "string") { + metadata[key.slice("ai.telemetry.metadata.".length)] = value; + } + } + + // #region @crumbs + const metadataKeyCount = Object.keys(metadata).length; + if (metadataKeyCount > 0) { + crumb("llm metadata built", { + spanId: event.spanId, + runId: event.runId, + responseModel, + metadataKeyCount, + metadataKeys: Object.keys(metadata), + fromRunTags: event.runTags?.length ?? 0, + fromTelemetry: metadataKeyCount - (event.runTags?.filter((t) => t.includes(":")).length ?? 0), + }); + } + // #endregion @crumbs + // Set _llmUsage side-channel for dual-write to llm_usage_v1 const llmUsage: LlmUsageData = { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", @@ -147,6 +197,7 @@ function enrichLlmCost(event: CreateEventInput): void { outputCost: cost.outputCost, totalCost: cost.totalCost, costDetails: cost.costDetails, + metadata, }; event._llmUsage = llmUsage; diff --git a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql index 65bd9c68ebd..5d7f879fdd3 100644 --- a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql +++ b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql @@ -27,13 +27,16 @@ CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 total_cost Decimal64(12) DEFAULT 0, cost_details Map(LowCardinality(String), Decimal64(12)), + metadata Map(LowCardinality(String), String), + start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)), duration UInt64 DEFAULT 0 CODEC(ZSTD(1)), inserted_at DateTime64(3) DEFAULT now64(3), INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_metadata_keys mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(inserted_at) diff --git a/internal-packages/clickhouse/src/llmUsage.ts b/internal-packages/clickhouse/src/llmUsage.ts index fd9deccb562..e9423962ac4 100644 --- a/internal-packages/clickhouse/src/llmUsage.ts +++ b/internal-packages/clickhouse/src/llmUsage.ts @@ -28,6 +28,8 @@ export const LlmUsageV1Input = z.object({ total_cost: z.number(), cost_details: z.record(z.string(), z.number()), + metadata: z.record(z.string(), z.string()), + start_time: z.string(), duration: z.string(), }); diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index b6e9547db06..d45002f6715 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -2370,6 +2370,21 @@ export class ClickHousePrinter { // Try to resolve column names through table context const resolvedChain = this.resolveFieldChain(chainWithPrefix); + // For Map columns, convert dot-notation to bracket syntax: + // metadata.user -> metadata['user'] + if (resolvedChain.length > 1) { + const rootColumnSchema = this.resolveFieldToColumnSchema([node.chain[0]]); + if (rootColumnSchema?.type.startsWith("Map(")) { + const rootCol = this.printIdentifierOrIndex(resolvedChain[0]); + const mapKeys = resolvedChain.slice(1); + let result = rootCol; + for (const key of mapKeys) { + result = `${result}[${this.context.addValue(String(key))}]`; + } + return result; + } + } + // Print each chain element let result = resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join("."); diff --git a/packages/core/src/v3/taskContext/otelProcessors.ts b/packages/core/src/v3/taskContext/otelProcessors.ts index 096f7c0ce76..1c0958d655d 100644 --- a/packages/core/src/v3/taskContext/otelProcessors.ts +++ b/packages/core/src/v3/taskContext/otelProcessors.ts @@ -30,6 +30,12 @@ export class TaskContextSpanProcessor implements SpanProcessor { span.setAttributes( flattenAttributes(taskContext.attributes, SemanticInternalAttributes.METADATA) ); + + // Set run tags as a proper array attribute (not flattened) so it arrives + // as an OTEL ArrayValue and can be extracted on the server side. + if (!taskContext.isRunDisabled && taskContext.ctx.run.tags?.length) { + span.setAttribute(SemanticInternalAttributes.RUN_TAGS, taskContext.ctx.run.tags); + } } if (!isPartialSpan(span) && !skipPartialSpan(span)) { From dcaaab3b301d4fdc3ad49629c9d9abddd71d4d81 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Thu, 12 Mar 2026 10:15:35 +0000 Subject: [PATCH 04/15] New AI streamText span sidebar --- .../components/runs/v3/ai/AIChatMessages.tsx | 253 +++++++++ .../components/runs/v3/ai/AIModelSummary.tsx | 125 +++++ .../components/runs/v3/ai/AISpanDetails.tsx | 202 ++++++++ .../runs/v3/ai/AIToolsInventory.tsx | 83 +++ .../runs/v3/ai/extractAISpanData.ts | 485 ++++++++++++++++++ .../webapp/app/components/runs/v3/ai/index.ts | 3 + .../webapp/app/components/runs/v3/ai/types.ts | 100 ++++ .../app/presenters/v3/SpanPresenter.server.ts | 14 + .../route.tsx | 50 +- apps/webapp/app/tailwind.css | 4 + .../clickhouseEventRepository.server.ts | 31 +- .../app/v3/eventRepository/common.server.ts | 5 +- .../app/v3/llmPricingRegistry.server.ts | 11 - apps/webapp/app/v3/otlpExporter.server.ts | 46 +- .../v3/utils/enrichCreatableEvents.server.ts | 50 -- internal-packages/llm-pricing/src/registry.ts | 11 - 16 files changed, 1377 insertions(+), 96 deletions(-) create mode 100644 apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts create mode 100644 apps/webapp/app/components/runs/v3/ai/index.ts create mode 100644 apps/webapp/app/components/runs/v3/ai/types.ts diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx new file mode 100644 index 00000000000..9fe39482c86 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -0,0 +1,253 @@ +import { lazy, Suspense, useState } from "react"; +import { CodeBlock } from "~/components/code/CodeBlock"; +import type { DisplayItem, ToolUse } from "./types"; + +// Lazy load streamdown to avoid SSR issues +const StreamdownRenderer = lazy(() => + import("streamdown").then((mod) => ({ + default: ({ children }: { children: string }) => ( + + {children} + + ), + })) +); + +export function AIChatMessages({ items }: { items: DisplayItem[] }) { + return ( +
+ {items.map((item, i) => { + switch (item.type) { + case "system": + return ; + case "user": + return ; + case "tool-use": + return ; + case "assistant": + return ; + } + })} +
+ ); +} + +// --------------------------------------------------------------------------- +// Section header (shared across all sections) +// --------------------------------------------------------------------------- + +function SectionHeader({ + label, + right, +}: { + label: string; + right?: React.ReactNode; +}) { + return ( +
+ {label} + {right &&
{right}
} +
+ ); +} + +// --------------------------------------------------------------------------- +// System +// --------------------------------------------------------------------------- + +function SystemSection({ text }: { text: string }) { + const [expanded, setExpanded] = useState(false); + const isLong = text.length > 150; + const preview = isLong ? text.slice(0, 150) + "..." : text; + + return ( +
+ setExpanded(!expanded)} + className="text-[10px] text-text-link hover:underline" + > + {expanded ? "Collapse" : "Expand"} + + ) : undefined + } + /> + 
+        {expanded || !isLong ? text : preview}
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// User +// --------------------------------------------------------------------------- + +function UserSection({ text }: { text: string }) { + return ( +
+ +

{text}

+
+ ); +} + +// --------------------------------------------------------------------------- +// Assistant response (with markdown/raw toggle) +// --------------------------------------------------------------------------- + +export function AssistantResponse({ + text, + headerLabel = "Assistant", +}: { + text: string; + headerLabel?: string; +}) { + const [mode, setMode] = useState<"rendered" | "raw">("rendered"); + + return ( +
+ + + +
+ } + /> + {mode === "rendered" ? ( +
+ {text}}> + {text} + +
+ ) : ( + + )} +
+ ); +} + +// --------------------------------------------------------------------------- +// Tool use (merged calls + results) +// --------------------------------------------------------------------------- + +function ToolUseSection({ tools }: { tools: ToolUse[] }) { + return ( +
+ + {tools.map((tool) => ( + + ))} +
+ ); +} + +type ToolTab = "input" | "output" | "details"; + +function ToolUseRow({ tool }: { tool: ToolUse }) { + const hasInput = tool.inputJson !== "{}"; + const hasResult = !!tool.resultOutput; + const hasDetails = !!tool.description || !!tool.parametersJson; + + const availableTabs: ToolTab[] = [ + ...(hasInput ? (["input"] as const) : []), + ...(hasResult ? (["output"] as const) : []), + ...(hasDetails ? (["details"] as const) : []), + ]; + + const defaultTab: ToolTab | null = hasInput ? "input" : null; + const [activeTab, setActiveTab] = useState(defaultTab); + + function handleTabClick(tab: ToolTab) { + setActiveTab(activeTab === tab ? null : tab); + } + + return ( +
+
+ {tool.toolName} + {tool.resultSummary && ( + {tool.resultSummary} + )} +
+ + {availableTabs.length > 0 && ( + <> +
+ {availableTabs.map((tab) => ( + + ))} +
+ + {activeTab === "input" && hasInput && ( +
+ +
+ )} + + {activeTab === "output" && hasResult && ( +
+ +
+ )} + + {activeTab === "details" && hasDetails && ( +
+ {tool.description && ( +

{tool.description}

+ )} + {tool.parametersJson && ( +
+ + Parameters schema + + +
+ )} +
+ )} + + )} +
+ ); +} diff --git a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx new file mode 100644 index 00000000000..9e627c844d3 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx @@ -0,0 +1,125 @@ +import { formatCurrencyAccurate } from "~/utils/numberFormatter"; +import type { AISpanData } from "./types"; + +export function AITagsRow({ aiData }: { aiData: AISpanData }) { + return ( +
+ {aiData.model} + {aiData.provider !== "unknown" && {aiData.provider}} + {aiData.finishReason && {aiData.finishReason}} + {aiData.serviceTier && tier: {aiData.serviceTier}} + {aiData.toolChoice && tools: {aiData.toolChoice}} + {aiData.toolCount != null && aiData.toolCount > 0 && ( + + {aiData.toolCount} {aiData.toolCount === 1 ? "tool" : "tools"} + + )} + {aiData.messageCount != null && ( + + {aiData.messageCount} {aiData.messageCount === 1 ? "msg" : "msgs"} + + )} + {aiData.telemetryMetadata && + Object.entries(aiData.telemetryMetadata).map(([key, value]) => ( + + {key}: {value} + + ))} +
+ ); +} + +export function AIStatsSummary({ aiData }: { aiData: AISpanData }) { + return ( +
+ Stats + +
+ + + {aiData.cachedTokens != null && aiData.cachedTokens > 0 && ( + + )} + {aiData.reasoningTokens != null && aiData.reasoningTokens > 0 && ( + + )} + + + {aiData.totalCost != null && ( + + )} + {aiData.msToFirstChunk != null && ( + + )} + {aiData.tokensPerSecond != null && ( + + )} +
+
+ ); +} + +function MetricRow({ + label, + value, + unit, + bold, + border, +}: { + label: string; + value: string; + unit?: string; + bold?: boolean; + border?: boolean; +}) { + return ( +
+ {label} + + {value} + {unit && {unit}} + +
+ ); +} + +function formatTtfc(ms: number): string { + if (ms >= 10_000) { + return `${(ms / 1000).toFixed(1)}s`; + } + return `${Math.round(ms)}ms`; +} + +function Pill({ + children, + variant = "default", +}: { + children: React.ReactNode; + variant?: "default" | "dimmed"; +}) { + return ( + + {children} + + ); +} diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx new file mode 100644 index 00000000000..988de788167 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -0,0 +1,202 @@ +import { useState } from "react"; +import { Clipboard, ClipboardCheck } from "lucide-react"; +import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import type { AISpanData, DisplayItem } from "./types"; +import { AITagsRow, AIStatsSummary } from "./AIModelSummary"; +import { AIChatMessages, AssistantResponse } from "./AIChatMessages"; +import { AIToolsInventory } from "./AIToolsInventory"; + +type AITab = "overview" | "messages" | "tools"; + +export function AISpanDetails({ + aiData, + rawProperties, +}: { + aiData: AISpanData; + rawProperties?: string; +}) { + const [tab, setTab] = useState("overview"); + const hasTools = + (aiData.toolDefinitions && aiData.toolDefinitions.length > 0) || aiData.toolCount != null; + + return ( +
+ {/* Tab bar */} +
+ + setTab("overview")} + shortcut={{ key: "o" }} + > + Overview + + setTab("messages")} + shortcut={{ key: "m" }} + > + Messages + + {hasTools && ( + setTab("tools")} + shortcut={{ key: "t" }} + > + Tools{aiData.toolCount != null ? ` (${aiData.toolCount})` : ""} + + )} + +
+ + {/* Tab content */} +
+ {tab === "overview" && } + {tab === "messages" && } + {tab === "tools" && } +
+ + {/* Footer: Copy raw */} + {rawProperties && } +
+ ); +} + +function OverviewTab({ aiData }: { aiData: AISpanData }) { + const { userText, outputText, outputToolNames } = extractInputOutput(aiData); + + return ( +
+ {/* Tags + Stats */} + + + + {/* Input (last user prompt) */} + {userText && ( +
+ + Input + +

{userText}

+
+ )} + + {/* Output (assistant response or tool calls) */} + {outputText && } + {outputToolNames.length > 0 && !outputText && ( +
+ + Output + +

+ Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} + {outputToolNames.join(", ")} +

+
+ )} +
+ ); +} + +function MessagesTab({ aiData }: { aiData: AISpanData }) { + return ( +
+
+ {aiData.items && aiData.items.length > 0 && } + {aiData.responseText && !hasAssistantItem(aiData.items) && ( + + )} +
+
+ ); +} + +function ToolsTab({ aiData }: { aiData: AISpanData }) { + return ; +} + +function CopyRawFooter({ rawProperties }: { rawProperties: string }) { + const [copied, setCopied] = useState(false); + + function handleCopy() { + navigator.clipboard.writeText(rawProperties); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } + + return ( +
+ +
+ ); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function extractInputOutput(aiData: AISpanData): { + userText: string | undefined; + outputText: string | undefined; + outputToolNames: string[]; +} { + let userText: string | undefined; + let outputText: string | undefined; + const outputToolNames: string[] = []; + + if (aiData.items) { + // Find the last user message + for (let i = aiData.items.length - 1; i >= 0; i--) { + if (aiData.items[i].type === "user") { + userText = (aiData.items[i] as { type: "user"; text: string }).text; + break; + } + } + + // Find the last assistant or tool-use item as the output + for (let i = aiData.items.length - 1; i >= 0; i--) { + const item = aiData.items[i]; + if (item.type === "assistant") { + outputText = item.text; + break; + } + if (item.type === "tool-use") { + for (const tool of item.tools) { + outputToolNames.push(tool.toolName); + } + break; + } + } + } + + // Fall back to responseText if no assistant item found + if (!outputText && aiData.responseText) { + outputText = aiData.responseText; + } + + return { userText, outputText, outputToolNames }; +} + +function hasAssistantItem(items: DisplayItem[] | undefined): boolean { + if (!items) return false; + return items.some((item) => item.type === "assistant"); +} diff --git a/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx new file mode 100644 index 00000000000..5130256f85b --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx @@ -0,0 +1,83 @@ +import { useState } from "react"; +import { CodeBlock } from "~/components/code/CodeBlock"; +import type { AISpanData, ToolDefinition } from "./types"; + +export function AIToolsInventory({ aiData }: { aiData: AISpanData }) { + const defs = aiData.toolDefinitions ?? []; + const calledNames = getCalledToolNames(aiData); + + if (defs.length === 0) { + return ( +
+ No tool definitions available for this span. +
+ ); + } + + return ( +
+ {defs.map((def) => { + const wasCalled = calledNames.has(def.name); + return ; + })} +
+ ); +} + +function ToolDefRow({ def, wasCalled }: { def: ToolDefinition; wasCalled: boolean }) { + const [showSchema, setShowSchema] = useState(false); + + return ( +
+
+
+ {def.name} + {wasCalled ? "called" : "not called"} +
+ + {def.description && ( +

{def.description}

+ )} + + {def.parametersJson && ( +
+ + {showSchema && ( +
+ +
+ )} +
+ )} +
+ ); +} + +function getCalledToolNames(aiData: AISpanData): Set { + const names = new Set(); + if (!aiData.items) return names; + + for (const item of aiData.items) { + if (item.type === "tool-use") { + for (const tool of item.tools) { + names.add(tool.toolName); + } + } + } + + return names; +} diff --git a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts new file mode 100644 index 00000000000..c9f0562afa4 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts @@ -0,0 +1,485 @@ +import type { AISpanData, DisplayItem, ToolDefinition, ToolUse } from "./types"; + +/** + * Extracts structured AI span data from unflattened OTEL span properties. + * + * Works with the nested object produced by `unflattenAttributes()` — expects + * keys like `gen_ai.response.model`, `ai.prompt.messages`, `trigger.llm.total_cost`, etc. + * + * @param properties Unflattened span properties object + * @param durationMs Span duration in milliseconds + * @returns Structured AI data, or undefined if this isn't an AI generation span + */ +export function extractAISpanData( + properties: Record, + durationMs: number +): AISpanData | undefined { + const genAi = properties.gen_ai; + if (!genAi || typeof genAi !== "object") return undefined; + + const g = genAi as Record; + const ai = rec(properties.ai); + const trigger = rec(properties.trigger); + + const gResponse = rec(g.response); + const gRequest = rec(g.request); + const gUsage = rec(g.usage); + const gOperation = rec(g.operation); + const aiModel = rec(ai.model); + const aiResponse = rec(ai.response); + const aiPrompt = rec(ai.prompt); + const aiUsage = rec(ai.usage); + const triggerLlm = rec(trigger.llm); + + const model = str(gResponse.model) ?? str(gRequest.model) ?? str(aiModel.id); + if (!model) return undefined; + + // Prefer ai.usage (richer) over gen_ai.usage + const inputTokens = num(aiUsage.inputTokens) ?? num(gUsage.input_tokens) ?? 0; + const outputTokens = num(aiUsage.outputTokens) ?? num(gUsage.output_tokens) ?? 0; + const totalTokens = num(aiUsage.totalTokens) ?? inputTokens + outputTokens; + + const tokensPerSecond = + num(aiResponse.avgOutputTokensPerSecond) ?? + (outputTokens > 0 && durationMs > 0 + ? Math.round((outputTokens / (durationMs / 1000)) * 10) / 10 + : undefined); + + const toolDefs = parseToolDefinitions(aiPrompt.tools); + const providerMeta = parseProviderMetadata(aiResponse.providerMetadata); + const aiTelemetry = rec(ai.telemetry); + const telemetryMeta = extractTelemetryMetadata(aiTelemetry.metadata); + + return { + model, + provider: str(g.system) ?? "unknown", + operationName: str(gOperation.name) ?? str(ai.operationId) ?? "", + finishReason: str(aiResponse.finishReason), + serviceTier: providerMeta?.serviceTier, + toolChoice: parseToolChoice(aiPrompt.toolChoice), + toolCount: toolDefs?.length, + messageCount: countMessages(aiPrompt.messages), + telemetryMetadata: telemetryMeta, + inputTokens, + outputTokens, + totalTokens, + cachedTokens: num(aiUsage.cachedInputTokens) ?? num(gUsage.cache_read_input_tokens), + reasoningTokens: num(aiUsage.reasoningTokens) ?? num(gUsage.reasoning_tokens), + tokensPerSecond, + msToFirstChunk: num(aiResponse.msToFirstChunk), + durationMs, + inputCost: num(triggerLlm.input_cost), + outputCost: num(triggerLlm.output_cost), + totalCost: num(triggerLlm.total_cost), + responseText: str(aiResponse.text) || undefined, + toolDefinitions: toolDefs, + items: buildDisplayItems(aiPrompt.messages, aiResponse.toolCalls, toolDefs), + }; +} + +// --------------------------------------------------------------------------- +// Primitive helpers +// --------------------------------------------------------------------------- + +function rec(v: unknown): Record { + return v && typeof v === "object" ? (v as Record) : {}; +} + +function str(v: unknown): string | undefined { + return typeof v === "string" ? v : undefined; +} + +function num(v: unknown): number | undefined { + return typeof v === "number" ? v : undefined; +} + +// --------------------------------------------------------------------------- +// Message → DisplayItem transformation +// --------------------------------------------------------------------------- + +type RawMessage = { + role: string; + content: unknown; + toolCallId?: string; + name?: string; +}; + +/** + * Build display items from prompt messages and optionally response tool calls. + * - Parses ai.prompt.messages and merges consecutive tool-call + tool-result pairs + * - If ai.response.toolCalls is present (finishReason=tool-calls), appends those too + */ +function buildDisplayItems( + messagesRaw: unknown, + responseToolCallsRaw: unknown, + toolDefs?: ToolDefinition[] +): DisplayItem[] | undefined { + const items = parseMessagesToDisplayItems(messagesRaw); + const responseToolCalls = parseResponseToolCalls(responseToolCallsRaw); + + if (!items && !responseToolCalls) return undefined; + + const result = items ?? []; + + if (responseToolCalls && responseToolCalls.length > 0) { + result.push({ type: "tool-use", tools: responseToolCalls }); + } + + if (toolDefs && toolDefs.length > 0) { + const defsByName = new Map(toolDefs.map((d) => [d.name, d])); + for (const item of result) { + if (item.type === "tool-use") { + for (const tool of item.tools) { + const def = defsByName.get(tool.toolName); + if (def) { + tool.description = def.description; + tool.parametersJson = def.parametersJson; + } + } + } + } + } + + return result.length > 0 ? result : undefined; +} + +function parseMessagesToDisplayItems(raw: unknown): DisplayItem[] | undefined { + if (typeof raw !== "string") return undefined; + + let messages: RawMessage[]; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + messages = parsed.map((item: unknown) => { + const m = rec(item); + return { + role: str(m.role) ?? "user", + content: m.content, + toolCallId: str(m.toolCallId), + name: str(m.name), + }; + }); + } catch { + return undefined; + } + + const items: DisplayItem[] = []; + let i = 0; + + while (i < messages.length) { + const msg = messages[i]; + + if (msg.role === "system") { + items.push({ type: "system", text: extractTextContent(msg.content) }); + i++; + continue; + } + + if (msg.role === "user") { + items.push({ type: "user", text: extractTextContent(msg.content) }); + i++; + continue; + } + + // Assistant message — check if it contains tool calls + if (msg.role === "assistant") { + const toolCalls = extractToolCalls(msg.content); + + if (toolCalls.length > 0) { + // Collect subsequent tool result messages that match these tool calls + const toolCallIds = new Set(toolCalls.map((tc) => tc.toolCallId)); + let j = i + 1; + while (j < messages.length && messages[j].role === "tool") { + j++; + } + // Gather tool result messages between i+1 and j + const toolResultMsgs = messages.slice(i + 1, j); + + // Build ToolUse entries by pairing calls with results + const tools: ToolUse[] = toolCalls.map((tc) => { + const resultMsg = toolResultMsgs.find((m) => { + // Match by toolCallId in the message's content parts + const results = extractToolResults(m.content); + return results.some((r) => r.toolCallId === tc.toolCallId); + }); + + const result = resultMsg + ? extractToolResults(resultMsg.content).find( + (r) => r.toolCallId === tc.toolCallId + ) + : undefined; + + return { + toolCallId: tc.toolCallId, + toolName: tc.toolName, + inputJson: JSON.stringify(tc.input, null, 2), + resultSummary: result?.summary, + resultOutput: result?.formattedOutput, + }; + }); + + items.push({ type: "tool-use", tools }); + i = j; // skip past the tool result messages + continue; + } + + // Assistant message with just text + const text = extractTextContent(msg.content); + if (text) { + items.push({ type: "assistant", text }); + } + i++; + continue; + } + + // Skip any other message types (tool messages that weren't consumed above) + i++; + } + + return items.length > 0 ? items : undefined; +} + +// --------------------------------------------------------------------------- +// Response tool calls (from ai.response.toolCalls, used when finishReason=tool-calls) +// --------------------------------------------------------------------------- + +/** + * Parse ai.response.toolCalls JSON string into ToolUse entries. + * These are tool calls the model requested but haven't been executed yet in this span. + */ +function parseResponseToolCalls(raw: unknown): ToolUse[] | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + const tools: ToolUse[] = []; + for (const item of parsed) { + const tc = rec(item); + if (tc.type === "tool-call" || tc.toolName || tc.toolCallId) { + tools.push({ + toolCallId: str(tc.toolCallId) ?? "", + toolName: str(tc.toolName) ?? "", + inputJson: JSON.stringify( + tc.input && typeof tc.input === "object" ? tc.input : {}, + null, + 2 + ), + }); + } + } + return tools.length > 0 ? tools : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Content part extraction +// --------------------------------------------------------------------------- + +function extractTextContent(content: unknown): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + + const texts: string[] = []; + for (const raw of content) { + const p = rec(raw); + if (p.type === "text" && typeof p.text === "string") { + texts.push(p.text); + } else if (typeof p.text === "string") { + texts.push(p.text); + } + } + return texts.join("\n"); +} + +type ParsedToolCall = { + toolCallId: string; + toolName: string; + input: Record; +}; + +function extractToolCalls(content: unknown): ParsedToolCall[] { + if (!Array.isArray(content)) return []; + const calls: ParsedToolCall[] = []; + for (const raw of content) { + const p = rec(raw); + if (p.type === "tool-call") { + calls.push({ + toolCallId: str(p.toolCallId) ?? "", + toolName: str(p.toolName) ?? "", + input: p.input && typeof p.input === "object" ? (p.input as Record) : {}, + }); + } + } + return calls; +} + +type ParsedToolResult = { + toolCallId: string; + toolName: string; + summary: string; + formattedOutput: string; +}; + +function extractToolResults(content: unknown): ParsedToolResult[] { + if (!Array.isArray(content)) return []; + const results: ParsedToolResult[] = []; + for (const raw of content) { + const p = rec(raw); + if (p.type === "tool-result") { + const { summary, formattedOutput } = summarizeToolOutput(p.output); + results.push({ + toolCallId: str(p.toolCallId) ?? "", + toolName: str(p.toolName) ?? "", + summary, + formattedOutput, + }); + } + } + return results; +} + +/** + * Summarize a tool output into a short label and a formatted string for display. + * Handles the AI SDK's `{ type: "json", value: { status, contentType, body, truncated } }` shape. + */ +function summarizeToolOutput(output: unknown): { summary: string; formattedOutput: string } { + if (typeof output === "string") { + return { + summary: output.length > 80 ? output.slice(0, 80) + "..." : output, + formattedOutput: output, + }; + } + + if (!output || typeof output !== "object") { + return { summary: "result", formattedOutput: JSON.stringify(output, null, 2) }; + } + + const o = output as Record; + + // AI SDK wraps tool results as { type: "json", value: { status, contentType, body, ... } } + if (o.type === "json" && o.value && typeof o.value === "object") { + const v = o.value as Record; + const parts: string[] = []; + if (typeof v.status === "number") parts.push(`${v.status}`); + if (typeof v.contentType === "string") parts.push(v.contentType); + if (v.truncated === true) parts.push("truncated"); + return { + summary: parts.length > 0 ? parts.join(" · ") : "json result", + formattedOutput: JSON.stringify(v, null, 2), + }; + } + + return { summary: "result", formattedOutput: JSON.stringify(output, null, 2) }; +} + +// --------------------------------------------------------------------------- +// Tool definitions (from ai.prompt.tools) +// --------------------------------------------------------------------------- + +/** + * Parse ai.prompt.tools — after the array fix, this arrives as a JSON array string + * where each element is itself a JSON string of a tool definition. + */ +function parseToolDefinitions(raw: unknown): ToolDefinition[] | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + const defs: ToolDefinition[] = []; + for (const item of parsed) { + // Each item is either a JSON string or already an object + const obj = typeof item === "string" ? JSON.parse(item) : item; + if (!obj || typeof obj !== "object") continue; + const o = obj as Record; + const name = str(o.name); + if (!name) continue; + const schema = o.parameters ?? o.inputSchema; + defs.push({ + name, + description: str(o.description), + parametersJson: + schema && typeof schema === "object" + ? JSON.stringify(schema, null, 2) + : undefined, + }); + } + return defs.length > 0 ? defs : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Provider metadata (service tier, inference geo, etc.) +// --------------------------------------------------------------------------- + +function parseProviderMetadata( + raw: unknown +): { serviceTier?: string } | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== "object") return undefined; + + // Anthropic: { anthropic: { usage: { service_tier: "standard" } } } + const anthropic = rec(parsed.anthropic ?? parsed); + const usage = rec(anthropic.usage); + const serviceTier = str(usage.service_tier); + + return serviceTier ? { serviceTier } : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Tool choice parsing +// --------------------------------------------------------------------------- + +function parseToolChoice(raw: unknown): string | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (typeof parsed === "string") return parsed; + if (parsed && typeof parsed === "object" && typeof parsed.type === "string") { + return parsed.type; + } + return undefined; + } catch { + return raw || undefined; + } +} + +// --------------------------------------------------------------------------- +// Message count +// --------------------------------------------------------------------------- + +function countMessages(raw: unknown): number | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + return parsed.length > 0 ? parsed.length : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Telemetry metadata +// --------------------------------------------------------------------------- + +function extractTelemetryMetadata(raw: unknown): Record | undefined { + if (!raw || typeof raw !== "object") return undefined; + + const result: Record = {}; + for (const [key, value] of Object.entries(raw as Record)) { + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + result[key] = String(value); + } + } + + return Object.keys(result).length > 0 ? result : undefined; +} diff --git a/apps/webapp/app/components/runs/v3/ai/index.ts b/apps/webapp/app/components/runs/v3/ai/index.ts new file mode 100644 index 00000000000..7e33a46fb2c --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/index.ts @@ -0,0 +1,3 @@ +export { AISpanDetails } from "./AISpanDetails"; +export { extractAISpanData } from "./extractAISpanData"; +export type { AISpanData, DisplayItem, ToolUse } from "./types"; diff --git a/apps/webapp/app/components/runs/v3/ai/types.ts b/apps/webapp/app/components/runs/v3/ai/types.ts new file mode 100644 index 00000000000..11fdce4e606 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/types.ts @@ -0,0 +1,100 @@ +// --------------------------------------------------------------------------- +// Tool use (merged assistant tool-call + tool result) +// --------------------------------------------------------------------------- + +export type ToolDefinition = { + name: string; + description?: string; + /** JSON schema as formatted string */ + parametersJson?: string; +}; + +export type ToolUse = { + toolCallId: string; + toolName: string; + /** Tool description from the definition, if available */ + description?: string; + /** JSON schema of the tool's parameters, pretty-printed */ + parametersJson?: string; + /** Formatted input args as JSON string */ + inputJson: string; + /** Short summary of the result (e.g. "200 · text/html · truncated") */ + resultSummary?: string; + /** Full formatted result for display in a code block */ + resultOutput?: string; +}; + +// --------------------------------------------------------------------------- +// Display items — what the UI actually renders +// --------------------------------------------------------------------------- + +/** System prompt text (collapsible) */ +export type SystemItem = { + type: "system"; + text: string; +}; + +/** User message text */ +export type UserItem = { + type: "user"; + text: string; +}; + +/** One or more tool calls with their results, grouped */ +export type ToolUseItem = { + type: "tool-use"; + tools: ToolUse[]; +}; + +/** Final assistant text response */ +export type AssistantItem = { + type: "assistant"; + text: string; +}; + +export type DisplayItem = SystemItem | UserItem | ToolUseItem | AssistantItem; + +// --------------------------------------------------------------------------- +// Span-level AI data +// --------------------------------------------------------------------------- + +export type AISpanData = { + model: string; + provider: string; + operationName: string; + + // Categorical tags + finishReason?: string; + serviceTier?: string; + toolChoice?: string; + toolCount?: number; + messageCount?: number; + /** User-defined telemetry metadata (from ai.telemetry.metadata) */ + telemetryMetadata?: Record; + + // Token counts + inputTokens: number; + outputTokens: number; + totalTokens: number; + cachedTokens?: number; + reasoningTokens?: number; + + // Performance + tokensPerSecond?: number; + msToFirstChunk?: number; + durationMs: number; + + // Cost + inputCost?: number; + outputCost?: number; + totalCost?: number; + + // Response text (final assistant output) + responseText?: string; + + // Tool definitions (from ai.prompt.tools) + toolDefinitions?: ToolDefinition[]; + + // Display-ready message items (system, user, tool-use groups, assistant text) + items?: DisplayItem[]; +}; diff --git a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts index ce83c2e242b..9ad94745616 100644 --- a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts @@ -24,6 +24,7 @@ import { engine } from "~/v3/runEngine.server"; import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server"; import { IEventRepository, SpanDetail } from "~/v3/eventRepository/eventRepository.types"; import { safeJsonParse } from "~/utils/json"; +import { extractAISpanData } from "~/components/runs/v3/ai"; type Result = Awaited>; export type Span = NonNullable["span"]>; @@ -543,6 +544,13 @@ export class SpanPresenter extends BasePresenter { entity: span.entity, metadata: span.metadata, triggeredRuns, + aiData: + span.properties && typeof span.properties === "object" + ? extractAISpanData( + span.properties as Record, + span.duration / 1_000_000 + ) + : undefined, }; switch (span.entity.type) { @@ -665,6 +673,12 @@ export class SpanPresenter extends BasePresenter { }; } default: + if (data.aiData) { + return { + ...data, + entity: { type: "ai-generation" as const, object: data.aiData }, + }; + } return { ...data, entity: null }; } } diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index e46eaa5148f..8f7ae61b5d5 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -19,7 +19,7 @@ import { taskRunErrorEnhancer, } from "@trigger.dev/core/v3"; import { assertNever } from "assert-never"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; import { typedjson, useTypedFetcher } from "remix-typedjson"; import { ExitIcon } from "~/assets/icons/ExitIcon"; import { FlagIcon } from "~/assets/icons/RegionIcons"; @@ -60,6 +60,7 @@ import { RunIcon } from "~/components/runs/v3/RunIcon"; import { RunTag } from "~/components/runs/v3/RunTag"; import { TruncatedCopyableValue } from "~/components/primitives/TruncatedCopyableValue"; import { SpanEvents } from "~/components/runs/v3/SpanEvents"; +import { AISpanDetails } from "~/components/runs/v3/ai"; import { SpanTitle } from "~/components/runs/v3/SpanTitle"; import { TaskRunAttemptStatusCombo } from "~/components/runs/v3/TaskRunAttemptStatus"; import { @@ -252,6 +253,8 @@ function SpanBody({ span = applySpanOverrides(span, spanOverrides); + const isAiGeneration = span.entity?.type === "ai-generation"; + return (
@@ -276,9 +279,13 @@ function SpanBody({ /> )}
-
+ {isAiGeneration ? ( -
+ ) : ( +
+ +
+ )}
); } @@ -1155,6 +1162,35 @@ function RunError({ error }: { error: TaskRunError }) { } } +function CollapsibleProperties({ code }: { code: string }) { + const [open, setOpen] = useState(false); + return ( +
+ + {open && ( +
+ +
+ )} +
+ ); +} + function SpanEntity({ span }: { span: Span }) { const isAdmin = useHasAdminAccess(); @@ -1352,6 +1388,14 @@ function SpanEntity({ span }: { span: Span }) { /> ); } + case "ai-generation": { + return ( + + ); + } default: { assertNever(span.entity); } diff --git a/apps/webapp/app/tailwind.css b/apps/webapp/app/tailwind.css index 2e435a032b7..8ec29b91568 100644 --- a/apps/webapp/app/tailwind.css +++ b/apps/webapp/app/tailwind.css @@ -85,6 +85,10 @@ } @layer utilities { + .scrollbar-gutter-stable { + scrollbar-gutter: stable; + } + .animated-gradient-glow { position: relative; overflow: visible; diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 7a46b6b3bfb..531c3e307e9 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -8,9 +8,7 @@ import type { TaskEventV2Input, } from "@internal/clickhouse"; import { Attributes, startSpan, trace, Tracer } from "@internal/tracing"; -import { trail } from "agentcrumbs"; // @crumbs -const crumb = trail("webapp:llm-dual-write"); // @crumbs import { createJsonErrorObject } from "@trigger.dev/core/v3/errors"; import { serializeTraceparent } from "@trigger.dev/core/v3/isomorphic"; import { @@ -233,7 +231,6 @@ export class ClickhouseEventRepository implements IEventRepository { } async #flushLlmUsageBatch(flushId: string, rows: LlmUsageV1Input[]) { - crumb("flushing llm usage batch", { flushId, rows: rows.length }); // @crumbs const [insertError] = await this._clickhouse.llmUsage.insert(rows, { params: { @@ -242,12 +239,9 @@ export class ClickhouseEventRepository implements IEventRepository { }); if (insertError) { - crumb("llm usage batch insert failed", { flushId, error: String(insertError) }); // @crumbs throw insertError; } - crumb("llm usage batch inserted", { flushId, rows: rows.length }); // @crumbs - logger.info("ClickhouseEventRepository.flushLlmUsageBatch Inserted LLM usage batch", { rows: rows.length, }); @@ -312,7 +306,6 @@ export class ClickhouseEventRepository implements IEventRepository { .map((e) => this.#createLlmUsageInput(e)); if (llmUsageRows.length > 0) { - crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id, metadataKeys: llmUsageRows.map((r) => Object.keys(r.metadata)) }); // @crumbs this._llmUsageFlushScheduler.addToBatch(llmUsageRows); } } @@ -1381,19 +1374,21 @@ export class ClickhouseEventRepository implements IEventRepository { } } - if ( - (span.properties == null || - (typeof span.properties === "object" && Object.keys(span.properties).length === 0)) && - typeof record.attributes_text === "string" - ) { - const parsedAttributes = this.#parseAttributes(record.attributes_text); - const resourceAttributes = parsedAttributes["$resource"]; + if (typeof record.attributes_text === "string") { + const shouldUpdate = + span.properties == null || + (typeof span.properties === "object" && Object.keys(span.properties).length === 0) || + (record.kind === "SPAN" && record.status !== "PARTIAL"); - // Remove the $resource key from the attributes - delete parsedAttributes["$resource"]; + if (shouldUpdate) { + const parsedAttributes = this.#parseAttributes(record.attributes_text); + const resourceAttributes = parsedAttributes["$resource"]; - span.properties = parsedAttributes; - span.resourceProperties = resourceAttributes as Record | undefined; + delete parsedAttributes["$resource"]; + + span.properties = parsedAttributes; + span.resourceProperties = resourceAttributes as Record | undefined; + } } } diff --git a/apps/webapp/app/v3/eventRepository/common.server.ts b/apps/webapp/app/v3/eventRepository/common.server.ts index 2e3bdf37c50..3ba8a50c7f7 100644 --- a/apps/webapp/app/v3/eventRepository/common.server.ts +++ b/apps/webapp/app/v3/eventRepository/common.server.ts @@ -140,7 +140,8 @@ export function createExceptionPropertiesFromError(error: TaskRunError): Excepti } } -// removes keys that start with a $ sign. If there are no keys left, return undefined +// Removes internal/private attribute keys from span properties. +// Filters: "$" prefixed keys (private metadata) and "ctx." prefixed keys (Trigger.dev run context) export function removePrivateProperties( attributes: Attributes | undefined | null ): Attributes | undefined { @@ -151,7 +152,7 @@ export function removePrivateProperties( const result: Attributes = {}; for (const [key, value] of Object.entries(attributes)) { - if (key.startsWith("$")) { + if (key.startsWith("$") || key.startsWith("ctx.")) { continue; } diff --git a/apps/webapp/app/v3/llmPricingRegistry.server.ts b/apps/webapp/app/v3/llmPricingRegistry.server.ts index d05b1afc2c8..e90a84689f5 100644 --- a/apps/webapp/app/v3/llmPricingRegistry.server.ts +++ b/apps/webapp/app/v3/llmPricingRegistry.server.ts @@ -1,37 +1,28 @@ import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-pricing"; -import { trail } from "agentcrumbs"; // @crumbs import { prisma, $replica } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server"; -const crumb = trail("webapp:llm-registry"); // @crumbs - async function initRegistry(registry: ModelPricingRegistry) { if (env.LLM_PRICING_SEED_ON_STARTUP) { - crumb("seeding llm pricing on startup"); // @crumbs const result = await seedLlmPricing(prisma); - crumb("seed complete", { modelsCreated: result.modelsCreated, modelsSkipped: result.modelsSkipped }); // @crumbs } await registry.loadFromDatabase(); - crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs } export const llmPricingRegistry = singleton("llmPricingRegistry", () => { if (!env.LLM_COST_TRACKING_ENABLED) { - crumb("llm cost tracking disabled via env"); // @crumbs return null; } - crumb("initializing registry singleton"); // @crumbs const registry = new ModelPricingRegistry($replica); // Wire up the registry so enrichCreatableEvents can use it setLlmPricingRegistry(registry); initRegistry(registry).catch((err) => { - crumb("registry init failed", { error: String(err) }); // @crumbs console.error("Failed to initialize LLM pricing registry", err); }); @@ -41,10 +32,8 @@ export const llmPricingRegistry = singleton("llmPricingRegistry", () => { registry .reload() .then(() => { - crumb("registry reloaded"); // @crumbs }) .catch((err) => { - crumb("registry reload failed", { error: String(err) }); // @crumbs console.error("Failed to reload LLM pricing registry", err); }); }, reloadInterval); diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index b5b5fee70fc..0a86fb65ec9 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -395,7 +395,22 @@ function convertSpansToCreateableEvents( ); const runTags = extractArrayAttribute(span.attributes ?? [], SemanticInternalAttributes.RUN_TAGS); - if (runTags && runTags.length > 0) { crumbOtlp("extracted runTags from span", { runTags, spanId: binaryToHex(span.spanId) }); } // @crumbs + + // #region @crumbs + if (span.attributes) { + crumbOtlp("span raw OTEL attrs", { + spanName: span.name, + spanId: binaryToHex(span.spanId), + attrCount: span.attributes.length, + attrs: span.attributes.map((a) => ({ + key: a.key, + type: a.value?.stringValue !== undefined ? "string" : a.value?.intValue !== undefined ? "int" : a.value?.doubleValue !== undefined ? "double" : a.value?.boolValue !== undefined ? "bool" : a.value?.arrayValue ? "array" : a.value?.bytesValue ? "bytes" : "unknown", + ...(a.value?.arrayValue ? { arrayLen: a.value.arrayValue.values?.length } : {}), + ...(a.value?.stringValue !== undefined ? { strLen: a.value.stringValue.length } : {}), + })), + }); + } + // #endregion @crumbs const properties = truncateAttributes( @@ -715,6 +730,8 @@ function convertKeyValueItemsToMap( ? attribute.value.boolValue : isBytesValue(attribute.value) ? binaryToHex(attribute.value.bytesValue) + : isArrayValue(attribute.value) + ? serializeArrayValue(attribute.value.arrayValue!.values) : undefined; return map; @@ -751,6 +768,8 @@ function convertSelectedKeyValueItemsToMap( ? attribute.value.boolValue : isBytesValue(attribute.value) ? binaryToHex(attribute.value.bytesValue) + : isArrayValue(attribute.value) + ? serializeArrayValue(attribute.value.arrayValue!.values) : undefined; return map; @@ -1064,6 +1083,31 @@ function isBytesValue(value: AnyValue | undefined): value is { bytesValue: Buffe return Buffer.isBuffer(value.bytesValue); } +function isArrayValue( + value: AnyValue | undefined +): value is { arrayValue: { values: AnyValue[] } } { + if (!value) return false; + + return value.arrayValue != null && Array.isArray(value.arrayValue.values); +} + +/** + * Serialize an OTEL array value into a JSON string. + * For arrays of strings, produces a JSON array: `["item1","item2"]` + * For mixed types, extracts primitives and serializes. + */ +function serializeArrayValue(values: AnyValue[]): string { + const items = values.map((v) => { + if (isStringValue(v)) return v.stringValue; + if (isIntValue(v)) return Number(v.intValue); + if (isDoubleValue(v)) return v.doubleValue; + if (isBoolValue(v)) return v.boolValue; + return null; + }); + + return JSON.stringify(items); +} + function binaryToHex(buffer: Buffer | string): string; function binaryToHex(buffer: Buffer | string | undefined): string | undefined; function binaryToHex(buffer: Buffer | string | undefined): string | undefined { diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index d76fdda5879..1c125060ae2 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -1,8 +1,5 @@ -import { trail } from "agentcrumbs"; // @crumbs import type { CreateEventInput, LlmUsageData } from "../eventRepository/eventRepository.types"; -const crumb = trail("webapp:llm-enrich"); // @crumbs - // Registry interface — matches ModelPricingRegistry from @internal/llm-pricing type CostRegistry = { isLoaded: boolean; @@ -48,25 +45,6 @@ function enrichLlmCost(event: CreateEventInput): void { const props = event.properties; if (!props) return; - // #region @crumbs - // Log all spans (not just gen_ai) that have conversation/chat/session/user context - if (!event.isPartial) { - const contextKeys = Object.entries(props).filter(([k]) => - k.startsWith("ai.telemetry.") || k.startsWith("gen_ai.conversation") || - k.startsWith("chat.") || k.includes("session") || k.includes("user") - ); - if (contextKeys.length > 0) { - crumb("span with context", { - spanId: event.spanId, - parentId: event.parentId, - runId: event.runId, - message: event.message, - contextAttrs: Object.fromEntries(contextKeys), - }); - } - } - // #endregion @crumbs - // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED) const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); if (!enrichableKinds.has(event.kind as string)) return; @@ -86,38 +64,25 @@ function enrichLlmCost(event: CreateEventInput): void { : null; if (!responseModel) { - // #region @crumbs - const genAiModel = props["gen_ai.response.model"]; - const aiModel = props["ai.model.id"]; - if (genAiModel || aiModel) { - crumb("responseModel null despite gen_ai attrs", { genAiModel, genAiModelType: typeof genAiModel, aiModel, spanId: event.spanId }); - } - // #endregion @crumbs return; } - crumb("llm span detected", { responseModel, kind: event.kind, spanId: event.spanId }); // @crumbs - // Extract usage details, normalizing attribute names const usageDetails = extractUsageDetails(props); // Need at least some token usage const hasTokens = Object.values(usageDetails).some((v) => v > 0); if (!hasTokens) { - crumb("llm span skipped: no tokens", { responseModel, spanId: event.spanId }); // @crumbs return; } if (!_registry?.isLoaded) { - crumb("llm span skipped: registry not loaded", { responseModel }); // @crumbs return; } const cost = _registry.calculateCost(responseModel, usageDetails); if (!cost) return; - crumb("llm cost enriched", { responseModel, totalCost: cost.totalCost, matchedModel: cost.matchedModelName, spanId: event.spanId }); // @crumbs - // Add trigger.llm.* attributes to the span event.properties = { ...props, @@ -165,21 +130,6 @@ function enrichLlmCost(event: CreateEventInput): void { } } - // #region @crumbs - const metadataKeyCount = Object.keys(metadata).length; - if (metadataKeyCount > 0) { - crumb("llm metadata built", { - spanId: event.spanId, - runId: event.runId, - responseModel, - metadataKeyCount, - metadataKeys: Object.keys(metadata), - fromRunTags: event.runTags?.length ?? 0, - fromTelemetry: metadataKeyCount - (event.runTags?.filter((t) => t.includes(":")).length ?? 0), - }); - } - // #endregion @crumbs - // Set _llmUsage side-channel for dual-write to llm_usage_v1 const llmUsage: LlmUsageData = { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 506a9360bd2..5b340eb4034 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -1,5 +1,4 @@ import type { PrismaClient } from "@trigger.dev/database"; -import { trail } from "agentcrumbs"; // @crumbs import type { LlmModelWithPricing, LlmCostResult, @@ -7,8 +6,6 @@ import type { PricingCondition, } from "./types.js"; -const crumb = trail("webapp:llm-pricing"); // @crumbs - type CompiledPattern = { regex: RegExp; model: LlmModelWithPricing; @@ -48,10 +45,7 @@ export class ModelPricingRegistry { orderBy: [{ startDate: "desc" }], }); - crumb("loaded models from db", { count: models.length }); // @crumbs - const compiled: CompiledPattern[] = []; - let skippedCount = 0; // @crumbs for (const model of models) { try { @@ -80,17 +74,14 @@ export class ModelPricingRegistry { }, }); } catch { - skippedCount++; // @crumbs // Skip models with invalid regex patterns console.warn(`Invalid regex pattern for model ${model.modelName}: ${model.matchPattern}`); - crumb("invalid regex pattern", { modelName: model.modelName, pattern: model.matchPattern }); // @crumbs } } this._patterns = compiled; this._exactMatchCache.clear(); this._loaded = true; - crumb("registry loaded", { patterns: compiled.length, skipped: skippedCount }); // @crumbs } async reload(): Promise { @@ -108,14 +99,12 @@ export class ModelPricingRegistry { for (const { regex, model } of this._patterns) { if (regex.test(responseModel)) { this._exactMatchCache.set(responseModel, model); - crumb("model matched", { responseModel, matchedModel: model.modelName }); // @crumbs return model; } } // Cache miss this._exactMatchCache.set(responseModel, null); - crumb("model not matched", { responseModel }); // @crumbs return null; } From ad96713146df8af05299538fb1d6dd221dfc5c20 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 09:05:45 +0000 Subject: [PATCH 05/15] A bunch of improvements around extracting AI data New model admin dashboard, test model strings, add and edit models, view missing models and easily add them. Also extract cost data from ai gateway provider response metadata, better enrichment. --- .../components/runs/v3/ai/AIModelSummary.tsx | 12 +- .../runs/v3/ai/extractAISpanData.ts | 44 +- .../webapp/app/components/runs/v3/ai/types.ts | 3 + apps/webapp/app/env.server.ts | 6 + .../routes/admin.api.v1.llm-models.missing.ts | 33 + .../app/routes/admin.llm-models.$modelId.tsx | 449 +++++ .../app/routes/admin.llm-models._index.tsx | 346 ++++ .../admin.llm-models.missing.$model.tsx | 467 +++++ .../admin.llm-models.missing._index.tsx | 158 ++ .../app/routes/admin.llm-models.new.tsx | 397 +++++ apps/webapp/app/routes/admin.tsx | 4 + .../services/admin/missingLlmModels.server.ts | 127 ++ .../app/services/clickhouseInstance.server.ts | 28 + .../v3/utils/enrichCreatableEvents.server.ts | 138 +- apps/webapp/package.json | 1 + apps/webapp/seed-ai-spans.mts | 1564 +++++++++++++++++ internal-packages/llm-pricing/src/registry.ts | 12 + 17 files changed, 3747 insertions(+), 42 deletions(-) create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts create mode 100644 apps/webapp/app/routes/admin.llm-models.$modelId.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models._index.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models.missing.$model.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models.missing._index.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models.new.tsx create mode 100644 apps/webapp/app/services/admin/missingLlmModels.server.ts create mode 100644 apps/webapp/seed-ai-spans.mts diff --git a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx index 9e627c844d3..a33b90ef899 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx @@ -6,6 +6,9 @@ export function AITagsRow({ aiData }: { aiData: AISpanData }) {
{aiData.model} {aiData.provider !== "unknown" && {aiData.provider}} + {aiData.resolvedProvider && ( + via {aiData.resolvedProvider} + )} {aiData.finishReason && {aiData.finishReason}} {aiData.serviceTier && tier: {aiData.serviceTier}} {aiData.toolChoice && tools: {aiData.toolChoice}} @@ -38,7 +41,14 @@ export function AIStatsSummary({ aiData }: { aiData: AISpanData }) { {aiData.cachedTokens != null && aiData.cachedTokens > 0 && ( - + + )} + {aiData.cacheCreationTokens != null && aiData.cacheCreationTokens > 0 && ( + )} {aiData.reasoningTokens != null && aiData.reasoningTokens > 0 && ( v ?? process.env.CLICKHOUSE_URL), + EVENTS_CLICKHOUSE_URL: z .string() .optional() diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts new file mode 100644 index 00000000000..5ca7077e1cc --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts @@ -0,0 +1,33 @@ +import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { getMissingLlmModels } from "~/services/admin/missingLlmModels.server"; + +async function requireAdmin(request: Request) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + throw json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + throw json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + return user; +} + +export async function loader({ request }: LoaderFunctionArgs) { + await requireAdmin(request); + + const url = new URL(request.url); + const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); + + if (isNaN(lookbackHours) || lookbackHours < 1 || lookbackHours > 720) { + return json({ error: "lookbackHours must be between 1 and 720" }, { status: 400 }); + } + + const models = await getMissingLlmModels({ lookbackHours }); + + return json({ models, lookbackHours }); +} diff --git a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx new file mode 100644 index 00000000000..4e72731cc3e --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx @@ -0,0 +1,449 @@ +import { Form, useNavigate } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { useState } from "react"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Input } from "~/components/primitives/Input"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const model = await prisma.llmModel.findUnique({ + where: { friendlyId: params.modelId }, + include: { + pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } }, + }, + }); + + if (!model) throw new Response("Model not found", { status: 404 }); + + // Convert Prisma Decimal to plain numbers for serialization + const serialized = { + ...model, + pricingTiers: model.pricingTiers.map((t) => ({ + ...t, + prices: t.prices.map((p) => ({ ...p, price: Number(p.price) })), + })), + }; + + return typedjson({ model: serialized }); +}; + +const SaveSchema = z.object({ + modelName: z.string().min(1), + matchPattern: z.string().min(1), + pricingTiersJson: z.string(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const friendlyId = params.modelId!; + const existing = await prisma.llmModel.findUnique({ where: { friendlyId } }); + if (!existing) throw new Response("Model not found", { status: 404 }); + const modelId = existing.id; + + const formData = await request.formData(); + const _action = formData.get("_action"); + + if (_action === "delete") { + await prisma.llmModel.delete({ where: { id: modelId } }); + await llmPricingRegistry?.reload(); + return redirect("/admin/llm-models"); + } + + if (_action === "save") { + const raw = Object.fromEntries(formData); + const parsed = SaveSchema.safeParse(raw); + + if (!parsed.success) { + return typedjson({ error: "Invalid form data", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, pricingTiersJson } = parsed.data; + + // Validate regex — strip (?i) POSIX flag since our registry handles it + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return typedjson({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + + // Parse tiers + let pricingTiers: Array<{ + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; + }>; + try { + pricingTiers = JSON.parse(pricingTiersJson); + } catch { + return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); + } + + // Update model + await prisma.llmModel.update({ + where: { id: modelId }, + data: { modelName, matchPattern }, + }); + + // Replace tiers + await prisma.llmPricingTier.deleteMany({ where: { modelId } }); + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId, + usageType, + price, + })), + }, + }, + }); + } + + await llmPricingRegistry?.reload(); + return typedjson({ success: true }); + } + + return typedjson({ error: "Unknown action" }, { status: 400 }); +} + +export default function AdminLlmModelDetailRoute() { + const { model } = useTypedLoaderData(); + const navigate = useNavigate(); + + const [modelName, setModelName] = useState(model.modelName); + const [matchPattern, setMatchPattern] = useState(model.matchPattern); + const [testInput, setTestInput] = useState(""); + const [tiers, setTiers] = useState(() => + model.pricingTiers.map((t) => ({ + name: t.name, + isDefault: t.isDefault, + priority: t.priority, + conditions: (t.conditions ?? []) as Array<{ + usageDetailPattern: string; + operator: string; + value: number; + }>, + prices: Object.fromEntries(t.prices.map((p) => [p.usageType, p.price])), + })) + ); + + // Test regex match + let testResult: boolean | null = null; + if (testInput) { + try { + const pattern = matchPattern.startsWith("(?i)") + ? matchPattern.slice(4) + : matchPattern; + testResult = new RegExp(pattern, "i").test(testInput); + } catch { + testResult = null; + } + } + + return ( +
+
+
+

{model.modelName}

+
+ + {model.source ?? "default"} + + + Back to list + +
+
+ +
+ + + +
+ {/* Model fields */} +
+ + setModelName(e.target.value)} + variant="medium" + fullWidth + /> +
+ +
+ + setMatchPattern(e.target.value)} + variant="medium" + fullWidth + className="font-mono text-xs" + /> +
+ + {/* Test pattern */} +
+ +
+ setTestInput(e.target.value)} + placeholder="Type a model name to test..." + variant="medium" + fullWidth + /> + {testInput && ( + + {testResult ? "Match" : "No match"} + + )} +
+
+ + {/* Pricing tiers */} +
+
+ + +
+ + {tiers.map((tier, tierIdx) => ( + { + const next = [...tiers]; + next[tierIdx] = updated; + setTiers(next); + }} + onRemove={() => setTiers(tiers.filter((_, i) => i !== tierIdx))} + /> + ))} +
+ + {/* Actions */} +
+ + + Cancel + +
+
+
+ + {/* Delete section */} +
+
{ + if (!confirm(`Delete model "${model.modelName}"?`)) e.preventDefault(); + }}> + + +
+
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Tier editor sub-component +// --------------------------------------------------------------------------- + +type TierData = { + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; +}; + +const COMMON_USAGE_TYPES = [ + "input", + "output", + "input_cached_tokens", + "cache_creation_input_tokens", + "reasoning_tokens", +]; + +function TierEditor({ + tier, + onChange, + onRemove, +}: { + tier: TierData; + onChange: (t: TierData) => void; + onRemove: () => void; +}) { + const [newUsageType, setNewUsageType] = useState(""); + + return ( +
+
+
+ onChange({ ...tier, name: e.target.value })} + placeholder="Tier name" + /> + + +
+ +
+ + {/* Prices */} +
+ + Prices (per token) + +
+ {Object.entries(tier.prices).map(([usageType, price]) => ( +
+ {usageType} + { + const val = parseFloat(e.target.value); + if (!isNaN(val)) { + onChange({ + ...tier, + prices: { ...tier.prices, [usageType]: val }, + }); + } + }} + /> + +
+ ))} +
+ + {/* Add price */} +
+ + {newUsageType && ( + + )} +
+
+
+ ); +} diff --git a/apps/webapp/app/routes/admin.llm-models._index.tsx b/apps/webapp/app/routes/admin.llm-models._index.tsx new file mode 100644 index 00000000000..2bfa46d0c8a --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models._index.tsx @@ -0,0 +1,346 @@ +import { MagnifyingGlassIcon } from "@heroicons/react/20/solid"; +import { Form, useFetcher, Link } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Input } from "~/components/primitives/Input"; +import { PaginationControls } from "~/components/primitives/Pagination"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { + Table, + TableBlankRow, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { createSearchParams } from "~/utils/searchParams"; +import { seedLlmPricing } from "@internal/llm-pricing"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +const PAGE_SIZE = 50; + +const SearchParams = z.object({ + page: z.coerce.number().optional(), + search: z.string().optional(), +}); + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const searchParams = createSearchParams(request.url, SearchParams); + if (!searchParams.success) throw new Error(searchParams.error); + const { page: rawPage, search } = searchParams.params.getAll(); + const page = rawPage ?? 1; + + const where = { + projectId: null as string | null, + ...(search ? { modelName: { contains: search, mode: "insensitive" as const } } : {}), + }; + + const [rawModels, total] = await Promise.all([ + prisma.llmModel.findMany({ + where, + include: { + pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } }, + }, + orderBy: { modelName: "asc" }, + skip: (page - 1) * PAGE_SIZE, + take: PAGE_SIZE, + }), + prisma.llmModel.count({ where }), + ]); + + // Convert Prisma Decimal to plain numbers for serialization + const models = rawModels.map((m) => ({ + ...m, + pricingTiers: m.pricingTiers.map((t) => ({ + ...t, + prices: t.prices.map((p) => ({ ...p, price: Number(p.price) })), + })), + })); + + return typedjson({ + models, + total, + page, + pageCount: Math.ceil(total / PAGE_SIZE), + filters: { search }, + }); +}; + +export async function action({ request }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const formData = await request.formData(); + const _action = formData.get("_action"); + + if (_action === "seed") { + console.log("[admin] seed action started"); + const result = await seedLlmPricing(prisma); + console.log(`[admin] seed complete: ${result.modelsCreated} created, ${result.modelsSkipped} skipped`); + await llmPricingRegistry?.reload(); + console.log("[admin] registry reloaded after seed"); + return typedjson({ + success: true, + message: `Seeded: ${result.modelsCreated} created, ${result.modelsSkipped} skipped`, + }); + } + + if (_action === "reload") { + console.log("[admin] reload action started"); + await llmPricingRegistry?.reload(); + console.log("[admin] registry reloaded"); + return typedjson({ success: true, message: "Registry reloaded" }); + } + + if (_action === "test") { + const modelString = formData.get("modelString"); + if (typeof modelString !== "string" || !modelString) { + return typedjson({ testResult: null }); + } + + // Use the registry's match() which handles prefix stripping automatically + const matched = llmPricingRegistry?.match(modelString) ?? null; + + return typedjson({ + testResult: { + modelString, + match: matched + ? { friendlyId: matched.friendlyId, modelName: matched.modelName } + : null, + }, + }); + } + + if (_action === "delete") { + const modelId = formData.get("modelId"); + if (typeof modelId === "string") { + await prisma.llmModel.delete({ where: { id: modelId } }); + await llmPricingRegistry?.reload(); + } + return typedjson({ success: true }); + } + + return typedjson({ error: "Unknown action" }, { status: 400 }); +} + +export default function AdminLlmModelsRoute() { + const { models, filters, page, pageCount, total } = + useTypedLoaderData(); + const seedFetcher = useFetcher(); + const reloadFetcher = useFetcher(); + const testFetcher = useFetcher<{ + testResult?: { + modelString: string; + match: { friendlyId: string; modelName: string } | null; + } | null; + }>(); + + const testResult = testFetcher.data?.testResult; + + return ( +
+
+
+
+ + +
+ +
+ + + + + + + + + + + + Missing models + + + + Add model + +
+
+ + {/* Model tester */} +
+ + + + + + + {testResult !== undefined && testResult !== null && ( +
+ + Testing: {testResult.modelString} + + {testResult.match ? ( +
+ Match:{" "} + + {testResult.match.modelName} + +
+ ) : ( +
+ No match found — this model has no pricing data +
+ )} +
+ )} +
+ +
+ + {total} global models (page {page} of {pageCount}) + + +
+ + + + + Model Name + Source + Input $/tok + Output $/tok + Other prices + + + + {models.length === 0 ? ( + + No models found + + ) : ( + models.map((model) => { + // Get default tier prices + const defaultTier = + model.pricingTiers.find((t) => t.isDefault) ?? model.pricingTiers[0]; + const priceMap = defaultTier + ? Object.fromEntries(defaultTier.prices.map((p) => [p.usageType, p.price])) + : {}; + const inputPrice = priceMap["input"]; + const outputPrice = priceMap["output"]; + const otherPrices = defaultTier + ? defaultTier.prices.filter( + (p) => p.usageType !== "input" && p.usageType !== "output" + ) + : []; + + return ( + + + + {model.modelName} + + + + + {model.source ?? "default"} + + + + + {inputPrice != null ? formatPrice(inputPrice) : "-"} + + + + + {outputPrice != null ? formatPrice(outputPrice) : "-"} + + + + {otherPrices.length > 0 ? ( + p.usageType).join(", ")}> + +{otherPrices.length} more + + ) : ( + - + )} + + + ); + }) + )} + +
+ + +
+
+ ); +} + +/** Format a per-token price as $/M tokens for readability */ +function formatPrice(perToken: number): string { + const perMillion = perToken * 1_000_000; + if (perMillion >= 1) return `$${perMillion.toFixed(2)}/M`; + if (perMillion >= 0.01) return `$${perMillion.toFixed(4)}/M`; + return `$${perMillion.toFixed(6)}/M`; +} diff --git a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx new file mode 100644 index 00000000000..7b6e83bf939 --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx @@ -0,0 +1,467 @@ +import { useState } from "react"; +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { + getMissingModelSamples, + type MissingModelSample, +} from "~/services/admin/missingLlmModels.server"; + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + // Model name is base64url-encoded in the URL param + const modelName = decodeURIComponent(params.model ?? ""); + if (!modelName) throw new Response("Missing model param", { status: 400 }); + + const url = new URL(request.url); + const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); + + let samples: MissingModelSample[] = []; + let error: string | undefined; + + try { + samples = await getMissingModelSamples({ model: modelName, lookbackHours, limit: 10 }); + } catch (e) { + error = e instanceof Error ? e.message : "Failed to query ClickHouse"; + } + + return typedjson({ modelName, samples, lookbackHours, error }); +}; + +export default function AdminMissingModelDetailRoute() { + const { modelName, samples, lookbackHours, error } = useTypedLoaderData(); + const [copied, setCopied] = useState(false); + const [expandedSpans, setExpandedSpans] = useState>(new Set()); + + const providerCosts = extractProviderCosts(samples); + const prompt = buildPrompt(modelName, samples, providerCosts); + + function handleCopy() { + navigator.clipboard.writeText(prompt).then(() => { + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }); + } + + function toggleSpan(spanId: string) { + setExpandedSpans((prev) => { + const next = new Set(prev); + if (next.has(spanId)) next.delete(spanId); + else next.add(spanId); + return next; + }); + } + + // Extract key token fields from the first sample for quick summary + const tokenSummary = samples.length > 0 ? extractTokenTypes(samples) : []; + + return ( +
+
+ {/* Header */} +
+
+

{modelName}

+ + Missing pricing — {samples.length} sample span{samples.length !== 1 ? "s" : ""} from + last {lookbackHours}h + +
+
+ + Add pricing + + + Back to missing + +
+
+ + {error && ( +
+ {error} +
+ )} + + {/* Token types summary */} + {tokenSummary.length > 0 && ( +
+ + Token types seen across samples + +
+ {tokenSummary.map((t) => ( + + {t.key} + + {t.min === t.max ? t.min.toLocaleString() : `${t.min.toLocaleString()}-${t.max.toLocaleString()}`} + + + ))} +
+ + These are the token usage types that need pricing entries (at minimum: input, output). + +
+ )} + + {/* Provider-reported costs */} + {providerCosts.length > 0 && ( +
+ + Provider-reported cost data found in {providerCosts.length} span{providerCosts.length !== 1 ? "s" : ""} + +
+ {providerCosts.map((c, i) => ( +
+ {c.source} + ${c.cost.toFixed(6)} + + ({c.inputTokens.toLocaleString()} in + {c.outputTokens.toLocaleString()} out) + +
+ ))} +
+ {providerCosts[0]?.estimatedInputPrice != null && ( +
+ + Estimated per-token rates (assuming ~3x output/input ratio): + +
+ input: {providerCosts[0].estimatedInputPrice.toExponential(4)} + output: {providerCosts[0].estimatedOutputPrice!.toExponential(4)} +
+ + Cross-reference with the provider's pricing page before using these estimates. + +
+ )} +
+ )} + + {/* Prompt section */} +
+
+ + Claude Code prompt — paste this to have it add pricing for this model + + +
+ 
+            {prompt}
+
+
+ + {/* Sample spans */} +
+ + Sample spans ({samples.length}) + + {samples.map((s) => { + const expanded = expandedSpans.has(s.span_id); + let parsedAttrs: Record | null = null; + try { + parsedAttrs = JSON.parse(s.attributes_text); + } catch { + // ignore + } + + return ( +
+ + {expanded && parsedAttrs && ( +
+ 
+                      {JSON.stringify(parsedAttrs, null, 2)}
+
+
+ )} +
+ ); + })} +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Extract unique token usage types across all samples +// --------------------------------------------------------------------------- + +type TokenTypeSummary = { key: string; min: number; max: number }; + +function extractTokenTypes(samples: MissingModelSample[]): TokenTypeSummary[] { + const stats = new Map(); + + for (const s of samples) { + let attrs: Record; + try { + attrs = JSON.parse(s.attributes_text); + } catch { + continue; + } + + // Collect from gen_ai.usage.* + const genAiUsage = getNestedObj(attrs, ["gen_ai", "usage"]); + if (genAiUsage) { + for (const [k, v] of Object.entries(genAiUsage)) { + if (typeof v === "number" && v > 0) { + const existing = stats.get(`gen_ai.usage.${k}`); + if (existing) { + existing.min = Math.min(existing.min, v); + existing.max = Math.max(existing.max, v); + } else { + stats.set(`gen_ai.usage.${k}`, { min: v, max: v }); + } + } + } + } + + // Collect from ai.usage.* + const aiUsage = getNestedObj(attrs, ["ai", "usage"]); + if (aiUsage) { + for (const [k, v] of Object.entries(aiUsage)) { + if (typeof v === "number" && v > 0) { + const existing = stats.get(`ai.usage.${k}`); + if (existing) { + existing.min = Math.min(existing.min, v); + existing.max = Math.max(existing.max, v); + } else { + stats.set(`ai.usage.${k}`, { min: v, max: v }); + } + } + } + } + } + + return Array.from(stats.entries()) + .map(([key, { min, max }]) => ({ key, min, max })) + .sort((a, b) => a.key.localeCompare(b.key)); +} + +function getNestedObj( + obj: Record, + path: string[] +): Record | null { + let current: unknown = obj; + for (const key of path) { + if (!current || typeof current !== "object") return null; + current = (current as Record)[key]; + } + return current && typeof current === "object" ? (current as Record) : null; +} + +// --------------------------------------------------------------------------- +// Extract provider-reported costs from providerMetadata +// --------------------------------------------------------------------------- + +type ProviderCostInfo = { + source: string; // "gateway" or "openrouter" + cost: number; + inputTokens: number; + outputTokens: number; + estimatedInputPrice?: number; // per-token estimate + estimatedOutputPrice?: number; // per-token estimate +}; + +function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[] { + const costs: ProviderCostInfo[] = []; + + for (const s of samples) { + let attrs: Record; + try { + attrs = JSON.parse(s.attributes_text); + } catch { + continue; + } + + // Parse providerMetadata — could be nested or stringified + let providerMeta: Record | null = null; + const aiResponse = getNestedObj(attrs, ["ai", "response"]); + const rawMeta = aiResponse?.providerMetadata; + if (typeof rawMeta === "string") { + try { providerMeta = JSON.parse(rawMeta); } catch {} + } else if (rawMeta && typeof rawMeta === "object") { + providerMeta = rawMeta as Record; + } + if (!providerMeta) continue; + + // Get token counts + const genAiUsage = getNestedObj(attrs, ["gen_ai", "usage"]); + const inputTokens = Number(genAiUsage?.input_tokens ?? 0); + const outputTokens = Number(genAiUsage?.output_tokens ?? 0); + if (inputTokens === 0 && outputTokens === 0) continue; + + // Gateway: { gateway: { cost: "0.0006615" } } + const gw = getNestedObj(providerMeta, ["gateway"]); + if (gw) { + const cost = parseFloat(String(gw.cost ?? "0")); + if (cost > 0) { + costs.push({ source: "gateway", cost, inputTokens, outputTokens }); + continue; + } + } + + // OpenRouter: { openrouter: { usage: { cost: 0.000135 } } } + const or = getNestedObj(providerMeta, ["openrouter"]); + const orUsage = or ? getNestedObj(or, ["usage"]) : null; + if (orUsage) { + const cost = Number(orUsage.cost ?? 0); + if (cost > 0) { + costs.push({ source: "openrouter", cost, inputTokens, outputTokens }); + continue; + } + } + } + + // Estimate per-token prices from aggregate costs if we have enough data + if (costs.length > 0) { + // Use least-squares to estimate input/output price from cost = input*pi + output*po + // With 2+ samples we can solve; with 1 we can only estimate a blended rate + const totalInput = costs.reduce((s, c) => s + c.inputTokens, 0); + const totalOutput = costs.reduce((s, c) => s + c.outputTokens, 0); + const totalCost = costs.reduce((s, c) => s + c.cost, 0); + + if (totalInput > 0 && totalOutput > 0) { + // Simple approach: assume output is 2-5x input price (common ratio) + // Use ratio r where output_price = r * input_price + // totalCost = input_price * (totalInput + r * totalOutput) + // Try r=3 (common for many models) + const r = 3; + const estimatedInputPrice = totalCost / (totalInput + r * totalOutput); + const estimatedOutputPrice = estimatedInputPrice * r; + + for (const c of costs) { + c.estimatedInputPrice = estimatedInputPrice; + c.estimatedOutputPrice = estimatedOutputPrice; + } + } + } + + return costs; +} + +// --------------------------------------------------------------------------- +// Prompt builder — focused on figuring out pricing, not API mechanics +// --------------------------------------------------------------------------- + +function buildPrompt(modelName: string, samples: MissingModelSample[], providerCosts: ProviderCostInfo[]): string { + const hasPrefix = modelName.includes("/"); + const prefix = hasPrefix ? modelName.split("/")[0] : null; + const baseName = hasPrefix ? modelName.split("/").slice(1).join("/") : modelName; + + // Extract token types from samples + const tokenTypes = extractTokenTypes(samples); + const tokenTypeList = tokenTypes.length > 0 + ? tokenTypes.map((t) => ` - ${t.key}: ${t.min === t.max ? t.min : `${t.min}-${t.max}`}`).join("\n") + : " (no token data found in samples)"; + + // Get a compact sample of attributes for context + let sampleAttrs = ""; + if (samples.length > 0) { + try { + const attrs = JSON.parse(samples[0].attributes_text); + // Extract just the relevant fields + const compact: Record = {}; + if (attrs.gen_ai) compact.gen_ai = attrs.gen_ai; + if (attrs.ai?.usage) compact["ai.usage"] = attrs.ai.usage; + if (attrs.ai?.response?.providerMetadata) { + compact["ai.response.providerMetadata"] = attrs.ai.response.providerMetadata; + } + sampleAttrs = JSON.stringify(compact, null, 2); + } catch { + // ignore + } + } + + // Build suggested regex + const escapedBase = baseName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const suggestedPattern = prefix + ? `(?i)^(${prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/)?(${escapedBase})$` + : `(?i)^(${escapedBase})$`; + + return `I need to add LLM pricing for the model "${modelName}". + +## Model info +- Full model string from spans: \`${modelName}\` +- Base model name: \`${baseName}\`${prefix ? `\n- Provider prefix: \`${prefix}\`` : ""} +- This model appears in production spans but has no pricing data. + +## Token types seen in spans +${tokenTypeList} + +## What I need you to do + +1. **Look up pricing**: Find the current per-token pricing for \`${baseName}\` from the provider's official pricing page. Search the web if needed. + +2. **Present the pricing to me** in the following format so I can review before adding: + +\`\`\` +Model name: ${baseName} +Match pattern: ${suggestedPattern} +Pricing tier: Standard + +Prices (per token): + input: + output: + (add any additional token types if applicable) +\`\`\` + +**IMPORTANT: Do NOT call the admin API or create the model yourself.** Just research the pricing and present it to me. I will add it via the admin dashboard or ask you to proceed once I've reviewed. + +## Pricing research notes + +- All prices should be in **cost per token** (NOT per million). To convert: divide $/M by 1,000,000. + - Example: $3.00/M tokens = 0.000003 per token +- The \`matchPattern\` regex should match the model name both with and without the provider prefix. + - Suggested: \`${suggestedPattern}\` + - This matches both \`${baseName}\` and \`${modelName}\` +- Based on the token types seen in spans, check if the provider charges differently for: + - \`input\` and \`output\` — always required + - \`input_cached_tokens\` — if the provider offers prompt caching discounts + - \`cache_creation_input_tokens\` — if there's a cache write cost + - \`reasoning_tokens\` — if the model has chain-of-thought/reasoning tokens${providerCosts.length > 0 ? ` + +## Provider-reported costs (from ${providerCosts[0].source}) +The gateway/router is reporting costs for this model. Use these to cross-reference your pricing: +${providerCosts.map((c) => `- $${c.cost.toFixed(6)} for ${c.inputTokens.toLocaleString()} input + ${c.outputTokens.toLocaleString()} output tokens`).join("\n")}${providerCosts[0].estimatedInputPrice != null ? ` +- Estimated per-token rates (rough, assuming ~3x output/input ratio): + - input: ${providerCosts[0].estimatedInputPrice.toExponential(4)} (${(providerCosts[0].estimatedInputPrice * 1_000_000).toFixed(4)} $/M) + - output: ${providerCosts[0].estimatedOutputPrice!.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice! * 1_000_000).toFixed(4)} $/M) +- Verify these against the official pricing page before using.` : ""}` : ""}${sampleAttrs ? ` + +## Sample span attributes (first span) +\`\`\`json +${sampleAttrs} +\`\`\`` : ""}`; +} diff --git a/apps/webapp/app/routes/admin.llm-models.missing._index.tsx b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx new file mode 100644 index 00000000000..b778b9a3dc9 --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx @@ -0,0 +1,158 @@ +import { useSearchParams } from "@remix-run/react"; +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { LinkButton } from "~/components/primitives/Buttons"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { + Table, + TableBlankRow, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { getMissingLlmModels } from "~/services/admin/missingLlmModels.server"; + +const LOOKBACK_OPTIONS = [ + { label: "1 hour", value: 1 }, + { label: "6 hours", value: 6 }, + { label: "24 hours", value: 24 }, + { label: "7 days", value: 168 }, + { label: "30 days", value: 720 }, +]; + +const SearchParams = z.object({ + lookbackHours: z.coerce.number().optional(), +}); + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const url = new URL(request.url); + const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); + + let models: Awaited> = []; + let error: string | undefined; + + try { + models = await getMissingLlmModels({ lookbackHours }); + } catch (e) { + error = e instanceof Error ? e.message : "Failed to query ClickHouse"; + } + + return typedjson({ models, lookbackHours, error }); +}; + +export default function AdminLlmModelsMissingRoute() { + const { models, lookbackHours, error } = useTypedLoaderData(); + const [searchParams, setSearchParams] = useSearchParams(); + + return ( +
+
+
+

Missing LLM Models

+ + Back to models + +
+ + + Models appearing in spans without cost enrichment. These models need pricing data added. + + + {/* Lookback selector */} +
+ Lookback: + {LOOKBACK_OPTIONS.map((opt) => ( + + {opt.label} + + ))} +
+ + {error && ( +
+ {error} +
+ )} + + + {models.length} unpriced model{models.length !== 1 ? "s" : ""} found in the last{" "} + {lookbackHours < 24 + ? `${lookbackHours}h` + : lookbackHours < 168 + ? `${lookbackHours / 24}d` + : `${Math.round(lookbackHours / 24)}d`} + + + + + + Model Name + Provider + Span Count + Actions + + + + {models.length === 0 ? ( + + All models have pricing data + + ) : ( + models.map((m) => ( + + )) + )} + +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Row component with link to detail page +// --------------------------------------------------------------------------- + +function MissingModelRow({ model: m }: { model: { model: string; system: string; count: number } }) { + return ( + + + + {m.model} + + + + {m.system || "-"} + + + {m.count.toLocaleString()} + + + + Details + + + + ); +} diff --git a/apps/webapp/app/routes/admin.llm-models.new.tsx b/apps/webapp/app/routes/admin.llm-models.new.tsx new file mode 100644 index 00000000000..b4cd957b740 --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.new.tsx @@ -0,0 +1,397 @@ +import { Form, useActionData, useSearchParams } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson } from "remix-typedjson"; +import { z } from "zod"; +import { useState } from "react"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Input } from "~/components/primitives/Input"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { generateFriendlyId } from "~/v3/friendlyIdentifiers"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + return typedjson({}); +}; + +const CreateSchema = z.object({ + modelName: z.string().min(1), + matchPattern: z.string().min(1), + pricingTiersJson: z.string(), +}); + +export async function action({ request }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const formData = await request.formData(); + const raw = Object.fromEntries(formData); + console.log("[admin] create model form data:", JSON.stringify(raw).slice(0, 500)); + const parsed = CreateSchema.safeParse(raw); + + if (!parsed.success) { + console.log("[admin] create model validation error:", JSON.stringify(parsed.error.issues)); + return typedjson({ error: "Invalid form data", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, pricingTiersJson } = parsed.data; + + // Validate regex — strip (?i) POSIX flag since our registry handles it + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return typedjson({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + + let pricingTiers: Array<{ + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; + }>; + try { + pricingTiers = JSON.parse(pricingTiersJson); + } catch { + return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); + } + + const model = await prisma.llmModel.create({ + data: { + friendlyId: generateFriendlyId("llm_model"), + modelName, + matchPattern, + source: "admin", + }, + }); + + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + await llmPricingRegistry?.reload(); + return redirect(`/admin/llm-models/${model.friendlyId}`); +} + +export default function AdminLlmModelNewRoute() { + const actionData = useActionData<{ error?: string; details?: unknown[] }>(); + const [params] = useSearchParams(); + const initialModelName = params.get("modelName") ?? ""; + const [modelName, setModelName] = useState(initialModelName); + const [matchPattern, setMatchPattern] = useState(""); + const [testInput, setTestInput] = useState(""); + const [tiers, setTiers] = useState([ + { name: "Standard", isDefault: true, priority: 0, conditions: [], prices: { input: 0, output: 0 } }, + ]); + + let testResult: boolean | null = null; + if (testInput && matchPattern) { + try { + const pattern = matchPattern.startsWith("(?i)") + ? matchPattern.slice(4) + : matchPattern; + testResult = new RegExp(pattern, "i").test(testInput); + } catch { + testResult = null; + } + } + + // Auto-generate match pattern from model name + function autoPattern() { + if (modelName) { + const escaped = modelName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + setMatchPattern(`(?i)^(${escaped})$`); + } + } + + return ( +
+
+
+

New LLM Model

+ + Back to list + +
+ +
+ + +
+
+ + setModelName(e.target.value)} + variant="medium" + fullWidth + placeholder="e.g. gemini-3-flash" + /> +
+ +
+
+ + +
+ setMatchPattern(e.target.value)} + variant="medium" + fullWidth + className="font-mono text-xs" + placeholder="(?i)^(google/)?(gemini-3-flash)$" + /> +
+ +
+ +
+ setTestInput(e.target.value)} + placeholder="Type a model name to test..." + variant="medium" + fullWidth + /> + {testInput && ( + + {testResult ? "Match" : "No match"} + + )} +
+
+ + {/* Pricing tiers */} +
+
+ + +
+ + {tiers.map((tier, tierIdx) => ( + { + const next = [...tiers]; + next[tierIdx] = updated; + setTiers(next); + }} + onRemove={() => setTiers(tiers.filter((_, i) => i !== tierIdx))} + /> + ))} +
+ + {actionData?.error && ( +
+ {actionData.error} + {actionData.details && ( + 
+                    {JSON.stringify(actionData.details, null, 2)}
+
+ )} +
+ )} + +
+ + + Cancel + +
+
+
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Shared tier editor (duplicated from detail page — could be extracted later) +// --------------------------------------------------------------------------- + +type TierData = { + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; +}; + +const COMMON_USAGE_TYPES = [ + "input", + "output", + "input_cached_tokens", + "cache_creation_input_tokens", + "reasoning_tokens", +]; + +function TierEditor({ + tier, + onChange, + onRemove, +}: { + tier: TierData; + onChange: (t: TierData) => void; + onRemove: () => void; +}) { + const [newUsageType, setNewUsageType] = useState(""); + + return ( +
+
+
+ onChange({ ...tier, name: e.target.value })} + placeholder="Tier name" + /> + + +
+ +
+ +
+ Prices (per token) +
+ {Object.entries(tier.prices).map(([usageType, price]) => ( +
+ {usageType} + { + const val = parseFloat(e.target.value); + if (!isNaN(val)) { + onChange({ ...tier, prices: { ...tier.prices, [usageType]: val } }); + } + }} + /> + +
+ ))} +
+ +
+ + {newUsageType && ( + + )} +
+
+
+ ); +} diff --git a/apps/webapp/app/routes/admin.tsx b/apps/webapp/app/routes/admin.tsx index ac8e56c855e..34792e66ee5 100644 --- a/apps/webapp/app/routes/admin.tsx +++ b/apps/webapp/app/routes/admin.tsx @@ -32,6 +32,10 @@ export default function Page() { label: "Concurrency", to: "/admin/concurrency", }, + { + label: "LLM Models", + to: "/admin/llm-models", + }, ]} layoutId={"admin"} /> diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts new file mode 100644 index 00000000000..01237ef2b57 --- /dev/null +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -0,0 +1,127 @@ +import { adminClickhouseClient } from "~/services/clickhouseInstance.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export type MissingLlmModel = { + model: string; + system: string; + count: number; +}; + +export async function getMissingLlmModels(opts: { + lookbackHours?: number; +} = {}): Promise { + const lookbackHours = opts.lookbackHours ?? 24; + const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000); + + // queryBuilderFast returns a factory function — call it to get the builder + const createBuilder = adminClickhouseClient.reader.queryBuilderFast<{ + model: string; + system: string; + cnt: string; + }>({ + name: "missingLlmModels", + table: "trigger_dev.task_events_v2", + columns: [ + { name: "model", expression: "attributes.gen_ai.response.model.:String" }, + { name: "system", expression: "attributes.gen_ai.system.:String" }, + { name: "cnt", expression: "count()" }, + ], + }); + const qb = createBuilder(); + + // Partition pruning on inserted_at (partition key is toDate(inserted_at)) + qb.where("inserted_at >= {since: DateTime64(3)}", { + since: formatDateTime(since), + }); + + // Only spans that have a model set + qb.where("attributes.gen_ai.response.model.:String != {empty: String}", { empty: "" }); + + // Only spans that were NOT cost-enriched (trigger.llm.total_cost is NULL) + qb.where("attributes.trigger.llm.total_cost.:Float64 IS NULL", {}); + + // Only completed spans + qb.where("kind = {kind: String}", { kind: "SPAN" }); + qb.where("status = {status: String}", { status: "OK" }); + + qb.groupBy("model, system"); + qb.orderBy("cnt DESC"); + qb.limit(100); + + const [err, rows] = await qb.execute(); + + if (err) { + throw err; + } + + if (!rows) { + return []; + } + + const candidates = rows + .filter((r) => r.model) + .map((r) => ({ + model: r.model, + system: r.system, + count: parseInt(r.cnt, 10), + })); + + if (candidates.length === 0) return []; + + // Filter out models that now have pricing in the database (added after spans were inserted). + // The registry's match() handles prefix stripping for gateway/openrouter models. + return candidates.filter((c) => !llmPricingRegistry?.match(c.model)); +} + +export type MissingModelSample = { + span_id: string; + run_id: string; + message: string; + attributes_text: string; + duration: string; + start_time: string; +}; + +export async function getMissingModelSamples(opts: { + model: string; + lookbackHours?: number; + limit?: number; +}): Promise { + const lookbackHours = opts.lookbackHours ?? 24; + const limit = opts.limit ?? 10; + const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000); + + const createBuilder = adminClickhouseClient.reader.queryBuilderFast({ + name: "missingModelSamples", + table: "trigger_dev.task_events_v2", + columns: [ + "span_id", + "run_id", + "message", + "attributes_text", + "duration", + "start_time", + ], + }); + const qb = createBuilder(); + + qb.where("inserted_at >= {since: DateTime64(3)}", { since: formatDateTime(since) }); + qb.where("attributes.gen_ai.response.model.:String = {model: String}", { model: opts.model }); + qb.where("attributes.trigger.llm.total_cost.:Float64 IS NULL", {}); + qb.where("kind = {kind: String}", { kind: "SPAN" }); + qb.where("status = {status: String}", { status: "OK" }); + qb.orderBy("start_time DESC"); + qb.limit(limit); + + const [err, rows] = await qb.execute(); + + if (err) { + throw err; + } + + return rows ?? []; +} + +function formatDateTime(date: Date): string { + return date.toISOString().replace("T", " ").replace("Z", ""); +} diff --git a/apps/webapp/app/services/clickhouseInstance.server.ts b/apps/webapp/app/services/clickhouseInstance.server.ts index 61494811a0e..9c4941671f3 100644 --- a/apps/webapp/app/services/clickhouseInstance.server.ts +++ b/apps/webapp/app/services/clickhouseInstance.server.ts @@ -71,6 +71,34 @@ function initializeLogsClickhouseClient() { }); } +export const adminClickhouseClient = singleton( + "adminClickhouseClient", + initializeAdminClickhouseClient +); + +function initializeAdminClickhouseClient() { + if (!env.ADMIN_CLICKHOUSE_URL) { + throw new Error("ADMIN_CLICKHOUSE_URL is not set"); + } + + const url = new URL(env.ADMIN_CLICKHOUSE_URL); + url.searchParams.delete("secure"); + + return new ClickHouse({ + url: url.toString(), + name: "admin-clickhouse", + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); +} + export const queryClickhouseClient = singleton( "queryClickhouseClient", initializeQueryClickhouseClient diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 1c125060ae2..a062e491de3 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -76,42 +76,66 @@ function enrichLlmCost(event: CreateEventInput): void { return; } - if (!_registry?.isLoaded) { - return; - } - - const cost = _registry.calculateCost(responseModel, usageDetails); - if (!cost) return; - - // Add trigger.llm.* attributes to the span - event.properties = { - ...props, - "trigger.llm.input_cost": cost.inputCost, - "trigger.llm.output_cost": cost.outputCost, - "trigger.llm.total_cost": cost.totalCost, - "trigger.llm.matched_model": cost.matchedModelName, - "trigger.llm.matched_model_id": cost.matchedModelId, - "trigger.llm.pricing_tier": cost.pricingTierName, - "trigger.llm.pricing_tier_id": cost.pricingTierId, - }; - - // Add style accessories for model, tokens, and cost + // Add style accessories for model and tokens (even without cost data) const inputTokens = usageDetails["input"] ?? 0; const outputTokens = usageDetails["output"] ?? 0; const totalTokens = inputTokens + outputTokens; + const pillItems: Array<{ text: string; icon: string }> = [ + { text: responseModel, icon: "tabler-cube" }, + { text: formatTokenCount(totalTokens), icon: "tabler-hash" }, + ]; + + // Try cost enrichment if the registry is loaded. + // The registry handles prefix stripping (e.g. "mistral/mistral-large-3" → "mistral-large-3") + // for gateway/openrouter models automatically in its match() method. + let cost: ReturnType["calculateCost"]> | null = null; + if (_registry?.isLoaded) { + cost = _registry.calculateCost(responseModel, usageDetails); + } + + // Fallback: extract cost from provider metadata (gateway/openrouter report per-request cost) + let providerCost: { totalCost: number; source: string } | null = null; + if (!cost) { + providerCost = extractProviderCost(props); + } + + if (cost) { + // Add trigger.llm.* attributes to the span from our pricing registry + event.properties = { + ...props, + "trigger.llm.input_cost": cost.inputCost, + "trigger.llm.output_cost": cost.outputCost, + "trigger.llm.total_cost": cost.totalCost, + "trigger.llm.matched_model": cost.matchedModelName, + "trigger.llm.matched_model_id": cost.matchedModelId, + "trigger.llm.pricing_tier": cost.pricingTierName, + "trigger.llm.pricing_tier_id": cost.pricingTierId, + }; + + pillItems.push({ text: formatCost(cost.totalCost), icon: "tabler-currency-dollar" }); + } else if (providerCost) { + // Use provider-reported cost as fallback (no input/output breakdown available) + event.properties = { + ...props, + "trigger.llm.total_cost": providerCost.totalCost, + "trigger.llm.cost_source": providerCost.source, + }; + + pillItems.push({ text: formatCost(providerCost.totalCost), icon: "tabler-currency-dollar" }); + } + event.style = { ...event.style, accessory: { style: "pills", - items: [ - { text: responseModel, icon: "tabler-cube" }, - { text: formatTokenCount(totalTokens), icon: "tabler-hash" }, - { text: formatCost(cost.totalCost), icon: "tabler-currency-dollar" }, - ], + items: pillItems, }, }; + // Only write llm_usage when cost data is available + if (!cost && !providerCost) return; + // Build metadata map from run tags and ai.telemetry.metadata.* const metadata: Record = {}; @@ -135,18 +159,18 @@ function enrichLlmCost(event: CreateEventInput): void { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", requestModel: (props["gen_ai.request.model"] as string) ?? responseModel, responseModel, - matchedModelId: cost.matchedModelId, + matchedModelId: cost?.matchedModelId ?? "", operationName: (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? "", - pricingTierId: cost.pricingTierId, - pricingTierName: cost.pricingTierName, + pricingTierId: cost?.pricingTierId ?? (providerCost ? `provider:${providerCost.source}` : ""), + pricingTierName: cost?.pricingTierName ?? (providerCost ? `${providerCost.source} reported` : ""), inputTokens: usageDetails["input"] ?? 0, outputTokens: usageDetails["output"] ?? 0, totalTokens: Object.values(usageDetails).reduce((sum, v) => sum + v, 0), usageDetails, - inputCost: cost.inputCost, - outputCost: cost.outputCost, - totalCost: cost.totalCost, - costDetails: cost.costDetails, + inputCost: cost?.inputCost ?? 0, + outputCost: cost?.outputCost ?? 0, + totalCost: cost?.totalCost ?? providerCost?.totalCost ?? 0, + costDetails: cost?.costDetails ?? {}, metadata, }; @@ -197,6 +221,15 @@ function enrichStyle(event: CreateEventInput) { // GenAI System check const system = props["gen_ai.system"]; if (typeof system === "string") { + // For gateway/openrouter, derive the icon from the model's provider prefix + // e.g. "mistral/mistral-large-3" → "mistral", "anthropic/claude-..." → "anthropic" + if (system === "gateway" || system === "openrouter") { + const modelId = props["gen_ai.request.model"] ?? props["ai.model.id"]; + if (typeof modelId === "string" && modelId.includes("/")) { + const provider = modelId.split("/")[0].replace(/-/g, ""); + return { ...baseStyle, icon: `tabler-brand-${provider}` }; + } + } return { ...baseStyle, icon: `tabler-brand-${system.split(".")[0]}` }; } @@ -225,6 +258,47 @@ function formatTokenCount(tokens: number): string { return tokens.toString(); } +/** + * Extract provider-reported cost from ai.response.providerMetadata. + * Gateway and OpenRouter include per-request cost in their metadata. + */ +function extractProviderCost( + props: Record +): { totalCost: number; source: string } | null { + const rawMeta = props["ai.response.providerMetadata"]; + if (typeof rawMeta !== "string") return null; + + let meta: Record; + try { + meta = JSON.parse(rawMeta); + } catch { + return null; + } + + if (!meta || typeof meta !== "object") return null; + + // Gateway: { gateway: { cost: "0.0006615" } } + const gateway = meta.gateway; + if (gateway && typeof gateway === "object") { + const gw = gateway as Record; + const cost = parseFloat(String(gw.cost ?? "0")); + if (cost > 0) return { totalCost: cost, source: "gateway" }; + } + + // OpenRouter: { openrouter: { usage: { cost: 0.000135 } } } + const openrouter = meta.openrouter; + if (openrouter && typeof openrouter === "object") { + const or = openrouter as Record; + const usage = or.usage; + if (usage && typeof usage === "object") { + const cost = Number((usage as Record).cost ?? 0); + if (cost > 0) return { totalCost: cost, source: "openrouter" }; + } + } + + return null; +} + function formatCost(cost: number): string { if (cost >= 1) return `$${cost.toFixed(2)}`; if (cost >= 0.01) return `$${cost.toFixed(4)}`; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index fce7f2769c1..48994376066 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -16,6 +16,7 @@ "start:local": "cross-env node --max-old-space-size=8192 ./build/server.js", "typecheck": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsc --noEmit -p ./tsconfig.check.json", "db:seed": "tsx seed.mts", + "db:seed:ai-spans": "tsx seed-ai-spans.mts", "upload:sourcemaps": "bash ./upload-sourcemaps.sh", "test": "vitest --no-file-parallelism", "eval:dev": "evalite watch" diff --git a/apps/webapp/seed-ai-spans.mts b/apps/webapp/seed-ai-spans.mts new file mode 100644 index 00000000000..19a72c23f22 --- /dev/null +++ b/apps/webapp/seed-ai-spans.mts @@ -0,0 +1,1564 @@ +import { trail } from "agentcrumbs"; // @crumbs +const crumb = trail("webapp"); // @crumbs +import { prisma } from "./app/db.server"; +import { createOrganization } from "./app/models/organization.server"; +import { createProject } from "./app/models/project.server"; +import { ClickHouse } from "@internal/clickhouse"; +import type { TaskEventV2Input, LlmUsageV1Input } from "@internal/clickhouse"; +import { + generateTraceId, + generateSpanId, +} from "./app/v3/eventRepository/common.server"; +import { + enrichCreatableEvents, + setLlmPricingRegistry, +} from "./app/v3/utils/enrichCreatableEvents.server"; +import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-pricing"; +import { nanoid } from "nanoid"; +import { unflattenAttributes } from "@trigger.dev/core/v3/utils/flattenAttributes"; +import type { Attributes } from "@opentelemetry/api"; +import type { CreateEventInput } from "./app/v3/eventRepository/eventRepository.types"; + +const ORG_TITLE = "AI Spans Dev"; +const PROJECT_NAME = "ai-chat-demo"; +const TASK_SLUG = "ai-chat"; +const QUEUE_NAME = "task/ai-chat"; +const WORKER_VERSION = "seed-ai-spans-v1"; + +// --------------------------------------------------------------------------- +// ClickHouse formatting helpers (replicated from clickhouseEventRepository) +// --------------------------------------------------------------------------- + +function formatStartTime(startTimeNs: bigint): string { + const str = startTimeNs.toString(); + if (str.length !== 19) return str; + return str.substring(0, 10) + "." + str.substring(10); +} + +function formatDuration(value: number | bigint): string { + if (value < 0) return "0"; + if (typeof value === "bigint") return value.toString(); + return Math.floor(value).toString(); +} + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().replace("T", " ").replace("Z", ""); +} + +function removePrivateProperties(attributes: Attributes): Attributes | undefined { + const result: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (key.startsWith("$") || key.startsWith("ctx.")) continue; + result[key] = value; + } + return Object.keys(result).length === 0 ? undefined : result; +} + +function eventToClickhouseRow(event: CreateEventInput): TaskEventV2Input { + // kind + let kind: string; + if (event.kind === "UNSPECIFIED") kind = "ANCESTOR_OVERRIDE"; + else if (event.level === "TRACE") kind = "SPAN"; + else if (event.isDebug) kind = "DEBUG_EVENT"; + else kind = `LOG_${(event.level ?? "LOG").toString().toUpperCase()}`; + + // status + let status: string; + if (event.isPartial) status = "PARTIAL"; + else if (event.isError) status = "ERROR"; + else if (event.isCancelled) status = "CANCELLED"; + else status = "OK"; + + // attributes + const publicAttrs = removePrivateProperties(event.properties as Attributes); + const unflattened = publicAttrs ? unflattenAttributes(publicAttrs) : {}; + const attributes = + unflattened && typeof unflattened === "object" ? { ...unflattened } : {}; + + // metadata — mirrors createEventToTaskEventV1InputMetadata + const metadataObj: Record = {}; + if (event.style) { + metadataObj.style = unflattenAttributes(event.style as Attributes); + } + if (event.attemptNumber) { + metadataObj.attemptNumber = event.attemptNumber; + } + // Extract entity from properties (SemanticInternalAttributes) + const entityType = event.properties?.["$entity.type"]; + if (typeof entityType === "string") { + metadataObj.entity = { + entityType, + entityId: event.properties?.["$entity.id"] as string | undefined, + entityMetadata: event.properties?.["$entity.metadata"] as string | undefined, + }; + } + const metadata = JSON.stringify(metadataObj); + + return { + environment_id: event.environmentId, + organization_id: event.organizationId, + project_id: event.projectId, + task_identifier: event.taskSlug, + run_id: event.runId, + start_time: formatStartTime(BigInt(event.startTime)), + duration: formatDuration(event.duration ?? 0), + trace_id: event.traceId, + span_id: event.spanId, + parent_span_id: event.parentId ?? "", + message: event.message, + kind, + status, + attributes, + metadata, + expires_at: formatClickhouseDateTime( + new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) + ), + machine_id: "", + }; +} + +function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { + const llm = event._llmUsage!; + return { + organization_id: event.organizationId, + project_id: event.projectId, + environment_id: event.environmentId, + run_id: event.runId, + task_identifier: event.taskSlug, + trace_id: event.traceId, + span_id: event.spanId, + gen_ai_system: llm.genAiSystem, + request_model: llm.requestModel, + response_model: llm.responseModel, + matched_model_id: llm.matchedModelId, + operation_name: llm.operationName, + pricing_tier_id: llm.pricingTierId, + pricing_tier_name: llm.pricingTierName, + input_tokens: llm.inputTokens, + output_tokens: llm.outputTokens, + total_tokens: llm.totalTokens, + usage_details: llm.usageDetails, + input_cost: llm.inputCost, + output_cost: llm.outputCost, + total_cost: llm.totalCost, + cost_details: llm.costDetails, + metadata: llm.metadata, + start_time: formatStartTime(BigInt(event.startTime)), + duration: formatDuration(event.duration ?? 0), + }; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function seedAiSpans() { + crumb("seed started"); // @crumbs + console.log("Starting AI span seed...\n"); + + // 1. Find user + crumb("finding user"); // @crumbs + const user = await prisma.user.findUnique({ + where: { email: "local@trigger.dev" }, + }); + if (!user) { + console.error("User local@trigger.dev not found. Run `pnpm run db:seed` first."); + process.exit(1); + } + crumb("user found", { userId: user.id }); // @crumbs + + // 2. Find or create org + crumb("finding/creating org"); // @crumbs + let org = await prisma.organization.findFirst({ + where: { title: ORG_TITLE, members: { some: { userId: user.id } } }, + }); + if (!org) { + org = await createOrganization({ title: ORG_TITLE, userId: user.id, companySize: "1-10" }); + console.log(`Created org: ${org.title} (${org.slug})`); + } else { + console.log(`Org exists: ${org.title} (${org.slug})`); + } + crumb("org ready", { orgId: org.id, slug: org.slug }); // @crumbs + + // 3. Find or create project + crumb("finding/creating project"); // @crumbs + let project = await prisma.project.findFirst({ + where: { name: PROJECT_NAME, organizationId: org.id }, + }); + if (!project) { + project = await createProject({ + organizationSlug: org.slug, + name: PROJECT_NAME, + userId: user.id, + version: "v3", + }); + console.log(`Created project: ${project.name} (${project.externalRef})`); + } else { + console.log(`Project exists: ${project.name} (${project.externalRef})`); + } + crumb("project ready", { projectId: project.id, ref: project.externalRef }); // @crumbs + + // 4. Get DEVELOPMENT environment + crumb("finding dev environment"); // @crumbs + const runtimeEnv = await prisma.runtimeEnvironment.findFirst({ + where: { projectId: project.id, type: "DEVELOPMENT" }, + }); + if (!runtimeEnv) { + console.error("No DEVELOPMENT environment found for project."); + process.exit(1); + } + crumb("dev env found", { envId: runtimeEnv.id }); // @crumbs + + // 5. Upsert background worker + crumb("upserting worker/task/queue"); // @crumbs + const worker = await prisma.backgroundWorker.upsert({ + where: { + projectId_runtimeEnvironmentId_version: { + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + version: WORKER_VERSION, + }, + }, + update: {}, + create: { + friendlyId: `worker_${nanoid()}`, + engine: "V2", + contentHash: `seed-ai-spans-${Date.now()}`, + sdkVersion: "3.0.0", + cliVersion: "3.0.0", + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + version: WORKER_VERSION, + metadata: {}, + }, + }); + + // 6. Upsert task + await prisma.backgroundWorkerTask.upsert({ + where: { workerId_slug: { workerId: worker.id, slug: TASK_SLUG } }, + update: {}, + create: { + friendlyId: `task_${nanoid()}`, + slug: TASK_SLUG, + filePath: "src/trigger/ai-chat.ts", + exportName: "aiChat", + workerId: worker.id, + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + }, + }); + + // 7. Upsert queue + await prisma.taskQueue.upsert({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: runtimeEnv.id, + name: QUEUE_NAME, + }, + }, + update: {}, + create: { + friendlyId: `queue_${nanoid()}`, + name: QUEUE_NAME, + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + }, + }); + + crumb("infra upserts done"); // @crumbs + + // 8. Create the TaskRun + crumb("creating TaskRun"); // @crumbs + const traceId = generateTraceId(); + const rootSpanId = generateSpanId(); + const now = Date.now(); + // Spans start at `now` and extend into the future. completedAt must cover + // the full span tree so getSpan's start_time <= completedAt filter works. + const startedAt = new Date(now); + const completedAt = new Date(now + 150_000); // 2.5 min to cover all spans + + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${nanoid()}`, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + taskIdentifier: TASK_SLUG, + payload: JSON.stringify({ + message: "What is the current Federal Reserve interest rate?", + }), + payloadType: "application/json", + traceId, + spanId: rootSpanId, + runtimeEnvironmentId: runtimeEnv.id, + projectId: project.id, + organizationId: org.id, + queue: QUEUE_NAME, + lockedToVersionId: worker.id, + startedAt, + completedAt, + runTags: ["user:seed_user_42", "chat:seed_session"], + taskEventStore: "clickhouse_v2", + }, + }); + + crumb("TaskRun created", { runId: run.friendlyId, traceId }); // @crumbs + console.log(`Created TaskRun: ${run.friendlyId}`); + + // 9. Build span tree + crumb("building span tree"); // @crumbs + const events = buildAiSpanTree({ + traceId, + rootSpanId, + runId: run.friendlyId, + environmentId: runtimeEnv.id, + projectId: project.id, + organizationId: org.id, + taskSlug: TASK_SLUG, + baseTimeMs: now, + }); + + crumb("span tree built", { spanCount: events.length }); // @crumbs + console.log(`Built ${events.length} spans`); + + // 10. Seed LLM pricing and enrich + crumb("seeding LLM pricing"); // @crumbs + const seedResult = await seedLlmPricing(prisma); + console.log( + `LLM pricing: ${seedResult.modelsCreated} created, ${seedResult.modelsSkipped} skipped` + ); + + crumb("loading pricing registry"); // @crumbs + const registry = new ModelPricingRegistry(prisma); + setLlmPricingRegistry(registry); + await registry.loadFromDatabase(); + + crumb("enriching events"); // @crumbs + const enriched = enrichCreatableEvents(events); + + const enrichedCount = enriched.filter((e) => e._llmUsage != null).length; + const totalCost = enriched.reduce((sum, e) => sum + (e._llmUsage?.totalCost ?? 0), 0); + console.log( + `Enriched ${enrichedCount} spans with LLM cost (total: $${totalCost.toFixed(6)})` + ); + + crumb("enrichment done", { enrichedCount, totalCost }); // @crumbs + + // 11. Insert into ClickHouse + crumb("inserting into ClickHouse"); // @crumbs + const clickhouseUrl = process.env.CLICKHOUSE_URL ?? process.env.EVENTS_CLICKHOUSE_URL; + if (!clickhouseUrl) { + console.error("CLICKHOUSE_URL or EVENTS_CLICKHOUSE_URL not set"); + process.exit(1); + } + + const url = new URL(clickhouseUrl); + url.searchParams.delete("secure"); + const clickhouse = new ClickHouse({ url: url.toString() }); + + // Convert to ClickHouse rows and insert + const chRows = enriched.map(eventToClickhouseRow); + await clickhouse.taskEventsV2.insert(chRows); + + crumb("task events inserted", { rowCount: chRows.length }); // @crumbs + + // Insert LLM usage rows + const llmRows = enriched.filter((e) => e._llmUsage != null).map(eventToLlmUsageRow); + if (llmRows.length > 0) { + await clickhouse.llmUsage.insert(llmRows); + crumb("llm usage inserted", { rowCount: llmRows.length }); // @crumbs + } + + // 12. Output + console.log("\nDone!\n"); + console.log( + `Run URL: http://localhost:3030/orgs/${org.slug}/projects/${project.slug}/env/dev/runs/${run.friendlyId}` + ); + console.log(`Spans: ${events.length}`); + console.log(`LLM cost enriched: ${enrichedCount}`); + console.log(`Total cost: $${totalCost.toFixed(6)}`); + crumb("seed complete"); // @crumbs + process.exit(0); +} + +// --------------------------------------------------------------------------- +// Span tree builder +// --------------------------------------------------------------------------- + +type SpanTreeParams = { + traceId: string; + rootSpanId: string; + runId: string; + environmentId: string; + projectId: string; + organizationId: string; + taskSlug: string; + baseTimeMs: number; +}; + +function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { + const { + traceId, + rootSpanId, + runId, + environmentId, + projectId, + organizationId, + taskSlug, + baseTimeMs, + } = params; + + const events: CreateEventInput[] = []; + const runTags = ["user:seed_user_42", "chat:seed_session"]; + + // Timing cursor — each span advances this + let cursor = baseTimeMs; + function next(durationMs: number) { + const start = cursor; + cursor += durationMs + 50; // 50ms gap between spans + return { startMs: start, durationMs }; + } + + function makeEvent(opts: { + message: string; + spanId: string; + parentId: string | undefined; + startMs: number; + durationMs: number; + properties: Record; + style?: Record; + attemptNumber?: number; + }): CreateEventInput { + const startNs = BigInt(opts.startMs) * BigInt(1_000_000); + const durationNs = opts.durationMs * 1_000_000; + return { + traceId, + spanId: opts.spanId, + parentId: opts.parentId, + message: opts.message, + kind: "INTERNAL" as any, + status: "OK" as any, + level: "TRACE" as any, + startTime: startNs, + duration: durationNs, + isError: false, + isPartial: false, + isCancelled: false, + isDebug: false, + runId, + environmentId, + projectId, + organizationId, + taskSlug, + properties: opts.properties, + metadata: undefined, + style: opts.style as any, + events: undefined, + runTags, + attemptNumber: opts.attemptNumber, + }; + } + + // --- Shared prompt content --- + const userMessage = "What is the current Federal Reserve interest rate?"; + const systemPrompt = "You are a helpful financial assistant with access to web search tools."; + const assistantResponse = + "The current Federal Reserve interest rate target range is 4.25% to 4.50%. This was set by the FOMC at their most recent meeting."; + const toolCallResult = JSON.stringify({ + status: 200, + contentType: "text/html", + body: "...Federal Reserve maintains the target range for the federal funds rate at 4-1/4 to 4-1/2 percent...", + truncated: true, + }); + const promptMessages = JSON.stringify([ + { role: "system", content: systemPrompt }, + { role: "user", content: userMessage }, + ]); + const toolDefs = JSON.stringify([ + JSON.stringify({ + type: "function", + name: "webSearch", + description: "Search the web for information", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + num: { type: "number" }, + }, + required: ["query"], + }, + }), + ]); + const toolCallsJson = JSON.stringify([ + { + id: "call_seed_001", + type: "function", + function: { + name: "webSearch", + arguments: '{"query":"federal reserve interest rate 2024","num":5}', + }, + }, + ]); + + // --- Span IDs --- + const attemptId = generateSpanId(); + const runFnId = generateSpanId(); + + // streamText sub-tree IDs + const streamWrapId = generateSpanId(); + const stream1Id = generateSpanId(); + const toolCall1Id = generateSpanId(); + const stream2Id = generateSpanId(); + + // generateText sub-tree IDs (Anthropic with cache) + const genTextWrapId = generateSpanId(); + const genTextDoId = generateSpanId(); + const toolCall2Id = generateSpanId(); + + // generateObject sub-tree IDs (gateway → xAI) + const genObjWrapId = generateSpanId(); + const genObjDoId = generateSpanId(); + + // generateObject sub-tree IDs (Google Gemini) + const genObjGeminiWrapId = generateSpanId(); + const genObjGeminiDoId = generateSpanId(); + + // ===================================================================== + // Structural spans: root → attempt → run() + // ===================================================================== + const rootStart = baseTimeMs; + const totalDuration = 120_000; // 2 minutes to cover all ~18 scenarios + + events.push( + makeEvent({ + message: taskSlug, + spanId: rootSpanId, + parentId: undefined, + startMs: rootStart, + durationMs: totalDuration, + properties: {}, + }) + ); + + events.push( + makeEvent({ + message: "Attempt 1", + spanId: attemptId, + parentId: rootSpanId, + startMs: rootStart + 30, + durationMs: totalDuration - 60, + properties: { "$entity.type": "attempt" }, + style: { icon: "attempt", variant: "cold" }, + attemptNumber: 1, + }) + ); + + events.push( + makeEvent({ + message: "run()", + spanId: runFnId, + parentId: attemptId, + startMs: rootStart + 60, + durationMs: totalDuration - 120, + properties: {}, + style: { icon: "task-fn-run" }, + attemptNumber: 1, + }) + ); + + // ===================================================================== + // 1) ai.streamText — OpenAI gpt-4o-mini with tool use (2 LLM calls) + // ===================================================================== + cursor = rootStart + 100; + const stWrap = next(9_500); + + events.push( + makeEvent({ + message: "ai.streamText", + spanId: streamWrapId, + parentId: runFnId, + ...stWrap, + properties: { + "ai.operationId": "ai.streamText", + "ai.model.id": "gpt-4o-mini", + "ai.model.provider": "openai.responses", + "ai.response.finishReason": "stop", + "ai.response.text": assistantResponse, + "ai.usage.inputTokens": 807, + "ai.usage.outputTokens": 242, + "ai.usage.totalTokens": 1049, + "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.streamText", + }, + }) + ); + + cursor = stWrap.startMs + 50; + const st1 = next(2_500); + events.push( + makeEvent({ + message: "ai.streamText.doStream", + spanId: stream1Id, + parentId: streamWrapId, + ...st1, + properties: { + "gen_ai.system": "openai.responses", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.usage.input_tokens": 284, + "gen_ai.usage.output_tokens": 55, + "ai.model.id": "gpt-4o-mini", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText.doStream", + "ai.prompt.messages": promptMessages, + "ai.prompt.tools": toolDefs, + "ai.prompt.toolChoice": '{"type":"auto"}', + "ai.settings.maxRetries": 2, + "ai.response.finishReason": "tool-calls", + "ai.response.toolCalls": toolCallsJson, + "ai.response.text": "", + "ai.response.id": "resp_seed_001", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.msToFirstChunk": 891.37, + "ai.response.msToFinish": 2321.12, + "ai.response.timestamp": new Date(st1.startMs + st1.durationMs).toISOString(), + "ai.usage.inputTokens": 284, + "ai.usage.outputTokens": 55, + "ai.usage.totalTokens": 339, + "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.streamText.doStream", + }, + }) + ); + + const tc1 = next(3_000); + events.push( + makeEvent({ + message: "ai.toolCall", + spanId: toolCall1Id, + parentId: streamWrapId, + ...tc1, + properties: { + "ai.operationId": "ai.toolCall", + "ai.toolCall.name": "webSearch", + "ai.toolCall.id": "call_seed_001", + "ai.toolCall.args": '{"query":"federal reserve interest rate 2024","num":5}', + "ai.toolCall.result": toolCallResult, + "operation.name": "ai.toolCall", + }, + }) + ); + + const st2 = next(3_500); + events.push( + makeEvent({ + message: "ai.streamText.doStream", + spanId: stream2Id, + parentId: streamWrapId, + ...st2, + properties: { + "gen_ai.system": "openai.responses", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.usage.input_tokens": 523, + "gen_ai.usage.output_tokens": 187, + "ai.model.id": "gpt-4o-mini", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText.doStream", + "ai.prompt.messages": promptMessages, + "ai.settings.maxRetries": 2, + "ai.response.finishReason": "stop", + "ai.response.text": assistantResponse, + "ai.response.reasoning": + "Let me analyze the Federal Reserve data to provide the current rate.", + "ai.response.id": "resp_seed_002", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.msToFirstChunk": 672.45, + "ai.response.msToFinish": 3412.89, + "ai.response.timestamp": new Date(st2.startMs + st2.durationMs).toISOString(), + "ai.usage.inputTokens": 523, + "ai.usage.outputTokens": 187, + "ai.usage.totalTokens": 710, + "ai.usage.reasoningTokens": 42, + "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.streamText.doStream", + }, + }) + ); + + // ===================================================================== + // 2) ai.generateText — Anthropic claude-haiku-4-5 with tool call + cache + // ===================================================================== + const gtWrap = next(4_200); + + events.push( + makeEvent({ + message: "ai.generateText", + spanId: genTextWrapId, + parentId: runFnId, + ...gtWrap, + properties: { + "ai.operationId": "ai.generateText", + "ai.model.id": "claude-haiku-4-5", + "ai.model.provider": "anthropic.messages", + "ai.response.finishReason": "stop", + "ai.response.text": "Based on the search results, the current rate is 4.25%-4.50%.", + "ai.usage.promptTokens": 9951, + "ai.usage.completionTokens": 803, + "ai.telemetry.metadata.agentName": "research-agent", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.generateText", + }, + }) + ); + + cursor = gtWrap.startMs + 50; + const gtDo = next(3_200); + events.push( + makeEvent({ + message: "ai.generateText.doGenerate", + spanId: genTextDoId, + parentId: genTextWrapId, + ...gtDo, + properties: { + "gen_ai.system": "anthropic.messages", + "gen_ai.request.model": "claude-haiku-4-5", + "gen_ai.response.model": "claude-haiku-4-5-20251001", + "gen_ai.usage.input_tokens": 9951, + "gen_ai.usage.output_tokens": 803, + "gen_ai.usage.cache_read_input_tokens": 8200, + "gen_ai.usage.cache_creation_input_tokens": 1751, + "ai.model.id": "claude-haiku-4-5", + "ai.model.provider": "anthropic.messages", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": promptMessages, + "ai.prompt.toolChoice": '{"type":"auto"}', + "ai.settings.maxRetries": 2, + "ai.response.finishReason": "tool-calls", + "ai.response.id": "msg_seed_003", + "ai.response.model": "claude-haiku-4-5-20251001", + "ai.response.text": + "I'll search for the latest Federal Reserve interest rate information.", + "ai.response.toolCalls": JSON.stringify([ + { + toolCallId: "toolu_seed_001", + toolName: "webSearch", + input: '{"query":"federal reserve interest rate current"}', + }, + ]), + "ai.response.providerMetadata": JSON.stringify({ + anthropic: { + usage: { + input_tokens: 9951, + output_tokens: 803, + cache_creation_input_tokens: 1751, + cache_read_input_tokens: 8200, + service_tier: "standard", + }, + }, + }), + "ai.response.timestamp": new Date(gtDo.startMs + gtDo.durationMs).toISOString(), + "ai.usage.promptTokens": 9951, + "ai.usage.completionTokens": 803, + "ai.telemetry.metadata.agentName": "research-agent", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.generateText.doGenerate", + }, + }) + ); + + const tc2 = next(500); + events.push( + makeEvent({ + message: "ai.toolCall", + spanId: toolCall2Id, + parentId: genTextWrapId, + ...tc2, + properties: { + "ai.operationId": "ai.toolCall", + "ai.toolCall.name": "webSearch", + "ai.toolCall.id": "toolu_seed_001", + "ai.toolCall.args": '{"query":"federal reserve interest rate current"}', + "ai.toolCall.result": + '[{"title":"Federal Reserve Board - Policy Rate","link":"https://federalreserve.gov/rates","snippet":"The target range is 4.25% to 4.50%"}]', + "operation.name": "ai.toolCall", + "resource.name": "ai-chat", + }, + }) + ); + + // ===================================================================== + // 3) ai.generateObject — Gateway → xAI/grok with structured output + // ===================================================================== + const goWrap = next(1_800); + + events.push( + makeEvent({ + message: "ai.generateObject", + spanId: genObjWrapId, + parentId: runFnId, + ...goWrap, + properties: { + "ai.operationId": "ai.generateObject", + "ai.model.id": "xai/grok-4.1-fast-non-reasoning", + "ai.model.provider": "gateway", + "ai.response.finishReason": "stop", + "ai.response.object": JSON.stringify({ + summary: "Fed rate at 4.25%-4.50%", + confidence: 0.95, + sources: ["federalreserve.gov"], + }), + "ai.telemetry.metadata.model": "xai/grok-4.1-fast-non-reasoning", + "ai.telemetry.metadata.schemaType": "schema", + "ai.telemetry.functionId": "generateObject", + "operation.name": "ai.generateObject", + }, + }) + ); + + cursor = goWrap.startMs + 50; + const goDo = next(1_600); + events.push( + makeEvent({ + message: "ai.generateObject.doGenerate", + spanId: genObjDoId, + parentId: genObjWrapId, + ...goDo, + properties: { + "gen_ai.system": "gateway", + "gen_ai.request.model": "xai/grok-4.1-fast-non-reasoning", + "gen_ai.response.model": "xai/grok-4.1-fast-non-reasoning", + "gen_ai.usage.input_tokens": 1629, + "gen_ai.usage.output_tokens": 158, + "ai.model.id": "xai/grok-4.1-fast-non-reasoning", + "ai.model.provider": "gateway", + "ai.operationId": "ai.generateObject.doGenerate", + "ai.prompt.messages": promptMessages, + "ai.settings.maxRetries": 3, + "ai.response.finishReason": "stop", + "ai.response.id": "aiobj_seed_001", + "ai.response.model": "xai/grok-4.1-fast-non-reasoning", + "ai.response.object": JSON.stringify({ + summary: "Fed rate at 4.25%-4.50%", + confidence: 0.95, + sources: ["federalreserve.gov"], + }), + "ai.response.providerMetadata": JSON.stringify({ + gateway: { + routing: { + originalModelId: "xai/grok-4.1-fast-non-reasoning", + resolvedProvider: "xai", + canonicalSlug: "xai/grok-4.1-fast-non-reasoning", + finalProvider: "xai", + modelAttemptCount: 1, + }, + cost: "0.0002905", + generationId: "gen_seed_001", + }, + }), + "ai.response.timestamp": new Date(goDo.startMs + goDo.durationMs).toISOString(), + "ai.usage.completionTokens": 158, + "ai.usage.promptTokens": 1629, + "ai.request.headers.user-agent": "ai/5.0.60", + "operation.name": "ai.generateObject.doGenerate", + }, + }) + ); + + // ===================================================================== + // 4) ai.generateObject — Google Gemini (generative-ai) with thinking tokens + // ===================================================================== + const goGemWrap = next(2_200); + + events.push( + makeEvent({ + message: "ai.generateObject", + spanId: genObjGeminiWrapId, + parentId: runFnId, + ...goGemWrap, + properties: { + "ai.operationId": "ai.generateObject", + "ai.model.id": "gemini-2.5-flash", + "ai.model.provider": "google.generative-ai", + "ai.response.finishReason": "stop", + "ai.response.object": JSON.stringify({ + category: "financial_data", + label: "interest_rate", + }), + "ai.telemetry.functionId": "classify-content", + "operation.name": "ai.generateObject", + }, + }) + ); + + cursor = goGemWrap.startMs + 50; + const goGemDo = next(2_000); + events.push( + makeEvent({ + message: "ai.generateObject.doGenerate", + spanId: genObjGeminiDoId, + parentId: genObjGeminiWrapId, + ...goGemDo, + properties: { + "gen_ai.system": "google.generative-ai", + "gen_ai.request.model": "gemini-2.5-flash", + "gen_ai.response.model": "gemini-2.5-flash", + "gen_ai.usage.input_tokens": 898, + "gen_ai.usage.output_tokens": 521, + "ai.model.id": "gemini-2.5-flash", + "ai.model.provider": "google.generative-ai", + "ai.operationId": "ai.generateObject.doGenerate", + "ai.prompt.messages": JSON.stringify([ + { + role: "user", + content: [ + { + type: "text", + text: "Classify this content: Federal Reserve interest rate analysis", + }, + ], + }, + ]), + "ai.settings.maxRetries": 3, + "ai.response.finishReason": "stop", + "ai.response.id": "aiobj_seed_gemini", + "ai.response.model": "gemini-2.5-flash", + "ai.response.object": JSON.stringify({ + category: "financial_data", + label: "interest_rate", + }), + "ai.response.providerMetadata": JSON.stringify({ + google: { + usageMetadata: { + thoughtsTokenCount: 510, + promptTokenCount: 898, + candidatesTokenCount: 11, + totalTokenCount: 1419, + }, + }, + }), + "ai.response.timestamp": new Date(goGemDo.startMs + goGemDo.durationMs).toISOString(), + "ai.usage.completionTokens": 521, + "ai.usage.promptTokens": 898, + "operation.name": "ai.generateObject.doGenerate", + }, + }) + ); + + // ===================================================================== + // Helper: add a wrapper + doGenerate/doStream pair + // ===================================================================== + function addLlmPair(opts: { + wrapperMsg: string; // e.g. "ai.generateText" + doMsg: string; // e.g. "ai.generateText.doGenerate" + system: string; + reqModel: string; + respModel: string; + inputTokens: number; + outputTokens: number; + finishReason: string; + wrapperDurationMs: number; + doDurationMs: number; + responseText?: string; + responseObject?: string; + responseReasoning?: string; + toolCallsJson?: string; + providerMetadata?: Record; + telemetryMetadata?: Record; + settings?: Record; + /** Use completionTokens/promptTokens instead of inputTokens/outputTokens */ + useCompletionStyle?: boolean; + cacheReadTokens?: number; + cacheCreationTokens?: number; + reasoningTokens?: number; + extraDoProps?: Record; + }) { + const wId = generateSpanId(); + const dId = generateSpanId(); + + const wrap = next(opts.wrapperDurationMs); + const wrapperProps: Record = { + "ai.operationId": opts.wrapperMsg, + "ai.model.id": opts.reqModel, + "ai.model.provider": opts.system, + "ai.response.finishReason": opts.finishReason, + "operation.name": opts.wrapperMsg, + }; + if (opts.responseText) wrapperProps["ai.response.text"] = opts.responseText; + if (opts.responseObject) wrapperProps["ai.response.object"] = opts.responseObject; + if (opts.telemetryMetadata) { + for (const [k, v] of Object.entries(opts.telemetryMetadata)) { + wrapperProps[`ai.telemetry.metadata.${k}`] = v; + } + } + + events.push(makeEvent({ message: opts.wrapperMsg, spanId: wId, parentId: runFnId, ...wrap, properties: wrapperProps })); + + cursor = wrap.startMs + 50; + const doTiming = next(opts.doDurationMs); + + const doProps: Record = { + "gen_ai.system": opts.system, + "gen_ai.request.model": opts.reqModel, + "gen_ai.response.model": opts.respModel, + "gen_ai.usage.input_tokens": opts.inputTokens, + "gen_ai.usage.output_tokens": opts.outputTokens, + "ai.model.id": opts.reqModel, + "ai.model.provider": opts.system, + "ai.operationId": opts.doMsg, + "ai.prompt.messages": promptMessages, + "ai.response.finishReason": opts.finishReason, + "ai.response.id": `resp_seed_${generateSpanId().slice(0, 8)}`, + "ai.response.model": opts.respModel, + "ai.response.timestamp": new Date(doTiming.startMs + doTiming.durationMs).toISOString(), + "operation.name": opts.doMsg, + }; + + // Token style + if (opts.useCompletionStyle) { + doProps["ai.usage.completionTokens"] = opts.outputTokens; + doProps["ai.usage.promptTokens"] = opts.inputTokens; + } else { + doProps["ai.usage.inputTokens"] = opts.inputTokens; + doProps["ai.usage.outputTokens"] = opts.outputTokens; + doProps["ai.usage.totalTokens"] = opts.inputTokens + opts.outputTokens; + } + + if (opts.responseText) doProps["ai.response.text"] = opts.responseText; + if (opts.responseObject) doProps["ai.response.object"] = opts.responseObject; + if (opts.responseReasoning) doProps["ai.response.reasoning"] = opts.responseReasoning; + if (opts.toolCallsJson) doProps["ai.response.toolCalls"] = opts.toolCallsJson; + if (opts.cacheReadTokens) { + doProps["gen_ai.usage.cache_read_input_tokens"] = opts.cacheReadTokens; + } + if (opts.cacheCreationTokens) { + doProps["gen_ai.usage.cache_creation_input_tokens"] = opts.cacheCreationTokens; + } + if (opts.reasoningTokens) { + doProps["ai.usage.reasoningTokens"] = opts.reasoningTokens; + } + if (opts.providerMetadata) { + doProps["ai.response.providerMetadata"] = JSON.stringify(opts.providerMetadata); + } + if (opts.settings) { + for (const [k, v] of Object.entries(opts.settings)) { + doProps[`ai.settings.${k}`] = v; + } + } + if (opts.telemetryMetadata) { + for (const [k, v] of Object.entries(opts.telemetryMetadata)) { + doProps[`ai.telemetry.metadata.${k}`] = v; + } + } + if (opts.extraDoProps) Object.assign(doProps, opts.extraDoProps); + + events.push(makeEvent({ message: opts.doMsg, spanId: dId, parentId: wId, ...doTiming, properties: doProps })); + + return { wrapperId: wId, doId: dId }; + } + + // Helper: add a tool call span + function addToolCall(parentId: string, name: string, args: string, result: string, durationMs = 500) { + const id = generateSpanId(); + const timing = next(durationMs); + events.push(makeEvent({ + message: "ai.toolCall", + spanId: id, + parentId, + ...timing, + properties: { + "ai.operationId": "ai.toolCall", + "ai.toolCall.name": name, + "ai.toolCall.id": `call_${generateSpanId().slice(0, 8)}`, + "ai.toolCall.args": args, + "ai.toolCall.result": result, + "operation.name": "ai.toolCall", + }, + })); + return id; + } + + // ===================================================================== + // 5) Gateway → Mistral mistral-large-3 + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "gateway", + reqModel: "mistral/mistral-large-3", + respModel: "mistral/mistral-large-3", + inputTokens: 1179, + outputTokens: 48, + finishReason: "stop", + wrapperDurationMs: 1_400, + doDurationMs: 1_200, + responseText: "The document discusses quarterly earnings guidance for tech sector.", + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "mistral/mistral-large-3", + resolvedProvider: "mistral", + resolvedProviderApiModelId: "mistral-large-latest", + canonicalSlug: "mistral/mistral-large-3", + finalProvider: "mistral", + modelAttemptCount: 1, + }, + cost: "0.0006615", + marketCost: "0.0006615", + generationId: "gen_seed_mistral_001", + }, + }, + extraDoProps: { "ai.request.headers.user-agent": "ai/5.0.60" }, + }); + + // ===================================================================== + // 6) Gateway → OpenAI gpt-5-mini (with fallback metadata) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "gateway", + reqModel: "openai/gpt-5-mini", + respModel: "openai/gpt-5-mini", + inputTokens: 2450, + outputTokens: 312, + finishReason: "stop", + wrapperDurationMs: 5_000, + doDurationMs: 4_800, + responseText: "NO", + useCompletionStyle: true, + providerMetadata: { + openai: { responseId: "resp_seed_gw_openai", serviceTier: "default" }, + gateway: { + routing: { + originalModelId: "openai/gpt-5-mini", + resolvedProvider: "openai", + resolvedProviderApiModelId: "gpt-5-mini-2025-08-07", + canonicalSlug: "openai/gpt-5-mini", + finalProvider: "openai", + fallbacksAvailable: ["azure"], + planningReasoning: "System credentials planned for: openai, azure. Total execution order: openai(system) → azure(system)", + modelAttemptCount: 1, + }, + cost: "0.000482", + generationId: "gen_seed_gpt5mini_001", + }, + }, + extraDoProps: { "ai.request.headers.user-agent": "ai/6.0.49" }, + }); + + // ===================================================================== + // 7) Gateway → DeepSeek deepseek-v3.2 (tool-calls) + // ===================================================================== + const ds = addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "gateway", + reqModel: "deepseek/deepseek-v3.2", + respModel: "deepseek/deepseek-v3.2", + inputTokens: 3200, + outputTokens: 420, + finishReason: "tool-calls", + wrapperDurationMs: 2_800, + doDurationMs: 2_500, + responseObject: JSON.stringify({ action: "search", query: "fed rate history" }), + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "deepseek/deepseek-v3.2", + resolvedProvider: "deepseek", + canonicalSlug: "deepseek/deepseek-v3.2", + finalProvider: "deepseek", + modelAttemptCount: 1, + }, + cost: "0.000156", + generationId: "gen_seed_deepseek_001", + }, + }, + }); + addToolCall(ds.wrapperId, "classifyContent", '{"text":"Federal Reserve rate analysis"}', '{"category":"finance","confidence":0.98}'); + + // ===================================================================== + // 8) Gateway → Anthropic claude-haiku via gateway prefix + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "gateway", + reqModel: "anthropic/claude-haiku-4-5-20251001", + respModel: "anthropic/claude-haiku-4-5-20251001", + inputTokens: 5400, + outputTokens: 220, + finishReason: "stop", + wrapperDurationMs: 1_800, + doDurationMs: 1_500, + responseText: "The content appears to be a standard financial news article. Classification: SAFE.", + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "anthropic/claude-haiku-4-5-20251001", + resolvedProvider: "anthropic", + canonicalSlug: "anthropic/claude-haiku-4-5-20251001", + finalProvider: "anthropic", + modelAttemptCount: 1, + }, + cost: "0.00312", + generationId: "gen_seed_gw_anthropic_001", + }, + }, + }); + + // ===================================================================== + // 9) Gateway → Google gemini-3-flash-preview (structured output) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "gateway", + reqModel: "google/gemini-3-flash-preview", + respModel: "google/gemini-3-flash-preview", + inputTokens: 720, + outputTokens: 85, + finishReason: "stop", + wrapperDurationMs: 1_200, + doDurationMs: 1_000, + responseObject: JSON.stringify({ sentiment: "neutral", topics: ["monetary_policy", "interest_rates"] }), + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "google/gemini-3-flash-preview", + resolvedProvider: "google", + canonicalSlug: "google/gemini-3-flash-preview", + finalProvider: "google", + modelAttemptCount: 1, + }, + cost: "0.0000803", + generationId: "gen_seed_gw_gemini_001", + }, + }, + }); + + // ===================================================================== + // 10) OpenRouter → x-ai/grok-4-fast (with reasoning_details) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "openrouter", + reqModel: "x-ai/grok-4-fast", + respModel: "x-ai/grok-4-fast", + inputTokens: 375, + outputTokens: 226, + finishReason: "stop", + wrapperDurationMs: 1_600, + doDurationMs: 1_400, + responseObject: JSON.stringify({ hook: "Breaking: Fed holds rates steady", isValidHook: true }), + useCompletionStyle: true, + telemetryMetadata: { model: "x-ai/grok-4-fast", schemaType: "schema", temperature: "1" }, + settings: { maxRetries: 2, temperature: 1 }, + providerMetadata: { + openrouter: { + provider: "xAI", + reasoning_details: [{ type: "reasoning.encrypted", data: "encrypted_seed_data..." }], + usage: { + promptTokens: 375, + promptTokensDetails: { cachedTokens: 343 }, + completionTokens: 226, + completionTokensDetails: { reasoningTokens: 210 }, + totalTokens: 601, + cost: 0.0001351845, + costDetails: { upstreamInferenceCost: 0.00013655 }, + }, + }, + }, + }); + + // ===================================================================== + // 11) OpenRouter → google/gemini-2.5-flash + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "openrouter", + reqModel: "google/gemini-2.5-flash", + respModel: "google/gemini-2.5-flash", + inputTokens: 1840, + outputTokens: 320, + finishReason: "stop", + wrapperDurationMs: 2_000, + doDurationMs: 1_800, + responseText: "Based on the latest FOMC minutes, the committee voted unanimously to maintain rates.", + useCompletionStyle: true, + providerMetadata: { + openrouter: { + provider: "Google AI Studio", + usage: { + promptTokens: 1840, + completionTokens: 320, + totalTokens: 2160, + cost: 0.000264, + costDetails: { upstreamInferenceCost: 0.000232 }, + }, + }, + }, + }); + + // ===================================================================== + // 12) OpenRouter → openai/gpt-4.1-mini (req ≠ resp model name) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "openrouter", + reqModel: "openai/gpt-4.1-mini", + respModel: "openai/gpt-4.1-mini-2025-04-14", + inputTokens: 890, + outputTokens: 145, + finishReason: "stop", + wrapperDurationMs: 1_400, + doDurationMs: 1_200, + responseObject: JSON.stringify({ summary: "Rate unchanged at 4.25-4.50%", date: "2024-12-18" }), + useCompletionStyle: true, + providerMetadata: { + openrouter: { + provider: "OpenAI", + usage: { + promptTokens: 890, + completionTokens: 145, + totalTokens: 1035, + cost: 0.0000518, + }, + }, + }, + }); + + // ===================================================================== + // 13) Azure → gpt-5 with tool-calls + // ===================================================================== + const az = addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "azure.responses", + reqModel: "gpt-5-2025-08-07", + respModel: "gpt-5-2025-08-07", + inputTokens: 2038, + outputTokens: 239, + finishReason: "tool-calls", + wrapperDurationMs: 3_500, + doDurationMs: 3_000, + responseText: "Let me look up the latest rate decision.", + toolCallsJson: JSON.stringify([{ + toolCallId: "call_azure_001", + toolName: "lookupRate", + input: '{"source":"federal_reserve","metric":"funds_rate"}', + }]), + providerMetadata: { + azure: { responseId: "resp_seed_azure_001", serviceTier: "default" }, + }, + }); + addToolCall(az.wrapperId, "lookupRate", '{"source":"federal_reserve","metric":"funds_rate"}', '{"rate":"4.25-4.50%","effectiveDate":"2024-12-18"}'); + + // ===================================================================== + // 14) Perplexity → sonar-pro + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "perplexity", + reqModel: "sonar-pro", + respModel: "sonar-pro", + inputTokens: 151, + outputTokens: 428, + finishReason: "stop", + wrapperDurationMs: 4_500, + doDurationMs: 4_200, + responseText: "According to the Federal Reserve's most recent announcement on December 18, 2024, the federal funds rate target range was maintained at 4.25% to 4.50%. This decision was made during the December FOMC meeting.", + }); + + // ===================================================================== + // 15) openai.chat → gpt-4o-mini (legacy chat completions, mode: "tool") + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "openai.chat", + reqModel: "gpt-4o-mini", + respModel: "gpt-4o-mini-2024-07-18", + inputTokens: 573, + outputTokens: 11, + finishReason: "stop", + wrapperDurationMs: 800, + doDurationMs: 600, + responseObject: JSON.stringify({ title: "Fed Rate Hold", emoji: "🏦" }), + settings: { maxRetries: 2, mode: "tool", temperature: 0.3 }, + providerMetadata: { + openai: { reasoningTokens: 0, cachedPromptTokens: 0 }, + }, + }); + + // ===================================================================== + // 16) Anthropic claude-sonnet-4-5 → streamText with reasoning + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.streamText", + doMsg: "ai.streamText.doStream", + system: "anthropic.messages", + reqModel: "claude-sonnet-4-5-20250929", + respModel: "claude-sonnet-4-5-20250929", + inputTokens: 15200, + outputTokens: 2840, + finishReason: "stop", + wrapperDurationMs: 12_000, + doDurationMs: 11_500, + responseText: "The Federal Reserve has maintained its target range for the federal funds rate at 4.25% to 4.50% since December 2024. This represents a pause in the rate-cutting cycle that began in September 2024. The FOMC has indicated it will continue to assess incoming data, the evolving outlook, and the balance of risks when considering further adjustments.", + responseReasoning: "The user is asking about the current Federal Reserve interest rate. Let me provide a comprehensive answer based on the most recent FOMC decision. I should include context about the rate trajectory and forward guidance.", + cacheReadTokens: 12400, + cacheCreationTokens: 2800, + providerMetadata: { + anthropic: { + usage: { + input_tokens: 15200, + output_tokens: 2840, + cache_creation_input_tokens: 2800, + cache_read_input_tokens: 12400, + service_tier: "standard", + inference_geo: "us-east-1", + }, + }, + }, + }); + + // ===================================================================== + // 17) google.vertex.chat → gemini-3.1-pro-preview with tool-calls + // ===================================================================== + const vt = addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "google.vertex.chat", + reqModel: "gemini-3.1-pro-preview", + respModel: "gemini-3.1-pro-preview", + inputTokens: 4200, + outputTokens: 680, + finishReason: "tool-calls", + wrapperDurationMs: 6_000, + doDurationMs: 5_500, + responseText: "I'll search for the latest FOMC decision and rate information.", + toolCallsJson: JSON.stringify([{ + toolCallId: "call_vertex_001", + toolName: "searchFOMC", + input: '{"query":"latest FOMC decision december 2024"}', + }]), + providerMetadata: { + google: { + usageMetadata: { + thoughtsTokenCount: 320, + promptTokenCount: 4200, + candidatesTokenCount: 680, + totalTokenCount: 5200, + }, + }, + }, + }); + addToolCall(vt.wrapperId, "searchFOMC", '{"query":"latest FOMC decision december 2024"}', '{"decision":"hold","rate":"4.25-4.50%","date":"2024-12-18","vote":"unanimous"}', 800); + + // ===================================================================== + // 18) openai.responses → gpt-5.4 with reasoning tokens + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.streamText", + doMsg: "ai.streamText.doStream", + system: "openai.responses", + reqModel: "gpt-5.4", + respModel: "gpt-5.4-2026-03-05", + inputTokens: 8900, + outputTokens: 1250, + finishReason: "stop", + wrapperDurationMs: 8_000, + doDurationMs: 7_500, + responseText: "The Federal Reserve's current target range for the federal funds rate is 4.25% to 4.50%, established at the December 2024 FOMC meeting. The committee has signaled a cautious approach to further rate cuts, citing persistent inflation concerns.", + responseReasoning: "I need to provide accurate, up-to-date information about the Federal Reserve interest rate. The last FOMC meeting was in December 2024 where they held rates steady after three consecutive cuts.", + reasoningTokens: 516, + providerMetadata: { + openai: { + responseId: "resp_seed_gpt54_001", + serviceTier: "default", + }, + }, + extraDoProps: { + "ai.response.msToFirstChunk": 1842.5, + "ai.response.msToFinish": 7234.8, + "ai.response.avgOutputTokensPerSecond": 172.8, + }, + }); + + // ===================================================================== + // 19) Cerebras cerebras-gpt-13b — no pricing, no provider cost + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "cerebras.chat", + reqModel: "cerebras-gpt-13b", + respModel: "cerebras-gpt-13b", + inputTokens: 450, + outputTokens: 120, + finishReason: "stop", + wrapperDurationMs: 600, + doDurationMs: 400, + responseText: "The Federal Reserve rate is currently at 4.25-4.50%.", + }); + + // ===================================================================== + // 20) Amazon Bedrock — no pricing in registry + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "amazon-bedrock", + reqModel: "us.anthropic.claude-sonnet-4-20250514-v1:0", + respModel: "us.anthropic.claude-sonnet-4-20250514-v1:0", + inputTokens: 3200, + outputTokens: 890, + finishReason: "stop", + wrapperDurationMs: 4_000, + doDurationMs: 3_500, + responseText: "Based on the latest FOMC statement, the target rate range remains at 4.25% to 4.50%.", + }); + + // ===================================================================== + // 21) Groq — fast inference, no pricing + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "groq.chat", + reqModel: "llama-4-scout-17b-16e-instruct", + respModel: "llama-4-scout-17b-16e-instruct", + inputTokens: 820, + outputTokens: 95, + finishReason: "stop", + wrapperDurationMs: 300, + doDurationMs: 200, + responseObject: JSON.stringify({ rate: "4.25-4.50%", source: "FOMC", date: "2024-12-18" }), + }); + + return events; +} + +// --------------------------------------------------------------------------- + +seedAiSpans() + .catch((e) => { + console.error("Seed failed:"); + console.error(e); + process.exit(1); + }) + .finally(async () => { + await prisma.$disconnect(); + }); diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 5b340eb4034..9ffb14eba49 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -103,6 +103,18 @@ export class ModelPricingRegistry { } } + // Fallback: strip provider prefix (e.g. "mistral/mistral-large-3" → "mistral-large-3") + // Gateway and OpenRouter prepend the provider to the model name. + if (responseModel.includes("/")) { + const stripped = responseModel.split("/").slice(1).join("/"); + for (const { regex, model } of this._patterns) { + if (regex.test(stripped)) { + this._exactMatchCache.set(responseModel, model); + return model; + } + } + } + // Cache miss this._exactMatchCache.set(responseModel, null); return null; From 6034d8b56ec52bfd1131fe230ce72d209e68c33b Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 11:30:15 +0000 Subject: [PATCH 06/15] Fixed a bunch of coderabbit issues --- .../admin.api.v1.llm-models.$modelId.ts | 89 ++++++++++--------- .../app/routes/admin.api.v1.llm-models.ts | 77 ++++++++-------- .../admin.llm-models.missing.$model.tsx | 6 +- .../services/admin/missingLlmModels.server.ts | 3 +- .../llm-pricing/scripts/sync-model-prices.sh | 2 +- .../llm-pricing/src/defaultPrices.ts | 2 +- internal-packages/llm-pricing/src/seed.ts | 55 ++++++------ 7 files changed, 123 insertions(+), 111 deletions(-) diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts index 8cf132daab9..4e8357c886c 100644 --- a/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts @@ -81,7 +81,13 @@ export async function action({ request, params }: ActionFunctionArgs) { return json({ error: "Method not allowed" }, { status: 405 }); } - const body = await request.json(); + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const parsed = UpdateModelSchema.safeParse(body); if (!parsed.success) { @@ -90,59 +96,56 @@ export async function action({ request, params }: ActionFunctionArgs) { const { modelName, matchPattern, startDate, pricingTiers } = parsed.data; - // Validate regex if provided + // Validate regex if provided — strip (?i) POSIX flag since our registry handles it if (matchPattern) { try { - new RegExp(matchPattern); + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); } catch { return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); } } - // Update model fields - const model = await prisma.llmModel.update({ - where: { id: modelId }, - data: { - ...(modelName !== undefined && { modelName }), - ...(matchPattern !== undefined && { matchPattern }), - ...(startDate !== undefined && { startDate: startDate ? new Date(startDate) : null }), - }, - }); - - // If pricing tiers provided, replace them entirely - if (pricingTiers) { - // Delete existing tiers (cascades to prices) - await prisma.llmPricingTier.deleteMany({ where: { modelId } }); - - // Create new tiers - for (const tier of pricingTiers) { - await prisma.llmPricingTier.create({ - data: { - modelId, - name: tier.name, - isDefault: tier.isDefault, - priority: tier.priority, - conditions: tier.conditions, - prices: { - create: Object.entries(tier.prices).map(([usageType, price]) => ({ - modelId, - usageType, - price, - })), + // Update model + tiers atomically + const updated = await prisma.$transaction(async (tx) => { + await tx.llmModel.update({ + where: { id: modelId }, + data: { + ...(modelName !== undefined && { modelName }), + ...(matchPattern !== undefined && { matchPattern }), + ...(startDate !== undefined && { startDate: startDate ? new Date(startDate) : null }), + }, + }); + + if (pricingTiers) { + await tx.llmPricingTier.deleteMany({ where: { modelId } }); + + for (const tier of pricingTiers) { + await tx.llmPricingTier.create({ + data: { + modelId, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId, + usageType, + price, + })), + }, }, - }, - }); + }); + } } - } - const updated = await prisma.llmModel.findUnique({ - where: { id: modelId }, - include: { - pricingTiers: { - include: { prices: true }, - orderBy: { priority: "asc" }, + return tx.llmModel.findUnique({ + where: { id: modelId }, + include: { + pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } }, }, - }, + }); }); return json({ model: updated }); diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts index 706c73549de..6305869c605 100644 --- a/apps/webapp/app/routes/admin.api.v1.llm-models.ts +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts @@ -75,7 +75,13 @@ export async function action({ request }: ActionFunctionArgs) { return json({ error: "Method not allowed" }, { status: 405 }); } - const body = await request.json(); + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const parsed = CreateModelSchema.safeParse(body); if (!parsed.success) { @@ -84,50 +90,51 @@ export async function action({ request }: ActionFunctionArgs) { const { modelName, matchPattern, startDate, source, pricingTiers } = parsed.data; - // Validate regex pattern + // Validate regex pattern — strip (?i) POSIX flag since our registry handles it try { - new RegExp(matchPattern); + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); } catch { return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); } - // Create model first, then tiers with explicit model connection - const model = await prisma.llmModel.create({ - data: { - friendlyId: generateFriendlyId("llm_model"), - modelName, - matchPattern, - startDate: startDate ? new Date(startDate) : null, - source, - }, - }); - - for (const tier of pricingTiers) { - await prisma.llmPricingTier.create({ + // Create model + tiers atomically + const created = await prisma.$transaction(async (tx) => { + const model = await tx.llmModel.create({ data: { - modelId: model.id, - name: tier.name, - isDefault: tier.isDefault, - priority: tier.priority, - conditions: tier.conditions, - prices: { - create: Object.entries(tier.prices).map(([usageType, price]) => ({ - modelId: model.id, - usageType, - price, - })), - }, + friendlyId: generateFriendlyId("llm_model"), + modelName, + matchPattern, + startDate: startDate ? new Date(startDate) : null, + source, }, }); - } - const created = await prisma.llmModel.findUnique({ - where: { id: model.id }, - include: { - pricingTiers: { - include: { prices: true }, + for (const tier of pricingTiers) { + await tx.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + return tx.llmModel.findUnique({ + where: { id: model.id }, + include: { + pricingTiers: { include: { prices: true } }, }, - }, + }); }); return json({ model: created }, { status: 201 }); diff --git a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx index 7b6e83bf939..fafa3c81235 100644 --- a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx +++ b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx @@ -16,7 +16,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await prisma.user.findUnique({ where: { id: userId } }); if (!user?.admin) return redirect("/"); - // Model name is base64url-encoded in the URL param + // Model name is URL-encoded in the URL param const modelName = decodeURIComponent(params.model ?? ""); if (!modelName) throw new Response("Missing model param", { status: 400 }); @@ -142,7 +142,7 @@ export default function AdminMissingModelDetailRoute() {
input: {providerCosts[0].estimatedInputPrice.toExponential(4)} - output: {providerCosts[0].estimatedOutputPrice!.toExponential(4)} + output: {providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)}
Cross-reference with the provider's pricing page before using these estimates. @@ -457,7 +457,7 @@ The gateway/router is reporting costs for this model. Use these to cross-referen ${providerCosts.map((c) => `- $${c.cost.toFixed(6)} for ${c.inputTokens.toLocaleString()} input + ${c.outputTokens.toLocaleString()} output tokens`).join("\n")}${providerCosts[0].estimatedInputPrice != null ? ` - Estimated per-token rates (rough, assuming ~3x output/input ratio): - input: ${providerCosts[0].estimatedInputPrice.toExponential(4)} (${(providerCosts[0].estimatedInputPrice * 1_000_000).toFixed(4)} $/M) - - output: ${providerCosts[0].estimatedOutputPrice!.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice! * 1_000_000).toFixed(4)} $/M) + - output: ${providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice ?? 0 * 1_000_000).toFixed(4)} $/M) - Verify these against the official pricing page before using.` : ""}` : ""}${sampleAttrs ? ` ## Sample span attributes (first span) diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts index 01237ef2b57..0ad06e7e2d9 100644 --- a/apps/webapp/app/services/admin/missingLlmModels.server.ts +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -70,7 +70,8 @@ export async function getMissingLlmModels(opts: { // Filter out models that now have pricing in the database (added after spans were inserted). // The registry's match() handles prefix stripping for gateway/openrouter models. - return candidates.filter((c) => !llmPricingRegistry?.match(c.model)); + if (!llmPricingRegistry?.isLoaded) return candidates; + return candidates.filter((c) => !llmPricingRegistry.match(c.model)); } export type MissingModelSample = { diff --git a/internal-packages/llm-pricing/scripts/sync-model-prices.sh b/internal-packages/llm-pricing/scripts/sync-model-prices.sh index 9bb6af11cbb..d72aa6714c6 100755 --- a/internal-packages/llm-pricing/scripts/sync-model-prices.sh +++ b/internal-packages/llm-pricing/scripts/sync-model-prices.sh @@ -50,7 +50,7 @@ echo "Generating defaultPrices.ts..." node -e " const data = JSON.parse(require('fs').readFileSync('$JSON_TARGET', 'utf-8')); const stripped = data.map(e => ({ - modelName: e.modelName, + modelName: e.modelName.trim(), matchPattern: e.matchPattern, startDate: e.createdAt, pricingTiers: e.pricingTiers.map(t => ({ diff --git a/internal-packages/llm-pricing/src/defaultPrices.ts b/internal-packages/llm-pricing/src/defaultPrices.ts index 91b8a6f89f6..689944a6432 100644 --- a/internal-packages/llm-pricing/src/defaultPrices.ts +++ b/internal-packages/llm-pricing/src/defaultPrices.ts @@ -1109,7 +1109,7 @@ export const defaultModelPrices: DefaultModelDefinition[] = [ ] }, { - "modelName": " gpt-4-preview", + "modelName": "gpt-4-preview", "matchPattern": "(?i)^(openai/)?(gpt-4-preview)$", "startDate": "2024-04-23T10:37:17.092Z", "pricingTiers": [ diff --git a/internal-packages/llm-pricing/src/seed.ts b/internal-packages/llm-pricing/src/seed.ts index b4f95373eff..d068c62a66d 100644 --- a/internal-packages/llm-pricing/src/seed.ts +++ b/internal-packages/llm-pricing/src/seed.ts @@ -23,36 +23,37 @@ export async function seedLlmPricing(prisma: PrismaClient): Promise<{ continue; } - // Create model first - const model = await prisma.llmModel.create({ - data: { - friendlyId: generateFriendlyId("llm_model"), - modelName: modelDef.modelName, - matchPattern: modelDef.matchPattern, - startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, - source: "default", - }, - }); - - // Create tiers and prices with explicit model connection - for (const tier of modelDef.pricingTiers) { - await prisma.llmPricingTier.create({ + // Create model + tiers atomically so partial models can't be left behind + await prisma.$transaction(async (tx) => { + const model = await tx.llmModel.create({ data: { - modelId: model.id, - name: tier.name, - isDefault: tier.isDefault, - priority: tier.priority, - conditions: tier.conditions, - prices: { - create: Object.entries(tier.prices).map(([usageType, price]) => ({ - modelId: model.id, - usageType, - price, - })), - }, + friendlyId: generateFriendlyId("llm_model"), + modelName: modelDef.modelName.trim(), + matchPattern: modelDef.matchPattern, + startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, + source: "default", }, }); - } + + for (const tier of modelDef.pricingTiers) { + await tx.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + }); modelsCreated++; } From 5aa5e6b9a15c09079b761ec255d33295ed86d1e9 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 12:27:17 +0000 Subject: [PATCH 07/15] fix some typescript errors --- .../runs/v3/ai/extractAISpanData.ts | 9 +++---- .../app/routes/admin.llm-models.$modelId.tsx | 6 ++--- .../app/routes/admin.llm-models._index.tsx | 4 ++-- .../admin.llm-models.missing.$model.tsx | 24 ++++++++++--------- .../admin.llm-models.missing._index.tsx | 2 +- .../app/routes/admin.llm-models.new.tsx | 6 ++--- .../services/admin/missingLlmModels.server.ts | 5 ++-- .../v3/utils/enrichCreatableEvents.server.ts | 6 ++--- internal-packages/llm-pricing/src/registry.ts | 6 ++--- internal-packages/tsql/src/query/schema.ts | 4 ++++ 10 files changed, 40 insertions(+), 32 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts index de96c073a28..82c1014d5ab 100644 --- a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts +++ b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts @@ -426,7 +426,7 @@ function parseProviderMetadata( ): { serviceTier?: string; resolvedProvider?: string; gatewayCost?: string } | undefined { if (typeof raw !== "string") return undefined; try { - const parsed = JSON.parse(raw); + const parsed = JSON.parse(raw) as Record; if (!parsed || typeof parsed !== "object") return undefined; let serviceTier: string | undefined; @@ -469,12 +469,13 @@ function parseToolChoice(raw: unknown): string | undefined { try { const parsed = JSON.parse(raw); if (typeof parsed === "string") return parsed; - if (parsed && typeof parsed === "object" && typeof parsed.type === "string") { - return parsed.type; + if (parsed && typeof parsed === "object") { + const obj = parsed as Record; + if (typeof obj.type === "string") return obj.type; } return undefined; } catch { - return raw || undefined; + return undefined; } } diff --git a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx index 4e72731cc3e..7f7fadaf436 100644 --- a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx +++ b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx @@ -14,7 +14,7 @@ import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const model = await prisma.llmModel.findUnique({ where: { friendlyId: params.modelId }, @@ -46,7 +46,7 @@ const SaveSchema = z.object({ export async function action({ request, params }: ActionFunctionArgs) { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const friendlyId = params.modelId!; const existing = await prisma.llmModel.findUnique({ where: { friendlyId } }); @@ -89,7 +89,7 @@ export async function action({ request, params }: ActionFunctionArgs) { prices: Record; }>; try { - pricingTiers = JSON.parse(pricingTiersJson); + pricingTiers = JSON.parse(pricingTiersJson) as typeof pricingTiers; } catch { return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); } diff --git a/apps/webapp/app/routes/admin.llm-models._index.tsx b/apps/webapp/app/routes/admin.llm-models._index.tsx index 2bfa46d0c8a..fb2f6fdc491 100644 --- a/apps/webapp/app/routes/admin.llm-models._index.tsx +++ b/apps/webapp/app/routes/admin.llm-models._index.tsx @@ -33,7 +33,7 @@ const SearchParams = z.object({ export const loader = async ({ request }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const searchParams = createSearchParams(request.url, SearchParams); if (!searchParams.success) throw new Error(searchParams.error); @@ -79,7 +79,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { export async function action({ request }: ActionFunctionArgs) { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const formData = await request.formData(); const _action = formData.get("_action"); diff --git a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx index fafa3c81235..78cb1c4fc91 100644 --- a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx +++ b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx @@ -14,7 +14,7 @@ import { export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); // Model name is URL-encoded in the URL param const modelName = decodeURIComponent(params.model ?? ""); @@ -142,7 +142,7 @@ export default function AdminMissingModelDetailRoute() {
input: {providerCosts[0].estimatedInputPrice.toExponential(4)} - output: {providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)} + output: {(providerCosts[0].estimatedOutputPrice ?? 0).toExponential(4)}
Cross-reference with the provider's pricing page before using these estimates. @@ -176,7 +176,7 @@ export default function AdminMissingModelDetailRoute() { const expanded = expandedSpans.has(s.span_id); let parsedAttrs: Record | null = null; try { - parsedAttrs = JSON.parse(s.attributes_text); + parsedAttrs = JSON.parse(s.attributes_text) as Record; } catch { // ignore } @@ -226,7 +226,7 @@ function extractTokenTypes(samples: MissingModelSample[]): TokenTypeSummary[] { for (const s of samples) { let attrs: Record; try { - attrs = JSON.parse(s.attributes_text); + attrs = JSON.parse(s.attributes_text) as Record; } catch { continue; } @@ -300,7 +300,7 @@ function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[] for (const s of samples) { let attrs: Record; try { - attrs = JSON.parse(s.attributes_text); + attrs = JSON.parse(s.attributes_text) as Record; } catch { continue; } @@ -310,7 +310,7 @@ function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[] const aiResponse = getNestedObj(attrs, ["ai", "response"]); const rawMeta = aiResponse?.providerMetadata; if (typeof rawMeta === "string") { - try { providerMeta = JSON.parse(rawMeta); } catch {} + try { providerMeta = JSON.parse(rawMeta) as Record; } catch {} } else if (rawMeta && typeof rawMeta === "object") { providerMeta = rawMeta as Record; } @@ -390,13 +390,15 @@ function buildPrompt(modelName: string, samples: MissingModelSample[], providerC let sampleAttrs = ""; if (samples.length > 0) { try { - const attrs = JSON.parse(samples[0].attributes_text); + const attrs = JSON.parse(samples[0].attributes_text) as Record; + const ai = attrs.ai as Record | undefined; + const aiResponse = (ai?.response ?? {}) as Record; // Extract just the relevant fields const compact: Record = {}; if (attrs.gen_ai) compact.gen_ai = attrs.gen_ai; - if (attrs.ai?.usage) compact["ai.usage"] = attrs.ai.usage; - if (attrs.ai?.response?.providerMetadata) { - compact["ai.response.providerMetadata"] = attrs.ai.response.providerMetadata; + if (ai?.usage) compact["ai.usage"] = ai.usage; + if (aiResponse.providerMetadata) { + compact["ai.response.providerMetadata"] = aiResponse.providerMetadata; } sampleAttrs = JSON.stringify(compact, null, 2); } catch { @@ -457,7 +459,7 @@ The gateway/router is reporting costs for this model. Use these to cross-referen ${providerCosts.map((c) => `- $${c.cost.toFixed(6)} for ${c.inputTokens.toLocaleString()} input + ${c.outputTokens.toLocaleString()} output tokens`).join("\n")}${providerCosts[0].estimatedInputPrice != null ? ` - Estimated per-token rates (rough, assuming ~3x output/input ratio): - input: ${providerCosts[0].estimatedInputPrice.toExponential(4)} (${(providerCosts[0].estimatedInputPrice * 1_000_000).toFixed(4)} $/M) - - output: ${providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice ?? 0 * 1_000_000).toFixed(4)} $/M) + - output: ${(providerCosts[0].estimatedOutputPrice ?? 0).toExponential(4)} (${((providerCosts[0].estimatedOutputPrice ?? 0) * 1_000_000).toFixed(4)} $/M) - Verify these against the official pricing page before using.` : ""}` : ""}${sampleAttrs ? ` ## Sample span attributes (first span) diff --git a/apps/webapp/app/routes/admin.llm-models.missing._index.tsx b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx index b778b9a3dc9..fd933cd22e9 100644 --- a/apps/webapp/app/routes/admin.llm-models.missing._index.tsx +++ b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx @@ -33,7 +33,7 @@ const SearchParams = z.object({ export const loader = async ({ request }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const url = new URL(request.url); const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); diff --git a/apps/webapp/app/routes/admin.llm-models.new.tsx b/apps/webapp/app/routes/admin.llm-models.new.tsx index b4cd957b740..20c6e1461f2 100644 --- a/apps/webapp/app/routes/admin.llm-models.new.tsx +++ b/apps/webapp/app/routes/admin.llm-models.new.tsx @@ -14,7 +14,7 @@ import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; export const loader = async ({ request }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); return typedjson({}); }; @@ -27,7 +27,7 @@ const CreateSchema = z.object({ export async function action({ request }: ActionFunctionArgs) { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const formData = await request.formData(); const raw = Object.fromEntries(formData); @@ -57,7 +57,7 @@ export async function action({ request }: ActionFunctionArgs) { prices: Record; }>; try { - pricingTiers = JSON.parse(pricingTiersJson); + pricingTiers = JSON.parse(pricingTiersJson) as typeof pricingTiers; } catch { return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); } diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts index 0ad06e7e2d9..7ce6bc2ab7e 100644 --- a/apps/webapp/app/services/admin/missingLlmModels.server.ts +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -70,8 +70,9 @@ export async function getMissingLlmModels(opts: { // Filter out models that now have pricing in the database (added after spans were inserted). // The registry's match() handles prefix stripping for gateway/openrouter models. - if (!llmPricingRegistry?.isLoaded) return candidates; - return candidates.filter((c) => !llmPricingRegistry.match(c.model)); + if (!llmPricingRegistry || !llmPricingRegistry.isLoaded) return candidates; + const registry = llmPricingRegistry; + return candidates.filter((c) => !registry.match(c.model)); } export type MissingModelSample = { diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index a062e491de3..6b25bae2c5c 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -126,12 +126,12 @@ function enrichLlmCost(event: CreateEventInput): void { } event.style = { - ...event.style, + ...(event.style as Record | undefined), accessory: { style: "pills", items: pillItems, }, - }; + } as unknown as typeof event.style; // Only write llm_usage when cost data is available if (!cost && !providerCost) return; @@ -270,7 +270,7 @@ function extractProviderCost( let meta: Record; try { - meta = JSON.parse(rawMeta); + meta = JSON.parse(rawMeta) as Record; } catch { return null; } diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 9ffb14eba49..f6563521a25 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -1,4 +1,4 @@ -import type { PrismaClient } from "@trigger.dev/database"; +import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; import type { LlmModelWithPricing, LlmCostResult, @@ -20,12 +20,12 @@ function compilePattern(pattern: string): RegExp { } export class ModelPricingRegistry { - private _prisma: PrismaClient; + private _prisma: PrismaClient | PrismaReplicaClient; private _patterns: CompiledPattern[] = []; private _exactMatchCache: Map = new Map(); private _loaded = false; - constructor(prisma: PrismaClient) { + constructor(prisma: PrismaClient | PrismaReplicaClient) { this._prisma = prisma; } diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts index 2ea81091aad..00a28382de5 100644 --- a/internal-packages/tsql/src/query/schema.ts +++ b/internal-packages/tsql/src/query/schema.ts @@ -23,6 +23,9 @@ export type ClickHouseType = | "Date32" | "DateTime" | "DateTime64" + | "DateTime64(3)" + | "DateTime64(9)" + | "Decimal64(12)" | "UUID" | "Bool" | "JSON" @@ -281,6 +284,7 @@ export type ColumnFormatType = | "runId" | "runStatus" | "duration" + | "durationNs" | "durationSeconds" | "costInDollars" | "cost" From 8bf339d369910e4f0110988120838819312d74cf Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 13:55:13 +0000 Subject: [PATCH 08/15] some fixes --- apps/webapp/test/otlpExporter.test.ts | 13 +++++++++++-- internal-packages/llm-pricing/package.json | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index 2a569b74d62..e7b047ac7dc 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -86,7 +86,7 @@ describe("OTLPExporter", () => { const event = $events[0]; expect(event.message).toBe("Responses API with 'gpt-4o'"); - expect(event.style).toEqual({ + expect(event.style).toMatchObject({ icon: "tabler-brand-openai", }); }); @@ -164,9 +164,18 @@ describe("OTLPExporter", () => { const event = $events[0]; expect(event.message).toBe("Responses API with gpt-4o"); - expect(event.style).toEqual({ + expect(event.style).toMatchObject({ icon: "tabler-brand-openai", }); + // Enrichment also adds model/token pills as accessories + const style = event.style as Record; + expect(style.accessory).toMatchObject({ + style: "pills", + items: expect.arrayContaining([ + expect.objectContaining({ text: "gpt-4o-2024-08-06" }), + expect.objectContaining({ text: "724" }), + ]), + }); }); it("should handle missing properties gracefully", () => { diff --git a/internal-packages/llm-pricing/package.json b/internal-packages/llm-pricing/package.json index 2ac51427dd6..8cf9e366f2c 100644 --- a/internal-packages/llm-pricing/package.json +++ b/internal-packages/llm-pricing/package.json @@ -11,8 +11,8 @@ }, "scripts": { "typecheck": "tsc --noEmit", - "generate": "./scripts/sync-model-prices.sh", - "sync-prices": "./scripts/sync-model-prices.sh", - "sync-prices:check": "./scripts/sync-model-prices.sh --check" + "generate": "echo 'defaultPrices.ts is pre-committed — run sync-prices to update'", + "sync-prices": "bash scripts/sync-model-prices.sh", + "sync-prices:check": "bash scripts/sync-model-prices.sh --check" } } From 622bee1997e8575db1e35ee7a6bccf0abdd54a19 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 14:23:37 +0000 Subject: [PATCH 09/15] A bunch of fixes --- .../components/runs/v3/ai/AISpanDetails.tsx | 21 +-- .../app/routes/admin.llm-models.$modelId.tsx | 14 +- apps/webapp/app/v3/querySchemas.ts | 1 - .../v3/utils/enrichCreatableEvents.server.ts | 4 +- apps/webapp/test/otlpExporter.test.ts | 64 +++++++ .../llm-pricing/src/registry.test.ts | 159 ++++++++++++++++++ internal-packages/llm-pricing/src/registry.ts | 18 +- 7 files changed, 255 insertions(+), 26 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index 988de788167..73dfa6d5bc6 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -16,8 +16,7 @@ export function AISpanDetails({ rawProperties?: string; }) { const [tab, setTab] = useState("overview"); - const hasTools = - (aiData.toolDefinitions && aiData.toolDefinitions.length > 0) || aiData.toolCount != null; + const toolCount = aiData.toolCount ?? aiData.toolDefinitions?.length ?? 0; return (
@@ -40,16 +39,14 @@ export function AISpanDetails({ > Messages - {hasTools && ( - setTab("tools")} - shortcut={{ key: "t" }} - > - Tools{aiData.toolCount != null ? ` (${aiData.toolCount})` : ""} - - )} + setTab("tools")} + shortcut={{ key: "t" }} + > + Tools{toolCount > 0 ? ` (${toolCount})` : ""} +
diff --git a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx index 7f7fadaf436..e37491a1b4f 100644 --- a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx +++ b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx @@ -1,4 +1,4 @@ -import { Form, useNavigate } from "@remix-run/react"; +import { Form, useActionData, useNavigate } from "@remix-run/react"; import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; import { redirect } from "@remix-run/server-runtime"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; @@ -130,6 +130,7 @@ export async function action({ request, params }: ActionFunctionArgs) { export default function AdminLlmModelDetailRoute() { const { model } = useTypedLoaderData(); + const actionData = useActionData<{ success?: boolean; error?: string; details?: unknown[] }>(); const navigate = useNavigate(); const [modelName, setModelName] = useState(model.modelName); @@ -273,6 +274,17 @@ export default function AdminLlmModelDetailRoute() { ))}
+ {actionData?.error && ( +
+ {actionData.error} + {actionData.details && ( + 
+                    {JSON.stringify(actionData.details, null, 2)}
+
+ )} +
+ )} + {/* Actions */}
{/* Tab content */} -
+
{tab === "overview" && } {tab === "messages" && } {tab === "tools" && }
- {/* Footer: Copy raw */} - {rawProperties && } + {/* Footer: Copy raw (admin only) */} + {isAdmin && rawProperties && }
); } @@ -126,23 +129,15 @@ function CopyRawFooter({ rawProperties }: { rawProperties: string }) { } return ( -
- + Copy raw properties +
); } From 421911816b263edf4c4258d549e4df227a3da4cc Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 16:25:04 +0000 Subject: [PATCH 11/15] Nicer Tools tab icon and tabs scroll behaviour --- .../app/components/runs/v3/ai/AISpanDetails.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index b057411e202..917062acad3 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -24,7 +24,7 @@ export function AISpanDetails({ return (
{/* Tab bar */} -
+
setTab("tools")} shortcut={{ key: "t" }} > - Tools{toolCount > 0 ? ` (${toolCount})` : ""} + + Tools + {toolCount > 0 && ( + + {toolCount} + + )} +
From 226de08c23410e5af6b471e9ebcdb737c0316453 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 17:07:17 +0000 Subject: [PATCH 12/15] Display details as a property table instead of pills --- .../components/runs/v3/ai/AIChatMessages.tsx | 3 +- .../components/runs/v3/ai/AIModelSummary.tsx | 106 +++++++----------- .../components/runs/v3/ai/AISpanDetails.tsx | 11 +- 3 files changed, 48 insertions(+), 72 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index 9fe39482c86..5c783258228 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -1,5 +1,6 @@ import { lazy, Suspense, useState } from "react"; import { CodeBlock } from "~/components/code/CodeBlock"; +import { Header3 } from "~/components/primitives/Headers"; import type { DisplayItem, ToolUse } from "./types"; // Lazy load streamdown to avoid SSR issues @@ -45,7 +46,7 @@ function SectionHeader({ }) { return (
- {label} + {label} {right &&
{right}
}
); diff --git a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx index a33b90ef899..62341fc9041 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx @@ -1,33 +1,36 @@ import { formatCurrencyAccurate } from "~/utils/numberFormatter"; +import { Header3 } from "~/components/primitives/Headers"; import type { AISpanData } from "./types"; export function AITagsRow({ aiData }: { aiData: AISpanData }) { return ( -
- {aiData.model} - {aiData.provider !== "unknown" && {aiData.provider}} - {aiData.resolvedProvider && ( - via {aiData.resolvedProvider} - )} - {aiData.finishReason && {aiData.finishReason}} - {aiData.serviceTier && tier: {aiData.serviceTier}} - {aiData.toolChoice && tools: {aiData.toolChoice}} - {aiData.toolCount != null && aiData.toolCount > 0 && ( - - {aiData.toolCount} {aiData.toolCount === 1 ? "tool" : "tools"} - - )} - {aiData.messageCount != null && ( - - {aiData.messageCount} {aiData.messageCount === 1 ? "msg" : "msgs"} - - )} - {aiData.telemetryMetadata && - Object.entries(aiData.telemetryMetadata).map(([key, value]) => ( - - {key}: {value} - - ))} +
+
+ + {aiData.provider !== "unknown" && } + {aiData.resolvedProvider && ( + + )} + {aiData.finishReason && } + {aiData.serviceTier && } + {aiData.toolChoice && } + {aiData.toolCount != null && aiData.toolCount > 0 && ( + + )} + {aiData.messageCount != null && ( + + )} + {aiData.telemetryMetadata && + Object.entries(aiData.telemetryMetadata).map(([key, value]) => ( + + ))} +
); } @@ -35,13 +38,16 @@ export function AITagsRow({ aiData }: { aiData: AISpanData }) { export function AIStatsSummary({ aiData }: { aiData: AISpanData }) { return (
- Stats - -
+ Stats +
{aiData.cachedTokens != null && aiData.cachedTokens > 0 && ( - + )} {aiData.cacheCreationTokens != null && aiData.cacheCreationTokens > 0 && ( )} - + {aiData.totalCost != null && ( @@ -84,22 +84,20 @@ function MetricRow({ value, unit, bold, - border, }: { label: string; value: string; unit?: string; bold?: boolean; - border?: boolean; }) { return ( -
+
{label} - + {value} {unit && {unit}} @@ -113,23 +111,3 @@ function formatTtfc(ms: number): string { } return `${Math.round(ms)}ms`; } - -function Pill({ - children, - variant = "default", -}: { - children: React.ReactNode; - variant?: "default" | "dimmed"; -}) { - return ( - - {children} - - ); -} diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index 917062acad3..1c8a64fc0f8 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -1,6 +1,7 @@ import { CheckIcon, ClipboardDocumentIcon } from "@heroicons/react/20/solid"; import { useState } from "react"; import { Button } from "~/components/primitives/Buttons"; +import { Header3 } from "~/components/primitives/Headers"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import { useHasAdminAccess } from "~/hooks/useUser"; import { AIChatMessages, AssistantResponse } from "./AIChatMessages"; @@ -77,7 +78,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { const { userText, outputText, outputToolNames } = extractInputOutput(aiData); return ( -
+
{/* Tags + Stats */} @@ -85,9 +86,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { {/* Input (last user prompt) */} {userText && (
- - Input - + Input

{userText}

)} @@ -96,9 +95,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { {outputText && } {outputToolNames.length > 0 && !outputText && (
- - Output - + Output

Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} {outputToolNames.join(", ")} From 218725ab44d2652fb818e06c4213d3725d54cfc0 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 17:11:53 +0000 Subject: [PATCH 13/15] Use proper paragraph component --- apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx index 5130256f85b..a329698dd5e 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx @@ -1,6 +1,7 @@ import { useState } from "react"; import { CodeBlock } from "~/components/code/CodeBlock"; import type { AISpanData, ToolDefinition } from "./types"; +import { Paragraph } from "~/components/primitives/Paragraph"; export function AIToolsInventory({ aiData }: { aiData: AISpanData }) { const defs = aiData.toolDefinitions ?? []; @@ -8,8 +9,8 @@ export function AIToolsInventory({ aiData }: { aiData: AISpanData }) { if (defs.length === 0) { return ( -

- No tool definitions available for this span. +
+ No tool definitions available for this span.
); } From 463f972250fcf1fa7586df388cc232fc340d616f Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 17:47:13 +0000 Subject: [PATCH 14/15] Style improvements to the messages --- .../components/runs/v3/ai/AIChatMessages.tsx | 99 ++++++++++++------- .../components/runs/v3/ai/AISpanDetails.tsx | 23 +++-- 2 files changed, 79 insertions(+), 43 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index 5c783258228..a50ef8ae806 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -1,6 +1,9 @@ +import { CheckIcon, ClipboardDocumentIcon, CodeBracketSquareIcon } from "@heroicons/react/20/solid"; import { lazy, Suspense, useState } from "react"; import { CodeBlock } from "~/components/code/CodeBlock"; +import { Button } from "~/components/primitives/Buttons"; import { Header3 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; import type { DisplayItem, ToolUse } from "./types"; // Lazy load streamdown to avoid SSR issues @@ -16,7 +19,7 @@ const StreamdownRenderer = lazy(() => export function AIChatMessages({ items }: { items: DisplayItem[] }) { return ( -
+
{items.map((item, i) => { switch (item.type) { case "system": @@ -37,13 +40,7 @@ export function AIChatMessages({ items }: { items: DisplayItem[] }) { // Section header (shared across all sections) // --------------------------------------------------------------------------- -function SectionHeader({ - label, - right, -}: { - label: string; - right?: React.ReactNode; -}) { +function SectionHeader({ label, right }: { label: string; right?: React.ReactNode }) { return (
{label} @@ -52,6 +49,14 @@ function SectionHeader({ ); } +export function ChatBubble({ children }: { children: React.ReactNode }) { + return ( +
+ {children} +
+ ); +} + // --------------------------------------------------------------------------- // System // --------------------------------------------------------------------------- @@ -62,7 +67,7 @@ function SystemSection({ text }: { text: string }) { const preview = isLong ? text.slice(0, 150) + "..." : text; return ( -
+
- 
-        {expanded || !isLong ? text : preview}
-
+ + + {expanded || !isLong ? text : preview} + +
); } @@ -89,9 +96,11 @@ function SystemSection({ text }: { text: string }) { function UserSection({ text }: { text: string }) { return ( -
+
-

{text}

+ + {text} +
); } @@ -108,36 +117,54 @@ export function AssistantResponse({ headerLabel?: string; }) { const [mode, setMode] = useState<"rendered" | "raw">("rendered"); + const [copied, setCopied] = useState(false); + + function handleCopy() { + navigator.clipboard.writeText(text); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } return ( -
+
- - + +
} /> {mode === "rendered" ? ( -
- {text}}> - {text} - -
+ + + {text}}> + {text} + + + ) : ( - + )}
); @@ -151,9 +178,13 @@ function ToolUseSection({ tools }: { tools: ToolUse[] }) { return (
- {tools.map((tool) => ( - - ))} + +
+ {tools.map((tool) => ( + + ))} +
+
); } @@ -228,9 +259,9 @@ function ToolUseRow({ tool }: { tool: ToolUse }) { )} {activeTab === "details" && hasDetails && ( -
+
{tool.description && ( -

{tool.description}

+

{tool.description}

)} {tool.parametersJson && (
diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index 1c8a64fc0f8..4b64da7db38 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -2,9 +2,10 @@ import { CheckIcon, ClipboardDocumentIcon } from "@heroicons/react/20/solid"; import { useState } from "react"; import { Button } from "~/components/primitives/Buttons"; import { Header3 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import { useHasAdminAccess } from "~/hooks/useUser"; -import { AIChatMessages, AssistantResponse } from "./AIChatMessages"; +import { AIChatMessages, AssistantResponse, ChatBubble } from "./AIChatMessages"; import { AIStatsSummary, AITagsRow } from "./AIModelSummary"; import { AIToolsInventory } from "./AIToolsInventory"; import type { AISpanData, DisplayItem } from "./types"; @@ -85,21 +86,25 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { {/* Input (last user prompt) */} {userText && ( -
+
Input -

{userText}

+ + {userText} +
)} {/* Output (assistant response or tool calls) */} {outputText && } {outputToolNames.length > 0 && !outputText && ( -
+
Output -

- Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} - {outputToolNames.join(", ")} -

+ + + Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} + {outputToolNames.join(", ")} + +
)}
@@ -109,7 +114,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { function MessagesTab({ aiData }: { aiData: AISpanData }) { return (
-
+
{aiData.items && aiData.items.length > 0 && } {aiData.responseText && !hasAssistantItem(aiData.items) && ( From 9fecede48ad32efd38a94fb16559eb602ae9072d Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sat, 14 Mar 2026 09:55:26 +0000 Subject: [PATCH 15/15] Renamed llm_usage_v1 to llm_metrics_v1, added some additional fields to track as well --- .server-changes/llm-cost-tracking.md | 2 +- .../ExamplesContent.tsx | 24 +++---- .../route.tsx | 6 ++ .../clickhouseEventRepository.server.ts | 67 ++++++++++--------- .../eventRepository/eventRepository.types.ts | 11 ++- apps/webapp/app/v3/querySchemas.ts | 61 +++++++++++++---- .../v3/utils/enrichCreatableEvents.server.ts | 39 ++++++++--- apps/webapp/seed-ai-spans.mts | 23 ++++--- apps/webapp/seed.mts | 60 +++++++++++++++++ apps/webapp/test/otlpExporter.test.ts | 30 ++++----- .../schema/024_create_llm_usage_v1.sql | 22 +++++- internal-packages/clickhouse/src/index.ts | 8 +-- .../src/{llmUsage.ts => llmMetrics.ts} | 21 ++++-- 13 files changed, 269 insertions(+), 105 deletions(-) rename internal-packages/clickhouse/src/{llmUsage.ts => llmMetrics.ts} (64%) diff --git a/.server-changes/llm-cost-tracking.md b/.server-changes/llm-cost-tracking.md index b68302731a7..7567aae7d1b 100644 --- a/.server-changes/llm-cost-tracking.md +++ b/.server-changes/llm-cost-tracking.md @@ -3,4 +3,4 @@ area: webapp type: feature --- -Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_usage_v1` ClickHouse table. +Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_metrics_v1` ClickHouse table that captures usage, cost, performance (TTFC, tokens/sec), and behavioral (finish reason, operation type) metrics. diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index 1235b443348..f0c3c1d616f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -126,12 +126,12 @@ LIMIT 100`, SUM(total_cost) AS total_cost, SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens -FROM llm_usage +FROM llm_metrics WHERE start_time > now() - INTERVAL 7 DAY GROUP BY response_model ORDER BY total_cost DESC`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM cost over time", @@ -139,12 +139,12 @@ ORDER BY total_cost DESC`, query: `SELECT timeBucket(), SUM(total_cost) AS total_cost -FROM llm_usage +FROM llm_metrics GROUP BY timeBucket ORDER BY timeBucket LIMIT 1000`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "Most expensive runs by LLM cost (top 50)", @@ -155,12 +155,12 @@ LIMIT 1000`, SUM(total_cost) AS llm_cost, SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens -FROM llm_usage +FROM llm_metrics GROUP BY run_id, task_identifier ORDER BY llm_cost DESC LIMIT 50`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM calls by provider", @@ -169,11 +169,11 @@ LIMIT 50`, gen_ai_system, count() AS call_count, SUM(total_cost) AS total_cost -FROM llm_usage +FROM llm_metrics GROUP BY gen_ai_system ORDER BY total_cost DESC`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM cost by user", @@ -184,13 +184,13 @@ ORDER BY total_cost DESC`, SUM(total_cost) AS total_cost, SUM(total_tokens) AS total_tokens, count() AS call_count -FROM llm_usage +FROM llm_metrics WHERE metadata.userId != '' GROUP BY metadata.userId ORDER BY total_cost DESC LIMIT 50`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM cost by metadata key", @@ -202,11 +202,11 @@ LIMIT 50`, total_cost, total_tokens, run_id -FROM llm_usage +FROM llm_metrics ORDER BY start_time DESC LIMIT 20`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, ]; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx index 29d9e246e2d..ee69419e1b7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx @@ -120,6 +120,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typedjson({ ...result, regions: regionsResult.regions }); } catch (error) { + logger.error("Failed to load test page", { + taskParam, + error: error instanceof Error ? error.message : error, + stack: error instanceof Error ? error.stack : undefined, + }); + return redirectWithErrorMessage( v3TestPath({ slug: organizationSlug }, { slug: projectParam }, environment), request, diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 531c3e307e9..86f3560669c 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -1,6 +1,6 @@ import type { ClickHouse, - LlmUsageV1Input, + LlmMetricsV1Input, TaskEventDetailedSummaryV1Result, TaskEventDetailsV1Result, TaskEventSummaryV1Result, @@ -96,7 +96,7 @@ export class ClickhouseEventRepository implements IEventRepository { private _clickhouse: ClickHouse; private _config: ClickhouseEventRepositoryConfig; private readonly _flushScheduler: DynamicFlushScheduler; - private readonly _llmUsageFlushScheduler: DynamicFlushScheduler; + private readonly _llmMetricsFlushScheduler: DynamicFlushScheduler; private _tracer: Tracer; private _version: "v1" | "v2"; @@ -122,10 +122,10 @@ export class ClickhouseEventRepository implements IEventRepository { }, }); - this._llmUsageFlushScheduler = new DynamicFlushScheduler({ + this._llmMetricsFlushScheduler = new DynamicFlushScheduler({ batchSize: 5000, flushInterval: 2000, - callback: this.#flushLlmUsageBatch.bind(this), + callback: this.#flushLlmMetricsBatch.bind(this), minConcurrency: 1, maxConcurrency: 2, maxBatchSize: 10000, @@ -230,9 +230,9 @@ export class ClickhouseEventRepository implements IEventRepository { }); } - async #flushLlmUsageBatch(flushId: string, rows: LlmUsageV1Input[]) { + async #flushLlmMetricsBatch(flushId: string, rows: LlmMetricsV1Input[]) { - const [insertError] = await this._clickhouse.llmUsage.insert(rows, { + const [insertError] = await this._clickhouse.llmMetrics.insert(rows, { params: { clickhouse_settings: this.#getClickhouseInsertSettings(), }, @@ -242,13 +242,13 @@ export class ClickhouseEventRepository implements IEventRepository { throw insertError; } - logger.info("ClickhouseEventRepository.flushLlmUsageBatch Inserted LLM usage batch", { + logger.info("ClickhouseEventRepository.flushLlmMetricsBatch Inserted LLM metrics batch", { rows: rows.length, }); } - #createLlmUsageInput(event: CreateEventInput): LlmUsageV1Input { - const llmUsage = event._llmUsage!; + #createLlmMetricsInput(event: CreateEventInput): LlmMetricsV1Input { + const llmMetrics = event._llmMetrics!; return { organization_id: event.organizationId, @@ -258,22 +258,27 @@ export class ClickhouseEventRepository implements IEventRepository { task_identifier: event.taskSlug, trace_id: event.traceId, span_id: event.spanId, - gen_ai_system: llmUsage.genAiSystem, - request_model: llmUsage.requestModel, - response_model: llmUsage.responseModel, - matched_model_id: llmUsage.matchedModelId, - operation_name: llmUsage.operationName, - pricing_tier_id: llmUsage.pricingTierId, - pricing_tier_name: llmUsage.pricingTierName, - input_tokens: llmUsage.inputTokens, - output_tokens: llmUsage.outputTokens, - total_tokens: llmUsage.totalTokens, - usage_details: llmUsage.usageDetails, - input_cost: llmUsage.inputCost, - output_cost: llmUsage.outputCost, - total_cost: llmUsage.totalCost, - cost_details: llmUsage.costDetails, - metadata: llmUsage.metadata, + gen_ai_system: llmMetrics.genAiSystem, + request_model: llmMetrics.requestModel, + response_model: llmMetrics.responseModel, + matched_model_id: llmMetrics.matchedModelId, + operation_id: llmMetrics.operationId, + finish_reason: llmMetrics.finishReason, + cost_source: llmMetrics.costSource, + pricing_tier_id: llmMetrics.pricingTierId, + pricing_tier_name: llmMetrics.pricingTierName, + input_tokens: llmMetrics.inputTokens, + output_tokens: llmMetrics.outputTokens, + total_tokens: llmMetrics.totalTokens, + usage_details: llmMetrics.usageDetails, + input_cost: llmMetrics.inputCost, + output_cost: llmMetrics.outputCost, + total_cost: llmMetrics.totalCost, + cost_details: llmMetrics.costDetails, + provider_cost: llmMetrics.providerCost, + ms_to_first_chunk: llmMetrics.msToFirstChunk, + tokens_per_second: llmMetrics.tokensPerSecond, + metadata: llmMetrics.metadata, start_time: this.#clampAndFormatStartTime(event.startTime.toString()), duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0), }; @@ -300,13 +305,13 @@ export class ClickhouseEventRepository implements IEventRepository { async insertMany(events: CreateEventInput[]): Promise { this.addToBatch(events.flatMap((event) => this.createEventToTaskEventV1Input(event))); - // Dual-write LLM usage records for spans with cost enrichment - const llmUsageRows = events - .filter((e) => e._llmUsage != null) - .map((e) => this.#createLlmUsageInput(e)); + // Dual-write LLM metrics records for spans with cost enrichment + const llmMetricsRows = events + .filter((e) => e._llmMetrics != null) + .map((e) => this.#createLlmMetricsInput(e)); - if (llmUsageRows.length > 0) { - this._llmUsageFlushScheduler.addToBatch(llmUsageRows); + if (llmMetricsRows.length > 0) { + this._llmMetricsFlushScheduler.addToBatch(llmMetricsRows); } } diff --git a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts index 69590dc9493..fcef0010ee0 100644 --- a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts +++ b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts @@ -21,12 +21,14 @@ export type { ExceptionEventProperties }; // Event Creation Types // ============================================================================ -export type LlmUsageData = { +export type LlmMetricsData = { genAiSystem: string; requestModel: string; responseModel: string; matchedModelId: string; - operationName: string; + operationId: string; + finishReason: string; + costSource: string; pricingTierId: string; pricingTierName: string; inputTokens: number; @@ -37,6 +39,9 @@ export type LlmUsageData = { outputCost: number; totalCost: number; costDetails: Record; + providerCost: number; + msToFirstChunk: number; + tokensPerSecond: number; metadata: Record; }; @@ -78,7 +83,7 @@ export type CreateEventInput = Omit< machineId?: string; runTags?: string[]; /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */ - _llmUsage?: LlmUsageData; + _llmMetrics?: LlmMetricsData; }; export type CreatableEventKind = TaskEventKind; diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 7ec23285d0c..b4f4fceb80c 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -600,12 +600,12 @@ export const metricsSchema: TableSchema = { * All available schemas for the query editor */ /** - * Schema definition for the llm_usage table (trigger_dev.llm_usage_v1) + * Schema definition for the llm_metrics table (trigger_dev.llm_metrics_v1) */ -export const llmUsageSchema: TableSchema = { - name: "llm_usage", - clickhouseName: "trigger_dev.llm_usage_v1", - description: "LLM token usage and cost data from GenAI spans", +export const llmMetricsSchema: TableSchema = { + name: "llm_metrics", + clickhouseName: "trigger_dev.llm_metrics_v1", + description: "LLM metrics: token usage, cost, performance, and behavior from GenAI spans", timeConstraint: "start_time", tenantColumns: { organizationId: "organization_id", @@ -669,11 +669,26 @@ export const llmUsageSchema: TableSchema = { coreColumn: true, }), }, - operation_name: { - name: "operation_name", + operation_id: { + name: "operation_id", ...column("LowCardinality(String)", { - description: "Operation type (e.g. chat, completion)", - example: "chat", + description: "Operation type (e.g. ai.streamText.doStream, ai.generateText.doGenerate)", + example: "ai.streamText.doStream", + }), + }, + finish_reason: { + name: "finish_reason", + ...column("LowCardinality(String)", { + description: "Why the LLM stopped generating (e.g. stop, tool-calls, length)", + example: "stop", + coreColumn: true, + }), + }, + cost_source: { + name: "cost_source", + ...column("LowCardinality(String)", { + description: "Where cost data came from (registry, gateway, openrouter)", + example: "registry", }), }, input_tokens: { @@ -700,14 +715,14 @@ export const llmUsageSchema: TableSchema = { input_cost: { name: "input_cost", ...column("Decimal64(12)", { - description: "Input cost in USD", + description: "Input cost in USD (from pricing registry)", customRenderType: "costInDollars", }), }, output_cost: { name: "output_cost", ...column("Decimal64(12)", { - description: "Output cost in USD", + description: "Output cost in USD (from pricing registry)", customRenderType: "costInDollars", }), }, @@ -719,6 +734,28 @@ export const llmUsageSchema: TableSchema = { coreColumn: true, }), }, + provider_cost: { + name: "provider_cost", + ...column("Decimal64(12)", { + description: "Provider-reported cost in USD (from gateway or openrouter)", + customRenderType: "costInDollars", + }), + }, + ms_to_first_chunk: { + name: "ms_to_first_chunk", + ...column("Float64", { + description: "Time to first chunk in milliseconds (TTFC)", + example: "245.3", + coreColumn: true, + }), + }, + tokens_per_second: { + name: "tokens_per_second", + ...column("Float64", { + description: "Average output tokens per second", + example: "72.5", + }), + }, pricing_tier_name: { name: "pricing_tier_name", ...column("LowCardinality(String)", { @@ -751,7 +788,7 @@ export const llmUsageSchema: TableSchema = { }, }; -export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmUsageSchema]; +export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmMetricsSchema]; /** * Default query for the query editor diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 29be7ae6fad..b9ed86aa874 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -1,4 +1,4 @@ -import type { CreateEventInput, LlmUsageData } from "../eventRepository/eventRepository.types"; +import type { CreateEventInput, LlmMetricsData } from "../eventRepository/eventRepository.types"; // Registry interface — matches ModelPricingRegistry from @internal/llm-pricing type CostRegistry = { @@ -36,12 +36,12 @@ function enrichCreatableEvent(event: CreateEventInput): CreateEventInput { event.message = message; event.style = enrichStyle(event); - enrichLlmCost(event); + enrichLlmMetrics(event); return event; } -function enrichLlmCost(event: CreateEventInput): void { +function enrichLlmMetrics(event: CreateEventInput): void { const props = event.properties; if (!props) return; @@ -133,7 +133,7 @@ function enrichLlmCost(event: CreateEventInput): void { }, } as unknown as typeof event.style; - // Only write llm_usage when cost data is available + // Only write llm_metrics when cost data is available if (!cost && !providerCost) return; // Build metadata map from run tags and ai.telemetry.metadata.* @@ -154,13 +154,33 @@ function enrichLlmCost(event: CreateEventInput): void { } } - // Set _llmUsage side-channel for dual-write to llm_usage_v1 - const llmUsage: LlmUsageData = { + // Extract new performance/behavioral fields + const finishReason = typeof props["ai.response.finishReason"] === "string" + ? props["ai.response.finishReason"] + : typeof props["gen_ai.response.finish_reasons"] === "string" + ? props["gen_ai.response.finish_reasons"] + : ""; + const operationId = typeof props["ai.operationId"] === "string" + ? props["ai.operationId"] + : (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? ""; + const msToFirstChunk = typeof props["ai.response.msToFirstChunk"] === "number" + ? props["ai.response.msToFirstChunk"] + : 0; + const avgTokensPerSec = typeof props["ai.response.avgOutputTokensPerSecond"] === "number" + ? props["ai.response.avgOutputTokensPerSecond"] + : 0; + const costSource = cost ? "registry" : providerCost ? providerCost.source : ""; + const providerCostValue = providerCost?.totalCost ?? 0; + + // Set _llmMetrics side-channel for dual-write to llm_metrics_v1 + const llmMetrics: LlmMetricsData = { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", requestModel: (props["gen_ai.request.model"] as string) ?? responseModel, responseModel, matchedModelId: cost?.matchedModelId ?? "", - operationName: (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? "", + operationId, + finishReason, + costSource, pricingTierId: cost?.pricingTierId ?? (providerCost ? `provider:${providerCost.source}` : ""), pricingTierName: cost?.pricingTierName ?? (providerCost ? `${providerCost.source} reported` : ""), inputTokens: usageDetails["input"] ?? 0, @@ -171,10 +191,13 @@ function enrichLlmCost(event: CreateEventInput): void { outputCost: cost?.outputCost ?? 0, totalCost: cost?.totalCost ?? providerCost?.totalCost ?? 0, costDetails: cost?.costDetails ?? {}, + providerCost: providerCostValue, + msToFirstChunk, + tokensPerSecond: avgTokensPerSec, metadata, }; - event._llmUsage = llmUsage; + event._llmMetrics = llmMetrics; } function extractUsageDetails(props: Record): Record { diff --git a/apps/webapp/seed-ai-spans.mts b/apps/webapp/seed-ai-spans.mts index 19a72c23f22..ef6a8f4b6ec 100644 --- a/apps/webapp/seed-ai-spans.mts +++ b/apps/webapp/seed-ai-spans.mts @@ -4,7 +4,7 @@ import { prisma } from "./app/db.server"; import { createOrganization } from "./app/models/organization.server"; import { createProject } from "./app/models/project.server"; import { ClickHouse } from "@internal/clickhouse"; -import type { TaskEventV2Input, LlmUsageV1Input } from "@internal/clickhouse"; +import type { TaskEventV2Input, LlmMetricsV1Input } from "@internal/clickhouse"; import { generateTraceId, generateSpanId, @@ -117,8 +117,8 @@ function eventToClickhouseRow(event: CreateEventInput): TaskEventV2Input { }; } -function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { - const llm = event._llmUsage!; +function eventToLlmMetricsRow(event: CreateEventInput): LlmMetricsV1Input { + const llm = event._llmMetrics!; return { organization_id: event.organizationId, project_id: event.projectId, @@ -131,7 +131,9 @@ function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { request_model: llm.requestModel, response_model: llm.responseModel, matched_model_id: llm.matchedModelId, - operation_name: llm.operationName, + operation_id: llm.operationId, + finish_reason: llm.finishReason, + cost_source: llm.costSource, pricing_tier_id: llm.pricingTierId, pricing_tier_name: llm.pricingTierName, input_tokens: llm.inputTokens, @@ -142,6 +144,9 @@ function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { output_cost: llm.outputCost, total_cost: llm.totalCost, cost_details: llm.costDetails, + provider_cost: llm.providerCost, + ms_to_first_chunk: llm.msToFirstChunk, + tokens_per_second: llm.tokensPerSecond, metadata: llm.metadata, start_time: formatStartTime(BigInt(event.startTime)), duration: formatDuration(event.duration ?? 0), @@ -335,8 +340,8 @@ async function seedAiSpans() { crumb("enriching events"); // @crumbs const enriched = enrichCreatableEvents(events); - const enrichedCount = enriched.filter((e) => e._llmUsage != null).length; - const totalCost = enriched.reduce((sum, e) => sum + (e._llmUsage?.totalCost ?? 0), 0); + const enrichedCount = enriched.filter((e) => e._llmMetrics != null).length; + const totalCost = enriched.reduce((sum, e) => sum + (e._llmMetrics?.totalCost ?? 0), 0); console.log( `Enriched ${enrichedCount} spans with LLM cost (total: $${totalCost.toFixed(6)})` ); @@ -362,10 +367,10 @@ async function seedAiSpans() { crumb("task events inserted", { rowCount: chRows.length }); // @crumbs // Insert LLM usage rows - const llmRows = enriched.filter((e) => e._llmUsage != null).map(eventToLlmUsageRow); + const llmRows = enriched.filter((e) => e._llmMetrics != null).map(eventToLlmMetricsRow); if (llmRows.length > 0) { - await clickhouse.llmUsage.insert(llmRows); - crumb("llm usage inserted", { rowCount: llmRows.length }); // @crumbs + await clickhouse.llmMetrics.insert(llmRows); + crumb("llm metrics inserted", { rowCount: llmRows.length }); // @crumbs } // 12. Output diff --git a/apps/webapp/seed.mts b/apps/webapp/seed.mts index aa08eaaeec0..9eb30cd2503 100644 --- a/apps/webapp/seed.mts +++ b/apps/webapp/seed.mts @@ -75,6 +75,7 @@ async function seed() { } await createBatchLimitOrgs(user); + await ensureDefaultWorkerGroup(); console.log("\n🎉 Seed complete!\n"); console.log("Summary:"); @@ -249,3 +250,62 @@ async function findOrCreateProject( return { project, environments }; } + +async function ensureDefaultWorkerGroup() { + // Check if the feature flag already exists + const existingFlag = await prisma.featureFlag.findUnique({ + where: { key: "defaultWorkerInstanceGroupId" }, + }); + + if (existingFlag) { + console.log(`✅ Default worker instance group already configured`); + return; + } + + // Check if a managed worker group already exists + let workerGroup = await prisma.workerInstanceGroup.findFirst({ + where: { type: "MANAGED" }, + }); + + if (!workerGroup) { + console.log("Creating default worker instance group..."); + + const { createHash, randomBytes } = await import("crypto"); + const tokenValue = `tr_wgt_${randomBytes(20).toString("hex")}`; + const tokenHash = createHash("sha256").update(tokenValue).digest("hex"); + + const token = await prisma.workerGroupToken.create({ + data: { tokenHash }, + }); + + workerGroup = await prisma.workerInstanceGroup.create({ + data: { + type: "MANAGED", + name: "local-dev", + masterQueue: "local-dev", + description: "Local development worker group", + tokenId: token.id, + }, + }); + + console.log(`✅ Created worker instance group: ${workerGroup.name} (${workerGroup.id})`); + } else { + console.log( + `✅ Worker instance group already exists: ${workerGroup.name} (${workerGroup.id})` + ); + } + + // Set the feature flag + await prisma.featureFlag.upsert({ + where: { key: "defaultWorkerInstanceGroupId" }, + create: { + key: "defaultWorkerInstanceGroupId", + value: workerGroup.id, + }, + update: { + value: workerGroup.id, + }, + }); + + console.log(`✅ Set defaultWorkerInstanceGroupId feature flag`); +} diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index 6ba54243faf..0194ae6bf74 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -505,20 +505,20 @@ describe("OTLPExporter", () => { }); }); - it("should set _llmUsage side-channel for dual-write", () => { + it("should set _llmMetrics side-channel for dual-write", () => { const events = [makeGenAiEvent()]; // @ts-expect-error const $events = enrichCreatableEvents(events); const event = $events[0]; - expect(event._llmUsage).toBeDefined(); - expect(event._llmUsage.genAiSystem).toBe("openai"); - expect(event._llmUsage.responseModel).toBe("gpt-4o-2024-08-06"); - expect(event._llmUsage.inputTokens).toBe(702); - expect(event._llmUsage.outputTokens).toBe(22); - expect(event._llmUsage.totalCost).toBeCloseTo(0.001975); - expect(event._llmUsage.operationName).toBe("ai.streamText.doStream"); + expect(event._llmMetrics).toBeDefined(); + expect(event._llmMetrics.genAiSystem).toBe("openai"); + expect(event._llmMetrics.responseModel).toBe("gpt-4o-2024-08-06"); + expect(event._llmMetrics.inputTokens).toBe(702); + expect(event._llmMetrics.outputTokens).toBe(22); + expect(event._llmMetrics.totalCost).toBeCloseTo(0.001975); + expect(event._llmMetrics.operationId).toBe("ai.streamText.doStream"); }); it("should skip partial spans", () => { @@ -528,7 +528,7 @@ describe("OTLPExporter", () => { // @ts-expect-error const $events = enrichCreatableEvents(events); expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); - expect($events[0]._llmUsage).toBeUndefined(); + expect($events[0]._llmMetrics).toBeUndefined(); }); it("should skip spans without gen_ai.response.model or gen_ai.request.model", () => { @@ -635,8 +635,8 @@ describe("OTLPExporter", () => { // @ts-expect-error const $events = enrichCreatableEvents(events); - expect($events[0]._llmUsage.inputTokens).toBe(500); - expect($events[0]._llmUsage.outputTokens).toBe(100); + expect($events[0]._llmMetrics.inputTokens).toBe(500); + expect($events[0]._llmMetrics.outputTokens).toBe(100); }); it("should prefer gen_ai.usage.total_tokens over input+output sum", () => { @@ -659,9 +659,9 @@ describe("OTLPExporter", () => { }); // LLM usage should also use the explicit total - expect(event._llmUsage.totalTokens).toBe(200); - expect(event._llmUsage.inputTokens).toBe(100); - expect(event._llmUsage.outputTokens).toBe(50); + expect(event._llmMetrics.totalTokens).toBe(200); + expect(event._llmMetrics.inputTokens).toBe(100); + expect(event._llmMetrics.outputTokens).toBe(50); }); it("should fall back to input+output when total_tokens is absent", () => { @@ -680,7 +680,7 @@ describe("OTLPExporter", () => { text: "375", icon: "tabler-hash", }); - expect(event._llmUsage.totalTokens).toBe(375); + expect(event._llmMetrics.totalTokens).toBe(375); }); it("should use total_tokens when only total is present without input/output breakdown", () => { diff --git a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql index 5d7f879fdd3..7cfbc0cea98 100644 --- a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql +++ b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql @@ -1,6 +1,7 @@ -- +goose Up -CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 +CREATE TABLE IF NOT EXISTS trigger_dev.llm_metrics_v1 ( + -- Tenant context organization_id LowCardinality(String), project_id LowCardinality(String), environment_id String CODEC(ZSTD(1)), @@ -9,30 +10,45 @@ CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 trace_id String CODEC(ZSTD(1)), span_id String CODEC(ZSTD(1)), + -- Model & provider gen_ai_system LowCardinality(String), request_model String CODEC(ZSTD(1)), response_model String CODEC(ZSTD(1)), matched_model_id String CODEC(ZSTD(1)), - operation_name LowCardinality(String), + operation_id LowCardinality(String), + finish_reason LowCardinality(String), + cost_source LowCardinality(String), + + -- Pricing pricing_tier_id String CODEC(ZSTD(1)), pricing_tier_name LowCardinality(String), + -- Token usage input_tokens UInt64 DEFAULT 0, output_tokens UInt64 DEFAULT 0, total_tokens UInt64 DEFAULT 0, usage_details Map(LowCardinality(String), UInt64), + -- Cost input_cost Decimal64(12) DEFAULT 0, output_cost Decimal64(12) DEFAULT 0, total_cost Decimal64(12) DEFAULT 0, cost_details Map(LowCardinality(String), Decimal64(12)), + provider_cost Decimal64(12) DEFAULT 0, + + -- Performance + ms_to_first_chunk Float64 DEFAULT 0, + tokens_per_second Float64 DEFAULT 0, + -- Attribution metadata Map(LowCardinality(String), String), + -- Timing start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)), duration UInt64 DEFAULT 0 CODEC(ZSTD(1)), inserted_at DateTime64(3) DEFAULT now64(3), + -- Indexes INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1, @@ -45,4 +61,4 @@ TTL toDateTime(inserted_at) + INTERVAL 365 DAY SETTINGS ttl_only_drop_parts = 1; -- +goose Down -DROP TABLE IF EXISTS trigger_dev.llm_usage_v1; +DROP TABLE IF EXISTS trigger_dev.llm_metrics_v1; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 336abf3761f..18e52483627 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -27,7 +27,7 @@ import { getLogsSearchListQueryBuilder, } from "./taskEvents.js"; import { insertMetrics } from "./metrics.js"; -import { insertLlmUsage } from "./llmUsage.js"; +import { insertLlmMetrics } from "./llmMetrics.js"; import { getErrorGroups, getErrorInstances, @@ -45,7 +45,7 @@ import type { Agent as HttpsAgent } from "https"; export type * from "./taskRuns.js"; export type * from "./taskEvents.js"; export type * from "./metrics.js"; -export type * from "./llmUsage.js"; +export type * from "./llmMetrics.js"; export type * from "./errors.js"; export type * from "./client/queryBuilder.js"; @@ -227,9 +227,9 @@ export class ClickHouse { }; } - get llmUsage() { + get llmMetrics() { return { - insert: insertLlmUsage(this.writer), + insert: insertLlmMetrics(this.writer), }; } diff --git a/internal-packages/clickhouse/src/llmUsage.ts b/internal-packages/clickhouse/src/llmMetrics.ts similarity index 64% rename from internal-packages/clickhouse/src/llmUsage.ts rename to internal-packages/clickhouse/src/llmMetrics.ts index e9423962ac4..1f830b707d8 100644 --- a/internal-packages/clickhouse/src/llmUsage.ts +++ b/internal-packages/clickhouse/src/llmMetrics.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { ClickhouseWriter } from "./client/types.js"; -export const LlmUsageV1Input = z.object({ +export const LlmMetricsV1Input = z.object({ organization_id: z.string(), project_id: z.string(), environment_id: z.string(), @@ -14,7 +14,10 @@ export const LlmUsageV1Input = z.object({ request_model: z.string(), response_model: z.string(), matched_model_id: z.string(), - operation_name: z.string(), + operation_id: z.string(), + finish_reason: z.string(), + cost_source: z.string(), + pricing_tier_id: z.string(), pricing_tier_name: z.string(), @@ -27,6 +30,10 @@ export const LlmUsageV1Input = z.object({ output_cost: z.number(), total_cost: z.number(), cost_details: z.record(z.string(), z.number()), + provider_cost: z.number(), + + ms_to_first_chunk: z.number(), + tokens_per_second: z.number(), metadata: z.record(z.string(), z.string()), @@ -34,11 +41,11 @@ export const LlmUsageV1Input = z.object({ duration: z.string(), }); -export type LlmUsageV1Input = z.input; +export type LlmMetricsV1Input = z.input; -export function insertLlmUsage(ch: ClickhouseWriter) { - return ch.insertUnsafe({ - name: "insertLlmUsage", - table: "trigger_dev.llm_usage_v1", +export function insertLlmMetrics(ch: ClickhouseWriter) { + return ch.insertUnsafe({ + name: "insertLlmMetrics", + table: "trigger_dev.llm_metrics_v1", }); }