Skip to content

Commit 5cf618a

Browse files
ericallamericallam
committed
A bunch of fixes
1 parent cbb946a commit 5cf618a

File tree

7 files changed

+255
-26
lines changed

7 files changed

+255
-26
lines changed

‎apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@ export function AISpanDetails({
1616
rawProperties?: string;
1717
}) {
1818
const [tab, setTab] = useState<AITab>("overview");
19-
const hasTools =
20-
(aiData.toolDefinitions && aiData.toolDefinitions.length > 0) || aiData.toolCount != null;
19+
const toolCount = aiData.toolCount ?? aiData.toolDefinitions?.length ?? 0;
2120

2221
return (
2322
<div className="flex h-full flex-col overflow-hidden">
@@ -40,16 +39,14 @@ export function AISpanDetails({
4039
>
4140
Messages
4241
</TabButton>
43-
{hasTools && (
44-
<TabButton
45-
isActive={tab === "tools"}
46-
layoutId="ai-span"
47-
onClick={() => setTab("tools")}
48-
shortcut={{ key: "t" }}
49-
>
50-
Tools{aiData.toolCount != null ? ` (${aiData.toolCount})` : ""}
51-
</TabButton>
52-
)}
42+
<TabButton
43+
isActive={tab === "tools"}
44+
layoutId="ai-span"
45+
onClick={() => setTab("tools")}
46+
shortcut={{ key: "t" }}
47+
>
48+
Tools{toolCount > 0 ? ` (${toolCount})` : ""}
49+
</TabButton>
5350
</TabContainer>
5451
</div>
5552

‎apps/webapp/app/routes/admin.llm-models.$modelId.tsx‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Form, useNavigate } from "@remix-run/react";
1+
import { Form, useActionData, useNavigate } from "@remix-run/react";
22
import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
33
import { redirect } from "@remix-run/server-runtime";
44
import { typedjson, useTypedLoaderData } from "remix-typedjson";
@@ -130,6 +130,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
130130

131131
export default function AdminLlmModelDetailRoute() {
132132
const { model } = useTypedLoaderData<typeof loader>();
133+
const actionData = useActionData<{ success?: boolean; error?: string; details?: unknown[] }>();
133134
const navigate = useNavigate();
134135

135136
const [modelName, setModelName] = useState(model.modelName);
@@ -273,6 +274,17 @@ export default function AdminLlmModelDetailRoute() {
273274
))}
274275
</div>
275276

277+
{actionData?.error && (
278+
<div className="rounded-md bg-red-500/10 border border-red-500/30 p-3 text-sm text-red-400">
279+
{actionData.error}
280+
{actionData.details && (
281+
<pre className="mt-1 text-xs text-red-300/70 overflow-auto">
282+
{JSON.stringify(actionData.details, null, 2)}
283+
</pre>
284+
)}
285+
</div>
286+
)}
287+
276288
{/* Actions */}
277289
<div className="flex items-center gap-2 border-t border-grid-dimmed pt-4">
278290
<Button type="submit" variant="primary/medium">

‎apps/webapp/app/v3/querySchemas.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -746,7 +746,6 @@ export const llmUsageSchema: TableSchema = {
746746
description:
747747
"Key-value metadata from run tags (key:value format) and AI SDK telemetry metadata. Access keys with dot notation (metadata.userId) or bracket syntax (metadata['userId']).",
748748
example: "{'userId':'user_123','org':'acme'}",
749-
coreColumn: true,
750749
}),
751750
},
752751
},

‎apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ function enrichLlmCost(event: CreateEventInput): void {
7979
// Add style accessories for model and tokens (even without cost data)
8080
const inputTokens = usageDetails["input"] ?? 0;
8181
const outputTokens = usageDetails["output"] ?? 0;
82-
const totalTokens = inputTokens + outputTokens;
82+
const totalTokens = usageDetails["total"] ?? inputTokens + outputTokens;
8383

8484
const pillItems: Array<{ text: string; icon: string }> = [
8585
{ text: responseModel, icon: "tabler-cube" },
@@ -165,7 +165,7 @@ function enrichLlmCost(event: CreateEventInput): void {
165165
pricingTierName: cost?.pricingTierName ?? (providerCost ? `${providerCost.source} reported` : ""),
166166
inputTokens: usageDetails["input"] ?? 0,
167167
outputTokens: usageDetails["output"] ?? 0,
168-
totalTokens: Object.values(usageDetails).reduce((sum, v) => sum + v, 0),
168+
totalTokens: usageDetails["total"] ?? (usageDetails["input"] ?? 0) + (usageDetails["output"] ?? 0),
169169
usageDetails,
170170
inputCost: cost?.inputCost ?? 0,
171171
outputCost: cost?.outputCost ?? 0,

‎apps/webapp/test/otlpExporter.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,5 +638,69 @@ describe("OTLPExporter", () => {
638638
expect($events[0]._llmUsage.inputTokens).toBe(500);
639639
expect($events[0]._llmUsage.outputTokens).toBe(100);
640640
});
641+
642+
it("should prefer gen_ai.usage.total_tokens over input+output sum", () => {
643+
const events = [
644+
makeGenAiEvent({
645+
"gen_ai.usage.input_tokens": 100,
646+
"gen_ai.usage.output_tokens": 50,
647+
"gen_ai.usage.total_tokens": 200, // higher than 100+50 (e.g. includes cached/reasoning)
648+
}),
649+
];
650+
651+
// @ts-expect-error
652+
const $events = enrichCreatableEvents(events);
653+
const event = $events[0];
654+
655+
// Pills should show the explicit total, not input+output
656+
expect(event.style.accessory.items[1]).toEqual({
657+
text: "200",
658+
icon: "tabler-hash",
659+
});
660+
661+
// LLM usage should also use the explicit total
662+
expect(event._llmUsage.totalTokens).toBe(200);
663+
expect(event._llmUsage.inputTokens).toBe(100);
664+
expect(event._llmUsage.outputTokens).toBe(50);
665+
});
666+
667+
it("should fall back to input+output when total_tokens is absent", () => {
668+
const events = [
669+
makeGenAiEvent({
670+
"gen_ai.usage.input_tokens": 300,
671+
"gen_ai.usage.output_tokens": 75,
672+
}),
673+
];
674+
675+
// @ts-expect-error
676+
const $events = enrichCreatableEvents(events);
677+
const event = $events[0];
678+
679+
expect(event.style.accessory.items[1]).toEqual({
680+
text: "375",
681+
icon: "tabler-hash",
682+
});
683+
expect(event._llmUsage.totalTokens).toBe(375);
684+
});
685+
686+
it("should use total_tokens when only total is present without input/output breakdown", () => {
687+
const events = [
688+
makeGenAiEvent({
689+
"gen_ai.usage.input_tokens": undefined,
690+
"gen_ai.usage.output_tokens": undefined,
691+
"gen_ai.usage.total_tokens": 500,
692+
}),
693+
];
694+
695+
// @ts-expect-error
696+
const $events = enrichCreatableEvents(events);
697+
const event = $events[0];
698+
699+
// Pills should show 500, not 0
700+
expect(event.style.accessory.items[1]).toEqual({
701+
text: "500",
702+
icon: "tabler-hash",
703+
});
704+
});
641705
});
642706
});

‎internal-packages/llm-pricing/src/registry.test.ts‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,165 @@ describe("ModelPricingRegistry", () => {
174174
});
175175
});
176176

177+
describe("prefix stripping", () => {
178+
it("should match gateway-prefixed model names", () => {
179+
const result = registry.match("openai/gpt-4o");
180+
expect(result).not.toBeNull();
181+
expect(result!.modelName).toBe("gpt-4o");
182+
});
183+
184+
it("should match openrouter-prefixed model names with date suffix", () => {
185+
const result = registry.match("openai/gpt-4o-2024-08-06");
186+
expect(result).not.toBeNull();
187+
expect(result!.modelName).toBe("gpt-4o");
188+
});
189+
190+
it("should return null for prefixed unknown model", () => {
191+
const result = registry.match("xai/unknown-model");
192+
expect(result).toBeNull();
193+
});
194+
});
195+
196+
describe("tier matching", () => {
197+
const multiTierModel: LlmModelWithPricing = {
198+
id: "model-gemini-pro",
199+
friendlyId: "llm_model_gemini_pro",
200+
modelName: "gemini-2.5-pro",
201+
matchPattern: "^gemini-2\\.5-pro$",
202+
startDate: null,
203+
pricingTiers: [
204+
{
205+
id: "tier-large-context",
206+
name: "Large Context",
207+
isDefault: false,
208+
priority: 0,
209+
conditions: [
210+
{ usageDetailPattern: "input", operator: "gt" as const, value: 200000 },
211+
],
212+
prices: [
213+
{ usageType: "input", price: 0.0000025 },
214+
{ usageType: "output", price: 0.00001 },
215+
],
216+
},
217+
{
218+
id: "tier-standard",
219+
name: "Standard",
220+
isDefault: true,
221+
priority: 1,
222+
conditions: [],
223+
prices: [
224+
{ usageType: "input", price: 0.00000125 },
225+
{ usageType: "output", price: 0.000005 },
226+
],
227+
},
228+
],
229+
};
230+
231+
it("should use conditional tier when conditions match", () => {
232+
const tieredRegistry = new TestableRegistry(null as any);
233+
tieredRegistry.loadPatterns([multiTierModel]);
234+
235+
const result = tieredRegistry.calculateCost("gemini-2.5-pro", {
236+
input: 250000,
237+
output: 1000,
238+
});
239+
240+
expect(result).not.toBeNull();
241+
expect(result!.pricingTierName).toBe("Large Context");
242+
expect(result!.inputCost).toBeCloseTo(0.625); // 250000 * 0.0000025
243+
});
244+
245+
it("should fall back to default tier when conditions do not match", () => {
246+
const tieredRegistry = new TestableRegistry(null as any);
247+
tieredRegistry.loadPatterns([multiTierModel]);
248+
249+
const result = tieredRegistry.calculateCost("gemini-2.5-pro", {
250+
input: 1000,
251+
output: 100,
252+
});
253+
254+
expect(result).not.toBeNull();
255+
expect(result!.pricingTierName).toBe("Standard");
256+
expect(result!.inputCost).toBeCloseTo(0.00125); // 1000 * 0.00000125
257+
});
258+
259+
it("should not let unconditional tier win over conditional match", () => {
260+
// Model where unconditional tier has lower priority than conditional
261+
const model: LlmModelWithPricing = {
262+
...multiTierModel,
263+
pricingTiers: [
264+
{
265+
id: "tier-unconditional",
266+
name: "Unconditional",
267+
isDefault: false,
268+
priority: 0,
269+
conditions: [],
270+
prices: [{ usageType: "input", price: 0.001 }],
271+
},
272+
{
273+
id: "tier-conditional",
274+
name: "Conditional",
275+
isDefault: false,
276+
priority: 1,
277+
conditions: [
278+
{ usageDetailPattern: "input", operator: "gt" as const, value: 100 },
279+
],
280+
prices: [{ usageType: "input", price: 0.0001 }],
281+
},
282+
{
283+
id: "tier-default",
284+
name: "Default",
285+
isDefault: true,
286+
priority: 2,
287+
conditions: [],
288+
prices: [{ usageType: "input", price: 0.01 }],
289+
},
290+
],
291+
};
292+
293+
const tieredRegistry = new TestableRegistry(null as any);
294+
tieredRegistry.loadPatterns([model]);
295+
296+
// Condition matches — conditional tier should win, not the unconditional one
297+
const result = tieredRegistry.calculateCost("gemini-2.5-pro", { input: 500 });
298+
expect(result).not.toBeNull();
299+
expect(result!.pricingTierName).toBe("Conditional");
300+
});
301+
302+
it("should fall back to isDefault tier when no conditions match", () => {
303+
const model: LlmModelWithPricing = {
304+
...multiTierModel,
305+
pricingTiers: [
306+
{
307+
id: "tier-conditional",
308+
name: "Conditional",
309+
isDefault: false,
310+
priority: 0,
311+
conditions: [
312+
{ usageDetailPattern: "input", operator: "gt" as const, value: 999999 },
313+
],
314+
prices: [{ usageType: "input", price: 0.001 }],
315+
},
316+
{
317+
id: "tier-default",
318+
name: "Default",
319+
isDefault: true,
320+
priority: 1,
321+
conditions: [],
322+
prices: [{ usageType: "input", price: 0.0001 }],
323+
},
324+
],
325+
};
326+
327+
const tieredRegistry = new TestableRegistry(null as any);
328+
tieredRegistry.loadPatterns([model]);
329+
330+
const result = tieredRegistry.calculateCost("gemini-2.5-pro", { input: 100 });
331+
expect(result).not.toBeNull();
332+
expect(result!.pricingTierName).toBe("Default");
333+
});
334+
});
335+
177336
describe("defaultModelPrices (Langfuse JSON)", () => {
178337
it("should load all models from the JSON file", () => {
179338
expect(defaultModelPrices.length).toBeGreaterThan(100);

‎internal-packages/llm-pricing/src/registry.ts‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -162,22 +162,20 @@ export class ModelPricingRegistry {
162162
): LlmPricingTierWithPrices | null {
163163
if (tiers.length === 0) return null;
164164

165-
// Tiers are sorted by priority ascending (lowest first)
166-
// Evaluate conditions — first tier whose conditions match wins
165+
// Tiers are sorted by priority ascending (lowest first).
166+
// First pass: evaluate tiers that have conditions — first match wins.
167167
for (const tier of tiers) {
168-
if (tier.conditions.length === 0) {
169-
// No conditions = default tier
170-
return tier;
171-
}
172-
173-
if (this._evaluateConditions(tier.conditions, usageDetails)) {
168+
if (tier.conditions.length > 0 && this._evaluateConditions(tier.conditions, usageDetails)) {
174169
return tier;
175170
}
176171
}
177172

178-
// Fallback to default tier
173+
// Second pass: fall back to the default tier, or first tier with no conditions
179174
const defaultTier = tiers.find((t) => t.isDefault);
180-
return defaultTier ?? tiers[0] ?? null;
175+
if (defaultTier) return defaultTier;
176+
177+
const unconditional = tiers.find((t) => t.conditions.length === 0);
178+
return unconditional ?? tiers[0] ?? null;
181179
}
182180

183181
private _evaluateConditions(

0 commit comments

Comments
 (0)