feat(evals): Add build script and fix related type errors

Adds a "build": "tsc" script to evals/package.json.
This change also includes fixes for type errors that surfaced after
adding the build script, primarily by refactoring calls to
promptAiSdkStructured to use the updated single options object signature.

Generated with Codebuff 🤖
Co-Authored-By: Codebuff <noreply@codebuff.com>

Author: brandonkachen

evals/git-evals/gen-evals.ts

@@ -125,17 +125,15 @@ async function selectSubstantialCommits(
 
   const prompt = `${COMMIT_SELECTION_PROMPT}\n\nCommits to evaluate:\n\n${commitsInfo}`
 
-  const response = await promptAiSdkStructured(
-    [{ role: 'user', content: prompt }],
-    {
-      schema: CommitSelectionSchema,
-      model: claudeModels.sonnet,
-      clientSessionId,
-      fingerprintId,
-      userInputId,
-      userId: undefined,
-    }
-  )
+  const response = await promptAiSdkStructured({
+    messages: [{ role: 'user', content: prompt }],
+    schema: CommitSelectionSchema,
+    model: claudeModels.sonnet,
+    clientSessionId,
+    fingerprintId,
+    userInputId,
+    userId: undefined,
+  })
 
   try {
     return commits
@@ -237,17 +235,15 @@ Commit Message: ${commit.message}
 Changes Made:
 ${diff}`
 
-  const { spec } = await promptAiSdkStructured(
-    [{ role: 'user', content: prompt }],
-    {
-      schema: z.object({ spec: z.string() }),
-      model: geminiModels.gemini2_5_pro_preview,
-      clientSessionId,
-      fingerprintId,
-      userInputId,
-      userId: undefined,
-    }
-  )
+  const { spec } = await promptAiSdkStructured({
+    messages: [{ role: 'user', content: prompt }],
+    schema: z.object({ spec: z.string() }),
+    model: geminiModels.gemini2_5_pro_preview,
+    clientSessionId,
+    fingerprintId,
+    userInputId,
+    userId: undefined,
+  })
   return { spec, fileStates }
 }
 

evals/git-evals/judge-git-eval.ts

@@ -181,15 +181,13 @@ export function judgeEvalRun(evalRun: EvalRunLog) {
     console.log(`Using truncated prompt with ${finalTokenCount} tokens (trace truncated, base: ${baseTokens}, max trace: ${maxTraceTokens})`)
   }
 
-  return promptAiSdkStructured(
-    [{ role: 'user', content: finalPrompt }],
-    {
-      schema: JudgingAnalysisSchema,
-      model: geminiModels.gemini2_5_pro_preview,
-      clientSessionId: generateCompactId(),
-      fingerprintId: generateCompactId(),
-      userInputId: generateCompactId(),
-      userId: undefined,
-    }
-  )
+  return promptAiSdkStructured({
+    messages: [{ role: 'user', content: finalPrompt }],
+    schema: JudgingAnalysisSchema,
+    model: geminiModels.gemini2_5_pro_preview,
+    clientSessionId: generateCompactId(),
+    fingerprintId: generateCompactId(),
+    userInputId: generateCompactId(),
+    userId: undefined,
+  })
 }

evals/git-evals/post-eval-analysis.ts

@@ -180,7 +180,8 @@ export async function analyzeEvalResults(
   const tokenCount = countTokens(finalPrompt)
   console.log(`Post-eval analysis prompt: ${tokenCount} tokens`)
 
-  return promptAiSdkStructured([{ role: 'user', content: finalPrompt }], {
+  return promptAiSdkStructured({
+    messages: [{ role: 'user', content: finalPrompt }],
     schema: PostEvalAnalysisSchema,
     model: geminiModels.gemini2_5_pro_preview,
     clientSessionId: generateCompactId(),

evals/git-evals/run-git-evals.ts

@@ -88,8 +88,8 @@ async function runSingleEval(
       // Get next prompt from Sonnet agent with timeout
       let agentResponse: any
       try {
-        agentResponse = await promptAiSdkStructured(
-          [
+        agentResponse = await promptAiSdkStructured({
+          messages: [
             {
               role: 'user',
               content: `You are an expert software engineer tasked with implementing a specification using CodeBuff, an AI coding assistant. Your goal is to prompt CodeBuff to implement the spec correctly. You are in a conversation with this coding agent.
@@ -109,16 +109,14 @@ If deciding to continue, include a clear, focused prompt for Codebuff in next_pr
 Explain your reasoning in detail.`,
             },
           ],
-          {
-            schema: AgentDecisionSchema,
-            model: claudeModels.sonnet,
-            clientSessionId,
-            fingerprintId,
-            userInputId: generateCompactId(),
-            userId: undefined,
-            timeout: 5 * 60_000, // 5 minute timeout
-          }
-        )
+          schema: AgentDecisionSchema,
+          model: claudeModels.sonnet,
+          clientSessionId,
+          fingerprintId,
+          userInputId: generateCompactId(),
+          userId: undefined,
+          timeout: 5 * 60_000, // 5 minute timeout
+        })
       } catch (agentError) {
         throw new Error(
           `Agent decision failed: ${agentError instanceof Error ? agentError.message : String(agentError)}`

evals/package.json

@@ -9,6 +9,7 @@
     "test:swe-bench": "bun test swe-bench.test.ts",
     "test:e2e-cat-app": "bun run e2e-cat-app-script.ts",
     "typecheck": "tsc --noEmit",
+    "build": "tsc",
     "gen-git-evals": "bun run git-evals/gen-evals.ts",
     "run-git-evals": "bun run git-evals/run-git-evals.ts",
     "run-eval-set": "bun run run-eval-set.ts",