SDK & Claude Code evals (#246)

Co-authored-by: James Grugett <jahooma@gmail.com>

Author: charleslien

.github/workflows/npm-app-release-prod.yml

@@ -96,7 +96,7 @@ jobs:
       new-version: ${{ needs.prepare-and-commit-prod.outputs.new_version }}
       artifact-name: updated-package
       checkout-ref: ${{ github.sha }}
-      env-overrides: '{"NEXT_PUBLIC_BACKEND_URL": "manicode-backend.onrender.com", "NEXT_PUBLIC_CB_ENVIRONMENT": "prod"}'
+      env-overrides: '{"NEXT_PUBLIC_CODEBUFF_BACKEND_URL": "manicode-backend.onrender.com", "NEXT_PUBLIC_CB_ENVIRONMENT": "prod"}'
     secrets: inherit
 
   # Create GitHub release with all binaries

.github/workflows/npm-app-release-staging.yml

@@ -134,7 +134,7 @@ jobs:
       new-version: ${{ needs.prepare-and-commit-staging.outputs.new_version }}
       artifact-name: updated-staging-package
       checkout-ref: ${{ github.event.pull_request.head.sha }}
-      env-overrides: '{"NEXT_PUBLIC_CB_ENVIRONMENT": "prod", "NEXT_PUBLIC_BACKEND_URL": "backend-pr-221-we0m.onrender.com"}'
+      env-overrides: '{"NEXT_PUBLIC_CB_ENVIRONMENT": "prod", "NEXT_PUBLIC_CODEBUFF_BACKEND_URL": "backend-pr-221-we0m.onrender.com"}'
     secrets: inherit
 
   # Create GitHub prerelease with all binaries

common/src/util/referral.ts

@@ -1,2 +1,2 @@
 export const getReferralLink = (referralCode: string): string =>
-  `${process.env.NEXT_PUBLIC_APP_URL}/referrals/${referralCode}`
+  `${process.env.NEXT_PUBLIC_CODEBUFF_APP_URL}/referrals/${referralCode}`

evals/bun.lock

@@ -38,22 +38,7 @@ function buildAnalysisPrompt(
     .join('\n\n---\n\n')
 
   // Build Codebuff changes section
-  const codebuffChanges = evalRun.fileStates
-    .map((state) => {
-      const diff = createPatch(state.path, state.preContent, state.postContent)
-      let content = `File: ${state.path}\n\nUnified Diff (Codebuff's Changes):\n${diff}`
-
-      if (includeBeforeContent) {
-        content += `\n\nPre-commit content:\n${state.preContent}`
-      }
-
-      if (includeAfterContent) {
-        content += `\n\nPost-commit content (Codebuff's Attempt):\n${state.postContent}`
-      }
-
-      return content
-    })
-    .join('\n\n---\n\n')
+  const codebuffChanges = evalRun.gitDiff
 
   // Build trace section
   const traceContent =

evals/git-evals/judge-git-eval.ts

@@ -82,7 +82,7 @@ Judge Analysis: ${judging.analysis}
 Strengths: ${judging.strengths.join('; ')}
 Weaknesses: ${judging.weaknesses.join('; ')}
 
-Files Changed by Codebuff: ${run.fileStates.map((f) => f.path).join(', ') || 'None'}
+Files Changed by Codebuff: ${run.gitDiff || 'None'}
 Ground Truth Files: ${run.eval_commit.fileStates.map((f) => f.path).join(', ')}
 
 Trace Summary: ${run.trace.length} conversation turns

evals/git-evals/post-eval-analysis.ts

@@ -65,6 +65,10 @@ class RunEvalSetCommand extends Command {
       description: 'Number of concurrent evals to run',
       min: 1,
     }),
+    'coding-agent': Flags.string({
+      description: 'Coding agent to use',
+      default: 'codebuff',
+    }),
     help: Flags.help({ char: 'h' }),
   }
 
@@ -83,6 +87,7 @@ async function runEvalSet(options: {
   insert: boolean
   title?: string
   concurrency?: number
+  'coding-agent': string
 }): Promise<void> {
   const {
     'output-dir': outputDir,
@@ -91,8 +96,14 @@ async function runEvalSet(options: {
     mock: mockEval,
     insert: shouldInsert,
     title,
+    'coding-agent': codingAgentstr,
   } = options
 
+  if (!['codebuff', 'claude'].includes(codingAgentstr)) {
+    throw new Error(`Invalid coding agent: ${codingAgentstr}`)
+  }
+  const codingAgent = codingAgentstr as 'codebuff' | 'claude'
+
   console.log('Starting eval set run...')
   console.log(`Output directory: ${outputDir}`)
 
@@ -148,7 +159,7 @@ async function runEvalSet(options: {
         : await runGitEvals(
             config.evalDataPath,
             config.outputDir,
-            config.agentType,
+            codingAgent,
             config.limit,
             options.concurrency === 1,
           )