evals: do not throw error on invalid tool call

Author: charleslien

backend/src/xml-stream-parser.ts

@@ -83,9 +83,9 @@ export async function* processStreamWithTags(
         },
       )
       const shortenedContents =
-        contents.length < 50
+        contents.length < 200
           ? contents
-          : contents.slice(0, 20) + '...' + contents.slice(-20)
+          : contents.slice(0, 100) + '...' + contents.slice(-100)
       const errorMessage = `Invalid JSON: ${JSON.stringify(shortenedContents)}\nError: ${error.message}`
       onResponseChunk({
         type: 'error',

common/src/util/messages.ts

@@ -1,7 +1,6 @@
-import { has, isEqual } from 'lodash'
+import { cloneDeep, has, isEqual } from 'lodash'
 
 import { buildArray } from './array'
-import { deepCopy } from './object'
 import { getToolCallString } from '../tools/utils'
 
 import type {
@@ -23,7 +22,7 @@ export function toContentString(msg: ModelMessage): string {
 export function withCacheControl<
   T extends { providerOptions?: ProviderMetadata },
 >(obj: T): T {
-  const wrapper = deepCopy(obj)
+  const wrapper = cloneDeep(obj)
   if (!wrapper.providerOptions) {
     wrapper.providerOptions = {}
   }
@@ -41,7 +40,7 @@ export function withCacheControl<
 export function withoutCacheControl<
   T extends { providerOptions?: ProviderMetadata },
 >(obj: T): T {
-  const wrapper = deepCopy(obj)
+  const wrapper = cloneDeep(obj)
   if (has(wrapper.providerOptions?.anthropic?.cacheControl, 'type')) {
     delete wrapper.providerOptions?.anthropic?.cacheControl?.type
   }

common/src/util/object.ts

@@ -129,7 +129,3 @@ export function errorToObject(value: any) {
   }
   return value
 }
-
-export function deepCopy<T>(obj: T): T {
-  return JSON.parse(JSON.stringify(obj)) as T
-}

evals/git-evals/run-git-evals.ts

@@ -7,6 +7,7 @@ import { promptAiSdkStructured } from '@codebuff/backend/llm-apis/vercel-ai-sdk/
 import { models } from '@codebuff/common/constants'
 import { withTimeout } from '@codebuff/common/util/promise'
 import { generateCompactId } from '@codebuff/common/util/string'
+import { cloneDeep } from 'lodash'
 import pLimit from 'p-limit'
 
 import { resetRepoToCommit } from '../scaffolding'
@@ -487,7 +488,7 @@ export async function runGitEvals(
                     `Completed eval for commit ${testRepoName} - ${evalCommit.spec.split('\n')[0]}`,
                   )
                   if (!logToStdout) {
-                    const finalResult = message.result
+                    const finalResult = cloneDeep(message.result)
                     for (const cbTrace of finalResult.trace) {
                       delete (cbTrace as any).steps
                     }

evals/git-evals/runners/codebuff.ts

@@ -62,7 +62,7 @@ export class CodebuffRunner implements Runner {
       prompt,
       handleEvent: (event) => {
         if (event.type === 'error') {
-          throw new Error(event.message)
+          console.log('\n\n' + JSON.stringify(event, null, 2))
         }
         if (event.type === 'text') {
           if (toolResults.length > 0) {