cancel all evals when ctrl-c

Author: charleslien

evals/git-evals/run-eval-set.ts

@@ -10,6 +10,7 @@ import {
   mockRunGitEvals,
   runGitEvals,
   setGlobalConcurrencyLimit,
+  terminateAllEvalChildren,
 } from './run-git-evals'
 
 import type { EvalConfig, EvalResult } from './types'
@@ -107,6 +108,17 @@ async function runEvalSet(options: {
   console.log('Starting eval set run...')
   console.log(`Output directory: ${outputDir}`)
 
+  // Set up signal handlers to clean up child processes
+  const signalHandler = async (signal: string) => {
+    console.log(`\nReceived ${signal}, cleaning up evaluation processes...`)
+    await terminateAllEvalChildren()
+    console.log('Cleanup complete.')
+    process.exit(signal === 'SIGINT' ? 130 : 143)
+  }
+
+  process.on('SIGINT', () => signalHandler('SIGINT'))
+  process.on('SIGTERM', () => signalHandler('SIGTERM'))
+
   setGlobalConcurrencyLimit(options.concurrency ?? 5)
 
   // Define the eval configurations

evals/git-evals/run-git-evals.ts

@@ -29,6 +29,7 @@ import type {
   EvalData,
 } from './types'
 import type { z } from 'zod/v4'
+import type { ChildProcess } from 'child_process'
 
 disableLiveUserInputCheck()
 
@@ -257,7 +258,7 @@ function getCodebuffFileStates(
     cwd: projectPath,
     stdio: ['ignore', 'pipe', 'pipe'],
   })
-  
+
   // Get diff of staged files to include new files
   return execFileSync('git', ['diff', '--staged'], {
     cwd: projectPath,
@@ -274,17 +275,88 @@ export function mockRunGitEvals(path: string) {
 // Global concurrency limiter that can be shared across multiple repository evaluations
 let globalConcurrencyLimiter: ReturnType<typeof pLimit> | null = null
 
+// Track all active child processes for cleanup
+const activeChildProcesses = new Set<ChildProcess>()
+let isCleaningUp = false
+
 export function setGlobalConcurrencyLimit(limit: number) {
   globalConcurrencyLimiter = pLimit(limit)
 }
 
+/**
+ * Terminates all active evaluation child processes
+ */
+export async function terminateAllEvalChildren(): Promise<void> {
+  if (isCleaningUp || activeChildProcesses.size === 0) {
+    return
+  }
+
+  isCleaningUp = true
+  console.log(
+    `\nTerminating ${activeChildProcesses.size} active evaluation processes...`,
+  )
+
+  const killPromises = Array.from(activeChildProcesses).map(async (child) => {
+    if (!child.pid || child.killed) {
+      return
+    }
+
+    try {
+      // First try graceful termination
+      if (process.platform === 'win32') {
+        // Windows: kill process tree
+        execFileSync('taskkill', ['/PID', String(child.pid), '/T'], {
+          stdio: 'ignore',
+          timeout: 3000,
+        })
+      } else {
+        // POSIX: kill process group
+        process.kill(-child.pid, 'SIGTERM')
+      }
+
+      // Wait a bit for graceful shutdown
+      await new Promise((resolve) => setTimeout(resolve, 2000))
+
+      // Force kill if still alive
+      if (!child.killed) {
+        if (process.platform === 'win32') {
+          execFileSync('taskkill', ['/F', '/PID', String(child.pid), '/T'], {
+            stdio: 'ignore',
+            timeout: 1000,
+          })
+        } else {
+          process.kill(-child.pid, 'SIGKILL')
+        }
+      }
+    } catch (error) {
+      // Process may have already exited
+      console.warn(`Failed to kill process ${child.pid}:`, error)
+    }
+  })
+
+  await Promise.allSettled(killPromises)
+  activeChildProcesses.clear()
+  isCleaningUp = false
+}
+
 export async function runGitEvals(
   evalDataPath: string,
   outputDir: string,
   codingAgent: 'codebuff' | 'claude',
   limit?: number,
   logToStdout: boolean = false,
 ): Promise<FullEvalLog> {
+  // Set up signal handlers if this is the main module
+  if (require.main === module) {
+    const signalHandler = async (signal: string) => {
+      console.log(`\nReceived ${signal}, cleaning up...`)
+      await terminateAllEvalChildren()
+      process.exit(signal === 'SIGINT' ? 130 : 143)
+    }
+
+    process.on('SIGINT', () => signalHandler('SIGINT'))
+    process.on('SIGTERM', () => signalHandler('SIGTERM'))
+  }
   console.log(`Loading eval data from: ${evalDataPath}`)
   const evalData = JSON.parse(
     fs.readFileSync(evalDataPath, 'utf-8'),
@@ -379,9 +451,16 @@ export async function runGitEvals(
                 fingerprintId,
                 codingAgent,
               ],
-              { stdio: ['pipe', 'pipe', 'pipe', 'ipc'], env: process.env },
+              {
+                stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+                env: process.env,
+                detached: true, // Create new process group for proper signal handling
+              },
             )
 
+            // Track child process for cleanup
+            activeChildProcesses.add(child)
+
             child.stdout?.pipe(logStream)
             child.stderr?.pipe(logStream)
 
@@ -421,7 +500,13 @@ export async function runGitEvals(
             )
 
             child.on('exit', (code) => {
-              logStream.end()
+              // Remove from tracking
+              activeChildProcesses.delete(child)
+
+              if (!logToStdout && logStream !== process.stdout) {
+                logStream.end()
+              }
+
               if (code !== 0) {
                 console.error(
                   `Eval process for ${evalCommit.sha} exited with code ${code}. See logs at ${logPath}`,

evals/git-evals/run-single-eval-process.ts

@@ -13,6 +13,17 @@ import { runSingleEval } from './run-git-evals'
 import type { EvalCommit } from './types'
 
 async function main() {
+  // Set up signal handlers for graceful shutdown
+  let shouldExit = false
+  const signalHandler = (signal: string) => {
+    console.log(`Child process received ${signal}, exiting gracefully...`)
+    shouldExit = true
+    process.exit(0)
+  }
+
+  process.on('SIGINT', () => signalHandler('SIGINT'))
+  process.on('SIGTERM', () => signalHandler('SIGTERM'))
+
   const [
     evalCommitFilePath,
     projectPath,
@@ -49,13 +60,24 @@ async function main() {
     recreateShell(projectPath)
     setWorkingDirectory(projectPath)
 
+    // Check if we should exit early due to signal
+    if (shouldExit) {
+      process.exit(0)
+    }
+
     const result = await runSingleEval(
       evalCommit,
       projectPath,
       clientSessionId,
       fingerprintId,
       codingAgent as any,
     )
+
+    // Check again after long-running operation
+    if (shouldExit) {
+      process.exit(0)
+    }
+
     console.log('Final result:', { result })
     if (process.send) {
       process.send({ type: 'result', result })
@@ -71,9 +93,11 @@ async function main() {
       })
     }
   } finally {
+    // Exit more quickly if signal received, otherwise wait briefly
+    const exitDelay = shouldExit ? 100 : 2000
     setTimeout(() => {
       process.exit(0)
-    }, 2000)
+    }, exitDelay)
   }
 }