Super-warm starts in deployed tasks proof

Author: ericallam

apps/supervisor/src/util.ts

@@ -1,5 +1,15 @@
 import { isMacOS, isWindows } from "std-env";
 
+export function normalizeDockerHostUrl(url: string) {
+  const $url = new URL(url);
+
+  if ($url.host === "localhost") {
+    $url.host = getDockerHostDomain();
+  }
+
+  return $url.toString();
+}
+
 export function getDockerHostDomain() {
   return isMacOS || isWindows ? "host.docker.internal" : "localhost";
 }

apps/supervisor/src/workloadManager/docker.ts

@@ -5,7 +5,7 @@ import {
   type WorkloadManagerOptions,
 } from "./types.js";
 import { env } from "../env.js";
-import { getDockerHostDomain, getRunnerId } from "../util.js";
+import { getDockerHostDomain, getRunnerId, normalizeDockerHostUrl } from "../util.js";
 import Docker from "dockerode";
 import { tryCatch } from "@trigger.dev/core";
 
@@ -78,7 +78,7 @@ export class DockerWorkloadManager implements WorkloadManager {
     ];
 
     if (this.opts.warmStartUrl) {
-      envVars.push(`TRIGGER_WARM_START_URL=${this.opts.warmStartUrl}`);
+      envVars.push(`TRIGGER_WARM_START_URL=${normalizeDockerHostUrl(this.opts.warmStartUrl)}`);
     }
 
     if (this.opts.metadataUrl) {

packages/cli-v3/src/entryPoints/managed/execution.ts

@@ -115,7 +115,9 @@ export class RunExecution {
    * Kills the current execution.
    */
   public async kill({ exitExecution = true }: { exitExecution?: boolean } = {}) {
-    await this.taskRunProcess?.kill("SIGKILL");
+    if (this.taskRunProcess) {
+      await this.taskRunProcessProvider.handleProcessAbort(this.taskRunProcess);
+    }
 
     if (exitExecution) {
       this.shutdown("kill");
@@ -135,13 +137,9 @@ export class RunExecution {
       throw new Error("prepareForExecution called after process was already created");
     }
 
-    this.taskRunProcess = this.taskRunProcessProvider.getProcess({
-      taskRunEnv: opts.taskRunEnv,
-      isWarmStart: true,
-    });
-
-    // Attach event handlers to the process
-    this.attachTaskRunProcessHandlers(this.taskRunProcess);
+    // Store the environment for later use, don't create process yet
+    // The process will be created when needed in executeRun
+    this.currentTaskRunEnv = opts.taskRunEnv;
 
     return this;
   }
@@ -182,15 +180,16 @@ export class RunExecution {
   }
 
   /**
-   * Returns true if no run has been started yet and the process is prepared for the next run.
+   * Returns true if no run has been started yet and we're prepared for the next run.
    */
   get canExecute(): boolean {
     // If we've ever had a run ID, this execution can't be reused
     if (this._runFriendlyId) {
       return false;
     }
 
-    return !!this.taskRunProcess?.isPreparedForNextRun;
+    // We can execute if we have the task run environment ready
+    return !!this.currentTaskRunEnv;
   }
 
   /**
@@ -557,35 +556,35 @@ export class RunExecution {
     metrics: TaskRunExecutionMetrics;
     isWarmStart?: boolean;
   }) {
-    // For immediate retries, we need to ensure the task run process is prepared for the next attempt
-    if (
-      this.runFriendlyId &&
-      this.taskRunProcess &&
-      !this.taskRunProcess.isPreparedForNextAttempt
-    ) {
-      this.sendDebugLog("killing existing task run process before executing next attempt");
-      await this.kill({ exitExecution: false }).catch(() => {});
-    }
+    const isImmediateRetry = !!this.runFriendlyId;
 
-    // To skip this step and eagerly create the task run process, run prepareForExecution first
-    if (!this.taskRunProcess || !this.taskRunProcess.isPreparedForNextRun) {
-      this.taskRunProcess = this.taskRunProcessProvider.getProcess({
-        taskRunEnv: { ...envVars, TRIGGER_PROJECT_REF: execution.project.ref },
-        isWarmStart,
-      });
-      this.attachTaskRunProcessHandlers(this.taskRunProcess);
+    if (isImmediateRetry) {
+      await this.taskRunProcessProvider.handleImmediateRetry();
     }
 
+    const taskRunEnv = this.currentTaskRunEnv ?? envVars;
+
+    this.taskRunProcess = await this.taskRunProcessProvider.getProcess({
+      taskRunEnv: { ...taskRunEnv, TRIGGER_PROJECT_REF: execution.project.ref },
+      isWarmStart,
+    });
+
+    this.attachTaskRunProcessHandlers(this.taskRunProcess);
+
     this.sendDebugLog("executing task run process", { runId: execution.run.id });
 
-    // Set up an abort handler that will cleanup the task run process
-    this.executionAbortController.signal.addEventListener("abort", async () => {
+    const abortHandler = async () => {
       this.sendDebugLog("execution aborted during task run, cleaning up process", {
         runId: execution.run.id,
       });
 
-      await this.taskRunProcess?.cleanup(true);
-    });
+      if (this.taskRunProcess) {
+        await this.taskRunProcessProvider.handleProcessAbort(this.taskRunProcess);
+      }
+    };
+
+    // Set up an abort handler that will cleanup the task run process
+    this.executionAbortController.signal.addEventListener("abort", abortHandler);
 
     const completion = await this.taskRunProcess.execute(
       {
@@ -600,18 +599,19 @@ export class RunExecution {
       isWarmStart
     );
 
+    this.executionAbortController.signal.removeEventListener("abort", abortHandler);
+
     // If we get here, the task completed normally
     this.sendDebugLog("completed run attempt", { attemptSuccess: completion.ok });
 
-    // Return the process to the provider for potential reuse
-    this.taskRunProcessProvider.returnProcess(this.taskRunProcess);
-
-    // The execution has finished, so we can cleanup the task run process if not being reused
-    const [error] = await tryCatch(this.taskRunProcess.cleanup(true));
+    // Return the process to the provider - this handles all cleanup logic
+    const [returnError] = await tryCatch(
+      this.taskRunProcessProvider.returnProcess(this.taskRunProcess)
+    );
 
-    if (error) {
-      this.sendDebugLog("failed to cleanup task run process, submitting completion anyway", {
-        error: error.message,
+    if (returnError) {
+      this.sendDebugLog("failed to return task run process, submitting completion anyway", {
+        error: returnError.message,
       });
     }
 

packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts

@@ -35,10 +35,26 @@ export class TaskRunProcessProvider {
     this.processKeepAliveMaxExecutionCount = opts.processKeepAliveMaxExecutionCount;
   }
 
+  async handleImmediateRetry(): Promise<void> {
+    if (!this.processKeepAliveEnabled) {
+      // For immediate retries, we need to ensure we have a clean process
+      if (this.persistentProcess) {
+        // If the process is not prepared for the next attempt, we need to get a fresh one
+        if (!this.persistentProcess.isPreparedForNextAttempt) {
+          this.sendDebugLog(
+            "existing task run process not prepared for retry, will get fresh process"
+          );
+          await this.persistentProcess.kill("SIGKILL");
+          this.persistentProcess = null;
+        }
+      }
+    }
+  }
+
   /**
    * Gets a TaskRunProcess, either by reusing an existing one or creating a new one
    */
-  getProcess(opts: GetProcessOptions): TaskRunProcess {
+  async getProcess(opts: GetProcessOptions): Promise<TaskRunProcess> {
     this.sendDebugLog("Getting TaskRunProcess", {
       processKeepAliveEnabled: this.processKeepAliveEnabled,
       hasPersistentProcess: !!this.persistentProcess,
@@ -47,16 +63,22 @@ export class TaskRunProcessProvider {
       isWarmStart: opts.isWarmStart,
     });
 
+    // If process keep-alive is disabled, always create a new process
+    if (!this.processKeepAliveEnabled) {
+      this.sendDebugLog("Creating new TaskRunProcess (keep-alive disabled)");
+      return this.createTaskRunProcess(opts);
+    }
+
     // If process keep-alive is enabled and we have a healthy persistent process, reuse it
-    if (this.processKeepAliveEnabled && this.shouldReusePersistentProcess()) {
+    if (this.shouldReusePersistentProcess()) {
       this.sendDebugLog("Reusing persistent TaskRunProcess", {
         executionCount: this.executionCount,
       });
 
       return this.persistentProcess!;
     }
 
-    // Create new process
+    // Create new process (keep-alive enabled but no reusable process available)
     this.sendDebugLog("Creating new TaskRunProcess", {
       hadPersistentProcess: !!this.persistentProcess,
       reason: this.processKeepAliveEnabled
@@ -66,41 +88,69 @@ export class TaskRunProcessProvider {
 
     // Clean up old persistent process if it exists
     if (this.persistentProcess) {
-      this.cleanupPersistentProcess();
+      await this.cleanupPersistentProcess();
     }
 
     const newProcess = this.createTaskRunProcess(opts);
     return newProcess;
   }
 
   /**
-   * Returns a process after execution, handling keep-alive logic
+   * Returns a process after execution, handling keep-alive logic and cleanup
    */
-  returnProcess(process: TaskRunProcess): void {
+  async returnProcess(process: TaskRunProcess): Promise<void> {
+    this.sendDebugLog("Returning TaskRunProcess", {
+      processKeepAliveEnabled: this.processKeepAliveEnabled,
+      executionCount: this.executionCount,
+      maxExecutionCount: this.processKeepAliveMaxExecutionCount,
+    });
+
     if (!this.processKeepAliveEnabled) {
-      this.sendDebugLog("Keep-alive disabled, not preserving process");
+      // Keep-alive disabled - immediately cleanup the process
+      this.sendDebugLog("Keep-alive disabled, cleaning up process immediately");
+      await process.cleanup(true);
       return;
     }
 
+    // Keep-alive enabled - check if we should keep the process alive
     if (this.shouldKeepProcessAlive(process)) {
       this.sendDebugLog("Keeping TaskRunProcess alive for next run", {
         executionCount: this.executionCount,
         maxExecutionCount: this.processKeepAliveMaxExecutionCount,
       });
 
+      // Call cleanup(false) to prepare for next run but keep process alive
+      await process.cleanup(false);
       this.persistentProcess = process;
       this.executionCount++;
     } else {
-      this.sendDebugLog("Not keeping TaskRunProcess alive", {
+      this.sendDebugLog("Not keeping TaskRunProcess alive, cleaning up", {
         executionCount: this.executionCount,
         maxExecutionCount: this.processKeepAliveMaxExecutionCount,
         isHealthy: this.isProcessHealthy(process),
       });
 
-      // Don't set as persistent, it will be cleaned up normally
+      // Cleanup the process completely
+      await process.cleanup(true);
     }
   }
 
+  /**
+   * Handles process abort/kill scenarios
+   */
+  async handleProcessAbort(process: TaskRunProcess): Promise<void> {
+    this.sendDebugLog("Handling process abort");
+
+    // If this was our persistent process, clear it
+    if (this.persistentProcess?.pid === process.pid) {
+      this.persistentProcess = null;
+      this.executionCount = 0;
+    }
+
+    // Kill the process
+    await process.cleanup(true);
+  }
+
   /**
    * Forces cleanup of any persistent process
    */
@@ -173,12 +223,12 @@ export class TaskRunProcessProvider {
     return process.isPreparedForNextRun || process.isPreparedForNextAttempt;
   }
 
-  private cleanupPersistentProcess(): void {
+  private async cleanupPersistentProcess(): Promise<void> {
     if (this.persistentProcess) {
       this.sendDebugLog("Cleaning up persistent TaskRunProcess");
 
       // Don't await this - let it cleanup in the background
-      this.persistentProcess.kill("SIGKILL").catch(() => {});
+      await this.persistentProcess.kill("SIGKILL").catch(() => {});
       this.persistentProcess = null;
       this.executionCount = 0;
     }