Support local scripts for private repo, add SSH helper

- SCP all scripts to instance instead of fetching from GitHub
- runner-setup.sh now checks SCRIPTS_DIR for local scripts first
- Add scripts/update-lmbda-ssh helper for debugging SSH
- Reduce boot status log spam (log every 30s with elapsed time)
- Add tmp/ to .gitignore

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: ryan-williams

.gitignore

@@ -173,3 +173,4 @@ pyrightconfig.json
 
 # Temportily ignore uv.lock
 uv.lock
+tmp/

scripts/update-lmbda-ssh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Update the lmbda SSH host IP from the current running Lambda instance
+# Usage: ./scripts/update-lmbda-ssh [instance_id]
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+SSH_CONF="$PROJECT_DIR/tmp/lmbda-ssh.conf"
+
+# Get instance IP (either by ID or first active instance)
+if [ -n "$1" ]; then
+    IP=$(python -m lambda_gha.cli get "$1" 2>/dev/null | jq -r '.data.ip // empty')
+else
+    IP=$(python -m lambda_gha.cli ls 2>/dev/null | jq -r '.data[0].ip // empty')
+fi
+
+if [ -z "$IP" ]; then
+    echo "No active Lambda instance found" >&2
+    exit 1
+fi
+
+# Create tmp dir if needed
+mkdir -p "$PROJECT_DIR/tmp"
+
+# Write SSH config fragment
+cat > "$SSH_CONF" <<EOF
+# Auto-generated by scripts/update-lmbda-ssh
+# Include this in ~/.ssh/config with: Include $SSH_CONF
+Host lmbda
+    HostName $IP
+    User ubuntu
+    IdentitiesOnly yes
+    IdentityFile ~/.ssh/lambda_ecdsa
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+EOF
+
+echo "Updated $SSH_CONF with IP: $IP"
+echo "Run: ssh lmbda"

src/lambda_gha/scripts/runner-setup.sh

@@ -28,16 +28,22 @@ BIN_DIR=/usr/local/bin
 RUNNER_STATE_DIR=/var/run/github-runner
 mkdir -p $RUNNER_STATE_DIR
 
-# Fetch shared functions from GitHub
-echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching shared functions from GitHub (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log
-FUNCTIONS_URL="https://raw.githubusercontent.com/Open-Athena/lambda-gha/${action_sha}/src/lambda_gha/templates/shared-functions.sh"
-if ! curl -sSL "$FUNCTIONS_URL" -o /tmp/shared-functions.sh && ! wget -q "$FUNCTIONS_URL" -O /tmp/shared-functions.sh; then
-  echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download shared functions" | tee -a /var/log/runner-setup.log
-  # Terminate via Lambda API
-  curl -s -X POST -H "Authorization: Bearer $LAMBDA_API_KEY" -H "Content-Type: application/json" \
-    -d "{\"instance_ids\": [\"$LAMBDA_INSTANCE_ID\"]}" \
-    "$LAMBDA_API_BASE/instance-operations/terminate" || true
-  exit 1
+# Get shared functions - from local dir if available (private repo), else fetch from GitHub
+SCRIPTS_DIR="${SCRIPTS_DIR:-}"
+if [ -n "$SCRIPTS_DIR" ] && [ -f "$SCRIPTS_DIR/shared-functions.sh" ]; then
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] Using local shared functions from $SCRIPTS_DIR" | tee -a /var/log/runner-setup.log
+  cp "$SCRIPTS_DIR/shared-functions.sh" /tmp/shared-functions.sh
+else
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] Fetching shared functions from GitHub (SHA: ${action_sha})" | tee -a /var/log/runner-setup.log
+  FUNCTIONS_URL="https://raw.githubusercontent.com/Open-Athena/lambda-gha/${action_sha}/src/lambda_gha/templates/shared-functions.sh"
+  if ! curl -sSL "$FUNCTIONS_URL" -o /tmp/shared-functions.sh && ! wget -q "$FUNCTIONS_URL" -O /tmp/shared-functions.sh; then
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: Failed to download shared functions" | tee -a /var/log/runner-setup.log
+    # Terminate via Lambda API
+    curl -s -X POST -H "Authorization: Bearer $LAMBDA_API_KEY" -H "Content-Type: application/json" \
+      -d "{\"instance_ids\": [\"$LAMBDA_INSTANCE_ID\"]}" \
+      "$LAMBDA_API_BASE/instance-operations/terminate" || true
+    exit 1
+  fi
 fi
 
 # Write shared functions that will be used by multiple scripts
@@ -134,12 +140,19 @@ else
 fi
 log "Downloaded runner binary"
 
-# Helper function to fetch scripts
+# Helper function to fetch scripts - uses local copy if available, else downloads
 fetch_script() {
   local script_name="$1"
-  local url="${BASE_URL}/${script_name}"
   local dest="${BIN_DIR}/${script_name}"
 
+  # Check for local copy first (private repo support)
+  if [ -n "$SCRIPTS_DIR" ] && [ -f "$SCRIPTS_DIR/$script_name" ]; then
+    cp "$SCRIPTS_DIR/$script_name" "$dest"
+    return 0
+  fi
+
+  # Fall back to downloading from GitHub
+  local url="${BASE_URL}/${script_name}"
   if command -v curl >/dev/null 2>&1; then
     curl -fsSL "$url" -o "$dest" || {
       log_error "Failed to fetch $script_name"
@@ -156,9 +169,12 @@ fetch_script() {
   fi
 }
 
-# Fetch job tracking scripts from GitHub
-# These scripts are called by GitHub runner hooks
-log "Fetching runner hook scripts"
+# Fetch job tracking scripts - from local if available (private repo), else GitHub
+if [ -n "$SCRIPTS_DIR" ]; then
+  log "Copying runner hook scripts from local $SCRIPTS_DIR"
+else
+  log "Fetching runner hook scripts from GitHub"
+fi
 BASE_URL="https://raw.githubusercontent.com/Open-Athena/lambda-gha/${action_sha}/src/lambda_gha/scripts"
 
 fetch_script "job-started-hook.sh"

src/lambda_gha/start.py

@@ -268,8 +268,10 @@ def wait_until_ready(self, ids: list[str], timeout: int = INSTANCE_POLL_TIMEOUT)
         start_time = time.time()
         pending = set(ids)
         details = {}
+        last_log_time = {}  # Track last log time per instance to reduce spam
 
         while pending and (time.time() - start_time) < timeout:
+            elapsed = int(time.time() - start_time)
             for instance_id in list(pending):
                 try:
                     result = self._api_request("GET", f"/instances/{instance_id}")
@@ -283,22 +285,30 @@ def wait_until_ready(self, ids: list[str], timeout: int = INSTANCE_POLL_TIMEOUT)
                             "status": status,
                         }
                         pending.remove(instance_id)
-                        print(f"Instance {instance_id} is ready: {instance.get('ip')}")
+                        print(f"[{elapsed}s] Instance {instance_id[:12]}... is ready: {instance.get('ip')}")
                     elif status in ("terminated", "terminating"):
                         raise RuntimeError(f"Instance {instance_id} terminated unexpectedly")
                     else:
-                        print(f"Instance {instance_id} status: {status}")
+                        # Log every 30s to reduce spam, but always log first status
+                        last_log = last_log_time.get(instance_id, 0)
+                        if elapsed - last_log >= 30 or last_log == 0:
+                            print(f"[{elapsed}s] Instance {instance_id[:12]}... status: {status}")
+                            last_log_time[instance_id] = elapsed
                 except requests.HTTPError as e:
                     if e.response.status_code == 404:
-                        print(f"Instance {instance_id} not found yet, retrying...")
+                        last_log = last_log_time.get(instance_id, 0)
+                        if elapsed - last_log >= 30 or last_log == 0:
+                            print(f"[{elapsed}s] Instance {instance_id[:12]}... not found yet, retrying...")
+                            last_log_time[instance_id] = elapsed
                     else:
                         raise
 
             if pending:
                 time.sleep(INSTANCE_POLL_INTERVAL)
 
         if pending:
-            raise TimeoutError(f"Instances did not become ready within {timeout}s: {pending}")
+            elapsed = int(time.time() - start_time)
+            raise TimeoutError(f"[{elapsed}s] Instances did not become ready within {timeout}s: {pending}")
 
         return details
 
@@ -399,39 +409,62 @@ def execute_setup_via_ssh(
             else:
                 raise RuntimeError(f"Failed to connect to {ip} via SSH after {max_retries} attempts")
 
-        # Read setup script from package (can't curl from private repo)
+        # Read all required scripts from package (can't curl from private repo)
         from importlib.resources import files
         scripts_dir = files("lambda_gha.scripts")
-        setup_script = (scripts_dir / "runner-setup.sh").read_text()
-
-        # Write script to temp file for SCP
-        script_file = tempfile.NamedTemporaryFile(mode='w', suffix='.sh', delete=False)
-        script_file.write(setup_script)
-        script_file.close()
-        os.chmod(script_file.name, stat.S_IRUSR | stat.S_IXUSR)  # 0500
+        templates_dir = files("lambda_gha.templates")
+
+        # Scripts to copy: (source, dest_name)
+        scripts_to_copy = [
+            (scripts_dir / "runner-setup.sh", "runner-setup.sh"),
+            (scripts_dir / "check-runner-termination.sh", "check-runner-termination.sh"),
+            (scripts_dir / "job-started-hook.sh", "job-started-hook.sh"),
+            (scripts_dir / "job-completed-hook.sh", "job-completed-hook.sh"),
+            (templates_dir / "shared-functions.sh", "shared-functions.sh"),
+        ]
 
-        # SCP the script to the instance
+        # SCP options
         scp_opts = ["-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"]
         if key_file:
             scp_opts.extend(["-i", key_file.name])
 
-        print(f"Copying setup script to instance...")
-        scp_result = subprocess.run(
-            ["scp"] + scp_opts + [script_file.name, f"{ssh_user}@{ip}:/tmp/runner-setup.sh"],
+        # Create scripts directory on instance
+        print(f"Creating scripts directory on instance...")
+        mkdir_result = subprocess.run(
+            ["ssh"] + ssh_opts + [f"{ssh_user}@{ip}", "mkdir -p /tmp/lambda-gha-scripts"],
             capture_output=True,
             text=True,
         )
-        if scp_result.returncode != 0:
-            raise RuntimeError(f"Failed to SCP script: {scp_result.stderr}")
-
-        # Build env export commands
+        if mkdir_result.returncode != 0:
+            raise RuntimeError(f"Failed to create scripts dir: {mkdir_result.stderr}")
+
+        # Copy all scripts
+        print(f"Copying {len(scripts_to_copy)} scripts to instance...")
+        for src_file, dest_name in scripts_to_copy:
+            content = src_file.read_text()
+            local_file = tempfile.NamedTemporaryFile(mode='w', suffix='.sh', delete=False)
+            local_file.write(content)
+            local_file.close()
+            os.chmod(local_file.name, stat.S_IRUSR | stat.S_IXUSR)
+
+            scp_result = subprocess.run(
+                ["scp"] + scp_opts + [local_file.name, f"{ssh_user}@{ip}:/tmp/lambda-gha-scripts/{dest_name}"],
+                capture_output=True,
+                text=True,
+            )
+            os.unlink(local_file.name)
+            if scp_result.returncode != 0:
+                raise RuntimeError(f"Failed to SCP {dest_name}: {scp_result.stderr}")
+
+        # Build env export commands (add SCRIPTS_DIR for local script access)
+        env_vars["SCRIPTS_DIR"] = "/tmp/lambda-gha-scripts"
         env_exports = "\n".join(f'export {k}="{v}"' for k, v in env_vars.items())
 
-        # Build the setup command: export vars, run script
+        # Build the setup command: export vars, run script from scripts dir
         setup_cmd = f'''
 {env_exports}
-chmod +x /tmp/runner-setup.sh
-sudo -E nohup /tmp/runner-setup.sh > /var/log/runner-setup.log 2>&1 &
+chmod +x /tmp/lambda-gha-scripts/*.sh
+sudo -E nohup /tmp/lambda-gha-scripts/runner-setup.sh > /var/log/runner-setup.log 2>&1 &
 '''
 
         print(f"Executing setup script...")

src/lambda_gha/templates/__init__.py

@@ -0,0 +1 @@
+# Templates package - contains shell script templates