feat: add eval framework for installer agent testing

.env.local.example

@@ -1,5 +1,9 @@
 # Local development configuration
 # Copy this to .env.local and fill in values
 
-# WorkOS Client ID for local development
-WORKOS_CLIENT_ID=client_xxx
+# Required for running evals
+ANTHROPIC_API_KEY=sk-ant-...
+
+# WorkOS credentials (optional for evals - placeholders used if missing)
+WORKOS_API_KEY=sk_test_...
+WORKOS_CLIENT_ID=client_...

.gitignore

@@ -28,3 +28,6 @@ src/version.ts
 .idea
 *.sublime-*
 dist/
+
+# Eval results
+tests/eval-results/

package.json

@@ -71,7 +71,7 @@
   "engines": {
     "node": ">=20.20"
   },
-  "packageManager": "pnpm@10.23.0+sha512.21c4e5698002ade97e4efe8b8b4a89a8de3c85a37919f957e7a0f30f38fbc5bbdd05980ffe29179b2fb6e6e691242e098d945d1601772cad0fef5fb6411e2a4b",
+  "packageManager": "pnpm@10.28.2",
   "scripts": {
     "clean": "rm -rf ./dist",
     "prebuild": "pnpm clean",
@@ -85,7 +85,10 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "eval": "tsx tests/evals/index.ts",
+    "eval:history": "tsx tests/evals/index.ts history",
+    "eval:compare": "tsx tests/evals/index.ts compare"
   },
   "author": "WorkOS",
   "license": "MIT"

skills/workos-authkit-tanstack-start/SKILL.md

@@ -12,9 +12,9 @@ description: Integrate WorkOS AuthKit with TanStack Start applications. Full-sta
    ├── Extract package name from install command
    └── README is source of truth for ALL code patterns
 
-2. Verify TanStack Start project
-   ├── @tanstack/start or @tanstack/react-start in package.json
-   └── app.config.ts exists (vinxi)
+2. Detect directory structure
+   ├── src/ (TanStack Start v1.132+, default)
+   └── app/ (legacy vinxi-based projects)
 
 3. Follow README install/setup exactly
    └── Do not invent commands or patterns
@@ -28,7 +28,7 @@ WebFetch: `https://github.com/workos/authkit-tanstack-start/blob/main/README.md`
 
 From README, extract:
 
-1. Package name from install command (e.g., `pnpm add @workos/...`)
+1. Package name: `@workos/authkit-tanstack-react-start`
 2. Use that exact name for all imports
 
 **README overrides this skill if conflict.**
@@ -37,9 +37,35 @@ From README, extract:
 
 - [ ] README fetched and package name extracted
 - [ ] `@tanstack/start` or `@tanstack/react-start` in package.json
-- [ ] `app.config.ts` exists
+- [ ] Identify directory structure: `src/` (modern) or `app/` (legacy)
 - [ ] Environment variables set (see below)
 
+## Directory Structure Detection
+
+**Modern TanStack Start (v1.132+)** uses `src/`:
+```
+src/
+├── start.ts              # Middleware config (CRITICAL)
+├── router.tsx            # Router setup
+├── routes/
+│   ├── __root.tsx        # Root layout
+│   ├── api.auth.callback.tsx  # OAuth callback (flat route)
+│   └── ...
+```
+
+**Legacy (vinxi-based)** uses `app/`:
+```
+app/
+├── start.ts or router.tsx
+├── routes/
+│   └── api/auth/callback.tsx  # OAuth callback (nested route)
+```
+
+**Detection:**
+```bash
+ls src/routes 2>/dev/null && echo "Modern (src/)" || echo "Legacy (app/)"
+```
+
 ## Environment Variables
 
 | Variable                 | Format       | Required |
@@ -51,56 +77,179 @@ From README, extract:
 
 Generate password if missing: `openssl rand -base64 32`
 
+Default redirect URI: `http://localhost:3000/api/auth/callback`
+
 ## Middleware Configuration (CRITICAL)
 
-**authkitMiddleware MUST be configured or auth will fail.**
+**authkitMiddleware MUST be configured or auth will fail silently.**
 
-Find file with `createRouter` (typically `app/router.tsx` or `app.tsx`).
+Create or update `src/start.ts` (or `app/start.ts` for legacy):
+
+```typescript
+import { authkitMiddleware } from '@workos/authkit-tanstack-react-start';
+
+export default {
+  requestMiddleware: [authkitMiddleware()],
+};
+```
+
+Alternative pattern with createStart:
+```typescript
+import { createStart } from '@tanstack/react-start';
+import { authkitMiddleware } from '@workos/authkit-tanstack-react-start';
+
+export default createStart({
+  requestMiddleware: [authkitMiddleware()],
+});
+```
 
 ### Verification Checklist
 
-- [ ] `authkitMiddleware` imported from SDK package
-- [ ] `middleware: [authkitMiddleware()]` in createRouter config
-- [ ] Array syntax used: `[authkitMiddleware()]` not `authkitMiddleware()`
+- [ ] `authkitMiddleware` imported from `@workos/authkit-tanstack-react-start`
+- [ ] Middleware in `requestMiddleware` array
+- [ ] File exports the config (default export or named `startInstance`)
+
+Verify: `grep -r "authkitMiddleware" src/ app/ 2>/dev/null`
 
-Verify: `grep "authkitMiddleware" app/router.tsx app.tsx src/router.tsx`
+## Callback Route (CRITICAL)
 
-## Logout Route Pattern
+Path must match `WORKOS_REDIRECT_URI`. For `/api/auth/callback`:
 
-Logout requires `signOut()` followed by redirect in a route loader. See README for exact implementation.
+**Modern (flat routes):** `src/routes/api.auth.callback.tsx`
+**Legacy (nested routes):** `app/routes/api/auth/callback.tsx`
 
-## Callback Route
+```typescript
+import { createFileRoute } from '@tanstack/react-router';
+import { handleCallbackRoute } from '@workos/authkit-tanstack-react-start';
 
-Path must match `WORKOS_REDIRECT_URI`. If URI is `/api/auth/callback`:
+export const Route = createFileRoute('/api/auth/callback')({
+  server: {
+    handlers: {
+      GET: handleCallbackRoute(),
+    },
+  },
+});
+```
+
+**Key points:**
+- Use `handleCallbackRoute()` - do not write custom OAuth logic
+- Route path string must match the URI path exactly
+- This is a server-only route (no component needed)
+
+## Protected Routes
+
+Use `getAuth()` in route loaders to check authentication:
+
+```typescript
+import { createFileRoute, redirect } from '@tanstack/react-router';
+import { getAuth, getSignInUrl } from '@workos/authkit-tanstack-react-start';
+
+export const Route = createFileRoute('/dashboard')({
+  loader: async () => {
+    const { user } = await getAuth();
+    if (!user) {
+      const signInUrl = await getSignInUrl();
+      throw redirect({ href: signInUrl });
+    }
+    return { user };
+  },
+  component: Dashboard,
+});
+```
 
-- File: `app/routes/api/auth/callback.tsx`
-- Use `handleAuth()` from SDK - do not write custom OAuth logic
+## Sign Out Route
+
+```typescript
+import { createFileRoute, redirect } from '@tanstack/react-router';
+import { signOut } from '@workos/authkit-tanstack-react-start';
+
+export const Route = createFileRoute('/signout')({
+  loader: async () => {
+    await signOut();
+    throw redirect({ href: '/' });
+  },
+});
+```
+
+## Client-Side Hooks (Optional)
+
+Only needed if you want reactive auth state in components.
+
+**1. Add AuthKitProvider to root:**
+
+```typescript
+// src/routes/__root.tsx
+import { AuthKitProvider } from '@workos/authkit-tanstack-react-start/client';
+
+function RootComponent() {
+  return (
+    <AuthKitProvider>
+      <Outlet />
+    </AuthKitProvider>
+  );
+}
+```
+
+**2. Use hooks in components:**
+
+```typescript
+import { useAuth } from '@workos/authkit-tanstack-react-start/client';
+
+function Profile() {
+  const { user, isLoading } = useAuth();
+  // ...
+}
+```
+
+**Note:** Server-side `getAuth()` is preferred for most use cases.
 
 ## Error Recovery
 
 ### "AuthKit middleware is not configured"
 
-**Cause:** `authkitMiddleware()` not added to router
-**Fix:** Add `middleware: [authkitMiddleware()]` to createRouter config
-**Verify:** `grep "authkitMiddleware" app/router.tsx app.tsx`
+**Cause:** `authkitMiddleware()` not in start.ts
+**Fix:** Create/update `src/start.ts` with middleware config
+**Verify:** `grep -r "authkitMiddleware" src/`
 
 ### "Module not found" for SDK
 
 **Cause:** Wrong package name or not installed
-**Fix:** Re-read README, extract correct package name, reinstall
-**Verify:** `ls node_modules/` + package name from README
+**Fix:** `pnpm add @workos/authkit-tanstack-react-start`
+**Verify:** `ls node_modules/@workos/authkit-tanstack-react-start`
 
 ### Callback 404
 
-**Cause:** Route path doesn't match WORKOS_REDIRECT_URI
-**Fix:** File path must mirror URI path under `app/routes/`
+**Cause:** Route file path doesn't match WORKOS_REDIRECT_URI
+**Fix:**
+- URI `/api/auth/callback` → file `src/routes/api.auth.callback.tsx` (flat) or `app/routes/api/auth/callback.tsx` (nested)
+- Route path string in `createFileRoute()` must match exactly
 
-### getAuth returns undefined
+### getAuth returns undefined user
 
-**Cause:** Middleware not configured
-**Fix:** Same as "AuthKit middleware not configured" above
+**Cause:** Middleware not configured or not running
+**Fix:** Ensure `authkitMiddleware()` is in start.ts requestMiddleware array
 
 ### "Cookie password too short"
 
 **Cause:** WORKOS_COOKIE_PASSWORD < 32 chars
 **Fix:** `openssl rand -base64 32`, update .env
+
+### Build fails with route type errors
+
+**Cause:** Route tree not regenerated after adding routes
+**Fix:** `pnpm dev` to regenerate `routeTree.gen.ts`
+
+## SDK Exports Reference
+
+**Server (main export):**
+- `authkitMiddleware()` - Request middleware
+- `handleCallbackRoute()` - OAuth callback handler
+- `getAuth()` - Get current session
+- `signOut()` - Sign out user
+- `getSignInUrl()` / `getSignUpUrl()` - Auth URLs
+- `switchToOrganization()` - Change org context
+
+**Client (`/client` subpath):**
+- `AuthKitProvider` - Context provider
+- `useAuth()` - Auth state hook
+- `useAccessToken()` - Token management

src/utils/exec-file.ts

@@ -0,0 +1,55 @@
+import { spawn } from 'node:child_process';
+
+export interface ExecResult {
+  status: number;
+  stdout: string;
+  stderr: string;
+}
+
+export interface ExecOptions {
+  cwd?: string;
+  timeout?: number;
+  env?: NodeJS.ProcessEnv;
+}
+
+/**
+ * Execute a command without throwing on non-zero exit codes.
+ * Returns { status, stdout, stderr } for all outcomes.
+ */
+export function execFileNoThrow(command: string, args: string[], options: ExecOptions = {}): Promise<ExecResult> {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, {
+      cwd: options.cwd,
+      env: options.env ?? process.env,
+      timeout: options.timeout,
+      shell: false,
+    });
+
+    let stdout = '';
+    let stderr = '';
+
+    child.stdout?.on('data', (data) => {
+      stdout += data.toString();
+    });
+
+    child.stderr?.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    child.on('close', (code) => {
+      resolve({
+        status: code ?? 1,
+        stdout,
+        stderr,
+      });
+    });
+
+    child.on('error', (err) => {
+      resolve({
+        status: 1,
+        stdout,
+        stderr: err.message,
+      });
+    });
+  });
+}