diff --git a/packages/cli/src/ai-context/context.ts b/packages/cli/src/ai-context/context.ts index da13c2a8b..fa75bee32 100644 --- a/packages/cli/src/ai-context/context.ts +++ b/packages/cli/src/ai-context/context.ts @@ -62,6 +62,10 @@ export const INVESTIGATE_REFERENCES = [ id: 'investigate-checks', description: 'Inspecting checks (`checks list`, `checks get`) and triggering on-demand runs', }, + { + id: 'investigate-test-sessions', + description: 'Run and inspect recorded test sessions, drill into test-session error groups, run RCA, and retrieve result assets', + }, ] as const export const COMMUNICATE_REFERENCES = [ @@ -99,7 +103,7 @@ export const ACTIONS = [ }, { id: 'investigate', - description: 'Access check status, analyze failures, and investigate errors.', + description: 'Access check and test-session status, analyze failures, and investigate errors.', references: INVESTIGATE_REFERENCES, }, { diff --git a/packages/cli/src/ai-context/references/investigate-test-sessions.md b/packages/cli/src/ai-context/references/investigate-test-sessions.md new file mode 100644 index 000000000..ff28c999c --- /dev/null +++ b/packages/cli/src/ai-context/references/investigate-test-sessions.md @@ -0,0 +1,125 @@ +# Investigating Test Sessions + +Use this flow when a user asks you to run a session, inspect recorded test-session results, investigate test-session error groups, run root cause analysis, or retrieve result logs, traces, videos, screenshots, and other assets. + +Read-only inspection commands execute immediately. `rca run` starts a new analysis, but it does not mutate checks or monitors. + +## Run or trigger a recorded session + +`checkly test`, `checkly trigger`, and `checkly pw-test` record results as a test session by default. Use `--no-record` only when the user explicitly does not want a recorded session. + +```bash +npx checkly test --test-session-name "Checkout regression" +npx checkly test --record --test-session-name "Checkout regression" +npx checkly trigger --tags production --test-session-name "Production smoke" +npx checkly pw-test --test-session-name "Playwright suite" +``` + +Useful flags: +- `--location ` or `--private-location ` - choose where checks run. +- `-e, --env KEY=value` or `--env-file ` - pass runtime environment variables. +- `--timeout ` - wait for session completion before the command exits. +- `-d, --detach` - keep cloud execution running if the local CLI is cancelled. +- `-r, --reporter json` - capture machine-readable run output, including test session IDs when available. + +If the command output includes a test session ID or link, use it directly with `test-sessions get`. + +## Find a test session + +```bash +npx checkly test-sessions list +npx checkly test-sessions list --status failed --limit 10 +npx checkly test-sessions list --branch main --provider github +npx checkly test-sessions list --search "checkout" +npx checkly test-sessions list --error-group +npx checkly test-sessions list --output json +``` + +Flags: +- `-l, --limit ` - number of sessions to return (1-100, default 20). +- `--cursor ` - fetch the next cursor page. +- `--from ` / `--to ` - filter by ISO date or Unix timestamp in seconds. +- `--status ` - `running`, `failed`, `passed`, or `cancelled`; repeat or comma-separate. +- `--provider ` - `github`, `vercel`, `api`, `trigger`, or `pw_reporter`; repeat or comma-separate. +- `--branch ` - filter by Git branch; repeat or comma-separate. +- `--user ` and `--no-users` - filter by commit owner/invoking user, or include sessions without either. +- `-s, --search ` - search test-session text fields. +- `--error-group ` - find sessions that include a specific test-session error group. +- `-o, --output ` - `table` (default), `json`, or `md`. + +JSON output uses the stable list envelope: + +```json +{ + "data": [], + "pagination": { + "nextId": null, + "length": 0 + } +} +``` + +## Inspect a test session + +```bash +npx checkly test-sessions get +npx checkly test-sessions get --watch +npx checkly test-sessions get --output json +npx checkly test-sessions get --error-groups-limit 20 +``` + +Flags: +- `-w, --watch` - wait for a running session to complete before rendering. +- `--error-groups-limit ` - number of error group IDs to show in detail output (default 5). +- `-o, --output ` - `detail` (default), `json`, or `md`. + +The detail view shows the session status, metadata, result rows, test-session result IDs, check IDs when available, and test-session error group IDs. Prefer `--watch` before investigating failures so you do not act on partial results. + +Use `--output json` when you need exact fields, result links, or asset URLs. For Playwright Check Suite results, inspect result payload fields such as Playwright result details, traces, videos, screenshots, reports, and links when present. + +## Inspect a test-session error group + +Use only error group IDs shown by `test-sessions get`. Test-session error groups are separate from check error groups, so do not pass them to `checks get --error-group`. + +```bash +npx checkly test-sessions get --error-group +npx checkly test-sessions get --error-group --full-error +npx checkly test-sessions get --error-group --output json +``` + +Flags: +- `--full-error` - print the complete raw error for the selected test-session error group. +- `-w, --watch` - wait for session completion before validating and fetching the error group. +- `-o, --output ` - `detail` (default), `json`, or `md`. + +Look for `rootCauseAnalyses` in JSON output. If one already exists, reuse it before starting a new RCA. + +## Run root cause analysis + +For test-session error groups, use `--test-session-error-group` (short alias `-te`). For regular check error groups, use `--error-group` (short alias `-e`). + +```bash +npx checkly rca run --test-session-error-group --watch +npx checkly rca run -te --user-context "Started after checkout deploy" --watch +npx checkly rca get --watch +``` + +Flags: +- `--user-context ` - add concise context that can improve the analysis. +- `-w, --watch` - wait for completion and print the analysis. +- `-o, --output ` - `detail` (default), `json`, or `md`; watch mode is only supported with detail output. + +If RCA is unavailable because of plan or entitlement limits, run `npx checkly account plan --output json` and report the relevant entitlement or upgrade URL. + +## Retrieve result assets + +There is no dedicated `test-sessions download` command. Use the JSON outputs and links exposed by the session or result payload. + +```bash +npx checkly test-sessions get --output json +npx checkly checks get --result --output json +``` + +Use `test-sessions get --output json` first. If a result includes public URLs or asset fields, download those URLs directly. If a result only gives `checkId` plus `testSessionResultId`, use `checks get --result --output json` to fetch detailed result data; terminal output summarizes available screenshots, traces, and videos, while JSON output exposes the underlying URLs/fields. + +Do not invent asset names or assume every result has the same artifact set. Some results have screenshots only, some have traces or videos, and some have no downloadable assets. diff --git a/packages/cli/src/ai-context/skill.md b/packages/cli/src/ai-context/skill.md index bebcb4639..9f28eac82 100644 --- a/packages/cli/src/ai-context/skill.md +++ b/packages/cli/src/ai-context/skill.md @@ -14,6 +14,8 @@ Then run `npx checkly skills ` to load up-to-date details for the action Use `npx checkly skills install` to install this skill into your project (supports Claude Code, Cursor, Codex and more). +For recorded test-session investigations, run `npx checkly skills investigate test-sessions`. + ## Progressive Disclosure via `npx checkly skills` The skill is structured for efficient context usage: diff --git a/skills/checkly/SKILL.md b/skills/checkly/SKILL.md index f1b7784d9..eef0d8986 100644 --- a/skills/checkly/SKILL.md +++ b/skills/checkly/SKILL.md @@ -14,6 +14,8 @@ Then run `npx checkly skills ` to load up-to-date details for the action Use `npx checkly skills install` to install this skill into your project (supports Claude Code, Cursor, Codex and more). +For recorded test-session investigations, run `npx checkly skills investigate test-sessions`. + ## Progressive Disclosure via `npx checkly skills` The skill is structured for efficient context usage: @@ -101,7 +103,7 @@ Learn how to initialize and set up a new Checkly CLI project from scratch. Learn how to create and manage monitoring checks using Checkly constructs and the CLI. ### `npx checkly skills investigate` -Access check status, analyze failures, and investigate errors. +Access check and test-session status, analyze failures, and investigate errors. ### `npx checkly skills communicate` Open incidents and lead customer communications via status pages.