From c1da7802592f6a55907156683700e525565fa9c6 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Fri, 20 Mar 2026 17:09:45 +0100 Subject: [PATCH 01/14] feat(cli): add AgenticCheck construct for AI-powered monitoring Adds the AgenticCheck construct that lets users define agentic checks in their Checkly project via checkly.config.ts. Agentic checks use a prompt string (no scripts/bundling) to define what the AI agent should verify. - Add AGENTIC to CheckTypes enum - Create AgenticCheck extending Check (not RuntimeCheck) - Validate prompt (required, max 10000 chars) and frequency (min 30) - Add construct export and analytics mapping - Add 6 tests covering synthesis, defaults, groups, and validation Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/constants.ts | 1 + .../__tests__/agentic-check.spec.ts | 199 ++++++++++++++++++ .../fixtures/agentic-check/package.json | 5 + .../test-cases/test-basic/checkly.config.js | 11 + .../test-cases/test-basic/test.check.js | 9 + .../test-check-defaults/checkly.config.js | 12 ++ .../test-check-defaults/test.check.js | 12 ++ .../test-group-mapping/checkly.config.js | 11 + .../test-group-mapping/test.check.js | 11 + .../checkly.config.js | 11 + .../test.check.js | 7 + .../checkly.config.js | 11 + .../test.check.js | 6 + .../checkly.config.js | 11 + .../test.check.js | 6 + packages/cli/src/constructs/agentic-check.ts | 88 ++++++++ packages/cli/src/constructs/index.ts | 1 + packages/cli/src/rest/analytics.ts | 2 + 18 files changed, 414 insertions(+) create mode 100644 packages/cli/src/constructs/__tests__/agentic-check.spec.ts create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/package.json create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/test.check.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/test.check.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/test.check.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/test.check.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/test.check.js create mode 100644 packages/cli/src/constructs/agentic-check.ts diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 2283e809f..88db279be 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -8,6 +8,7 @@ export const CheckTypes = { ICMP: 'ICMP', DNS: 'DNS', URL: 'URL', + AGENTIC: 'AGENTIC', } as const export type CheckType = typeof CheckTypes[keyof typeof CheckTypes] diff --git a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts new file mode 100644 index 000000000..1741adf72 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts @@ -0,0 +1,199 @@ +import path from 'node:path' + +import { describe, it, expect, afterAll, beforeAll } from 'vitest' + +import { FixtureSandbox } from '../../testing/fixture-sandbox' +import { ParseProjectOutput } from '../../commands/debug/parse-project' + +async function parseProject (fixt: FixtureSandbox, ...args: string[]): Promise { + const result = await fixt.run('npx', [ + 'checkly', + 'debug', + 'parse-project', + ...args, + ]) + + if (result.exitCode !== 0) { + // eslint-disable-next-line no-console + console.error('stderr', result.stderr) + // eslint-disable-next-line no-console + console.error('stdout', result.stdout) + } + + expect(result.exitCode).toBe(0) + + const output: ParseProjectOutput = JSON.parse(result.stdout) + + return output +} + +const DEFAULT_TEST_TIMEOUT = 30_000 + +describe('AgenticCheck', () => { + let fixt: FixtureSandbox + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'agentic-check'), + }) + }, 180_000) + + afterAll(async () => { + await fixt?.destroy() + }) + + it('should create a basic agentic check with prompt', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-basic/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'homepage-health', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + name: 'Homepage Health Check', + prompt: 'Navigate to https://example.com and verify the page loads correctly.', + activated: true, + frequency: 30, + locations: ['us-east-1', 'eu-west-1'], + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should apply default check settings', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-check-defaults/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'check-should-have-defaults', + type: 'check', + member: true, + payload: expect.objectContaining({ + tags: ['default tags'], + }), + }), + expect.objectContaining({ + logicalId: 'check-should-not-have-defaults', + type: 'check', + member: true, + payload: expect.objectContaining({ + tags: ['not default tags'], + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should support setting groups with `group`', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-group-mapping/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'test-group', + type: 'check-group', + member: true, + }), + expect.objectContaining({ + logicalId: 'check', + type: 'check', + member: true, + payload: expect.objectContaining({ + groupId: { + ref: 'test-group', + }, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when prompt is empty', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-missing-prompt/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"prompt" is required'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when prompt exceeds 10000 characters', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-prompt-too-long/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"prompt" must be at most 10000 characters'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when frequency is less than 30', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-frequency-too-low/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"frequency" must be at least 30'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/package.json b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/package.json new file mode 100644 index 000000000..6059a0f54 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/package.json @@ -0,0 +1,5 @@ +{ + "name": "agentic-check-fixture", + "type": "module", + "devDependencies": {} +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js new file mode 100644 index 000000000..7a83a410a --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js @@ -0,0 +1,9 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('homepage-health', { + name: 'Homepage Health Check', + prompt: 'Navigate to https://example.com and verify the page loads correctly.', + activated: true, + frequency: 30, + locations: ['us-east-1', 'eu-west-1'], +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/checkly.config.js new file mode 100644 index 000000000..b3f83a62b --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/checkly.config.js @@ -0,0 +1,12 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + tags: ['default tags'], + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/test.check.js new file mode 100644 index 000000000..c2b0ef542 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/test.check.js @@ -0,0 +1,12 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('check-should-have-defaults', { + name: 'Check With Defaults', + prompt: 'Verify the homepage loads.', +}) + +new AgenticCheck('check-should-not-have-defaults', { + name: 'Check Without Defaults', + prompt: 'Verify the homepage loads.', + tags: ['not default tags'], +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/test.check.js new file mode 100644 index 000000000..785b5b462 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/test.check.js @@ -0,0 +1,11 @@ +import { AgenticCheck, CheckGroupV2 } from 'checkly/constructs' + +const group = new CheckGroupV2('test-group', { + name: 'Test Group', +}) + +new AgenticCheck('check', { + name: 'Agentic Check in Group', + prompt: 'Verify the homepage loads.', + group, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/test.check.js new file mode 100644 index 000000000..63f6c01ac --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/test.check.js @@ -0,0 +1,7 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('low-frequency', { + name: 'Low Frequency Check', + prompt: 'Verify the homepage loads.', + frequency: 15, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/test.check.js new file mode 100644 index 000000000..32ce642de --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/test.check.js @@ -0,0 +1,6 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('missing-prompt', { + name: 'Missing Prompt Check', + prompt: '', +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/test.check.js new file mode 100644 index 000000000..2ddda6b85 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/test.check.js @@ -0,0 +1,6 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('long-prompt', { + name: 'Long Prompt Check', + prompt: 'x'.repeat(10001), +}) diff --git a/packages/cli/src/constructs/agentic-check.ts b/packages/cli/src/constructs/agentic-check.ts new file mode 100644 index 000000000..69967e30a --- /dev/null +++ b/packages/cli/src/constructs/agentic-check.ts @@ -0,0 +1,88 @@ +import { Check, CheckProps } from './check' +import { Session } from './project' +import { CheckTypes } from '../constants' +import { Diagnostics } from './diagnostics' +import { InvalidPropertyValueDiagnostic } from './construct-diagnostics' + +export interface AgenticCheckProps extends CheckProps { + /** + * The prompt that defines what the agentic check should verify. + * Maximum 10,000 characters. + */ + prompt: string +} + +/** + * Creates an Agentic Check that uses AI to monitor websites and applications. + * + * Agentic checks use a prompt to define what should be verified, without requiring + * traditional scripts. The AI agent interprets the prompt and performs the checks. + * + * @example + * ```typescript + * new AgenticCheck('homepage-health', { + * name: 'Homepage Health Check', + * prompt: ` + * Navigate to https://example.com and verify: + * 1. The page loads with a 200 status + * 2. The main heading is visible + * 3. No console errors are present + * `, + * activated: true, + * frequency: 30, + * locations: ['us-east-1', 'eu-west-1'], + * }) + * ``` + */ +export class AgenticCheck extends Check { + readonly prompt: string + + /** + * Constructs the Agentic Check instance. + * + * @param logicalId unique project-scoped resource name identification + * @param props check configuration properties + */ + constructor (logicalId: string, props: AgenticCheckProps) { + super(logicalId, props) + this.prompt = props.prompt + Session.registerConstruct(this) + this.addSubscriptions() + this.addPrivateLocationCheckAssignments() + } + + describe (): string { + return `AgenticCheck:${this.logicalId}` + } + + async validate (diagnostics: Diagnostics): Promise { + await super.validate(diagnostics) + + if (!this.prompt || this.prompt.trim().length === 0) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'prompt', + new Error('"prompt" is required and must not be empty.'), + )) + } else if (this.prompt.length > 10_000) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'prompt', + new Error(`"prompt" must be at most 10000 characters, got ${this.prompt.length}.`), + )) + } + + if (this.frequency !== undefined && this.frequency < 30) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'frequency', + new Error('"frequency" must be at least 30 for agentic checks.'), + )) + } + } + + synthesize () { + return { + ...super.synthesize(), + checkType: CheckTypes.AGENTIC, + prompt: this.prompt, + } + } +} diff --git a/packages/cli/src/constructs/index.ts b/packages/cli/src/constructs/index.ts index 914ffe95c..e2bee57f9 100644 --- a/packages/cli/src/constructs/index.ts +++ b/packages/cli/src/constructs/index.ts @@ -48,3 +48,4 @@ export * from './dns-request' export * from './icmp-monitor' export * from './icmp-assertion' export * from './icmp-request' +export * from './agentic-check' diff --git a/packages/cli/src/rest/analytics.ts b/packages/cli/src/rest/analytics.ts index ec540c08f..338a444a7 100644 --- a/packages/cli/src/rest/analytics.ts +++ b/packages/cli/src/rest/analytics.ts @@ -17,6 +17,7 @@ const checkTypeToPath: Record = { [CheckTypes.ICMP]: 'icmp', [CheckTypes.DNS]: 'dns', [CheckTypes.URL]: 'url-monitors', + [CheckTypes.AGENTIC]: 'agentic-checks', } // Default aggregated metrics per check type @@ -30,6 +31,7 @@ const defaultMetrics: Record = { [CheckTypes.DNS]: ['availability', 'total_avg', 'total_p50', 'total_p95', 'total_p99'], [CheckTypes.ICMP]: ['availability', 'packetLoss_avg', 'latencyAvg_avg', 'latencyAvg_p50', 'latencyAvg_p95', 'latencyAvg_p99'], [CheckTypes.HEARTBEAT]: ['availability'], + [CheckTypes.AGENTIC]: ['availability'], } export type GroupBy = 'runLocation' | 'statusCode' From a77f11725fd60e2599cdec9cc94df05cc75c1814 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Fri, 20 Mar 2026 17:30:25 +0100 Subject: [PATCH 02/14] feat(cli): force runParallel for agentic checks Agentic checks always run in parallel for better reporting UX. The construct omits runParallel from props (users can't override it) and hardcodes it to true in synthesize(). Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/constructs/__tests__/agentic-check.spec.ts | 1 + packages/cli/src/constructs/agentic-check.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts index 1741adf72..e1d9e91ef 100644 --- a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts @@ -66,6 +66,7 @@ describe('AgenticCheck', () => { activated: true, frequency: 30, locations: ['us-east-1', 'eu-west-1'], + runParallel: true, }), }), ]), diff --git a/packages/cli/src/constructs/agentic-check.ts b/packages/cli/src/constructs/agentic-check.ts index 69967e30a..66e58f135 100644 --- a/packages/cli/src/constructs/agentic-check.ts +++ b/packages/cli/src/constructs/agentic-check.ts @@ -4,7 +4,7 @@ import { CheckTypes } from '../constants' import { Diagnostics } from './diagnostics' import { InvalidPropertyValueDiagnostic } from './construct-diagnostics' -export interface AgenticCheckProps extends CheckProps { +export interface AgenticCheckProps extends Omit { /** * The prompt that defines what the agentic check should verify. * Maximum 10,000 characters. @@ -83,6 +83,7 @@ export class AgenticCheck extends Check { ...super.synthesize(), checkType: CheckTypes.AGENTIC, prompt: this.prompt, + runParallel: true, } } } From 5a240011f21652b7c9b96221a263b9efaa199901 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 12:13:43 +0200 Subject: [PATCH 03/14] Revert "feat(cli): force runParallel for agentic checks" This reverts commit a77f11725fd60e2599cdec9cc94df05cc75c1814. --- packages/cli/src/constructs/__tests__/agentic-check.spec.ts | 1 - packages/cli/src/constructs/agentic-check.ts | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts index e1d9e91ef..1741adf72 100644 --- a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts @@ -66,7 +66,6 @@ describe('AgenticCheck', () => { activated: true, frequency: 30, locations: ['us-east-1', 'eu-west-1'], - runParallel: true, }), }), ]), diff --git a/packages/cli/src/constructs/agentic-check.ts b/packages/cli/src/constructs/agentic-check.ts index 66e58f135..69967e30a 100644 --- a/packages/cli/src/constructs/agentic-check.ts +++ b/packages/cli/src/constructs/agentic-check.ts @@ -4,7 +4,7 @@ import { CheckTypes } from '../constants' import { Diagnostics } from './diagnostics' import { InvalidPropertyValueDiagnostic } from './construct-diagnostics' -export interface AgenticCheckProps extends Omit { +export interface AgenticCheckProps extends CheckProps { /** * The prompt that defines what the agentic check should verify. * Maximum 10,000 characters. @@ -83,7 +83,6 @@ export class AgenticCheck extends Check { ...super.synthesize(), checkType: CheckTypes.AGENTIC, prompt: this.prompt, - runParallel: true, } } } From 583e7999d6806972a8402dd1a20b7b8d203bc822 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 12:35:12 +0200 Subject: [PATCH 04/14] feat(cli): restrict AgenticCheck props to platform-supported fields Lock down AgenticCheckProps to only the subset of CheckProps the Checkly platform currently honors for agentic checks. Omits locations, privateLocations, runParallel, retryStrategy, shouldFail, doubleCheck, triggerIncident and groupId at the type level, and restricts frequency to the discrete set exposed in the webapp builder. Defensive overrides in the constructor ensure project-level config defaults for any of these fields are also ignored, so the construct never claims to support what the platform won't honor. These restrictions can be relaxed additively (without breaking changes) once the platform supports the corresponding capabilities for agentic checks. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../__tests__/agentic-check.spec.ts | 36 ++++++- .../test-cases/test-basic/test.check.js | 3 +- .../test-locations-override/checkly.config.js | 12 +++ .../test-locations-override/test.check.js | 6 ++ packages/cli/src/constructs/agentic-check.ts | 101 ++++++++++++++++-- 5 files changed, 144 insertions(+), 14 deletions(-) create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/test.check.js diff --git a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts index 1741adf72..af65899ee 100644 --- a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts @@ -64,8 +64,38 @@ describe('AgenticCheck', () => { name: 'Homepage Health Check', prompt: 'Navigate to https://example.com and verify the page loads correctly.', activated: true, + frequency: 60, + locations: ['us-east-1'], + runParallel: false, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should default to a single us-east-1 location and ignore project-level locations', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-locations-override/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'locations-overridden', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + locations: ['us-east-1'], frequency: 30, - locations: ['us-east-1', 'eu-west-1'], + runParallel: false, }), }), ]), @@ -178,7 +208,7 @@ describe('AgenticCheck', () => { })) }, DEFAULT_TEST_TIMEOUT) - it('should fail validation when frequency is less than 30', async () => { + it('should fail validation when frequency is not in the supported set', async () => { const output = await parseProject( fixt, '--config', @@ -190,7 +220,7 @@ describe('AgenticCheck', () => { fatal: true, observations: expect.arrayContaining([ expect.objectContaining({ - message: expect.stringContaining('"frequency" must be at least 30'), + message: expect.stringContaining('"frequency" must be one of 30, 60, 120, 180, 360, 720, 1440'), }), ]), }), diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js index 7a83a410a..62b7d1de0 100644 --- a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js @@ -4,6 +4,5 @@ new AgenticCheck('homepage-health', { name: 'Homepage Health Check', prompt: 'Navigate to https://example.com and verify the page loads correctly.', activated: true, - frequency: 30, - locations: ['us-east-1', 'eu-west-1'], + frequency: 60, }) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/checkly.config.js new file mode 100644 index 000000000..8a97a1718 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/checkly.config.js @@ -0,0 +1,12 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + locations: ['eu-west-1', 'ap-southeast-1'], + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/test.check.js new file mode 100644 index 000000000..31b8a6e27 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/test.check.js @@ -0,0 +1,6 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('locations-overridden', { + name: 'Agentic check with project-default locations', + prompt: 'Verify the homepage loads.', +}) diff --git a/packages/cli/src/constructs/agentic-check.ts b/packages/cli/src/constructs/agentic-check.ts index 69967e30a..a7a3c991b 100644 --- a/packages/cli/src/constructs/agentic-check.ts +++ b/packages/cli/src/constructs/agentic-check.ts @@ -1,22 +1,86 @@ import { Check, CheckProps } from './check' +import { Frequency } from './frequency' import { Session } from './project' import { CheckTypes } from '../constants' import { Diagnostics } from './diagnostics' import { InvalidPropertyValueDiagnostic } from './construct-diagnostics' -export interface AgenticCheckProps extends CheckProps { +/** + * Frequency values (in minutes) currently supported for agentic checks. + * Mirrors the values exposed in the Checkly webapp's agentic check builder. + */ +const ALLOWED_AGENTIC_FREQUENCIES = [30, 60, 120, 180, 360, 720, 1440] as const + +/** + * Frequencies (in minutes) currently supported for agentic checks: 30, 60, 120, + * 180, 360, 720 or 1440. The matching `Frequency` constants + * (`EVERY_30M`, `EVERY_1H`, `EVERY_2H`, `EVERY_3H`, `EVERY_6H`, `EVERY_12H`, + * `EVERY_24H`) are also accepted. + */ +export type AgenticCheckFrequency = + | 30 + | 60 + | 120 + | 180 + | 360 + | 720 + | 1440 + | Frequency + +/** + * The single location agentic checks currently run from. Until the platform + * supports running agentic checks from multiple locations, this value is + * forced server-side and the construct does not let users override it. + */ +const AGENTIC_CHECK_LOCATION = 'us-east-1' + +/** + * Configuration properties for {@link AgenticCheck}. + * + * Agentic checks intentionally expose only the subset of options that the + * Checkly platform currently supports for them. Properties such as + * `locations`, `privateLocations`, `runParallel`, `retryStrategy`, + * `shouldFail`, `doubleCheck`, `triggerIncident` and `groupId` are omitted + * because the platform does not yet honor them for agentic checks. They will + * be added back as additive, non-breaking changes once support lands. + */ +export interface AgenticCheckProps extends Omit { /** * The prompt that defines what the agentic check should verify. * Maximum 10,000 characters. */ prompt: string + + /** + * How often the check should run. Agentic checks currently support a + * restricted set of frequencies. Defaults to {@link Frequency.EVERY_30M}. + * + * @example + * ```typescript + * frequency: Frequency.EVERY_1H + * // or equivalently + * frequency: 60 + * ``` + */ + frequency?: AgenticCheckFrequency } /** * Creates an Agentic Check that uses AI to monitor websites and applications. * - * Agentic checks use a prompt to define what should be verified, without requiring - * traditional scripts. The AI agent interprets the prompt and performs the checks. + * Agentic checks use a prompt to define what should be verified, without + * requiring traditional scripts. The AI agent interprets the prompt and + * performs the checks. * * @example * ```typescript @@ -28,9 +92,6 @@ export interface AgenticCheckProps extends CheckProps { * 2. The main heading is visible * 3. No console errors are present * `, - * activated: true, - * frequency: 30, - * locations: ['us-east-1', 'eu-west-1'], * }) * ``` */ @@ -46,9 +107,27 @@ export class AgenticCheck extends Check { constructor (logicalId: string, props: AgenticCheckProps) { super(logicalId, props) this.prompt = props.prompt + + // Defensive overrides: even though these props are omitted from the type, + // `Check.applyConfigDefaults()` may pull them in from the project-level + // `checks` config defaults. Force them to the only values the platform + // currently honors so the construct never claims to support something + // it doesn't. + this.locations = [AGENTIC_CHECK_LOCATION] + this.privateLocations = undefined + this.runParallel = false + this.retryStrategy = undefined + this.shouldFail = undefined + this.doubleCheck = undefined + this.triggerIncident = undefined + + // Default frequency to 30m if the user did not specify one. + if (this.frequency === undefined) { + this.frequency = 30 + } + Session.registerConstruct(this) this.addSubscriptions() - this.addPrivateLocationCheckAssignments() } describe (): string { @@ -70,10 +149,14 @@ export class AgenticCheck extends Check { )) } - if (this.frequency !== undefined && this.frequency < 30) { + if (this.frequency !== undefined + && !(ALLOWED_AGENTIC_FREQUENCIES as readonly number[]).includes(this.frequency)) { diagnostics.add(new InvalidPropertyValueDiagnostic( 'frequency', - new Error('"frequency" must be at least 30 for agentic checks.'), + new Error( + `"frequency" must be one of ${ALLOWED_AGENTIC_FREQUENCIES.join(', ')} ` + + `for agentic checks, got ${this.frequency}.`, + ), )) } } From e24774d1604c0cd32a611e34c466da4c36a986c0 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 14:12:44 +0200 Subject: [PATCH 05/14] feat(cli): add agentRuntime to AgenticCheck for skills and env var access Adds an `agentRuntime` prop to AgenticCheck that declares the runtime context the agent has access to during execution: which skills it can use and which environment variables it is permitted to read. agentRuntime is the explicit security boundary for the agent. Anything not declared here is unavailable at runtime, which keeps the blast radius of any prompt injection as small as possible. Environment variables accept either bare names or `{ name, description }` objects; descriptions are passed to the agent so it can decide when to read each variable, and are validated to stay within the runner's 200-char limit. The construct always synthesizes a complete agentRuntime payload (with empty arrays when unset), so the backend has unambiguous source-of-truth semantics: omitted skills/env vars mean "the agent should not have them", not "preserve whatever was there before". The backend translates this clean shape into the existing agenticCheckData storage blob. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../__tests__/agentic-check.spec.ts | 87 +++++++++++ .../test-agent-runtime/checkly.config.js | 11 ++ .../test-agent-runtime/test.check.js | 18 +++ .../checkly.config.js | 11 ++ .../test.check.js | 11 ++ .../checkly.config.js | 11 ++ .../test.check.js | 9 ++ packages/cli/src/constructs/agentic-check.ts | 140 ++++++++++++++++++ 8 files changed, 298 insertions(+) create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/test.check.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/checkly.config.js create mode 100644 packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/test.check.js diff --git a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts index af65899ee..9bcfa3d84 100644 --- a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts @@ -67,6 +67,55 @@ describe('AgenticCheck', () => { frequency: 60, locations: ['us-east-1'], runParallel: false, + agentRuntime: { + skills: [], + environmentVariables: [], + }, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should expose agentRuntime skills and environment variables', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-agent-runtime/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'agent-runtime-check', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + agentRuntime: { + skills: ['checkly/playwright-skill'], + environmentVariables: [ + 'API_KEY', + { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, + ], + }, + }), + }), + expect.objectContaining({ + logicalId: 'default-agent-runtime', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + agentRuntime: { + skills: [], + environmentVariables: [], + }, }), }), ]), @@ -226,4 +275,42 @@ describe('AgenticCheck', () => { }), })) }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when an environment variable name is empty', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-empty-env-var-name/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"agentRuntime.environmentVariables[0]" must have a non-empty name'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when an environment variable description exceeds 200 characters', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-description-too-long/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"agentRuntime.environmentVariables[0].description" must be at most 200 characters'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) }) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js new file mode 100644 index 000000000..c96d3e7b5 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js @@ -0,0 +1,18 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('agent-runtime-check', { + name: 'Agent runtime check', + prompt: 'Sign in to the test account and verify the dashboard loads.', + agentRuntime: { + skills: ['checkly/playwright-skill'], + environmentVariables: [ + 'API_KEY', + { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, + ], + }, +}) + +new AgenticCheck('default-agent-runtime', { + name: 'Defaults agent runtime', + prompt: 'Verify the homepage loads.', +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/test.check.js new file mode 100644 index 000000000..d53b0f5e9 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/test.check.js @@ -0,0 +1,11 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('long-description', { + name: 'Description too long', + prompt: 'Verify the homepage loads.', + agentRuntime: { + environmentVariables: [ + { name: 'API_KEY', description: 'x'.repeat(201) }, + ], + }, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/test.check.js new file mode 100644 index 000000000..0969ace2d --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/test.check.js @@ -0,0 +1,9 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('empty-env-var', { + name: 'Bad env var entry', + prompt: 'Verify the homepage loads.', + agentRuntime: { + environmentVariables: [' '], + }, +}) diff --git a/packages/cli/src/constructs/agentic-check.ts b/packages/cli/src/constructs/agentic-check.ts index a7a3c991b..0df784fee 100644 --- a/packages/cli/src/constructs/agentic-check.ts +++ b/packages/cli/src/constructs/agentic-check.ts @@ -34,6 +34,79 @@ export type AgenticCheckFrequency = */ const AGENTIC_CHECK_LOCATION = 'us-east-1' +/** + * Maximum length of an environment variable description, in characters. + * Matches the truncation length applied by the agentic runner. + */ +const MAX_ENV_VAR_DESCRIPTION_LENGTH = 200 + +/** + * An environment variable the agent is permitted to read at runtime. + * + * Use the bare string form when the variable name is self-explanatory, or + * the object form to provide a `description` that helps the agent understand + * what the variable is for. Descriptions are passed to the model so it can + * make better decisions about when to read the variable. + * + * @example + * ```typescript + * 'API_KEY' + * { name: 'TOKEN_42', description: 'Feature flag service auth token' } + * ``` + */ +export type AgentRuntimeEnvironmentVariable = + | string + | { + /** The environment variable name. */ + name: string + /** + * Optional human-readable explanation of what the variable is for. + * Passed to the agent so it can decide when to read the variable. + * Truncated to {@link MAX_ENV_VAR_DESCRIPTION_LENGTH} characters. + */ + description?: string + } + +/** + * Configures the runtime context the agent has access to during a check. + * + * `agentRuntime` is the explicit allowlist of resources the agent may use + * at execution time. Anything not declared here is unavailable to the agent. + * Treat it as a security boundary: the smaller the runtime surface, the + * smaller the blast radius of any prompt injection. + */ +export interface AgentRuntime { + /** + * Skills the agent can use during execution. Each entry is a skill + * package name as accepted by `npx checkly skills install`. + * + * @example ['checkly/playwright-skill'] + */ + skills?: string[] + + /** + * Environment variables the agent is permitted to read at runtime. + * + * **Variables not listed here are not exposed to the agent**, even if + * they exist in the Checkly account. This is the primary defense against + * prompt injection: an attacker who controls content the agent reads + * cannot exfiltrate secrets the agent never had access to. + * + * Each entry is either a bare variable name, or an object with a + * `name` and an optional `description`. Descriptions help the agent + * understand what each variable is for. + * + * @example + * ```typescript + * environmentVariables: [ + * 'API_KEY', + * { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, + * ] + * ``` + */ + environmentVariables?: AgentRuntimeEnvironmentVariable[] +} + /** * Configuration properties for {@link AgenticCheck}. * @@ -73,6 +146,18 @@ export interface AgenticCheckProps extends Omit { + if (this.agentRuntime?.skills) { + for (const [index, skill] of this.agentRuntime.skills.entries()) { + if (typeof skill !== 'string' || skill.trim().length === 0) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'agentRuntime.skills', + new Error( + `"agentRuntime.skills[${index}]" must be a non-empty string.`, + ), + )) + } + } + } + + if (this.agentRuntime?.environmentVariables) { + for (const [index, entry] of this.agentRuntime.environmentVariables.entries()) { + const name = typeof entry === 'string' ? entry : entry?.name + if (typeof name !== 'string' || name.trim().length === 0) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'agentRuntime.environmentVariables', + new Error( + `"agentRuntime.environmentVariables[${index}]" must have a non-empty name.`, + ), + )) + continue + } + + if (typeof entry !== 'string' + && typeof entry.description === 'string' + && entry.description.length > MAX_ENV_VAR_DESCRIPTION_LENGTH) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'agentRuntime.environmentVariables', + new Error( + `"agentRuntime.environmentVariables[${index}].description" must be at most ` + + `${MAX_ENV_VAR_DESCRIPTION_LENGTH} characters, got ${entry.description.length}.`, + ), + )) + } + } + } } synthesize () { @@ -166,6 +298,14 @@ export class AgenticCheck extends Check { ...super.synthesize(), checkType: CheckTypes.AGENTIC, prompt: this.prompt, + // Always emit `agentRuntime` so the backend has an explicit, complete + // picture of the runtime surface the user wants. The CLI is the source + // of truth: omitted skills/env vars mean "the agent should not have + // them", not "preserve whatever was there before". + agentRuntime: { + skills: this.agentRuntime?.skills ?? [], + environmentVariables: this.agentRuntime?.environmentVariables ?? [], + }, } } } From 3b24c1cd7ac02c586a501ec4c354cff471d29167 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 22:55:14 +0200 Subject: [PATCH 06/14] feat(cli): add code generation support for AgenticCheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires AgenticCheck into the \`checkly import\` code generation flow so users can import an existing agentic check from the backend and get a construct file that matches the locked-down props type added earlier on this branch. Implementation: - New \`AgenticCheckCodegen\` in \`agentic-check-codegen.ts\` mirroring the Heartbeat/Browser codegen pattern. Places generated files under \`resources/agentic-checks/{name}\`, imports \`AgenticCheck\` from \`checkly/constructs\`, and emits a \`new AgenticCheck(...)\` expression. - The resource shape (\`AgenticCheckResource\`) carries \`prompt\` and the backend's \`agenticCheckData\` storage blob. A small helper reverse- translates that blob into the CLI construct's public shape: * \`agenticCheckData.skills\` → \`agentRuntime.skills\` * \`agenticCheckData.selectedEnvironmentVariables\` → \`agentRuntime.environmentVariables\` — mapping \`key\` back to \`name\` for object-form entries * \`agenticCheckData.assertionRules\` → deliberately not emitted (agent-generated, preserved server-side via the backend's merge logic, the construct intentionally does not expose them) An imported check with no skills and no selected env vars produces no \`agentRuntime\` block at all, to avoid noise in the generated code. - Registers the new codegen in \`CheckCodegen.constructor\` and wires \`AGENTIC\` into the dispatch switches in \`describe()\` and \`gencode()\` alongside the existing check types. \`buildCheckProps\` gains a narrow \`BuildCheckPropsOptions\` parameter — currently a single \`skipRetryStrategy\` flag — because \`retryStrategy\` is the only field that \`buildCheckProps\` emits unconditionally (null is rendered as \`RetryStrategyBuilder.noRetries()\`). The AgenticCheck construct's props type omits \`retryStrategy\` entirely, so emitting it would produce a generated file that does not type-check. The AgenticCheckCodegen passes \`skipRetryStrategy: true\` and clears \`locations\` from a resource copy before calling \`buildCheckProps\` — the two fixes together make the generated output round-trip cleanly through the construct's props type. Verified end-to-end by extending \`ai-context/context.fixtures.json\` with an example agentic check (prompt, skills, mixed-form env vars, sample assertionRules) and running \`npm run prepare:ai-context\`. The generated \`gen/resources/agentic-checks/example-agentic-check.check.ts\` contains only fields the construct accepts, reverse-translates env vars from \`key\` to \`name\`, omits \`assertionRules\`, and type-checks against the construct. Full CLI suite remains green (829 passing / 2 skipped / 60 files). No regressions in any other check type's codegen from the new optional options parameter. Covers item #3 on Simo's new-construct checklist. Items #5 (examples), #6 (E2E tests), and #8 (AI context reference doc) still pending on this branch. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../cli/src/ai-context/context.fixtures.json | 75 +++++++++ .../src/constructs/agentic-check-codegen.ts | 147 ++++++++++++++++++ packages/cli/src/constructs/check-codegen.ts | 33 +++- 3 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/constructs/agentic-check-codegen.ts diff --git a/packages/cli/src/ai-context/context.fixtures.json b/packages/cli/src/ai-context/context.fixtures.json index d5591564d..7760d97b8 100644 --- a/packages/cli/src/ai-context/context.fixtures.json +++ b/packages/cli/src/ai-context/context.fixtures.json @@ -824,6 +824,81 @@ "privateLocations": [] } }, + { + "logicalId": "example-agentic-check", + "physicalId": "dddddddd-dddd-dddd-dddd-dddddddddddd", + "type": "check", + "member": true, + "pending": true, + "payload": { + "id": "dddddddd-dddd-dddd-dddd-dddddddddddd", + "checkType": "AGENTIC", + "name": "Example Agentic Check", + "frequency": 60, + "frequencyOffset": 0, + "activated": true, + "muted": false, + "shouldFail": false, + "locations": [ + "us-east-1" + ], + "accountId": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "created_at": "2025-10-28T17:25:17.037Z", + "updated_at": null, + "tags": [ + "app:webshop" + ], + "alertSettings": { + "reminders": { + "amount": 0, + "interval": 5 + }, + "escalationType": "RUN_BASED", + "runBasedEscalation": { + "failedRunThreshold": 1 + }, + "timeBasedEscalation": { + "minutesFailingThreshold": 5 + }, + "parallelRunFailureThreshold": { + "enabled": false, + "percentage": 10 + } + }, + "useGlobalAlertSettings": true, + "groupId": null, + "groupOrder": null, + "retryStrategy": null, + "runParallel": false, + "triggerIncident": false, + "prompt": "Navigate to https://www.checklyhq.com and verify the homepage loads with the main heading visible. Then navigate to /pricing and confirm at least three plan tiers are displayed.", + "agenticCheckData": { + "skills": [ + "checkly/playwright-skill" + ], + "selectedEnvironmentVariables": [ + "ENVIRONMENT_URL", + { + "key": "TEST_USER_EMAIL", + "description": "Login email for the test account" + } + ], + "assertionRules": [ + { + "source": "JSON_BODY", + "property": "$.status", + "comparison": "EQUALS", + "target": "passing", + "order": 0, + "regex": null, + "description": "Overall check result status" + } + ] + }, + "alertChannelSubscriptions": [], + "privateLocations": [] + } + }, { "logicalId": "example-maintenance-window", "physicalId": 1000001, diff --git a/packages/cli/src/constructs/agentic-check-codegen.ts b/packages/cli/src/constructs/agentic-check-codegen.ts new file mode 100644 index 000000000..b24120577 --- /dev/null +++ b/packages/cli/src/constructs/agentic-check-codegen.ts @@ -0,0 +1,147 @@ +import { Codegen, Context } from './internal/codegen' +import { expr, ident, ObjectValueBuilder } from '../sourcegen' +import { buildCheckProps, CheckResource } from './check-codegen' + +/** + * Shape of a `selectedEnvironmentVariables` entry as stored on the backend + * inside `agenticCheckData`. The runner accepts two forms — a bare variable + * name, or an object with `key` and an optional `description`. The CLI + * construct uses `name` (not `key`), so the codegen translates object-form + * entries during emission. + */ +type StoredAgenticEnvironmentVariable = + | string + | { key: string, description?: string } + +/** + * Shape of `agenticCheckData` as stored on the backend and returned to the + * CLI during `checkly import`. Only fields the construct exposes are read. + * `assertionRules` is deliberately ignored — the agent generates those on + * the first run and the backend's deploy logic preserves them, so the CLI + * construct never needs to emit them. + */ +interface StoredAgenticCheckData { + skills?: string[] | null + selectedEnvironmentVariables?: StoredAgenticEnvironmentVariable[] | null + // Intentionally not consumed by codegen: + // assertionRules?: unknown[] + [key: string]: unknown +} + +export interface AgenticCheckResource extends CheckResource { + checkType: 'AGENTIC' + prompt: string + agenticCheckData?: StoredAgenticCheckData | null +} + +const construct = 'AgenticCheck' + +export class AgenticCheckCodegen extends Codegen { + describe (resource: AgenticCheckResource): string { + return `Agentic Check: ${resource.name}` + } + + gencode (logicalId: string, resource: AgenticCheckResource, context: Context): void { + const filePath = context.filePath('resources/agentic-checks', resource.name, { + tags: resource.tags, + unique: true, + }) + + const file = this.program.generatedConstructFile(filePath.fullPath) + + file.namedImport(construct, 'checkly/constructs') + + // The `AgenticCheck` construct's props type intentionally omits + // `locations`, `privateLocations`, `runParallel`, `retryStrategy`, + // `shouldFail`, `doubleCheck`, `triggerIncident` and `groupId` — the + // platform does not yet honor any of these for agentic checks (see the + // construct file for the full rationale). The codegen has to match + // that omission: if the generated file contained any of those keys, + // it would not type-check against the construct the moment a user + // opens it. + // + // Most of the omitted fields are already conditional in + // `buildCheckProps` (they're only emitted when non-false/non-empty), + // so we pass a resource copy with `locations` cleared — the rest + // never get populated for agentic checks today anyway, and if they + // ever do, the fall-through will surface as a typecheck error in the + // generated file which is the right place to catch it. + // `retryStrategy` is the exception: `buildCheckProps` always emits + // it, so we explicitly opt out via `skipRetryStrategy`. + const sanitizedResource: AgenticCheckResource = { + ...resource, + locations: undefined, + } + + file.section(expr(ident(construct), builder => { + builder.new(builder => { + builder.string(logicalId) + builder.object(builder => { + builder.string('prompt', resource.prompt) + + // Emit agentRuntime only when there's something meaningful to + // carry. An imported check with no skills and no selected env + // vars would otherwise produce an empty `agentRuntime: {}` block, + // which is noise in the generated code. + const agentRuntimeValue = buildAgentRuntimeObject(resource.agenticCheckData) + if (agentRuntimeValue !== undefined) { + builder.object('agentRuntime', agentRuntimeValue) + } + + buildCheckProps(this.program, file, builder, sanitizedResource, context, { + skipRetryStrategy: true, + }) + }) + }) + })) + } +} + +/** + * Build an `agentRuntime: { ... }` object literal for the codegen output, + * reverse-translating the backend's storage shape (`selectedEnvironmentVariables` + * with `key`) into the CLI construct's shape (`environmentVariables` with + * `name`). Returns `undefined` when the input contains nothing worth + * emitting, so the caller can skip the property entirely. + */ +function buildAgentRuntimeObject ( + data: StoredAgenticCheckData | null | undefined, +): ((builder: ObjectValueBuilder) => void) | undefined { + if (!data) return undefined + + const skills = Array.isArray(data.skills) ? data.skills.filter(s => typeof s === 'string' && s.length > 0) : [] + const storedEnvVars = Array.isArray(data.selectedEnvironmentVariables) + ? data.selectedEnvironmentVariables + : [] + + if (skills.length === 0 && storedEnvVars.length === 0) { + return undefined + } + + return builder => { + if (skills.length > 0) { + builder.array('skills', arrayBuilder => { + for (const skill of skills) { + arrayBuilder.string(skill) + } + }) + } + + if (storedEnvVars.length > 0) { + builder.array('environmentVariables', arrayBuilder => { + for (const entry of storedEnvVars) { + if (typeof entry === 'string') { + arrayBuilder.string(entry) + } else if (entry && typeof entry === 'object' && typeof entry.key === 'string') { + arrayBuilder.object(objectBuilder => { + objectBuilder.string('name', entry.key) + if (typeof entry.description === 'string' && entry.description.length > 0) { + objectBuilder.string('description', entry.description) + } + }) + } + } + }) + } + } +} diff --git a/packages/cli/src/constructs/check-codegen.ts b/packages/cli/src/constructs/check-codegen.ts index 5c4b262c3..e913f94b7 100644 --- a/packages/cli/src/constructs/check-codegen.ts +++ b/packages/cli/src/constructs/check-codegen.ts @@ -1,5 +1,6 @@ import { Codegen, Context } from './internal/codegen' import { Program, ObjectValueBuilder, GeneratedFile } from '../sourcegen' +import { AgenticCheckCodegen, AgenticCheckResource } from './agentic-check-codegen' import { AlertEscalationResource, valueForAlertEscalation } from './alert-escalation-policy-codegen' import { ApiCheckCodegen, ApiCheckResource } from './api-check-codegen' import { BrowserCheckCodegen, BrowserCheckResource } from './browser-check-codegen' @@ -38,12 +39,33 @@ export interface CheckResource { runParallel?: boolean } +/** + * Options controlling which common check fields `buildCheckProps` emits. + * + * The defaults match the historical behavior — every field is emitted if + * the resource provides it. Individual check types can opt out of specific + * fields when their construct's props type does not accept them. For + * example, `AgenticCheck` omits `retryStrategy` from its props, so its + * codegen passes `skipRetryStrategy: true` to avoid emitting code that + * would not type-check against the construct. + */ +export interface BuildCheckPropsOptions { + /** + * Skip emitting the `retryStrategy` property. Unlike most fields in + * `buildCheckProps`, `retryStrategy` is emitted unconditionally (null is + * rendered as `RetryStrategyBuilder.noRetries()`), so opting out requires + * an explicit flag. + */ + skipRetryStrategy?: boolean +} + export function buildCheckProps ( program: Program, genfile: GeneratedFile, builder: ObjectValueBuilder, resource: CheckResource, context: Context, + options: BuildCheckPropsOptions = {}, ): void { builder.string('name', resource.name, { order: -1000 }) @@ -176,7 +198,9 @@ export function buildCheckProps ( builder.boolean('testOnly', resource.testOnly) } - builder.value('retryStrategy', valueForRetryStrategy(genfile, resource.retryStrategy)) + if (!options.skipRetryStrategy) { + builder.value('retryStrategy', valueForRetryStrategy(genfile, resource.retryStrategy)) + } if (resource.runParallel !== undefined && resource.runParallel !== false) { builder.boolean('runParallel', resource.runParallel) @@ -214,6 +238,7 @@ export function buildRuntimeCheckProps ( } export class CheckCodegen extends Codegen { + agenticCheckCodegen: AgenticCheckCodegen apiCheckCodegen: ApiCheckCodegen browserCheckCodegen: BrowserCheckCodegen checkGroupCodegen: CheckGroupCodegen @@ -226,6 +251,7 @@ export class CheckCodegen extends Codegen { constructor (program: Program) { super(program) + this.agenticCheckCodegen = new AgenticCheckCodegen(program) this.apiCheckCodegen = new ApiCheckCodegen(program) this.browserCheckCodegen = new BrowserCheckCodegen(program) this.checkGroupCodegen = new CheckGroupCodegen(program) @@ -241,6 +267,8 @@ export class CheckCodegen extends Codegen { const { checkType } = resource switch (checkType) { + case 'AGENTIC': + return this.agenticCheckCodegen.describe(resource as AgenticCheckResource) case 'BROWSER': return this.browserCheckCodegen.describe(resource as BrowserCheckResource) case 'API': @@ -266,6 +294,9 @@ export class CheckCodegen extends Codegen { const { checkType } = resource switch (checkType) { + case 'AGENTIC': + this.agenticCheckCodegen.gencode(logicalId, resource as AgenticCheckResource, context) + return case 'BROWSER': this.browserCheckCodegen.gencode(logicalId, resource as BrowserCheckResource, context) return From 4fc6783994ad03f5d607ccdf568d64fd43d9b38a Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 23:33:30 +0200 Subject: [PATCH 07/14] test(cli): add unit tests for AgenticCheckCodegen Cover the codegen edge cases that the `prepare:ai-context` happy-path fixture doesn't exercise: missing/null/empty `agenticCheckData`, skills-only, env vars-only, mixed bare-string and object-form env var entries, the `key` -> `name` reverse translation, and the explicit suppression of `locations`, `retryStrategy`, and `assertionRules` from the generated source. Also drop a few defensive runtime checks in the codegen that the typed `StoredAgenticCheckData` interface already guarantees, and trim a comment block in `gencode()`. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../__tests__/agentic-check-codegen.spec.ts | 261 ++++++++++++++++++ .../src/constructs/agentic-check-codegen.ts | 37 +-- 2 files changed, 272 insertions(+), 26 deletions(-) create mode 100644 packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts diff --git a/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts new file mode 100644 index 000000000..d89671cb5 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts @@ -0,0 +1,261 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { AgenticCheckCodegen, AgenticCheckResource } from '../agentic-check-codegen' +import { Context } from '../internal/codegen/context' +import { Program } from '../../sourcegen' + +interface RenderEnv { + rootDirectory: string + cleanup: () => Promise +} + +async function createRenderEnv (): Promise { + const rootDirectory = await mkdtemp(path.join(tmpdir(), 'agentic-codegen-')) + return { + rootDirectory, + cleanup: () => rm(rootDirectory, { recursive: true, force: true }), + } +} + +async function renderResource (env: RenderEnv, resource: AgenticCheckResource): Promise { + const program = new Program({ + rootDirectory: env.rootDirectory, + constructFileSuffix: '.check', + specFileSuffix: '.spec', + language: 'typescript', + }) + + const codegen = new AgenticCheckCodegen(program) + const context = new Context() + + codegen.gencode(resource.id, resource, context) + + await program.realize() + + // The codegen registers exactly one generated file per call. Read it back. + const [filePath] = program.paths + if (filePath === undefined) { + throw new Error('AgenticCheckCodegen did not register any generated files') + } + + return readFile(filePath, 'utf8') +} + +const baseResource = (overrides: Partial = {}): AgenticCheckResource => ({ + id: 'agentic-check', + checkType: 'AGENTIC', + name: 'Agentic Check', + prompt: 'Verify the homepage loads.', + ...overrides, +}) + +describe('AgenticCheckCodegen', () => { + let env: RenderEnv + + beforeEach(async () => { + env = await createRenderEnv() + }) + + afterEach(async () => { + await env.cleanup() + }) + + describe('describe()', () => { + it('should describe the resource by name', () => { + const program = new Program({ + rootDirectory: env.rootDirectory, + constructFileSuffix: '.check', + specFileSuffix: '.spec', + language: 'typescript', + }) + const codegen = new AgenticCheckCodegen(program) + expect(codegen.describe(baseResource({ name: 'Homepage Check' }))) + .toEqual('Agentic Check: Homepage Check') + }) + }) + + describe('gencode()', () => { + it('should emit a minimal AgenticCheck with just a prompt', async () => { + const source = await renderResource(env, baseResource()) + + expect(source).toContain('import { AgenticCheck } from \'checkly/constructs\'') + expect(source).toContain('new AgenticCheck(\'agentic-check\'') + expect(source).toContain('name: \'Agentic Check\'') + expect(source).toContain('prompt: \'Verify the homepage loads.\'') + }) + + it('should not emit `locations` even when the backend returns one', async () => { + // The backend forces a single location for agentic checks today and + // the construct hardcodes it. Surfacing it in generated code would + // break type-checking against `AgenticCheckProps`. + const source = await renderResource(env, baseResource({ + locations: ['us-east-1'], + })) + + expect(source).not.toContain('locations') + }) + + it('should not emit `retryStrategy`', async () => { + // `AgenticCheckProps` omits `retryStrategy`, so generated code must + // never include it — not even the default `RetryStrategyBuilder.noRetries()` + // that other check types fall back to. + const source = await renderResource(env, baseResource({ + retryStrategy: { + type: 'FIXED', + baseBackoffSeconds: 60, + maxRetries: 2, + maxDurationSeconds: 600, + sameRegion: true, + }, + })) + + expect(source).not.toContain('retryStrategy') + expect(source).not.toContain('RetryStrategyBuilder') + }) + + it('should not emit `agentRuntime` when `agenticCheckData` is missing', async () => { + const source = await renderResource(env, baseResource()) + expect(source).not.toContain('agentRuntime') + }) + + it('should not emit `agentRuntime` when `agenticCheckData` is null', async () => { + const source = await renderResource(env, baseResource({ agenticCheckData: null })) + expect(source).not.toContain('agentRuntime') + }) + + it('should not emit `agentRuntime` when skills and env vars are both empty', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + skills: [], + selectedEnvironmentVariables: [], + }, + })) + expect(source).not.toContain('agentRuntime') + }) + + it('should emit `agentRuntime.skills` when skills are present', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + skills: ['checkly/playwright-skill', 'checkly/http-skill'], + }, + })) + + expect(source).toContain('agentRuntime: {') + expect(source).toContain('skills: [') + expect(source).toContain('\'checkly/playwright-skill\'') + expect(source).toContain('\'checkly/http-skill\'') + // No empty environmentVariables when none are selected. + expect(source).not.toContain('environmentVariables') + }) + + it('should drop empty skill names', async () => { + // Defensive — the backend should never store empty strings, but if it + // ever does we don't want them in generated code. + const source = await renderResource(env, baseResource({ + agenticCheckData: { + skills: ['checkly/playwright-skill', ''], + }, + })) + + expect(source).toContain('\'checkly/playwright-skill\'') + expect(source).not.toMatch(/skills:\s*\[[^\]]*''/m) + }) + + it('should emit `agentRuntime.environmentVariables` for bare-string entries', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: ['ENVIRONMENT_URL', 'API_KEY'], + }, + })) + + expect(source).toContain('environmentVariables: [') + expect(source).toContain('\'ENVIRONMENT_URL\'') + expect(source).toContain('\'API_KEY\'') + // Bare strings should not be expanded into objects. The only `name:` + // we expect in the file is the top-level `name: 'Agentic Check'`. + expect((source.match(/name:/g) ?? []).length).toEqual(1) + }) + + it('should reverse-translate `key` into `name` for object-form entries', async () => { + // The backend stores selected env vars as `{ key, description? }` but + // the construct exposes `{ name, description? }`. The codegen has to + // translate on the way out so the generated file matches the construct + // type. + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: [ + { key: 'TEST_USER_EMAIL', description: 'Login email for the test account' }, + ], + }, + })) + + expect(source).toContain('name: \'TEST_USER_EMAIL\'') + expect(source).toContain('description: \'Login email for the test account\'') + expect(source).not.toContain('key:') + }) + + it('should omit `description` when not provided on object-form entries', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: [ + { key: 'PLAIN_KEY' }, + ], + }, + })) + + expect(source).toContain('name: \'PLAIN_KEY\'') + expect(source).not.toContain('description') + }) + + it('should support a mix of bare-string and object-form env vars', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: [ + 'ENVIRONMENT_URL', + { key: 'TEST_USER_EMAIL', description: 'Login email' }, + ], + }, + })) + + expect(source).toContain('\'ENVIRONMENT_URL\'') + expect(source).toContain('name: \'TEST_USER_EMAIL\'') + expect(source).toContain('description: \'Login email\'') + }) + + it('should never emit `assertionRules`', async () => { + // Assertion rules are agent-generated and preserved server-side. The + // construct does not accept them, so generated code must not surface + // them even if the backend includes them in `agenticCheckData`. + const dataWithAssertions = { + skills: ['checkly/playwright-skill'], + assertionRules: [{ id: '1', expression: 'response.status === 200' }], + } + const source = await renderResource(env, baseResource({ + // assertionRules is intentionally absent from the typed shape; we + // want to verify the codegen ignores extras when the backend hands + // them over. + agenticCheckData: dataWithAssertions as AgenticCheckResource['agenticCheckData'], + })) + + expect(source).not.toContain('assertionRules') + }) + + it('should preserve common check fields emitted by buildCheckProps', async () => { + const source = await renderResource(env, baseResource({ + activated: true, + muted: true, + tags: ['app:webshop'], + frequency: 60, + })) + + expect(source).toContain('activated: true') + expect(source).toContain('muted: true') + expect(source).toContain('\'app:webshop\'') + expect(source).toContain('frequency: ') + }) + }) +}) diff --git a/packages/cli/src/constructs/agentic-check-codegen.ts b/packages/cli/src/constructs/agentic-check-codegen.ts index b24120577..2b9f41972 100644 --- a/packages/cli/src/constructs/agentic-check-codegen.ts +++ b/packages/cli/src/constructs/agentic-check-codegen.ts @@ -23,9 +23,6 @@ type StoredAgenticEnvironmentVariable = interface StoredAgenticCheckData { skills?: string[] | null selectedEnvironmentVariables?: StoredAgenticEnvironmentVariable[] | null - // Intentionally not consumed by codegen: - // assertionRules?: unknown[] - [key: string]: unknown } export interface AgenticCheckResource extends CheckResource { @@ -51,23 +48,13 @@ export class AgenticCheckCodegen extends Codegen { file.namedImport(construct, 'checkly/constructs') - // The `AgenticCheck` construct's props type intentionally omits - // `locations`, `privateLocations`, `runParallel`, `retryStrategy`, - // `shouldFail`, `doubleCheck`, `triggerIncident` and `groupId` — the - // platform does not yet honor any of these for agentic checks (see the - // construct file for the full rationale). The codegen has to match - // that omission: if the generated file contained any of those keys, - // it would not type-check against the construct the moment a user - // opens it. - // - // Most of the omitted fields are already conditional in - // `buildCheckProps` (they're only emitted when non-false/non-empty), - // so we pass a resource copy with `locations` cleared — the rest - // never get populated for agentic checks today anyway, and if they - // ever do, the fall-through will surface as a typecheck error in the - // generated file which is the right place to catch it. - // `retryStrategy` is the exception: `buildCheckProps` always emits - // it, so we explicitly opt out via `skipRetryStrategy`. + // `AgenticCheckProps` omits several fields that the platform does not + // yet honor (see `agentic-check.ts` for the full list and rationale). + // To keep the generated file type-checking against the construct, clear + // `locations` (which the construct hardcodes to a single value) and + // skip `retryStrategy` emission. The other omitted fields are already + // conditional in `buildCheckProps` and never populated for agentic + // checks today. const sanitizedResource: AgenticCheckResource = { ...resource, locations: undefined, @@ -109,10 +96,8 @@ function buildAgentRuntimeObject ( ): ((builder: ObjectValueBuilder) => void) | undefined { if (!data) return undefined - const skills = Array.isArray(data.skills) ? data.skills.filter(s => typeof s === 'string' && s.length > 0) : [] - const storedEnvVars = Array.isArray(data.selectedEnvironmentVariables) - ? data.selectedEnvironmentVariables - : [] + const skills = (data.skills ?? []).filter(s => s.length > 0) + const storedEnvVars = data.selectedEnvironmentVariables ?? [] if (skills.length === 0 && storedEnvVars.length === 0) { return undefined @@ -132,10 +117,10 @@ function buildAgentRuntimeObject ( for (const entry of storedEnvVars) { if (typeof entry === 'string') { arrayBuilder.string(entry) - } else if (entry && typeof entry === 'object' && typeof entry.key === 'string') { + } else { arrayBuilder.object(objectBuilder => { objectBuilder.string('name', entry.key) - if (typeof entry.description === 'string' && entry.description.length > 0) { + if (entry.description) { objectBuilder.string('description', entry.description) } }) From 06812d2a55042d4a1c85ed42974e53d3eb4a0caf Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 23:37:59 +0200 Subject: [PATCH 08/14] docs(cli): add AgenticCheck examples for AI context and boilerplate - Add an `AGENTIC_CHECK` entry to `EXAMPLE_CONFIGS` so the `prepare:ai-context` pipeline can interpolate the generated example into reference docs. - Simplify the `example-agentic-check` fixture to a single self-contained pricing-page check (no skills, no env vars). The unit tests added in the previous commit cover the more complex agentRuntime cases. - Add an `agentic.check.ts` (and the JS twin) to the boilerplate project so first-time CLI users see what an agentic check looks like alongside the existing API/URL/heartbeat examples. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../__checks__/agentic.check.js | 15 +++++++++++ .../__checks__/agentic.check.ts | 15 +++++++++++ .../cli/src/ai-context/context.fixtures.json | 26 +++---------------- packages/cli/src/ai-context/context.ts | 5 ++++ 4 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 examples/boilerplate-project-js/__checks__/agentic.check.js create mode 100644 examples/boilerplate-project/__checks__/agentic.check.ts diff --git a/examples/boilerplate-project-js/__checks__/agentic.check.js b/examples/boilerplate-project-js/__checks__/agentic.check.js new file mode 100644 index 000000000..01096c664 --- /dev/null +++ b/examples/boilerplate-project-js/__checks__/agentic.check.js @@ -0,0 +1,15 @@ +const { AgenticCheck } = require('checkly/constructs') + +// Agentic checks are AI-powered: instead of writing code, you describe what +// the check should do in natural language and the agent figures out how. +// Read more at: https://www.checklyhq.com/docs/agentic-checks/ + +new AgenticCheck('checkly-pricing-page', { + name: 'Checkly pricing page', + prompt: + 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.', + // Agentic checks currently run from a single location and follow their own + // frequency cadence (30, 60, 120, 180, 360, 720 or 1440 minutes). The + // construct hardcodes the location and validates the frequency for you. + frequency: 60, +}) diff --git a/examples/boilerplate-project/__checks__/agentic.check.ts b/examples/boilerplate-project/__checks__/agentic.check.ts new file mode 100644 index 000000000..81b5d18a3 --- /dev/null +++ b/examples/boilerplate-project/__checks__/agentic.check.ts @@ -0,0 +1,15 @@ +import { AgenticCheck } from 'checkly/constructs' + +// Agentic checks are AI-powered: instead of writing code, you describe what +// the check should do in natural language and the agent figures out how. +// Read more at: https://www.checklyhq.com/docs/agentic-checks/ + +new AgenticCheck('checkly-pricing-page', { + name: 'Checkly pricing page', + prompt: + 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.', + // Agentic checks currently run from a single location and follow their own + // frequency cadence (30, 60, 120, 180, 360, 720 or 1440 minutes). The + // construct hardcodes the location and validates the frequency for you. + frequency: 60, +}) diff --git a/packages/cli/src/ai-context/context.fixtures.json b/packages/cli/src/ai-context/context.fixtures.json index 7760d97b8..85f478d45 100644 --- a/packages/cli/src/ai-context/context.fixtures.json +++ b/packages/cli/src/ai-context/context.fixtures.json @@ -871,29 +871,11 @@ "retryStrategy": null, "runParallel": false, "triggerIncident": false, - "prompt": "Navigate to https://www.checklyhq.com and verify the homepage loads with the main heading visible. Then navigate to /pricing and confirm at least three plan tiers are displayed.", + "prompt": "Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.", "agenticCheckData": { - "skills": [ - "checkly/playwright-skill" - ], - "selectedEnvironmentVariables": [ - "ENVIRONMENT_URL", - { - "key": "TEST_USER_EMAIL", - "description": "Login email for the test account" - } - ], - "assertionRules": [ - { - "source": "JSON_BODY", - "property": "$.status", - "comparison": "EQUALS", - "target": "passing", - "order": 0, - "regex": null, - "description": "Overall check result status" - } - ] + "skills": [], + "selectedEnvironmentVariables": [], + "assertionRules": [] }, "alertChannelSubscriptions": [], "privateLocations": [] diff --git a/packages/cli/src/ai-context/context.ts b/packages/cli/src/ai-context/context.ts index 259f32d63..90b42994b 100644 --- a/packages/cli/src/ai-context/context.ts +++ b/packages/cli/src/ai-context/context.ts @@ -147,6 +147,11 @@ export default defineConfig({ `, reference: 'https://www.checklyhq.com/docs/constructs/project/', }, + AGENTIC_CHECK: { + templateString: '', + exampleConfigPath: 'resources/agentic-checks/example-agentic-check.check.ts', + reference: 'https://www.checklyhq.com/docs/constructs/agentic-check/', + }, BROWSER_CHECK: { templateString: '', exampleConfigPath: From d768e20c709d5ddf56ac480e5fc70c681b169f0e Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 23:45:00 +0200 Subject: [PATCH 09/14] test(cli): add e2e deploy round-trip for AgenticCheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `deploy-agentic-project` fixture that exercises both the plain-prompt form and the `agentRuntime` form, plus a corresponding `describe('deploy-agentic-project', ...)` block in `deploy.spec.ts`. The tests cover two scenarios: 1. `--preview` produces an import plan that lists both agentic checks under the `Create:` heading, exercising the codegen + deploy planner path without leaving any state behind. 2. `--force` actually creates the checks on the target account and reads them back via the public REST API to confirm that `checkType`, the hardcoded `us-east-1` location, and round-tripped tags all land correctly. Both tests require the e2e CI account to be entitled to agentic checks. If the account is not entitled, the tests will fail with a 402 — that's the signal to enable the entitlement, not to weaken the test. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/e2e/__tests__/deploy.spec.ts | 63 +++++++++++++++++++ .../deploy-agentic-project/agentic.check.ts | 28 +++++++++ .../deploy-agentic-project/checkly.config.ts | 6 ++ .../deploy-agentic-project/package.json | 7 +++ 4 files changed, 104 insertions(+) create mode 100644 packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts create mode 100644 packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/checkly.config.ts create mode 100644 packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/package.json diff --git a/packages/cli/e2e/__tests__/deploy.spec.ts b/packages/cli/e2e/__tests__/deploy.spec.ts index 4c2bee1cd..95a66f227 100644 --- a/packages/cli/e2e/__tests__/deploy.spec.ts +++ b/packages/cli/e2e/__tests__/deploy.spec.ts @@ -261,6 +261,69 @@ describe('deploy', { timeout: 45_000 }, () => { }) }) + describe('deploy-agentic-project', () => { + let fixt: FixtureSandbox + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'deploy-agentic-project'), + }) + }, 180_000) + + afterAll(async () => { + await fixt?.destroy() + }) + + it('Should preview an agentic check deployment', async () => { + // Use --preview so the test doesn't depend on the e2e account being + // entitled to actually create agentic checks. The plan still goes + // through full server-side validation, so we get coverage of the + // deploy round-trip without leaving resources behind. + const { stdout } = await runDeploy(fixt, ['--preview'], { + env: { + PROJECT_LOGICAL_ID: projectLogicalId, + PRIVATE_LOCATION_SLUG_NAME: privateLocationSlugname, + CHECKLY_CLI_VERSION: undefined, + }, + }) + + expect(stdout).toContain( + `Create: + AgenticCheck: agentic-pricing-check + AgenticCheck: agentic-runtime-check +`) + }) + + it('Should deploy and re-read an agentic check', async () => { + const { stderr, stdout } = await runDeploy(fixt, ['--force'], { + env: { + PROJECT_LOGICAL_ID: projectLogicalId, + PRIVATE_LOCATION_SLUG_NAME: privateLocationSlugname, + CHECKLY_CLI_VERSION: undefined, + }, + }) + + expect(stderr).toBe('') + expect(stdout).not.toContain('Notice: replacing version') + + const checks = await getAllResources('checks') + const pricingCheck = checks.find(({ name }: { name: string }) => + name === 'Agentic Pricing Check') + const runtimeCheck = checks.find(({ name }: { name: string }) => + name === 'Agentic Runtime Check') + + expect(pricingCheck).toBeDefined() + expect(pricingCheck.checkType).toEqual('AGENTIC') + // The construct hardcodes a single location for agentic checks. + expect(pricingCheck.locations).toEqual(['us-east-1']) + expect(pricingCheck.tags).toEqual(expect.arrayContaining(['e2e', 'agentic'])) + + expect(runtimeCheck).toBeDefined() + expect(runtimeCheck.checkType).toEqual('AGENTIC') + expect(runtimeCheck.locations).toEqual(['us-east-1']) + }) + }) + describe('test-only-project', () => { let fixt: FixtureSandbox diff --git a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts new file mode 100644 index 000000000..c89bbabeb --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts @@ -0,0 +1,28 @@ +/* eslint-disable */ +import { AgenticCheck, Frequency } from 'checkly/constructs' + +new AgenticCheck('agentic-pricing-check', { + name: 'Agentic Pricing Check', + prompt: + 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.', + activated: false, + muted: true, + frequency: Frequency.EVERY_1H, + tags: ['e2e', 'agentic'], +}) + +new AgenticCheck('agentic-runtime-check', { + name: 'Agentic Runtime Check', + prompt: + 'Navigate to https://www.checklyhq.com and verify the homepage loads with the main heading visible.', + activated: false, + muted: true, + frequency: 60, + agentRuntime: { + skills: ['checkly/playwright-skill'], + environmentVariables: [ + 'ENVIRONMENT_URL', + { name: 'TEST_USER_EMAIL', description: 'Login email for the test account' }, + ], + }, +}) diff --git a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/checkly.config.ts b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/checkly.config.ts new file mode 100644 index 000000000..178a7d075 --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/checkly.config.ts @@ -0,0 +1,6 @@ +const config = { + projectName: 'Deploy Agentic Project', + logicalId: process.env.PROJECT_LOGICAL_ID, + repoUrl: 'https://github.com/checkly/checkly-cli', +} +export default config diff --git a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/package.json b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/package.json new file mode 100644 index 000000000..424ed115c --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/package.json @@ -0,0 +1,7 @@ +{ + "name": "project", + "version": "1.0.0", + "dependencies": { + "jiti": "^2.6.1" + } +} From 3affc513ea50dead4e2828b4e368b1c32b0c8848 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Mon, 6 Apr 2026 23:50:06 +0200 Subject: [PATCH 10/14] docs(cli): add Agentic Check reference for the AI skill - Add `configure-agentic-checks` to the `REFERENCES` array so the `npx checkly skills` command and the generated `checkly.rules.md` expose the new construct alongside the other check types. - Add `configure-agentic-checks.md` covering the locked-down props, the restricted frequency set, the `agentRuntime` security boundary, the prompt-writing guidance, and the assertion-rules round-trip rules. The example is interpolated from `EXAMPLE_CONFIGS.AGENTIC_CHECK`. - Mention agentic checks in the `skill.md` frontmatter description so agents discover the construct from the skill summary alone. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/ai-context/context.ts | 4 ++ .../references/configure-agentic-checks.md | 45 +++++++++++++++++++ packages/cli/src/ai-context/skill.md | 2 +- 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/ai-context/references/configure-agentic-checks.md diff --git a/packages/cli/src/ai-context/context.ts b/packages/cli/src/ai-context/context.ts index 90b42994b..53be0fdb4 100644 --- a/packages/cli/src/ai-context/context.ts +++ b/packages/cli/src/ai-context/context.ts @@ -1,4 +1,8 @@ export const REFERENCES = [ + { + id: 'configure-agentic-checks', + description: 'Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists', + }, { id: 'configure-api-checks', description: 'Api Check construct (`ApiCheck`), assertions, and authentication setup scripts', diff --git a/packages/cli/src/ai-context/references/configure-agentic-checks.md b/packages/cli/src/ai-context/references/configure-agentic-checks.md new file mode 100644 index 000000000..0a780f7a7 --- /dev/null +++ b/packages/cli/src/ai-context/references/configure-agentic-checks.md @@ -0,0 +1,45 @@ +# Agentic Checks + +- Import the `AgenticCheck` construct from `checkly/constructs`. +- Agentic checks are AI-powered: instead of writing code, you describe what the check should do in natural language with the `prompt` property. The agent decides how to satisfy the prompt at runtime. +- Write prompts as concrete imperative steps, not vague goals. Tell the agent which URL to navigate to and what specific signals confirm success — for example, "Navigate to https://example.com/pricing and verify that at least three plan tiers are displayed", not "Check that pricing works". +- Keep prompts under 10000 characters. The construct will fail validation otherwise. +- **Frequency is restricted.** Only `30`, `60`, `120`, `180`, `360`, `720`, or `1440` minutes are accepted (matching `Frequency.EVERY_30M`, `EVERY_1H`, `EVERY_2H`, `EVERY_3H`, `EVERY_6H`, `EVERY_12H`, `EVERY_24H`). Anything else fails validation. +- **Locations are not configurable.** Agentic checks currently run from a single fixed location. The construct hardcodes it — do not pass `locations` or `privateLocations`. +- **Several common check fields are intentionally omitted** from `AgenticCheckProps`: `runParallel`, `retryStrategy`, `shouldFail`, `doubleCheck`, `triggerIncident`, and `groupId`. The platform does not yet honor these for agentic checks. Setting them in the construct is a TypeScript error. +- **Important:** The target URL must be publicly accessible. Checks run on Checkly's cloud infrastructure, not locally. If the user is developing against localhost, suggest a tunneling tool (ngrok, cloudflare tunnel) or a preview/staging deployment. +- **Plan-gated:** Agentic checks require the `AGENTIC_CHECKS` entitlement on the account. Run `npx checkly skills manage` to check entitlements before using. + +## `agentRuntime` — security boundary for skills and env vars + +`agentRuntime` is the explicit allowlist of resources the agent may use at execution time. Anything not declared in `agentRuntime` is **unavailable** to the agent. Treat it as a security boundary: the smaller the runtime surface, the smaller the blast radius of any prompt injection. + +```typescript +agentRuntime: { + // Skills the agent is allowed to load. Each entry is a skill ID + // installed on the account (see `npx checkly skills install`). + skills: ['checkly/playwright-skill'], + + // Environment variables the agent is allowed to read at runtime. + // Anything not listed here is hidden from the agent process — even + // if it's defined at the project or check level. + environmentVariables: [ + // Bare string form: variable name only. + 'ENVIRONMENT_URL', + // Object form: pair the variable with a description so the agent + // can decide when to read it. Descriptions are passed to the model + // and are truncated to 200 characters. + { name: 'TEST_USER_EMAIL', description: 'Login email for the test account' }, + ], +}, +``` + +- Only declare env vars the agent **needs**. Adding a variable to `environmentVariables` exposes it to the model and to anything the model invokes via skills. +- Descriptions are not just documentation — they steer the model's decisions. Use them to disambiguate variables that have non-obvious names. +- Skills must already be installed on the Checkly account (managed via the Checkly app or `npx checkly skills install`). The CLI does not validate the skill ID at deploy time. + +## Assertion rules + +The agent generates its own assertion rules on the first successful run, and the platform persists them server-side. **The CLI construct does not expose assertion rules** — do not try to set them. They survive across deploys: importing an existing agentic check via `checkly import` will not surface them in the generated TypeScript, and a subsequent deploy of that file will not erase them on the backend. + + diff --git a/packages/cli/src/ai-context/skill.md b/packages/cli/src/ai-context/skill.md index 1ba7133af..c9245af3b 100644 --- a/packages/cli/src/ai-context/skill.md +++ b/packages/cli/src/ai-context/skill.md @@ -1,6 +1,6 @@ --- name: checkly -description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. +description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with Agentic Checks, API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. allowed-tools: Bash(npx:checkly:*) Bash(npm:install:*) metadata: author: checkly From c8c922e121641460dafd54b8e7570bebf1d3f8f5 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Tue, 7 Apr 2026 08:09:05 +0200 Subject: [PATCH 11/14] fix: regen ai context and fix NO_RETRIES --- packages/cli/src/ai-context/references/manage.md | 2 +- skills/checkly/SKILL.md | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/ai-context/references/manage.md b/packages/cli/src/ai-context/references/manage.md index a14aca25a..97decd436 100644 --- a/packages/cli/src/ai-context/references/manage.md +++ b/packages/cli/src/ai-context/references/manage.md @@ -27,7 +27,7 @@ Entitlement keys follow the pattern `{CHECK_TYPE}_{FEATURE}` — match them to t | Check property | Entitlement pattern to search | What to do if disabled | |---|---|---| -| `retryStrategy` | `*_RETRY_STRATEGY_*` and `*_MAX_RETRIES_*` for your check type | Omit `retryStrategy` entirely, or use only `NO_RETRY` | +| `retryStrategy` | `*_RETRY_STRATEGY_*` and `*_MAX_RETRIES_*` for your check type | Omit `retryStrategy` entirely, or use only `NO_RETRIES` | | `runParallel` | `*_SCHEDULING_STRATEGY_PARALLEL` for your check type | Omit `runParallel` or set to `false` (use round-robin) | | `frequency` | `*_FREQ_*` for your check type | Use only frequencies where the entitlement is enabled | | `locations` | `locations.all` array | Use only locations where `available` is `true` | diff --git a/skills/checkly/SKILL.md b/skills/checkly/SKILL.md index 88dc86a96..4ba3ffb39 100644 --- a/skills/checkly/SKILL.md +++ b/skills/checkly/SKILL.md @@ -1,6 +1,6 @@ --- name: checkly -description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. +description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with Agentic Checks, API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. allowed-tools: Bash(npx:checkly:*) Bash(npm:install:*) metadata: author: checkly @@ -42,6 +42,9 @@ Learn how to initialize and set up a new Checkly CLI project from scratch. ### `npx checkly skills configure` Learn how to create and manage monitoring checks using Checkly constructs and the CLI. +#### `npx checkly skills configure agentic-checks` +Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists + #### `npx checkly skills configure api-checks` Api Check construct (`ApiCheck`), assertions, and authentication setup scripts From 29d72af5ec6593bb4619492374fd4f8ecf0f9a59 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Tue, 7 Apr 2026 08:29:18 +0200 Subject: [PATCH 12/14] docs(cli): correct AgenticCheck skill installation guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skills declared on `agentRuntime.skills` are installed on the runner via `npx skills add ` (https://skills.sh), not via the `npx checkly skills install` meta-skill installer that ships the SKILL.md file into a project. The previous JSDoc, reference doc, and example fixtures conflated the two. Updates: - `agentic-check.ts`: rewrite the `AgentRuntime.skills` JSDoc to describe the real installation path, list the three identifier forms skills.sh accepts (full URL, owner/repo, plain name), and note that the runner preloads the `playwright-cli` skill so users only need to declare extras. - `configure-agentic-checks.md`: same correction in the AI reference doc, with the example block expanded to show all three forms. - Test/example fixtures: switch the placeholder skill from the non-existent `'checkly/playwright-skill'` to `'addyosmani/web-quality-skills'` to make it visible that third-party skills published to skills.sh work — not just Checkly's own. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../deploy-agentic-project/agentic.check.ts | 2 +- .../references/configure-agentic-checks.md | 15 +++++++++++---- .../__tests__/agentic-check-codegen.spec.ts | 12 ++++++------ .../constructs/__tests__/agentic-check.spec.ts | 2 +- .../test-cases/test-agent-runtime/test.check.js | 2 +- packages/cli/src/constructs/agentic-check.ts | 15 ++++++++++++--- 6 files changed, 32 insertions(+), 16 deletions(-) diff --git a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts index c89bbabeb..16a2191aa 100644 --- a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts +++ b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts @@ -19,7 +19,7 @@ new AgenticCheck('agentic-runtime-check', { muted: true, frequency: 60, agentRuntime: { - skills: ['checkly/playwright-skill'], + skills: ['addyosmani/web-quality-skills'], environmentVariables: [ 'ENVIRONMENT_URL', { name: 'TEST_USER_EMAIL', description: 'Login email for the test account' }, diff --git a/packages/cli/src/ai-context/references/configure-agentic-checks.md b/packages/cli/src/ai-context/references/configure-agentic-checks.md index 0a780f7a7..bbd4604d3 100644 --- a/packages/cli/src/ai-context/references/configure-agentic-checks.md +++ b/packages/cli/src/ai-context/references/configure-agentic-checks.md @@ -16,9 +16,15 @@ ```typescript agentRuntime: { - // Skills the agent is allowed to load. Each entry is a skill ID - // installed on the account (see `npx checkly skills install`). - skills: ['checkly/playwright-skill'], + // Additional skills to load on top of the runner's defaults (the + // `playwright-cli` skill is preloaded automatically — you don't need + // to declare it). Each entry is passed verbatim to `npx skills add` + // on the runner, so any third-party skill published to https://skills.sh + // works — not just Checkly's own. Supported identifier forms: + // - full URL form: 'https://skills.sh/microsoft/playwright-cli/playwright-cli' + // - owner/repo form: 'addyosmani/web-quality-skills' + // - plain name: 'cost-optimization' + skills: ['addyosmani/web-quality-skills'], // Environment variables the agent is allowed to read at runtime. // Anything not listed here is hidden from the agent process — even @@ -36,7 +42,8 @@ agentRuntime: { - Only declare env vars the agent **needs**. Adding a variable to `environmentVariables` exposes it to the model and to anything the model invokes via skills. - Descriptions are not just documentation — they steer the model's decisions. Use them to disambiguate variables that have non-obvious names. -- Skills must already be installed on the Checkly account (managed via the Checkly app or `npx checkly skills install`). The CLI does not validate the skill ID at deploy time. +- The runner installs each skill via `npx skills add` at the start of every check run. The CLI does not validate the skill identifier at deploy time, so a typo will not surface until the first run. +- The `playwright-cli` skill is preloaded for every agentic check. Only declare additional skills here. ## Assertion rules diff --git a/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts index d89671cb5..25ffe7fce 100644 --- a/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts +++ b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts @@ -140,14 +140,14 @@ describe('AgenticCheckCodegen', () => { it('should emit `agentRuntime.skills` when skills are present', async () => { const source = await renderResource(env, baseResource({ agenticCheckData: { - skills: ['checkly/playwright-skill', 'checkly/http-skill'], + skills: ['addyosmani/web-quality-skills', 'cost-optimization'], }, })) expect(source).toContain('agentRuntime: {') expect(source).toContain('skills: [') - expect(source).toContain('\'checkly/playwright-skill\'') - expect(source).toContain('\'checkly/http-skill\'') + expect(source).toContain('\'addyosmani/web-quality-skills\'') + expect(source).toContain('\'cost-optimization\'') // No empty environmentVariables when none are selected. expect(source).not.toContain('environmentVariables') }) @@ -157,11 +157,11 @@ describe('AgenticCheckCodegen', () => { // ever does we don't want them in generated code. const source = await renderResource(env, baseResource({ agenticCheckData: { - skills: ['checkly/playwright-skill', ''], + skills: ['addyosmani/web-quality-skills', ''], }, })) - expect(source).toContain('\'checkly/playwright-skill\'') + expect(source).toContain('\'addyosmani/web-quality-skills\'') expect(source).not.toMatch(/skills:\s*\[[^\]]*''/m) }) @@ -231,7 +231,7 @@ describe('AgenticCheckCodegen', () => { // construct does not accept them, so generated code must not surface // them even if the backend includes them in `agenticCheckData`. const dataWithAssertions = { - skills: ['checkly/playwright-skill'], + skills: ['addyosmani/web-quality-skills'], assertionRules: [{ id: '1', expression: 'response.status === 200' }], } const source = await renderResource(env, baseResource({ diff --git a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts index 9bcfa3d84..10fc5dc50 100644 --- a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts @@ -98,7 +98,7 @@ describe('AgenticCheck', () => { payload: expect.objectContaining({ checkType: 'AGENTIC', agentRuntime: { - skills: ['checkly/playwright-skill'], + skills: ['addyosmani/web-quality-skills'], environmentVariables: [ 'API_KEY', { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js index c96d3e7b5..632c50eec 100644 --- a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js @@ -4,7 +4,7 @@ new AgenticCheck('agent-runtime-check', { name: 'Agent runtime check', prompt: 'Sign in to the test account and verify the dashboard loads.', agentRuntime: { - skills: ['checkly/playwright-skill'], + skills: ['addyosmani/web-quality-skills'], environmentVariables: [ 'API_KEY', { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, diff --git a/packages/cli/src/constructs/agentic-check.ts b/packages/cli/src/constructs/agentic-check.ts index 0df784fee..7093ae01d 100644 --- a/packages/cli/src/constructs/agentic-check.ts +++ b/packages/cli/src/constructs/agentic-check.ts @@ -77,10 +77,19 @@ export type AgentRuntimeEnvironmentVariable = */ export interface AgentRuntime { /** - * Skills the agent can use during execution. Each entry is a skill - * package name as accepted by `npx checkly skills install`. + * Additional skills to load into the agent's runtime, on top of the + * defaults the runner provides automatically (currently the + * `playwright-cli` skill is preloaded for browser automation). * - * @example ['checkly/playwright-skill'] + * Each entry is passed verbatim to `npx skills add` on the runner, so + * any third-party skill published to [skills.sh](https://skills.sh) + * works — not just Checkly's own. Supported identifier forms: + * + * - A full skills.sh URL — e.g. `'https://skills.sh/microsoft/playwright-cli/playwright-cli'` + * - A `/` shorthand — e.g. `'addyosmani/web-quality-skills'` + * - A plain skill name registered on skills.sh — e.g. `'cost-optimization'` + * + * @example ['addyosmani/web-quality-skills'] */ skills?: string[] From 8fed9947e03474fb6f9834324dd0104b5a864938 Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Tue, 7 Apr 2026 10:03:15 +0200 Subject: [PATCH 13/14] ci: temporary debug step to fingerprint the e2e Checkly account Adds a debug step before `npm run test:e2e` (ubuntu only) that prints SHA-256 hashes of the secret account ID/name plus a few non-secret fields from `/next/accounts/` and `/v1/accounts/me/entitlements`. The hashes let us verify which account the secrets resolve to without leaking the actual values in this public repo's CI logs. The entitlement block shows whether `maxAgenticChecks` bumps are landing on the account the e2e tests actually use, and the existing-AGENTIC count surfaces stale orphans that may be eating the budget. Compare locally with: node -e "console.log(require('crypto').createHash('sha256') \\ .update('').digest('hex'))" To remove once the right account is confirmed. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/test.yml | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8e4472a2f..16ef4acac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,6 +41,98 @@ jobs: - run: npm ci - run: npm run prepack - run: npm run test + # TEMP DEBUG (AI-114): print a non-secret fingerprint of the e2e + # account so we can verify which account the secrets resolve to + # without leaking the actual ID/name. Compare the SHA-256 hashes + # against `node -e "console.log(require('crypto').createHash('sha256').update('').digest('hex'))"` locally. + # Remove once the right account is confirmed. + - name: Debug — e2e account fingerprint (TEMP) + if: matrix.os == 'ubuntu-latest' + env: + CHECKLY_ACCOUNT_ID: ${{ secrets.E2E_CHECKLY_ACCOUNT_ID }} + CHECKLY_ACCOUNT_NAME: ${{ secrets.E2E_CHECKLY_ACCOUNT_NAME }} + CHECKLY_API_KEY: ${{ secrets.E2E_CHECKLY_API_KEY }} + run: | + set -e + echo "::group::E2E account fingerprint" + + ID_HASH=$(node -e "console.log(require('crypto').createHash('sha256').update(process.env.CHECKLY_ACCOUNT_ID || '').digest('hex'))") + NAME_HASH=$(node -e "console.log(require('crypto').createHash('sha256').update(process.env.CHECKLY_ACCOUNT_NAME || '').digest('hex'))") + echo "Secret CHECKLY_ACCOUNT_ID sha256: $ID_HASH" + echo "Secret CHECKLY_ACCOUNT_NAME sha256: $NAME_HASH" + + # Hit /next/accounts/ to confirm credentials are valid and + # to see what account the API actually responds with. The full + # response is captured into a variable (never echoed) and only + # the SHA-256 + a couple of non-secret fields are printed. + ACCOUNT_RESPONSE=$(curl -sf "https://api.checklyhq.com/next/accounts/$CHECKLY_ACCOUNT_ID" \ + -H "Authorization: Bearer $CHECKLY_API_KEY" \ + -H "x-checkly-account: $CHECKLY_ACCOUNT_ID" || echo '__REQUEST_FAILED__') + + if [ "$ACCOUNT_RESPONSE" = "__REQUEST_FAILED__" ]; then + echo "Account lookup FAILED — credentials may be invalid" + else + printf '%s' "$ACCOUNT_RESPONSE" | node -e " + let d='' + process.stdin.on('data',c=>d+=c).on('end',()=>{ + const a=JSON.parse(d) + const sha=s=>require('crypto').createHash('sha256').update(String(s ?? '')).digest('hex') + console.log(' /next/accounts response:') + console.log(' .id sha256: ' + sha(a.id)) + console.log(' .name sha256: ' + sha(a.name)) + console.log(' .plan: ' + (a.plan ?? '(none)')) + console.log(' .planDisplayName: ' + (a.planDisplayName ?? '(none)')) + console.log(' .runtimeId: ' + (a.runtimeId ?? '(none)')) + }) + " + fi + + # Show the live AGENTIC_CHECKS entitlement (this is what the + # deploy-time gate compares usage against). Useful to confirm + # whether the maxAgenticChecks bump landed on the right row. + ENTITLEMENTS_RESPONSE=$(curl -sf "https://api.checklyhq.com/v1/accounts/me/entitlements" \ + -H "Authorization: Bearer $CHECKLY_API_KEY" \ + -H "x-checkly-account: $CHECKLY_ACCOUNT_ID" || echo '__REQUEST_FAILED__') + + if [ "$ENTITLEMENTS_RESPONSE" = "__REQUEST_FAILED__" ]; then + echo "Entitlements lookup FAILED" + else + printf '%s' "$ENTITLEMENTS_RESPONSE" | node -e " + let d='' + process.stdin.on('data',c=>d+=c).on('end',()=>{ + const r=JSON.parse(d) + const ag=(r.entitlements||[]).find(e=>e.key==='AGENTIC_CHECKS') + console.log(' /v1/accounts/me/entitlements:') + console.log(' plan: ' + (r.plan ?? '(none)')) + if (ag) { + console.log(' AGENTIC_CHECKS.enabled: ' + ag.enabled) + console.log(' AGENTIC_CHECKS.quantity: ' + (ag.quantity ?? '(unlimited)')) + console.log(' AGENTIC_CHECKS.type: ' + ag.type) + } else { + console.log(' AGENTIC_CHECKS: (not present in entitlements)') + } + }) + " + fi + + # Count how many AGENTIC checks already exist in the account. + # If this is anywhere near the cap, stale orphans are eating + # the budget — not the bump that didn't take effect. + CHECKS_RESPONSE=$(curl -sf "https://api.checklyhq.com/v1/checks?limit=100" \ + -H "Authorization: Bearer $CHECKLY_API_KEY" \ + -H "x-checkly-account: $CHECKLY_ACCOUNT_ID" || echo '[]') + AGENTIC_COUNT=$(printf '%s' "$CHECKS_RESPONSE" | node -e " + let d='' + process.stdin.on('data',c=>d+=c).on('end',()=>{ + try { + const r=JSON.parse(d) + const list=Array.isArray(r) ? r : (r.data || []) + console.log(list.filter(c=>c.checkType==='AGENTIC').length) + } catch { console.log('parse-error') } + }) + ") + echo " Existing AGENTIC checks in account (first 100 of /v1/checks): $AGENTIC_COUNT" + echo "::endgroup::" - run: npm run test:e2e env: CHECKLY_ACCOUNT_NAME: ${{ secrets.E2E_CHECKLY_ACCOUNT_NAME }} From 13f698867bc957af0b0ec0ec566e907e3713758c Mon Sep 17 00:00:00 2001 From: Herve Labas Date: Tue, 7 Apr 2026 10:26:09 +0200 Subject: [PATCH 14/14] ci: remove temporary e2e account fingerprint debug step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fingerprint did its job — it surfaced that the secrets pointed at an account different from the one being bumped, and that the target account's `maxAgenticChecks` column was unset (the `add_agentic_check_quota_columns` migration only seeded HOBBY, STARTER, TEAM and TRIAL plan accounts, leaving CONTRACT plan rows with NULL). The right account has been bumped and the e2e tests should pass on the next run. Reverting the workflow back to its previous shape. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/test.yml | 92 -------------------------------------- 1 file changed, 92 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 16ef4acac..8e4472a2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,98 +41,6 @@ jobs: - run: npm ci - run: npm run prepack - run: npm run test - # TEMP DEBUG (AI-114): print a non-secret fingerprint of the e2e - # account so we can verify which account the secrets resolve to - # without leaking the actual ID/name. Compare the SHA-256 hashes - # against `node -e "console.log(require('crypto').createHash('sha256').update('').digest('hex'))"` locally. - # Remove once the right account is confirmed. - - name: Debug — e2e account fingerprint (TEMP) - if: matrix.os == 'ubuntu-latest' - env: - CHECKLY_ACCOUNT_ID: ${{ secrets.E2E_CHECKLY_ACCOUNT_ID }} - CHECKLY_ACCOUNT_NAME: ${{ secrets.E2E_CHECKLY_ACCOUNT_NAME }} - CHECKLY_API_KEY: ${{ secrets.E2E_CHECKLY_API_KEY }} - run: | - set -e - echo "::group::E2E account fingerprint" - - ID_HASH=$(node -e "console.log(require('crypto').createHash('sha256').update(process.env.CHECKLY_ACCOUNT_ID || '').digest('hex'))") - NAME_HASH=$(node -e "console.log(require('crypto').createHash('sha256').update(process.env.CHECKLY_ACCOUNT_NAME || '').digest('hex'))") - echo "Secret CHECKLY_ACCOUNT_ID sha256: $ID_HASH" - echo "Secret CHECKLY_ACCOUNT_NAME sha256: $NAME_HASH" - - # Hit /next/accounts/ to confirm credentials are valid and - # to see what account the API actually responds with. The full - # response is captured into a variable (never echoed) and only - # the SHA-256 + a couple of non-secret fields are printed. - ACCOUNT_RESPONSE=$(curl -sf "https://api.checklyhq.com/next/accounts/$CHECKLY_ACCOUNT_ID" \ - -H "Authorization: Bearer $CHECKLY_API_KEY" \ - -H "x-checkly-account: $CHECKLY_ACCOUNT_ID" || echo '__REQUEST_FAILED__') - - if [ "$ACCOUNT_RESPONSE" = "__REQUEST_FAILED__" ]; then - echo "Account lookup FAILED — credentials may be invalid" - else - printf '%s' "$ACCOUNT_RESPONSE" | node -e " - let d='' - process.stdin.on('data',c=>d+=c).on('end',()=>{ - const a=JSON.parse(d) - const sha=s=>require('crypto').createHash('sha256').update(String(s ?? '')).digest('hex') - console.log(' /next/accounts response:') - console.log(' .id sha256: ' + sha(a.id)) - console.log(' .name sha256: ' + sha(a.name)) - console.log(' .plan: ' + (a.plan ?? '(none)')) - console.log(' .planDisplayName: ' + (a.planDisplayName ?? '(none)')) - console.log(' .runtimeId: ' + (a.runtimeId ?? '(none)')) - }) - " - fi - - # Show the live AGENTIC_CHECKS entitlement (this is what the - # deploy-time gate compares usage against). Useful to confirm - # whether the maxAgenticChecks bump landed on the right row. - ENTITLEMENTS_RESPONSE=$(curl -sf "https://api.checklyhq.com/v1/accounts/me/entitlements" \ - -H "Authorization: Bearer $CHECKLY_API_KEY" \ - -H "x-checkly-account: $CHECKLY_ACCOUNT_ID" || echo '__REQUEST_FAILED__') - - if [ "$ENTITLEMENTS_RESPONSE" = "__REQUEST_FAILED__" ]; then - echo "Entitlements lookup FAILED" - else - printf '%s' "$ENTITLEMENTS_RESPONSE" | node -e " - let d='' - process.stdin.on('data',c=>d+=c).on('end',()=>{ - const r=JSON.parse(d) - const ag=(r.entitlements||[]).find(e=>e.key==='AGENTIC_CHECKS') - console.log(' /v1/accounts/me/entitlements:') - console.log(' plan: ' + (r.plan ?? '(none)')) - if (ag) { - console.log(' AGENTIC_CHECKS.enabled: ' + ag.enabled) - console.log(' AGENTIC_CHECKS.quantity: ' + (ag.quantity ?? '(unlimited)')) - console.log(' AGENTIC_CHECKS.type: ' + ag.type) - } else { - console.log(' AGENTIC_CHECKS: (not present in entitlements)') - } - }) - " - fi - - # Count how many AGENTIC checks already exist in the account. - # If this is anywhere near the cap, stale orphans are eating - # the budget — not the bump that didn't take effect. - CHECKS_RESPONSE=$(curl -sf "https://api.checklyhq.com/v1/checks?limit=100" \ - -H "Authorization: Bearer $CHECKLY_API_KEY" \ - -H "x-checkly-account: $CHECKLY_ACCOUNT_ID" || echo '[]') - AGENTIC_COUNT=$(printf '%s' "$CHECKS_RESPONSE" | node -e " - let d='' - process.stdin.on('data',c=>d+=c).on('end',()=>{ - try { - const r=JSON.parse(d) - const list=Array.isArray(r) ? r : (r.data || []) - console.log(list.filter(c=>c.checkType==='AGENTIC').length) - } catch { console.log('parse-error') } - }) - ") - echo " Existing AGENTIC checks in account (first 100 of /v1/checks): $AGENTIC_COUNT" - echo "::endgroup::" - run: npm run test:e2e env: CHECKLY_ACCOUNT_NAME: ${{ secrets.E2E_CHECKLY_ACCOUNT_NAME }}