diff --git a/examples/boilerplate-project-js/__checks__/agentic.check.js b/examples/boilerplate-project-js/__checks__/agentic.check.js new file mode 100644 index 000000000..01096c664 --- /dev/null +++ b/examples/boilerplate-project-js/__checks__/agentic.check.js @@ -0,0 +1,15 @@ +const { AgenticCheck } = require('checkly/constructs') + +// Agentic checks are AI-powered: instead of writing code, you describe what +// the check should do in natural language and the agent figures out how. +// Read more at: https://www.checklyhq.com/docs/agentic-checks/ + +new AgenticCheck('checkly-pricing-page', { + name: 'Checkly pricing page', + prompt: + 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.', + // Agentic checks currently run from a single location and follow their own + // frequency cadence (30, 60, 120, 180, 360, 720 or 1440 minutes). The + // construct hardcodes the location and validates the frequency for you. + frequency: 60, +}) diff --git a/examples/boilerplate-project/__checks__/agentic.check.ts b/examples/boilerplate-project/__checks__/agentic.check.ts new file mode 100644 index 000000000..81b5d18a3 --- /dev/null +++ b/examples/boilerplate-project/__checks__/agentic.check.ts @@ -0,0 +1,15 @@ +import { AgenticCheck } from 'checkly/constructs' + +// Agentic checks are AI-powered: instead of writing code, you describe what +// the check should do in natural language and the agent figures out how. +// Read more at: https://www.checklyhq.com/docs/agentic-checks/ + +new AgenticCheck('checkly-pricing-page', { + name: 'Checkly pricing page', + prompt: + 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.', + // Agentic checks currently run from a single location and follow their own + // frequency cadence (30, 60, 120, 180, 360, 720 or 1440 minutes). The + // construct hardcodes the location and validates the frequency for you. + frequency: 60, +}) diff --git a/packages/cli/e2e/__tests__/deploy.spec.ts b/packages/cli/e2e/__tests__/deploy.spec.ts index 4c2bee1cd..95a66f227 100644 --- a/packages/cli/e2e/__tests__/deploy.spec.ts +++ b/packages/cli/e2e/__tests__/deploy.spec.ts @@ -261,6 +261,69 @@ describe('deploy', { timeout: 45_000 }, () => { }) }) + describe('deploy-agentic-project', () => { + let fixt: FixtureSandbox + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'deploy-agentic-project'), + }) + }, 180_000) + + afterAll(async () => { + await fixt?.destroy() + }) + + it('Should preview an agentic check deployment', async () => { + // Use --preview so the test doesn't depend on the e2e account being + // entitled to actually create agentic checks. The plan still goes + // through full server-side validation, so we get coverage of the + // deploy round-trip without leaving resources behind. + const { stdout } = await runDeploy(fixt, ['--preview'], { + env: { + PROJECT_LOGICAL_ID: projectLogicalId, + PRIVATE_LOCATION_SLUG_NAME: privateLocationSlugname, + CHECKLY_CLI_VERSION: undefined, + }, + }) + + expect(stdout).toContain( + `Create: + AgenticCheck: agentic-pricing-check + AgenticCheck: agentic-runtime-check +`) + }) + + it('Should deploy and re-read an agentic check', async () => { + const { stderr, stdout } = await runDeploy(fixt, ['--force'], { + env: { + PROJECT_LOGICAL_ID: projectLogicalId, + PRIVATE_LOCATION_SLUG_NAME: privateLocationSlugname, + CHECKLY_CLI_VERSION: undefined, + }, + }) + + expect(stderr).toBe('') + expect(stdout).not.toContain('Notice: replacing version') + + const checks = await getAllResources('checks') + const pricingCheck = checks.find(({ name }: { name: string }) => + name === 'Agentic Pricing Check') + const runtimeCheck = checks.find(({ name }: { name: string }) => + name === 'Agentic Runtime Check') + + expect(pricingCheck).toBeDefined() + expect(pricingCheck.checkType).toEqual('AGENTIC') + // The construct hardcodes a single location for agentic checks. + expect(pricingCheck.locations).toEqual(['us-east-1']) + expect(pricingCheck.tags).toEqual(expect.arrayContaining(['e2e', 'agentic'])) + + expect(runtimeCheck).toBeDefined() + expect(runtimeCheck.checkType).toEqual('AGENTIC') + expect(runtimeCheck.locations).toEqual(['us-east-1']) + }) + }) + describe('test-only-project', () => { let fixt: FixtureSandbox diff --git a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts new file mode 100644 index 000000000..16a2191aa --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/agentic.check.ts @@ -0,0 +1,28 @@ +/* eslint-disable */ +import { AgenticCheck, Frequency } from 'checkly/constructs' + +new AgenticCheck('agentic-pricing-check', { + name: 'Agentic Pricing Check', + prompt: + 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.', + activated: false, + muted: true, + frequency: Frequency.EVERY_1H, + tags: ['e2e', 'agentic'], +}) + +new AgenticCheck('agentic-runtime-check', { + name: 'Agentic Runtime Check', + prompt: + 'Navigate to https://www.checklyhq.com and verify the homepage loads with the main heading visible.', + activated: false, + muted: true, + frequency: 60, + agentRuntime: { + skills: ['addyosmani/web-quality-skills'], + environmentVariables: [ + 'ENVIRONMENT_URL', + { name: 'TEST_USER_EMAIL', description: 'Login email for the test account' }, + ], + }, +}) diff --git a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/checkly.config.ts b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/checkly.config.ts new file mode 100644 index 000000000..178a7d075 --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/checkly.config.ts @@ -0,0 +1,6 @@ +const config = { + projectName: 'Deploy Agentic Project', + logicalId: process.env.PROJECT_LOGICAL_ID, + repoUrl: 'https://github.com/checkly/checkly-cli', +} +export default config diff --git a/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/package.json b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/package.json new file mode 100644 index 000000000..424ed115c --- /dev/null +++ b/packages/cli/e2e/__tests__/fixtures/deploy-agentic-project/package.json @@ -0,0 +1,7 @@ +{ + "name": "project", + "version": "1.0.0", + "dependencies": { + "jiti": "^2.6.1" + } +} diff --git a/packages/cli/src/ai-context/context.fixtures.json b/packages/cli/src/ai-context/context.fixtures.json index d5591564d..85f478d45 100644 --- a/packages/cli/src/ai-context/context.fixtures.json +++ b/packages/cli/src/ai-context/context.fixtures.json @@ -824,6 +824,63 @@ "privateLocations": [] } }, + { + "logicalId": "example-agentic-check", + "physicalId": "dddddddd-dddd-dddd-dddd-dddddddddddd", + "type": "check", + "member": true, + "pending": true, + "payload": { + "id": "dddddddd-dddd-dddd-dddd-dddddddddddd", + "checkType": "AGENTIC", + "name": "Example Agentic Check", + "frequency": 60, + "frequencyOffset": 0, + "activated": true, + "muted": false, + "shouldFail": false, + "locations": [ + "us-east-1" + ], + "accountId": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + "created_at": "2025-10-28T17:25:17.037Z", + "updated_at": null, + "tags": [ + "app:webshop" + ], + "alertSettings": { + "reminders": { + "amount": 0, + "interval": 5 + }, + "escalationType": "RUN_BASED", + "runBasedEscalation": { + "failedRunThreshold": 1 + }, + "timeBasedEscalation": { + "minutesFailingThreshold": 5 + }, + "parallelRunFailureThreshold": { + "enabled": false, + "percentage": 10 + } + }, + "useGlobalAlertSettings": true, + "groupId": null, + "groupOrder": null, + "retryStrategy": null, + "runParallel": false, + "triggerIncident": false, + "prompt": "Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.", + "agenticCheckData": { + "skills": [], + "selectedEnvironmentVariables": [], + "assertionRules": [] + }, + "alertChannelSubscriptions": [], + "privateLocations": [] + } + }, { "logicalId": "example-maintenance-window", "physicalId": 1000001, diff --git a/packages/cli/src/ai-context/context.ts b/packages/cli/src/ai-context/context.ts index 259f32d63..53be0fdb4 100644 --- a/packages/cli/src/ai-context/context.ts +++ b/packages/cli/src/ai-context/context.ts @@ -1,4 +1,8 @@ export const REFERENCES = [ + { + id: 'configure-agentic-checks', + description: 'Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists', + }, { id: 'configure-api-checks', description: 'Api Check construct (`ApiCheck`), assertions, and authentication setup scripts', @@ -147,6 +151,11 @@ export default defineConfig({ `, reference: 'https://www.checklyhq.com/docs/constructs/project/', }, + AGENTIC_CHECK: { + templateString: '', + exampleConfigPath: 'resources/agentic-checks/example-agentic-check.check.ts', + reference: 'https://www.checklyhq.com/docs/constructs/agentic-check/', + }, BROWSER_CHECK: { templateString: '', exampleConfigPath: diff --git a/packages/cli/src/ai-context/references/configure-agentic-checks.md b/packages/cli/src/ai-context/references/configure-agentic-checks.md new file mode 100644 index 000000000..bbd4604d3 --- /dev/null +++ b/packages/cli/src/ai-context/references/configure-agentic-checks.md @@ -0,0 +1,52 @@ +# Agentic Checks + +- Import the `AgenticCheck` construct from `checkly/constructs`. +- Agentic checks are AI-powered: instead of writing code, you describe what the check should do in natural language with the `prompt` property. The agent decides how to satisfy the prompt at runtime. +- Write prompts as concrete imperative steps, not vague goals. Tell the agent which URL to navigate to and what specific signals confirm success — for example, "Navigate to https://example.com/pricing and verify that at least three plan tiers are displayed", not "Check that pricing works". +- Keep prompts under 10000 characters. The construct will fail validation otherwise. +- **Frequency is restricted.** Only `30`, `60`, `120`, `180`, `360`, `720`, or `1440` minutes are accepted (matching `Frequency.EVERY_30M`, `EVERY_1H`, `EVERY_2H`, `EVERY_3H`, `EVERY_6H`, `EVERY_12H`, `EVERY_24H`). Anything else fails validation. +- **Locations are not configurable.** Agentic checks currently run from a single fixed location. The construct hardcodes it — do not pass `locations` or `privateLocations`. +- **Several common check fields are intentionally omitted** from `AgenticCheckProps`: `runParallel`, `retryStrategy`, `shouldFail`, `doubleCheck`, `triggerIncident`, and `groupId`. The platform does not yet honor these for agentic checks. Setting them in the construct is a TypeScript error. +- **Important:** The target URL must be publicly accessible. Checks run on Checkly's cloud infrastructure, not locally. If the user is developing against localhost, suggest a tunneling tool (ngrok, cloudflare tunnel) or a preview/staging deployment. +- **Plan-gated:** Agentic checks require the `AGENTIC_CHECKS` entitlement on the account. Run `npx checkly skills manage` to check entitlements before using. + +## `agentRuntime` — security boundary for skills and env vars + +`agentRuntime` is the explicit allowlist of resources the agent may use at execution time. Anything not declared in `agentRuntime` is **unavailable** to the agent. Treat it as a security boundary: the smaller the runtime surface, the smaller the blast radius of any prompt injection. + +```typescript +agentRuntime: { + // Additional skills to load on top of the runner's defaults (the + // `playwright-cli` skill is preloaded automatically — you don't need + // to declare it). Each entry is passed verbatim to `npx skills add` + // on the runner, so any third-party skill published to https://skills.sh + // works — not just Checkly's own. Supported identifier forms: + // - full URL form: 'https://skills.sh/microsoft/playwright-cli/playwright-cli' + // - owner/repo form: 'addyosmani/web-quality-skills' + // - plain name: 'cost-optimization' + skills: ['addyosmani/web-quality-skills'], + + // Environment variables the agent is allowed to read at runtime. + // Anything not listed here is hidden from the agent process — even + // if it's defined at the project or check level. + environmentVariables: [ + // Bare string form: variable name only. + 'ENVIRONMENT_URL', + // Object form: pair the variable with a description so the agent + // can decide when to read it. Descriptions are passed to the model + // and are truncated to 200 characters. + { name: 'TEST_USER_EMAIL', description: 'Login email for the test account' }, + ], +}, +``` + +- Only declare env vars the agent **needs**. Adding a variable to `environmentVariables` exposes it to the model and to anything the model invokes via skills. +- Descriptions are not just documentation — they steer the model's decisions. Use them to disambiguate variables that have non-obvious names. +- The runner installs each skill via `npx skills add` at the start of every check run. The CLI does not validate the skill identifier at deploy time, so a typo will not surface until the first run. +- The `playwright-cli` skill is preloaded for every agentic check. Only declare additional skills here. + +## Assertion rules + +The agent generates its own assertion rules on the first successful run, and the platform persists them server-side. **The CLI construct does not expose assertion rules** — do not try to set them. They survive across deploys: importing an existing agentic check via `checkly import` will not surface them in the generated TypeScript, and a subsequent deploy of that file will not erase them on the backend. + + diff --git a/packages/cli/src/ai-context/skill.md b/packages/cli/src/ai-context/skill.md index 1ba7133af..c9245af3b 100644 --- a/packages/cli/src/ai-context/skill.md +++ b/packages/cli/src/ai-context/skill.md @@ -1,6 +1,6 @@ --- name: checkly -description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. +description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with Agentic Checks, API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. allowed-tools: Bash(npx:checkly:*) Bash(npm:install:*) metadata: author: checkly diff --git a/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts new file mode 100644 index 000000000..25ffe7fce --- /dev/null +++ b/packages/cli/src/constructs/__tests__/agentic-check-codegen.spec.ts @@ -0,0 +1,261 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' + +import { AgenticCheckCodegen, AgenticCheckResource } from '../agentic-check-codegen' +import { Context } from '../internal/codegen/context' +import { Program } from '../../sourcegen' + +interface RenderEnv { + rootDirectory: string + cleanup: () => Promise +} + +async function createRenderEnv (): Promise { + const rootDirectory = await mkdtemp(path.join(tmpdir(), 'agentic-codegen-')) + return { + rootDirectory, + cleanup: () => rm(rootDirectory, { recursive: true, force: true }), + } +} + +async function renderResource (env: RenderEnv, resource: AgenticCheckResource): Promise { + const program = new Program({ + rootDirectory: env.rootDirectory, + constructFileSuffix: '.check', + specFileSuffix: '.spec', + language: 'typescript', + }) + + const codegen = new AgenticCheckCodegen(program) + const context = new Context() + + codegen.gencode(resource.id, resource, context) + + await program.realize() + + // The codegen registers exactly one generated file per call. Read it back. + const [filePath] = program.paths + if (filePath === undefined) { + throw new Error('AgenticCheckCodegen did not register any generated files') + } + + return readFile(filePath, 'utf8') +} + +const baseResource = (overrides: Partial = {}): AgenticCheckResource => ({ + id: 'agentic-check', + checkType: 'AGENTIC', + name: 'Agentic Check', + prompt: 'Verify the homepage loads.', + ...overrides, +}) + +describe('AgenticCheckCodegen', () => { + let env: RenderEnv + + beforeEach(async () => { + env = await createRenderEnv() + }) + + afterEach(async () => { + await env.cleanup() + }) + + describe('describe()', () => { + it('should describe the resource by name', () => { + const program = new Program({ + rootDirectory: env.rootDirectory, + constructFileSuffix: '.check', + specFileSuffix: '.spec', + language: 'typescript', + }) + const codegen = new AgenticCheckCodegen(program) + expect(codegen.describe(baseResource({ name: 'Homepage Check' }))) + .toEqual('Agentic Check: Homepage Check') + }) + }) + + describe('gencode()', () => { + it('should emit a minimal AgenticCheck with just a prompt', async () => { + const source = await renderResource(env, baseResource()) + + expect(source).toContain('import { AgenticCheck } from \'checkly/constructs\'') + expect(source).toContain('new AgenticCheck(\'agentic-check\'') + expect(source).toContain('name: \'Agentic Check\'') + expect(source).toContain('prompt: \'Verify the homepage loads.\'') + }) + + it('should not emit `locations` even when the backend returns one', async () => { + // The backend forces a single location for agentic checks today and + // the construct hardcodes it. Surfacing it in generated code would + // break type-checking against `AgenticCheckProps`. + const source = await renderResource(env, baseResource({ + locations: ['us-east-1'], + })) + + expect(source).not.toContain('locations') + }) + + it('should not emit `retryStrategy`', async () => { + // `AgenticCheckProps` omits `retryStrategy`, so generated code must + // never include it — not even the default `RetryStrategyBuilder.noRetries()` + // that other check types fall back to. + const source = await renderResource(env, baseResource({ + retryStrategy: { + type: 'FIXED', + baseBackoffSeconds: 60, + maxRetries: 2, + maxDurationSeconds: 600, + sameRegion: true, + }, + })) + + expect(source).not.toContain('retryStrategy') + expect(source).not.toContain('RetryStrategyBuilder') + }) + + it('should not emit `agentRuntime` when `agenticCheckData` is missing', async () => { + const source = await renderResource(env, baseResource()) + expect(source).not.toContain('agentRuntime') + }) + + it('should not emit `agentRuntime` when `agenticCheckData` is null', async () => { + const source = await renderResource(env, baseResource({ agenticCheckData: null })) + expect(source).not.toContain('agentRuntime') + }) + + it('should not emit `agentRuntime` when skills and env vars are both empty', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + skills: [], + selectedEnvironmentVariables: [], + }, + })) + expect(source).not.toContain('agentRuntime') + }) + + it('should emit `agentRuntime.skills` when skills are present', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + skills: ['addyosmani/web-quality-skills', 'cost-optimization'], + }, + })) + + expect(source).toContain('agentRuntime: {') + expect(source).toContain('skills: [') + expect(source).toContain('\'addyosmani/web-quality-skills\'') + expect(source).toContain('\'cost-optimization\'') + // No empty environmentVariables when none are selected. + expect(source).not.toContain('environmentVariables') + }) + + it('should drop empty skill names', async () => { + // Defensive — the backend should never store empty strings, but if it + // ever does we don't want them in generated code. + const source = await renderResource(env, baseResource({ + agenticCheckData: { + skills: ['addyosmani/web-quality-skills', ''], + }, + })) + + expect(source).toContain('\'addyosmani/web-quality-skills\'') + expect(source).not.toMatch(/skills:\s*\[[^\]]*''/m) + }) + + it('should emit `agentRuntime.environmentVariables` for bare-string entries', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: ['ENVIRONMENT_URL', 'API_KEY'], + }, + })) + + expect(source).toContain('environmentVariables: [') + expect(source).toContain('\'ENVIRONMENT_URL\'') + expect(source).toContain('\'API_KEY\'') + // Bare strings should not be expanded into objects. The only `name:` + // we expect in the file is the top-level `name: 'Agentic Check'`. + expect((source.match(/name:/g) ?? []).length).toEqual(1) + }) + + it('should reverse-translate `key` into `name` for object-form entries', async () => { + // The backend stores selected env vars as `{ key, description? }` but + // the construct exposes `{ name, description? }`. The codegen has to + // translate on the way out so the generated file matches the construct + // type. + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: [ + { key: 'TEST_USER_EMAIL', description: 'Login email for the test account' }, + ], + }, + })) + + expect(source).toContain('name: \'TEST_USER_EMAIL\'') + expect(source).toContain('description: \'Login email for the test account\'') + expect(source).not.toContain('key:') + }) + + it('should omit `description` when not provided on object-form entries', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: [ + { key: 'PLAIN_KEY' }, + ], + }, + })) + + expect(source).toContain('name: \'PLAIN_KEY\'') + expect(source).not.toContain('description') + }) + + it('should support a mix of bare-string and object-form env vars', async () => { + const source = await renderResource(env, baseResource({ + agenticCheckData: { + selectedEnvironmentVariables: [ + 'ENVIRONMENT_URL', + { key: 'TEST_USER_EMAIL', description: 'Login email' }, + ], + }, + })) + + expect(source).toContain('\'ENVIRONMENT_URL\'') + expect(source).toContain('name: \'TEST_USER_EMAIL\'') + expect(source).toContain('description: \'Login email\'') + }) + + it('should never emit `assertionRules`', async () => { + // Assertion rules are agent-generated and preserved server-side. The + // construct does not accept them, so generated code must not surface + // them even if the backend includes them in `agenticCheckData`. + const dataWithAssertions = { + skills: ['addyosmani/web-quality-skills'], + assertionRules: [{ id: '1', expression: 'response.status === 200' }], + } + const source = await renderResource(env, baseResource({ + // assertionRules is intentionally absent from the typed shape; we + // want to verify the codegen ignores extras when the backend hands + // them over. + agenticCheckData: dataWithAssertions as AgenticCheckResource['agenticCheckData'], + })) + + expect(source).not.toContain('assertionRules') + }) + + it('should preserve common check fields emitted by buildCheckProps', async () => { + const source = await renderResource(env, baseResource({ + activated: true, + muted: true, + tags: ['app:webshop'], + frequency: 60, + })) + + expect(source).toContain('activated: true') + expect(source).toContain('muted: true') + expect(source).toContain('\'app:webshop\'') + expect(source).toContain('frequency: ') + }) + }) +}) diff --git a/packages/cli/src/constructs/__tests__/agentic-check.spec.ts b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts new file mode 100644 index 000000000..10fc5dc50 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/agentic-check.spec.ts @@ -0,0 +1,316 @@ +import path from 'node:path' + +import { describe, it, expect, afterAll, beforeAll } from 'vitest' + +import { FixtureSandbox } from '../../testing/fixture-sandbox' +import { ParseProjectOutput } from '../../commands/debug/parse-project' + +async function parseProject (fixt: FixtureSandbox, ...args: string[]): Promise { + const result = await fixt.run('npx', [ + 'checkly', + 'debug', + 'parse-project', + ...args, + ]) + + if (result.exitCode !== 0) { + // eslint-disable-next-line no-console + console.error('stderr', result.stderr) + // eslint-disable-next-line no-console + console.error('stdout', result.stdout) + } + + expect(result.exitCode).toBe(0) + + const output: ParseProjectOutput = JSON.parse(result.stdout) + + return output +} + +const DEFAULT_TEST_TIMEOUT = 30_000 + +describe('AgenticCheck', () => { + let fixt: FixtureSandbox + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'agentic-check'), + }) + }, 180_000) + + afterAll(async () => { + await fixt?.destroy() + }) + + it('should create a basic agentic check with prompt', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-basic/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'homepage-health', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + name: 'Homepage Health Check', + prompt: 'Navigate to https://example.com and verify the page loads correctly.', + activated: true, + frequency: 60, + locations: ['us-east-1'], + runParallel: false, + agentRuntime: { + skills: [], + environmentVariables: [], + }, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should expose agentRuntime skills and environment variables', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-agent-runtime/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'agent-runtime-check', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + agentRuntime: { + skills: ['addyosmani/web-quality-skills'], + environmentVariables: [ + 'API_KEY', + { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, + ], + }, + }), + }), + expect.objectContaining({ + logicalId: 'default-agent-runtime', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + agentRuntime: { + skills: [], + environmentVariables: [], + }, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should default to a single us-east-1 location and ignore project-level locations', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-locations-override/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'locations-overridden', + type: 'check', + member: true, + payload: expect.objectContaining({ + checkType: 'AGENTIC', + locations: ['us-east-1'], + frequency: 30, + runParallel: false, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should apply default check settings', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-check-defaults/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'check-should-have-defaults', + type: 'check', + member: true, + payload: expect.objectContaining({ + tags: ['default tags'], + }), + }), + expect.objectContaining({ + logicalId: 'check-should-not-have-defaults', + type: 'check', + member: true, + payload: expect.objectContaining({ + tags: ['not default tags'], + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should support setting groups with `group`', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-group-mapping/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: false, + }), + payload: expect.objectContaining({ + resources: expect.arrayContaining([ + expect.objectContaining({ + logicalId: 'test-group', + type: 'check-group', + member: true, + }), + expect.objectContaining({ + logicalId: 'check', + type: 'check', + member: true, + payload: expect.objectContaining({ + groupId: { + ref: 'test-group', + }, + }), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when prompt is empty', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-missing-prompt/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"prompt" is required'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when prompt exceeds 10000 characters', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-prompt-too-long/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"prompt" must be at most 10000 characters'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when frequency is not in the supported set', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-frequency-too-low/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"frequency" must be one of 30, 60, 120, 180, 360, 720, 1440'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when an environment variable name is empty', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-empty-env-var-name/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"agentRuntime.environmentVariables[0]" must have a non-empty name'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) + + it('should fail validation when an environment variable description exceeds 200 characters', async () => { + const output = await parseProject( + fixt, + '--config', + fixt.abspath('test-cases/test-validation-description-too-long/checkly.config.js'), + ) + + expect(output).toEqual(expect.objectContaining({ + diagnostics: expect.objectContaining({ + fatal: true, + observations: expect.arrayContaining([ + expect.objectContaining({ + message: expect.stringContaining('"agentRuntime.environmentVariables[0].description" must be at most 200 characters'), + }), + ]), + }), + })) + }, DEFAULT_TEST_TIMEOUT) +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/package.json b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/package.json new file mode 100644 index 000000000..6059a0f54 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/package.json @@ -0,0 +1,5 @@ +{ + "name": "agentic-check-fixture", + "type": "module", + "devDependencies": {} +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js new file mode 100644 index 000000000..632c50eec --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-agent-runtime/test.check.js @@ -0,0 +1,18 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('agent-runtime-check', { + name: 'Agent runtime check', + prompt: 'Sign in to the test account and verify the dashboard loads.', + agentRuntime: { + skills: ['addyosmani/web-quality-skills'], + environmentVariables: [ + 'API_KEY', + { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, + ], + }, +}) + +new AgenticCheck('default-agent-runtime', { + name: 'Defaults agent runtime', + prompt: 'Verify the homepage loads.', +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js new file mode 100644 index 000000000..62b7d1de0 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-basic/test.check.js @@ -0,0 +1,8 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('homepage-health', { + name: 'Homepage Health Check', + prompt: 'Navigate to https://example.com and verify the page loads correctly.', + activated: true, + frequency: 60, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/checkly.config.js new file mode 100644 index 000000000..b3f83a62b --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/checkly.config.js @@ -0,0 +1,12 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + tags: ['default tags'], + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/test.check.js new file mode 100644 index 000000000..c2b0ef542 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-check-defaults/test.check.js @@ -0,0 +1,12 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('check-should-have-defaults', { + name: 'Check With Defaults', + prompt: 'Verify the homepage loads.', +}) + +new AgenticCheck('check-should-not-have-defaults', { + name: 'Check Without Defaults', + prompt: 'Verify the homepage loads.', + tags: ['not default tags'], +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/test.check.js new file mode 100644 index 000000000..785b5b462 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-group-mapping/test.check.js @@ -0,0 +1,11 @@ +import { AgenticCheck, CheckGroupV2 } from 'checkly/constructs' + +const group = new CheckGroupV2('test-group', { + name: 'Test Group', +}) + +new AgenticCheck('check', { + name: 'Agentic Check in Group', + prompt: 'Verify the homepage loads.', + group, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/checkly.config.js new file mode 100644 index 000000000..8a97a1718 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/checkly.config.js @@ -0,0 +1,12 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + locations: ['eu-west-1', 'ap-southeast-1'], + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/test.check.js new file mode 100644 index 000000000..31b8a6e27 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-locations-override/test.check.js @@ -0,0 +1,6 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('locations-overridden', { + name: 'Agentic check with project-default locations', + prompt: 'Verify the homepage loads.', +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/test.check.js new file mode 100644 index 000000000..d53b0f5e9 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-description-too-long/test.check.js @@ -0,0 +1,11 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('long-description', { + name: 'Description too long', + prompt: 'Verify the homepage loads.', + agentRuntime: { + environmentVariables: [ + { name: 'API_KEY', description: 'x'.repeat(201) }, + ], + }, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/test.check.js new file mode 100644 index 000000000..0969ace2d --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-empty-env-var-name/test.check.js @@ -0,0 +1,9 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('empty-env-var', { + name: 'Bad env var entry', + prompt: 'Verify the homepage loads.', + agentRuntime: { + environmentVariables: [' '], + }, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/test.check.js new file mode 100644 index 000000000..63f6c01ac --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-frequency-too-low/test.check.js @@ -0,0 +1,7 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('low-frequency', { + name: 'Low Frequency Check', + prompt: 'Verify the homepage loads.', + frequency: 15, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/test.check.js new file mode 100644 index 000000000..32ce642de --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-missing-prompt/test.check.js @@ -0,0 +1,6 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('missing-prompt', { + name: 'Missing Prompt Check', + prompt: '', +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/checkly.config.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/checkly.config.js new file mode 100644 index 000000000..45e764704 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/checkly.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Agentic Check Fixture', + logicalId: 'agentic-check-fixture', + checks: { + checkMatch: '**/*.check.js', + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/test.check.js b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/test.check.js new file mode 100644 index 000000000..2ddda6b85 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/agentic-check/test-cases/test-validation-prompt-too-long/test.check.js @@ -0,0 +1,6 @@ +import { AgenticCheck } from 'checkly/constructs' + +new AgenticCheck('long-prompt', { + name: 'Long Prompt Check', + prompt: 'x'.repeat(10001), +}) diff --git a/packages/cli/src/constructs/agentic-check-codegen.ts b/packages/cli/src/constructs/agentic-check-codegen.ts new file mode 100644 index 000000000..2b9f41972 --- /dev/null +++ b/packages/cli/src/constructs/agentic-check-codegen.ts @@ -0,0 +1,132 @@ +import { Codegen, Context } from './internal/codegen' +import { expr, ident, ObjectValueBuilder } from '../sourcegen' +import { buildCheckProps, CheckResource } from './check-codegen' + +/** + * Shape of a `selectedEnvironmentVariables` entry as stored on the backend + * inside `agenticCheckData`. The runner accepts two forms — a bare variable + * name, or an object with `key` and an optional `description`. The CLI + * construct uses `name` (not `key`), so the codegen translates object-form + * entries during emission. + */ +type StoredAgenticEnvironmentVariable = + | string + | { key: string, description?: string } + +/** + * Shape of `agenticCheckData` as stored on the backend and returned to the + * CLI during `checkly import`. Only fields the construct exposes are read. + * `assertionRules` is deliberately ignored — the agent generates those on + * the first run and the backend's deploy logic preserves them, so the CLI + * construct never needs to emit them. + */ +interface StoredAgenticCheckData { + skills?: string[] | null + selectedEnvironmentVariables?: StoredAgenticEnvironmentVariable[] | null +} + +export interface AgenticCheckResource extends CheckResource { + checkType: 'AGENTIC' + prompt: string + agenticCheckData?: StoredAgenticCheckData | null +} + +const construct = 'AgenticCheck' + +export class AgenticCheckCodegen extends Codegen { + describe (resource: AgenticCheckResource): string { + return `Agentic Check: ${resource.name}` + } + + gencode (logicalId: string, resource: AgenticCheckResource, context: Context): void { + const filePath = context.filePath('resources/agentic-checks', resource.name, { + tags: resource.tags, + unique: true, + }) + + const file = this.program.generatedConstructFile(filePath.fullPath) + + file.namedImport(construct, 'checkly/constructs') + + // `AgenticCheckProps` omits several fields that the platform does not + // yet honor (see `agentic-check.ts` for the full list and rationale). + // To keep the generated file type-checking against the construct, clear + // `locations` (which the construct hardcodes to a single value) and + // skip `retryStrategy` emission. The other omitted fields are already + // conditional in `buildCheckProps` and never populated for agentic + // checks today. + const sanitizedResource: AgenticCheckResource = { + ...resource, + locations: undefined, + } + + file.section(expr(ident(construct), builder => { + builder.new(builder => { + builder.string(logicalId) + builder.object(builder => { + builder.string('prompt', resource.prompt) + + // Emit agentRuntime only when there's something meaningful to + // carry. An imported check with no skills and no selected env + // vars would otherwise produce an empty `agentRuntime: {}` block, + // which is noise in the generated code. + const agentRuntimeValue = buildAgentRuntimeObject(resource.agenticCheckData) + if (agentRuntimeValue !== undefined) { + builder.object('agentRuntime', agentRuntimeValue) + } + + buildCheckProps(this.program, file, builder, sanitizedResource, context, { + skipRetryStrategy: true, + }) + }) + }) + })) + } +} + +/** + * Build an `agentRuntime: { ... }` object literal for the codegen output, + * reverse-translating the backend's storage shape (`selectedEnvironmentVariables` + * with `key`) into the CLI construct's shape (`environmentVariables` with + * `name`). Returns `undefined` when the input contains nothing worth + * emitting, so the caller can skip the property entirely. + */ +function buildAgentRuntimeObject ( + data: StoredAgenticCheckData | null | undefined, +): ((builder: ObjectValueBuilder) => void) | undefined { + if (!data) return undefined + + const skills = (data.skills ?? []).filter(s => s.length > 0) + const storedEnvVars = data.selectedEnvironmentVariables ?? [] + + if (skills.length === 0 && storedEnvVars.length === 0) { + return undefined + } + + return builder => { + if (skills.length > 0) { + builder.array('skills', arrayBuilder => { + for (const skill of skills) { + arrayBuilder.string(skill) + } + }) + } + + if (storedEnvVars.length > 0) { + builder.array('environmentVariables', arrayBuilder => { + for (const entry of storedEnvVars) { + if (typeof entry === 'string') { + arrayBuilder.string(entry) + } else { + arrayBuilder.object(objectBuilder => { + objectBuilder.string('name', entry.key) + if (entry.description) { + objectBuilder.string('description', entry.description) + } + }) + } + } + }) + } + } +} diff --git a/packages/cli/src/constructs/agentic-check.ts b/packages/cli/src/constructs/agentic-check.ts new file mode 100644 index 000000000..7093ae01d --- /dev/null +++ b/packages/cli/src/constructs/agentic-check.ts @@ -0,0 +1,320 @@ +import { Check, CheckProps } from './check' +import { Frequency } from './frequency' +import { Session } from './project' +import { CheckTypes } from '../constants' +import { Diagnostics } from './diagnostics' +import { InvalidPropertyValueDiagnostic } from './construct-diagnostics' + +/** + * Frequency values (in minutes) currently supported for agentic checks. + * Mirrors the values exposed in the Checkly webapp's agentic check builder. + */ +const ALLOWED_AGENTIC_FREQUENCIES = [30, 60, 120, 180, 360, 720, 1440] as const + +/** + * Frequencies (in minutes) currently supported for agentic checks: 30, 60, 120, + * 180, 360, 720 or 1440. The matching `Frequency` constants + * (`EVERY_30M`, `EVERY_1H`, `EVERY_2H`, `EVERY_3H`, `EVERY_6H`, `EVERY_12H`, + * `EVERY_24H`) are also accepted. + */ +export type AgenticCheckFrequency = + | 30 + | 60 + | 120 + | 180 + | 360 + | 720 + | 1440 + | Frequency + +/** + * The single location agentic checks currently run from. Until the platform + * supports running agentic checks from multiple locations, this value is + * forced server-side and the construct does not let users override it. + */ +const AGENTIC_CHECK_LOCATION = 'us-east-1' + +/** + * Maximum length of an environment variable description, in characters. + * Matches the truncation length applied by the agentic runner. + */ +const MAX_ENV_VAR_DESCRIPTION_LENGTH = 200 + +/** + * An environment variable the agent is permitted to read at runtime. + * + * Use the bare string form when the variable name is self-explanatory, or + * the object form to provide a `description` that helps the agent understand + * what the variable is for. Descriptions are passed to the model so it can + * make better decisions about when to read the variable. + * + * @example + * ```typescript + * 'API_KEY' + * { name: 'TOKEN_42', description: 'Feature flag service auth token' } + * ``` + */ +export type AgentRuntimeEnvironmentVariable = + | string + | { + /** The environment variable name. */ + name: string + /** + * Optional human-readable explanation of what the variable is for. + * Passed to the agent so it can decide when to read the variable. + * Truncated to {@link MAX_ENV_VAR_DESCRIPTION_LENGTH} characters. + */ + description?: string + } + +/** + * Configures the runtime context the agent has access to during a check. + * + * `agentRuntime` is the explicit allowlist of resources the agent may use + * at execution time. Anything not declared here is unavailable to the agent. + * Treat it as a security boundary: the smaller the runtime surface, the + * smaller the blast radius of any prompt injection. + */ +export interface AgentRuntime { + /** + * Additional skills to load into the agent's runtime, on top of the + * defaults the runner provides automatically (currently the + * `playwright-cli` skill is preloaded for browser automation). + * + * Each entry is passed verbatim to `npx skills add` on the runner, so + * any third-party skill published to [skills.sh](https://skills.sh) + * works — not just Checkly's own. Supported identifier forms: + * + * - A full skills.sh URL — e.g. `'https://skills.sh/microsoft/playwright-cli/playwright-cli'` + * - A `/` shorthand — e.g. `'addyosmani/web-quality-skills'` + * - A plain skill name registered on skills.sh — e.g. `'cost-optimization'` + * + * @example ['addyosmani/web-quality-skills'] + */ + skills?: string[] + + /** + * Environment variables the agent is permitted to read at runtime. + * + * **Variables not listed here are not exposed to the agent**, even if + * they exist in the Checkly account. This is the primary defense against + * prompt injection: an attacker who controls content the agent reads + * cannot exfiltrate secrets the agent never had access to. + * + * Each entry is either a bare variable name, or an object with a + * `name` and an optional `description`. Descriptions help the agent + * understand what each variable is for. + * + * @example + * ```typescript + * environmentVariables: [ + * 'API_KEY', + * { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' }, + * ] + * ``` + */ + environmentVariables?: AgentRuntimeEnvironmentVariable[] +} + +/** + * Configuration properties for {@link AgenticCheck}. + * + * Agentic checks intentionally expose only the subset of options that the + * Checkly platform currently supports for them. Properties such as + * `locations`, `privateLocations`, `runParallel`, `retryStrategy`, + * `shouldFail`, `doubleCheck`, `triggerIncident` and `groupId` are omitted + * because the platform does not yet honor them for agentic checks. They will + * be added back as additive, non-breaking changes once support lands. + */ +export interface AgenticCheckProps extends Omit { + /** + * The prompt that defines what the agentic check should verify. + * Maximum 10,000 characters. + */ + prompt: string + + /** + * How often the check should run. Agentic checks currently support a + * restricted set of frequencies. Defaults to {@link Frequency.EVERY_30M}. + * + * @example + * ```typescript + * frequency: Frequency.EVERY_1H + * // or equivalently + * frequency: 60 + * ``` + */ + frequency?: AgenticCheckFrequency + + /** + * Configures the runtime context the agent has access to during execution: + * which skills it can use, which environment variables it can read, and + * (in the future) other access surfaces such as network policies or tool + * allowlists. + * + * Treat `agentRuntime` as a security boundary. Anything not declared here + * is unavailable to the agent at runtime, which keeps the blast radius of + * any prompt injection as small as possible. + */ + agentRuntime?: AgentRuntime +} + +/** + * Creates an Agentic Check that uses AI to monitor websites and applications. + * + * Agentic checks use a prompt to define what should be verified, without + * requiring traditional scripts. The AI agent interprets the prompt and + * performs the checks. + * + * @example + * ```typescript + * new AgenticCheck('homepage-health', { + * name: 'Homepage Health Check', + * prompt: ` + * Navigate to https://example.com and verify: + * 1. The page loads with a 200 status + * 2. The main heading is visible + * 3. No console errors are present + * `, + * }) + * ``` + */ +export class AgenticCheck extends Check { + readonly prompt: string + readonly agentRuntime?: AgentRuntime + + /** + * Constructs the Agentic Check instance. + * + * @param logicalId unique project-scoped resource name identification + * @param props check configuration properties + */ + constructor (logicalId: string, props: AgenticCheckProps) { + super(logicalId, props) + this.prompt = props.prompt + this.agentRuntime = props.agentRuntime + + // Defensive overrides: even though these props are omitted from the type, + // `Check.applyConfigDefaults()` may pull them in from the project-level + // `checks` config defaults. Force them to the only values the platform + // currently honors so the construct never claims to support something + // it doesn't. + this.locations = [AGENTIC_CHECK_LOCATION] + this.privateLocations = undefined + this.runParallel = false + this.retryStrategy = undefined + this.shouldFail = undefined + this.doubleCheck = undefined + this.triggerIncident = undefined + + // Default frequency to 30m if the user did not specify one. + if (this.frequency === undefined) { + this.frequency = 30 + } + + Session.registerConstruct(this) + this.addSubscriptions() + } + + describe (): string { + return `AgenticCheck:${this.logicalId}` + } + + async validate (diagnostics: Diagnostics): Promise { + await super.validate(diagnostics) + + if (!this.prompt || this.prompt.trim().length === 0) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'prompt', + new Error('"prompt" is required and must not be empty.'), + )) + } else if (this.prompt.length > 10_000) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'prompt', + new Error(`"prompt" must be at most 10000 characters, got ${this.prompt.length}.`), + )) + } + + if (this.frequency !== undefined + && !(ALLOWED_AGENTIC_FREQUENCIES as readonly number[]).includes(this.frequency)) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'frequency', + new Error( + `"frequency" must be one of ${ALLOWED_AGENTIC_FREQUENCIES.join(', ')} ` + + `for agentic checks, got ${this.frequency}.`, + ), + )) + } + + this.validateAgentRuntime(diagnostics) + } + + // eslint-disable-next-line require-await + protected async validateAgentRuntime (diagnostics: Diagnostics): Promise { + if (this.agentRuntime?.skills) { + for (const [index, skill] of this.agentRuntime.skills.entries()) { + if (typeof skill !== 'string' || skill.trim().length === 0) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'agentRuntime.skills', + new Error( + `"agentRuntime.skills[${index}]" must be a non-empty string.`, + ), + )) + } + } + } + + if (this.agentRuntime?.environmentVariables) { + for (const [index, entry] of this.agentRuntime.environmentVariables.entries()) { + const name = typeof entry === 'string' ? entry : entry?.name + if (typeof name !== 'string' || name.trim().length === 0) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'agentRuntime.environmentVariables', + new Error( + `"agentRuntime.environmentVariables[${index}]" must have a non-empty name.`, + ), + )) + continue + } + + if (typeof entry !== 'string' + && typeof entry.description === 'string' + && entry.description.length > MAX_ENV_VAR_DESCRIPTION_LENGTH) { + diagnostics.add(new InvalidPropertyValueDiagnostic( + 'agentRuntime.environmentVariables', + new Error( + `"agentRuntime.environmentVariables[${index}].description" must be at most ` + + `${MAX_ENV_VAR_DESCRIPTION_LENGTH} characters, got ${entry.description.length}.`, + ), + )) + } + } + } + } + + synthesize () { + return { + ...super.synthesize(), + checkType: CheckTypes.AGENTIC, + prompt: this.prompt, + // Always emit `agentRuntime` so the backend has an explicit, complete + // picture of the runtime surface the user wants. The CLI is the source + // of truth: omitted skills/env vars mean "the agent should not have + // them", not "preserve whatever was there before". + agentRuntime: { + skills: this.agentRuntime?.skills ?? [], + environmentVariables: this.agentRuntime?.environmentVariables ?? [], + }, + } + } +} diff --git a/packages/cli/src/constructs/check-codegen.ts b/packages/cli/src/constructs/check-codegen.ts index 5c4b262c3..e913f94b7 100644 --- a/packages/cli/src/constructs/check-codegen.ts +++ b/packages/cli/src/constructs/check-codegen.ts @@ -1,5 +1,6 @@ import { Codegen, Context } from './internal/codegen' import { Program, ObjectValueBuilder, GeneratedFile } from '../sourcegen' +import { AgenticCheckCodegen, AgenticCheckResource } from './agentic-check-codegen' import { AlertEscalationResource, valueForAlertEscalation } from './alert-escalation-policy-codegen' import { ApiCheckCodegen, ApiCheckResource } from './api-check-codegen' import { BrowserCheckCodegen, BrowserCheckResource } from './browser-check-codegen' @@ -38,12 +39,33 @@ export interface CheckResource { runParallel?: boolean } +/** + * Options controlling which common check fields `buildCheckProps` emits. + * + * The defaults match the historical behavior — every field is emitted if + * the resource provides it. Individual check types can opt out of specific + * fields when their construct's props type does not accept them. For + * example, `AgenticCheck` omits `retryStrategy` from its props, so its + * codegen passes `skipRetryStrategy: true` to avoid emitting code that + * would not type-check against the construct. + */ +export interface BuildCheckPropsOptions { + /** + * Skip emitting the `retryStrategy` property. Unlike most fields in + * `buildCheckProps`, `retryStrategy` is emitted unconditionally (null is + * rendered as `RetryStrategyBuilder.noRetries()`), so opting out requires + * an explicit flag. + */ + skipRetryStrategy?: boolean +} + export function buildCheckProps ( program: Program, genfile: GeneratedFile, builder: ObjectValueBuilder, resource: CheckResource, context: Context, + options: BuildCheckPropsOptions = {}, ): void { builder.string('name', resource.name, { order: -1000 }) @@ -176,7 +198,9 @@ export function buildCheckProps ( builder.boolean('testOnly', resource.testOnly) } - builder.value('retryStrategy', valueForRetryStrategy(genfile, resource.retryStrategy)) + if (!options.skipRetryStrategy) { + builder.value('retryStrategy', valueForRetryStrategy(genfile, resource.retryStrategy)) + } if (resource.runParallel !== undefined && resource.runParallel !== false) { builder.boolean('runParallel', resource.runParallel) @@ -214,6 +238,7 @@ export function buildRuntimeCheckProps ( } export class CheckCodegen extends Codegen { + agenticCheckCodegen: AgenticCheckCodegen apiCheckCodegen: ApiCheckCodegen browserCheckCodegen: BrowserCheckCodegen checkGroupCodegen: CheckGroupCodegen @@ -226,6 +251,7 @@ export class CheckCodegen extends Codegen { constructor (program: Program) { super(program) + this.agenticCheckCodegen = new AgenticCheckCodegen(program) this.apiCheckCodegen = new ApiCheckCodegen(program) this.browserCheckCodegen = new BrowserCheckCodegen(program) this.checkGroupCodegen = new CheckGroupCodegen(program) @@ -241,6 +267,8 @@ export class CheckCodegen extends Codegen { const { checkType } = resource switch (checkType) { + case 'AGENTIC': + return this.agenticCheckCodegen.describe(resource as AgenticCheckResource) case 'BROWSER': return this.browserCheckCodegen.describe(resource as BrowserCheckResource) case 'API': @@ -266,6 +294,9 @@ export class CheckCodegen extends Codegen { const { checkType } = resource switch (checkType) { + case 'AGENTIC': + this.agenticCheckCodegen.gencode(logicalId, resource as AgenticCheckResource, context) + return case 'BROWSER': this.browserCheckCodegen.gencode(logicalId, resource as BrowserCheckResource, context) return diff --git a/packages/cli/src/constructs/index.ts b/packages/cli/src/constructs/index.ts index 914ffe95c..e2bee57f9 100644 --- a/packages/cli/src/constructs/index.ts +++ b/packages/cli/src/constructs/index.ts @@ -48,3 +48,4 @@ export * from './dns-request' export * from './icmp-monitor' export * from './icmp-assertion' export * from './icmp-request' +export * from './agentic-check' diff --git a/packages/cli/src/rest/analytics.ts b/packages/cli/src/rest/analytics.ts index 4d7279539..fb052fed4 100644 --- a/packages/cli/src/rest/analytics.ts +++ b/packages/cli/src/rest/analytics.ts @@ -19,6 +19,7 @@ const checkTypeToPath: Partial> = { [CheckTypes.ICMP]: 'icmp', [CheckTypes.DNS]: 'dns', [CheckTypes.URL]: 'url-monitors', + [CheckTypes.AGENTIC]: 'agentic-checks', } // Default aggregated metrics per check type @@ -32,6 +33,7 @@ const defaultMetrics: Partial> = { [CheckTypes.DNS]: ['availability', 'total_avg', 'total_p50', 'total_p95', 'total_p99'], [CheckTypes.ICMP]: ['availability', 'packetLoss_avg', 'latencyAvg_avg', 'latencyAvg_p50', 'latencyAvg_p95', 'latencyAvg_p99'], [CheckTypes.HEARTBEAT]: ['availability'], + [CheckTypes.AGENTIC]: ['availability'], } export type GroupBy = 'runLocation' | 'statusCode' diff --git a/skills/checkly/SKILL.md b/skills/checkly/SKILL.md index 88dc86a96..4ba3ffb39 100644 --- a/skills/checkly/SKILL.md +++ b/skills/checkly/SKILL.md @@ -1,6 +1,6 @@ --- name: checkly -description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. +description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with Agentic Checks, API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages. allowed-tools: Bash(npx:checkly:*) Bash(npm:install:*) metadata: author: checkly @@ -42,6 +42,9 @@ Learn how to initialize and set up a new Checkly CLI project from scratch. ### `npx checkly skills configure` Learn how to create and manage monitoring checks using Checkly constructs and the CLI. +#### `npx checkly skills configure agentic-checks` +Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists + #### `npx checkly skills configure api-checks` Api Check construct (`ApiCheck`), assertions, and authentication setup scripts