From 679650eaefbe4e23de5e3070cba45ea524a2350a Mon Sep 17 00:00:00 2001 From: stefan judis Date: Fri, 22 May 2026 14:58:56 +0200 Subject: [PATCH 1/7] chore: ignore env files and Playwright test output --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index aafcb34..1c7a0dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ node_modules/ .DS_Store *.log +.env +test-results From 8e26a5b877fbb40ce1e4d6ea8ad0608b6c02ef42 Mon Sep 17 00:00:00 2001 From: stefan judis Date: Fri, 22 May 2026 14:59:08 +0200 Subject: [PATCH 2/7] refactor: run skill sync check as PlaywrightCheck against local files Switches the skill sync check from MultiStepCheck (which compared two remote raw URLs) to PlaywrightCheck pointed at the repo's playwright.config.ts. The spec now diffs the checked-in skill files against the upstream source resolved from skills.config.ts via buildRawUrl, so the check fails the moment the deployed bundle drifts from upstream. Also groups the check under a new ai-context CheckGroupV2 so future AI-context checks share alerting and tagging. --- checks/ai-context.group.ts | 5 ++++ checks/checkly-skill-sync.check.ts | 16 +++++------ checks/checkly-skill-sync.spec.ts | 43 ++++++++++++++++++------------ playwright.config.ts | 6 +++++ tsconfig.json | 1 + 5 files changed, 45 insertions(+), 26 deletions(-) create mode 100644 checks/ai-context.group.ts create mode 100644 playwright.config.ts diff --git a/checks/ai-context.group.ts b/checks/ai-context.group.ts new file mode 100644 index 0000000..bb927cf --- /dev/null +++ b/checks/ai-context.group.ts @@ -0,0 +1,5 @@ +import { CheckGroupV2 } from "checkly/constructs"; + +export const aiContextGroup = new CheckGroupV2("ai-context", { + name: "checkly-plugin & AI-Context", +}); diff --git a/checks/checkly-skill-sync.check.ts b/checks/checkly-skill-sync.check.ts index d6e81f2..4609fa8 100644 --- a/checks/checkly-skill-sync.check.ts +++ b/checks/checkly-skill-sync.check.ts @@ -1,16 +1,14 @@ -import { MultiStepCheck, RetryStrategyBuilder } from "checkly/constructs"; +import { PlaywrightCheck } from "checkly/constructs"; +import { aiContextGroup } from "./ai-context.group.ts"; -new MultiStepCheck("checkly-skill-sync", { +new PlaywrightCheck("checkly-skill-sync", { name: "Checkly skill sync", description: "Compares the Checkly skill included in checkly/checkly-plugin with the source skill in checkly/checkly-cli.", - code: { - entrypoint: "./checkly-skill-sync.spec.ts", - }, + playwrightConfigPath: "../playwright.config.ts", + include: ["skills/checkly/**"], activated: true, - retryStrategy: RetryStrategyBuilder.singleRetry({ - baseBackoffSeconds: 60, - sameRegion: true, - }), runParallel: false, + // alertChannels: [AlertChannel.fromId(287691)], + group: aiContextGroup, }); diff --git a/checks/checkly-skill-sync.spec.ts b/checks/checkly-skill-sync.spec.ts index c4f4d8f..e1539e1 100644 --- a/checks/checkly-skill-sync.spec.ts +++ b/checks/checkly-skill-sync.spec.ts @@ -1,9 +1,11 @@ import { expect, test, type APIRequestContext } from "@playwright/test"; +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { config } from "../skills.config.ts"; +import { buildRawUrl } from "../scripts/sync.ts"; -const files = ["SKILL.md", "README.md"] as const; -const includedBaseUrl = - "https://raw.githubusercontent.com/checkly/checkly-plugin/main/skills/checkly"; -const originalBaseUrl = "https://raw.githubusercontent.com/checkly/checkly-cli/main/skills/checkly"; +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); async function fetchText(request: APIRequestContext, url: string) { const response = await request.get(url); @@ -13,17 +15,24 @@ async function fetchText(request: APIRequestContext, url: string) { return response.text(); } -test.describe("included Checkly skill", () => { - for (const file of files) { - test(`${file} matches the checkly-cli source`, async ({ request }) => { - const includedUrl = `${includedBaseUrl}/${file}`; - const originalUrl = `${originalBaseUrl}/${file}`; - const [included, original] = await Promise.all([ - fetchText(request, includedUrl), - fetchText(request, originalUrl), - ]); +for (const skill of config.skills) { + test.describe(`included ${skill.name} skill`, () => { + for (const file of skill.files) { + test(`${file} matches the ${skill.source.repo} source`, async ({ request }) => { + const localPath = resolve(repoRoot, "skills", skill.name, file); + const upstreamUrl = buildRawUrl({ + repo: skill.source.repo, + ref: skill.source.ref, + path: skill.source.path, + file, + }); + const [local, upstream] = await Promise.all([ + readFile(localPath, "utf8"), + fetchText(request, upstreamUrl), + ]); - expect(included, `${includedUrl} should match ${originalUrl}`).toBe(original); - }); - } -}); + expect(local, `${localPath} should match ${upstreamUrl}`).toBe(upstream); + }); + } + }); +} diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..a492f95 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: "./checks", + retries: process.env.CHECKLY ? 1 : 0, +}); diff --git a/tsconfig.json b/tsconfig.json index 00eb55b..aabb791 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,6 +16,7 @@ "include": [ "checkly.config.ts", "checks/**/*.ts", + "playwright.config.ts", "skills.config.ts", "scripts/**/*.ts", "test/**/*.ts" From 91b87da283e03faf532ccc802b9bd6ddfc8eb882 Mon Sep 17 00:00:00 2001 From: stefan judis Date: Fri, 22 May 2026 14:59:19 +0200 Subject: [PATCH 3/7] ci: add verify workflow for PRs and pushes to main Runs lint, format check, typecheck, and unit tests so PRs cannot merge red and pushes to main are guarded with the same gate as the local pre-commit hook. --- .github/workflows/verify.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/verify.yml diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..ca877bf --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,32 @@ +name: Verify + +on: + push: + branches: [main] + pull_request: + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Lint + run: npm run lint + + - name: Format check + run: npm run format:check + + - name: Typecheck + run: npm run typecheck + + - name: Unit tests + run: npm test From 84228437f30cca8cb39a0860c43b9096f6f99753 Mon Sep 17 00:00:00 2001 From: stefan judis Date: Fri, 22 May 2026 14:59:30 +0200 Subject: [PATCH 4/7] ci: add Checkly deploy workflow On push to main, re-runs validation as a safety net, executes "npx checkly test --record" against Checkly infra, then deploys with "npx checkly deploy --force". A deploy-checkly concurrency group prevents overlapping deploys from racing the API. Requires CHECKLY_API_KEY and CHECKLY_ACCOUNT_ID secrets in the repo Actions settings. --- .github/workflows/deploy.yml | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..527c4f6 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,44 @@ +name: Deploy Checkly monitoring + +on: + push: + branches: [main] + +concurrency: + group: deploy-checkly + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + env: + CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }} + CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Lint + run: npm run lint + + - name: Format check + run: npm run format:check + + - name: Typecheck + run: npm run typecheck + + - name: Unit tests + run: npm test + + - name: Run Checkly checks + run: npx checkly test --record + + - name: Deploy to Checkly + run: npx checkly deploy --force From 652e9ed8e4c10e3e9ad2fec0807ac5abbfdba26d Mon Sep 17 00:00:00 2001 From: stefan judis Date: Fri, 22 May 2026 15:02:53 +0200 Subject: [PATCH 5/7] chore: re-enable alert channel on skill sync check --- checks/checkly-skill-sync.check.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checks/checkly-skill-sync.check.ts b/checks/checkly-skill-sync.check.ts index 4609fa8..a8ceafa 100644 --- a/checks/checkly-skill-sync.check.ts +++ b/checks/checkly-skill-sync.check.ts @@ -1,4 +1,4 @@ -import { PlaywrightCheck } from "checkly/constructs"; +import { AlertChannel, PlaywrightCheck } from "checkly/constructs"; import { aiContextGroup } from "./ai-context.group.ts"; new PlaywrightCheck("checkly-skill-sync", { @@ -9,6 +9,6 @@ new PlaywrightCheck("checkly-skill-sync", { include: ["skills/checkly/**"], activated: true, runParallel: false, - // alertChannels: [AlertChannel.fromId(287691)], + alertChannels: [AlertChannel.fromId(287691)], group: aiContextGroup, }); From 8b484d03ddd7b06c2aabe82d240bef52aea5bcfd Mon Sep 17 00:00:00 2001 From: stefan judis Date: Fri, 22 May 2026 15:05:52 +0200 Subject: [PATCH 6/7] style: sort imports and quote skill name in describe title --- checks/checkly-skill-sync.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checks/checkly-skill-sync.spec.ts b/checks/checkly-skill-sync.spec.ts index e1539e1..0c2058a 100644 --- a/checks/checkly-skill-sync.spec.ts +++ b/checks/checkly-skill-sync.spec.ts @@ -2,8 +2,8 @@ import { expect, test, type APIRequestContext } from "@playwright/test"; import { readFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { config } from "../skills.config.ts"; import { buildRawUrl } from "../scripts/sync.ts"; +import { config } from "../skills.config.ts"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -16,7 +16,7 @@ async function fetchText(request: APIRequestContext, url: string) { } for (const skill of config.skills) { - test.describe(`included ${skill.name} skill`, () => { + test.describe(`included '${skill.name}' skill`, () => { for (const file of skill.files) { test(`${file} matches the ${skill.source.repo} source`, async ({ request }) => { const localPath = resolve(repoRoot, "skills", skill.name, file); From 3364baa39306ca0f577c1dc2aae5d31eaad12412 Mon Sep 17 00:00:00 2001 From: stefan judis Date: Fri, 22 May 2026 15:13:33 +0200 Subject: [PATCH 7/7] chore: rename Checkly project to "Checkly Plugin and AI Context" The project now covers more than skill-sync monitoring (see the new ai-context CheckGroupV2), so the name and logicalId no longer fit. Renames projectName and bumps logicalId to checkly-plugin-and-ai-context so future AI-context checks deploy under the same umbrella. Note: this orphans the old checkly-plugin-skills-monitoring project in the Checkly account on next deploy. Destroy it manually if no longer needed. --- checkly.config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checkly.config.ts b/checkly.config.ts index b8eab88..422f7a5 100644 --- a/checkly.config.ts +++ b/checkly.config.ts @@ -2,8 +2,8 @@ import { defineConfig } from "checkly"; import { Frequency } from "checkly/constructs"; export default defineConfig({ - projectName: "Checkly Plugin Skills Monitoring", - logicalId: "checkly-plugin-skills-monitoring", + projectName: "Checkly Plugin and AI Context", + logicalId: "checkly-plugin-and-ai-context", repoUrl: "https://github.com/checkly/checkly-plugin", checks: { activated: true,