diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
new file mode 100644
index 0000000..527c4f6
--- /dev/null
+++ b/.github/workflows/deploy.yml
@@ -0,0 +1,44 @@
+name: Deploy Checkly monitoring
+
+on:
+  push:
+    branches: [main]
+
+concurrency:
+  group: deploy-checkly
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    env:
+      CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
+      CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Format check
+        run: npm run format:check
+
+      - name: Typecheck
+        run: npm run typecheck
+
+      - name: Unit tests
+        run: npm test
+
+      - name: Run Checkly checks
+        run: npx checkly test --record
+
+      - name: Deploy to Checkly
+        run: npx checkly deploy --force
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
new file mode 100644
index 0000000..ca877bf
--- /dev/null
+++ b/.github/workflows/verify.yml
@@ -0,0 +1,32 @@
+name: Verify
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Format check
+        run: npm run format:check
+
+      - name: Typecheck
+        run: npm run typecheck
+
+      - name: Unit tests
+        run: npm test
diff --git a/.gitignore b/.gitignore
index aafcb34..1c7a0dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 node_modules/
 .DS_Store
 *.log
+.env
+test-results
diff --git a/checkly.config.ts b/checkly.config.ts
index b8eab88..422f7a5 100644
--- a/checkly.config.ts
+++ b/checkly.config.ts
@@ -2,8 +2,8 @@ import { defineConfig } from "checkly";
 import { Frequency } from "checkly/constructs";
 
 export default defineConfig({
-  projectName: "Checkly Plugin Skills Monitoring",
-  logicalId: "checkly-plugin-skills-monitoring",
+  projectName: "Checkly Plugin and AI Context",
+  logicalId: "checkly-plugin-and-ai-context",
   repoUrl: "https://github.com/checkly/checkly-plugin",
   checks: {
     activated: true,
diff --git a/checks/ai-context.group.ts b/checks/ai-context.group.ts
new file mode 100644
index 0000000..bb927cf
--- /dev/null
+++ b/checks/ai-context.group.ts
@@ -0,0 +1,5 @@
+import { CheckGroupV2 } from "checkly/constructs";
+
+export const aiContextGroup = new CheckGroupV2("ai-context", {
+  name: "checkly-plugin & AI-Context",
+});
diff --git a/checks/checkly-skill-sync.check.ts b/checks/checkly-skill-sync.check.ts
index d6e81f2..a8ceafa 100644
--- a/checks/checkly-skill-sync.check.ts
+++ b/checks/checkly-skill-sync.check.ts
@@ -1,16 +1,14 @@
-import { MultiStepCheck, RetryStrategyBuilder } from "checkly/constructs";
+import { AlertChannel, PlaywrightCheck } from "checkly/constructs";
+import { aiContextGroup } from "./ai-context.group.ts";
 
-new MultiStepCheck("checkly-skill-sync", {
+new PlaywrightCheck("checkly-skill-sync", {
   name: "Checkly skill sync",
   description:
     "Compares the Checkly skill included in checkly/checkly-plugin with the source skill in checkly/checkly-cli.",
-  code: {
-    entrypoint: "./checkly-skill-sync.spec.ts",
-  },
+  playwrightConfigPath: "../playwright.config.ts",
+  include: ["skills/checkly/**"],
   activated: true,
-  retryStrategy: RetryStrategyBuilder.singleRetry({
-    baseBackoffSeconds: 60,
-    sameRegion: true,
-  }),
   runParallel: false,
+  alertChannels: [AlertChannel.fromId(287691)],
+  group: aiContextGroup,
 });
diff --git a/checks/checkly-skill-sync.spec.ts b/checks/checkly-skill-sync.spec.ts
index c4f4d8f..0c2058a 100644
--- a/checks/checkly-skill-sync.spec.ts
+++ b/checks/checkly-skill-sync.spec.ts
@@ -1,9 +1,11 @@
 import { expect, test, type APIRequestContext } from "@playwright/test";
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { buildRawUrl } from "../scripts/sync.ts";
+import { config } from "../skills.config.ts";
 
-const files = ["SKILL.md", "README.md"] as const;
-const includedBaseUrl =
-  "https://raw.githubusercontent.com/checkly/checkly-plugin/main/skills/checkly";
-const originalBaseUrl = "https://raw.githubusercontent.com/checkly/checkly-cli/main/skills/checkly";
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 
 async function fetchText(request: APIRequestContext, url: string) {
   const response = await request.get(url);
@@ -13,17 +15,24 @@ async function fetchText(request: APIRequestContext, url: string) {
   return response.text();
 }
 
-test.describe("included Checkly skill", () => {
-  for (const file of files) {
-    test(`${file} matches the checkly-cli source`, async ({ request }) => {
-      const includedUrl = `${includedBaseUrl}/${file}`;
-      const originalUrl = `${originalBaseUrl}/${file}`;
-      const [included, original] = await Promise.all([
-        fetchText(request, includedUrl),
-        fetchText(request, originalUrl),
-      ]);
+for (const skill of config.skills) {
+  test.describe(`included '${skill.name}' skill`, () => {
+    for (const file of skill.files) {
+      test(`${file} matches the ${skill.source.repo} source`, async ({ request }) => {
+        const localPath = resolve(repoRoot, "skills", skill.name, file);
+        const upstreamUrl = buildRawUrl({
+          repo: skill.source.repo,
+          ref: skill.source.ref,
+          path: skill.source.path,
+          file,
+        });
+        const [local, upstream] = await Promise.all([
+          readFile(localPath, "utf8"),
+          fetchText(request, upstreamUrl),
+        ]);
 
-      expect(included, `${includedUrl} should match ${originalUrl}`).toBe(original);
-    });
-  }
-});
+        expect(local, `${localPath} should match ${upstreamUrl}`).toBe(upstream);
+      });
+    }
+  });
+}
diff --git a/playwright.config.ts b/playwright.config.ts
new file mode 100644
index 0000000..a492f95
--- /dev/null
+++ b/playwright.config.ts
@@ -0,0 +1,6 @@
+import { defineConfig } from "@playwright/test";
+
+export default defineConfig({
+  testDir: "./checks",
+  retries: process.env.CHECKLY ? 1 : 0,
+});
diff --git a/tsconfig.json b/tsconfig.json
index 00eb55b..aabb791 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -16,6 +16,7 @@
   "include": [
     "checkly.config.ts",
     "checks/**/*.ts",
+    "playwright.config.ts",
     "skills.config.ts",
     "scripts/**/*.ts",
     "test/**/*.ts"