diff --git a/.prettierignore b/.prettierignore index 4801a9547..bd211fdcd 100644 --- a/.prettierignore +++ b/.prettierignore @@ -22,3 +22,6 @@ bazel/map-size-tracking/test/size-golden.json **/test/goldens/**/*.md **/pnpm-lock.yaml + +# AI Skills intentionally not formatted to ensure validation failures. +ng-dev/ai/skills/fixtures/invalid-frontmatter-location/skills/test-skill/SKILL.md diff --git a/ng-dev/ai/BUILD.bazel b/ng-dev/ai/BUILD.bazel index 1ded3d988..dc9df06ef 100644 --- a/ng-dev/ai/BUILD.bazel +++ b/ng-dev/ai/BUILD.bazel @@ -14,6 +14,7 @@ ts_project( "//ng-dev:node_modules/cli-progress", "//ng-dev:node_modules/fast-glob", "//ng-dev:node_modules/yargs", + "//ng-dev/ai/skills", "//ng-dev/utils", ], ) diff --git a/ng-dev/ai/cli.ts b/ng-dev/ai/cli.ts index 78b97bd21..9f1b099a6 100644 --- a/ng-dev/ai/cli.ts +++ b/ng-dev/ai/cli.ts @@ -8,8 +8,9 @@ import {Argv} from 'yargs'; import {MigrateModule} from './migrate.js'; import {FixModule} from './fix.js'; +import {SkillsModule} from './skills/cli.js'; /** Build the parser for the AI commands. */ export function buildAiParser(localYargs: Argv) { - return localYargs.command(MigrateModule).command(FixModule); + return localYargs.command(MigrateModule).command(FixModule).command(SkillsModule); } diff --git a/ng-dev/ai/skills/BUILD.bazel b/ng-dev/ai/skills/BUILD.bazel new file mode 100644 index 000000000..40da01c7c --- /dev/null +++ b/ng-dev/ai/skills/BUILD.bazel @@ -0,0 +1,39 @@ +load("//tools:defaults.bzl", "jasmine_test", "ts_project") + +ts_project( + name = "skills", + srcs = glob( + ["**/*.ts"], + exclude = ["**/*.spec.ts"], + ), + visibility = ["//ng-dev:__subpackages__"], + deps = [ + "//ng-dev:node_modules/@types/node", + "//ng-dev:node_modules/@types/yargs", + "//ng-dev:node_modules/fast-glob", + "//ng-dev:node_modules/yaml", + "//ng-dev:node_modules/yargs", + "//ng-dev:node_modules/zod", + "//ng-dev/utils", + ], +) + +ts_project( + name = "test_lib", + testonly = True, + srcs = ["validate.spec.ts"], + deps = [ + ":skills", + "//ng-dev:node_modules/@types/jasmine", + "//ng-dev:node_modules/@types/node", + "//ng-dev/utils", + "//ng-dev/utils/testing", + ], +) + +jasmine_test( + name = "test", + data = glob(["fixtures/**"]) + [ + ":test_lib", + ], +) diff --git a/ng-dev/ai/skills/cli.ts b/ng-dev/ai/skills/cli.ts new file mode 100644 index 000000000..c2513b07a --- /dev/null +++ b/ng-dev/ai/skills/cli.ts @@ -0,0 +1,39 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Arguments, Argv, CommandModule} from 'yargs'; + +import {validateSkills} from './validate.js'; +import {determineRepoBaseDirFromCwd} from '../../utils/repo-directory.js'; + +interface Options { + baseDir: string; +} + +async function builder(yargs: Argv) { + return yargs.option('base-dir' as 'baseDir', { + type: 'string', + default: determineRepoBaseDirFromCwd(), + hidden: true, + description: 'The base directory to look for skills in', + }); +} + +async function handler({baseDir}: Arguments) { + process.exitCode = (await validateSkills(baseDir)).exitCode; +} + +/** + * Validates all skills found in the `skills/` directory. + */ +export const SkillsModule: CommandModule<{}, Options> = { + command: 'skills validate', + describe: 'Validate agent skills in the repository', + builder, + handler, +}; diff --git a/ng-dev/ai/skills/fixtures/complex-skill/skills/complex-skill/SKILL.md b/ng-dev/ai/skills/fixtures/complex-skill/skills/complex-skill/SKILL.md new file mode 100644 index 000000000..40de8e835 --- /dev/null +++ b/ng-dev/ai/skills/fixtures/complex-skill/skills/complex-skill/SKILL.md @@ -0,0 +1,11 @@ +--- +name: complex-skill +description: Valid description +license: MIT +compatibility: Any +metadata: + key: value +allowed-tools: tool1 +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/invalid-frontmatter-location/skills/test-skill/SKILL.md b/ng-dev/ai/skills/fixtures/invalid-frontmatter-location/skills/test-skill/SKILL.md new file mode 100644 index 000000000..fc11542af --- /dev/null +++ b/ng-dev/ai/skills/fixtures/invalid-frontmatter-location/skills/test-skill/SKILL.md @@ -0,0 +1,7 @@ + +--- +name: test-skill +description: Valid description +license: MIT +--- +Instruction diff --git a/ng-dev/ai/skills/fixtures/invalid-name-format/skills/InvalidName/SKILL.md b/ng-dev/ai/skills/fixtures/invalid-name-format/skills/InvalidName/SKILL.md new file mode 100644 index 000000000..c13fdb6e9 --- /dev/null +++ b/ng-dev/ai/skills/fixtures/invalid-name-format/skills/InvalidName/SKILL.md @@ -0,0 +1,7 @@ +--- +name: InvalidName +description: Valid description +license: MIT +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/invalid-schema/skills/bad-schema/SKILL.md b/ng-dev/ai/skills/fixtures/invalid-schema/skills/bad-schema/SKILL.md new file mode 100644 index 000000000..2ef2253eb --- /dev/null +++ b/ng-dev/ai/skills/fixtures/invalid-schema/skills/bad-schema/SKILL.md @@ -0,0 +1,6 @@ +--- +name: bad-schema +description: Valid description +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/missing-frontmatter/skills/bad-skill/SKILL.md b/ng-dev/ai/skills/fixtures/missing-frontmatter/skills/bad-skill/SKILL.md new file mode 100644 index 000000000..5c62b58e4 --- /dev/null +++ b/ng-dev/ai/skills/fixtures/missing-frontmatter/skills/bad-skill/SKILL.md @@ -0,0 +1 @@ +Just text diff --git a/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill1/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill1/SKILL.md new file mode 100644 index 000000000..124ebcd8b --- /dev/null +++ b/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill1/SKILL.md @@ -0,0 +1,7 @@ +--- +name: skill1 +description: Valid description +license: MIT +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill2/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill2/SKILL.md new file mode 100644 index 000000000..749c8030b --- /dev/null +++ b/ng-dev/ai/skills/fixtures/multiple-mixed/skills/skill2/SKILL.md @@ -0,0 +1,13 @@ +--- +name: skill2 +description: Invalid description +license: MIT +allowed-tools: + - should + - have + - been + - a + - string +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill1/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill1/SKILL.md new file mode 100644 index 000000000..124ebcd8b --- /dev/null +++ b/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill1/SKILL.md @@ -0,0 +1,7 @@ +--- +name: skill1 +description: Valid description +license: MIT +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill2/SKILL.md b/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill2/SKILL.md new file mode 100644 index 000000000..ff81b8924 --- /dev/null +++ b/ng-dev/ai/skills/fixtures/multiple-valid/skills/skill2/SKILL.md @@ -0,0 +1,7 @@ +--- +name: skill2 +description: Valid description +license: MIT +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/name-mismatch/skills/wrong-name/SKILL.md b/ng-dev/ai/skills/fixtures/name-mismatch/skills/wrong-name/SKILL.md new file mode 100644 index 000000000..11cc0cc48 --- /dev/null +++ b/ng-dev/ai/skills/fixtures/name-mismatch/skills/wrong-name/SKILL.md @@ -0,0 +1,7 @@ +--- +name: right-name +description: Valid description +license: MIT +--- + +Instruction diff --git a/ng-dev/ai/skills/fixtures/valid-skill/skills/test-skill/SKILL.md b/ng-dev/ai/skills/fixtures/valid-skill/skills/test-skill/SKILL.md new file mode 100644 index 000000000..5caf122ad --- /dev/null +++ b/ng-dev/ai/skills/fixtures/valid-skill/skills/test-skill/SKILL.md @@ -0,0 +1,7 @@ +--- +name: test-skill +description: Valid description +license: MIT +--- + +Instruction diff --git a/ng-dev/ai/skills/validate.spec.ts b/ng-dev/ai/skills/validate.spec.ts new file mode 100644 index 000000000..b68395ddc --- /dev/null +++ b/ng-dev/ai/skills/validate.spec.ts @@ -0,0 +1,71 @@ +import {join} from 'path'; +import {validateSkill, validateSkills} from './validate.js'; + +function getFixturePath(relativePath: string): string { + return join(process.cwd(), 'ng-dev/ai/skills/fixtures', relativePath); +} + +describe('validateSkills', () => { + it('should pass for valid skill', async () => { + const skillPath = getFixturePath('valid-skill/skills/test-skill/SKILL.md'); + const {failures} = await validateSkill(skillPath); + expect(failures.length).toBe(0); + }); + + it('should fail for missing frontmatter', async () => { + const skillPath = getFixturePath('missing-frontmatter/skills/bad-skill/SKILL.md'); + const {failures} = await validateSkill(skillPath); + + expect(failures.length).toBe(1); + expect(failures).toContain(jasmine.stringMatching('Missing or invalid frontmatter')); + }); + + it('should fail if frontmatter is not at the start of the file', async () => { + const skillPath = getFixturePath('invalid-frontmatter-location/skills/test-skill/SKILL.md'); + const {failures} = await validateSkill(skillPath); + + expect(failures).toContain(jasmine.stringMatching('Missing or invalid frontmatter')); + }); + + it('should fail for invalid schema (missing license)', async () => { + const skillPath = getFixturePath('invalid-schema/skills/bad-schema/SKILL.md'); + const {failures} = await validateSkill(skillPath); + + expect(failures.length).toBe(1); + expect(failures).toContain(jasmine.stringMatching('Schema validation failure')); + }); + + it('should fail for name mismatch', async () => { + const skillPath = getFixturePath('name-mismatch/skills/wrong-name/SKILL.md'); + const {failures} = await validateSkill(skillPath); + + expect(failures.length).toBe(1); + expect(failures).toContain(jasmine.stringMatching('Name mismatch')); + }); + + it('should fail for invalid name format', async () => { + const skillPath = getFixturePath('invalid-name-format/skills/InvalidName/SKILL.md'); + const {failures} = await validateSkill(skillPath); + + expect(failures.length).toBe(1); + expect(failures).toContain(jasmine.stringMatching('Schema validation failure')); + }); + + it('should pass with valid optional fields', async () => { + const skillPath = getFixturePath('complex-skill/skills/complex-skill/SKILL.md'); + const {failures} = await validateSkill(skillPath); + expect(failures.length).toBe(0); + }); + + it('should validate multiple skills', async () => { + const repoRoot = getFixturePath('multiple-valid'); + const {exitCode} = await validateSkills(repoRoot); + expect(exitCode).toBe(0); + }); + + it('should fail if one of multiple skills is invalid', async () => { + const repoRoot = getFixturePath('multiple-mixed'); + const {exitCode} = await validateSkills(repoRoot); + expect(exitCode).toBe(1); + }); +}); diff --git a/ng-dev/ai/skills/validate.ts b/ng-dev/ai/skills/validate.ts new file mode 100644 index 000000000..533fef6fa --- /dev/null +++ b/ng-dev/ai/skills/validate.ts @@ -0,0 +1,129 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {z} from 'zod'; +import {readFile} from 'node:fs/promises'; +import {join, basename} from 'node:path'; +import glob from 'fast-glob'; +import {parse} from 'yaml'; +import {bold, green, Log, red, yellow} from '../../utils/logging.js'; + +/** + * Validation schema for the SKILL.md frontmatter. + * Based on https://agentskills.io/specification + */ +const skillFrontmatterSchema = z.object({ + name: z + .string() + .min(1) + .max(64) + .regex(/^[a-z0-9-]+$/, 'Name must only contain lowercase alphanumeric characters and hyphens') + .refine( + (val) => !val.startsWith('-') && !val.endsWith('-') && !val.includes('--'), + 'Name must not start/end with hyphens or contain consecutive hyphens', + ), + description: z.string().min(1).max(1024), + license: z.string().min(1), + compatibility: z.string().min(1).max(500).optional(), + metadata: z.record(z.string(), z.string()).optional(), + 'allowed-tools': z + .string() + .transform((val) => + val + .split(',') + .map((s) => s.trim()) + .filter(Boolean), + ) + .optional(), +}); + +export interface ValidationResult { + name: string; + failures: string[]; +} + +/** Validates all skills found in the `skills/` directory. */ +export async function validateSkills( + repoRoot: string, +): Promise<{results: ValidationResult[]; exitCode: number}> { + let errorCount = 0; + const skillFiles = await glob('**/SKILL.md', {cwd: join(repoRoot, 'skills'), absolute: true}); + + if (skillFiles.length === 0) { + Log.info(` ${yellow('⚠')} No skills found in skills/ directory.`); + return {results: [], exitCode: 0}; + } + + Log.info(`Found ${skillFiles.length} skills. Validating...`); + + const validationResults = await Promise.all(skillFiles.map(validateSkill)); + + for (const result of validationResults.sort((a, b) => a.failures.length - b.failures.length)) { + if (result.failures.length > 0) { + Log.info(` ${red('✘')} ${bold(result.name)} (${join('skills', result.name, 'SKILL.md')})`); + result.failures.forEach((failure) => { + Log.info(` - ${failure}`); + errorCount++; + }); + } else { + Log.info(` ${green('✔')} ${bold(result.name)} (${join('skills', result.name, 'SKILL.md')})`); + } + } + + Log.info(); + if (errorCount > 0) { + Log.error(` ${red('✘')} Validation failed with ${errorCount} errors.`); + return {results: validationResults, exitCode: 1}; + } else { + Log.info(` ${green('✔')} All skills validated successfully.`); + return {results: validationResults, exitCode: 0}; + } +} + +/** Validates a single skill file. */ +export async function validateSkill(filePath: string): Promise { + const name = basename(join(filePath, '..')); + const failures: string[] = []; + + try { + const content = await readFile(filePath, {encoding: 'utf8'}); + const frontmatterRaw = content.match(/^---\n([\s\S]*?)\n---/); + + if (frontmatterRaw === null) { + failures.push('Missing or invalid frontmatter in SKILL.md'); + return {name, failures}; + } + + let frontmatter: unknown; + try { + frontmatter = parse(frontmatterRaw[1]); + } catch (e) { + failures.push(`Failed to parse YAML frontmatter: ${(e as Error).message}`); + return {name, failures}; + } + + const frontmatterData = skillFrontmatterSchema.safeParse(frontmatter); + if (frontmatterData.success === false) { + for (const issue of frontmatterData.error.issues) { + failures.push(`Schema validation failure: [${issue.path.join('.')}] ${issue.message}`); + } + return {name, failures}; + } + + // Check name match if data looks like an object with a name + if (frontmatterData.data?.name !== name) { + failures.push( + `Name mismatch. Expected "${name}", found "${frontmatterData.data?.name ?? ''}"`, + ); + } + } catch (e) { + failures.push(`Unexpected error: ${(e as Error).message}`); + } + + return {name, failures}; +} diff --git a/ng-dev/package.json b/ng-dev/package.json index 9c4bf46a1..304d56915 100644 --- a/ng-dev/package.json +++ b/ng-dev/package.json @@ -68,6 +68,7 @@ "utf-8-validate": "6.0.6", "which": "6.0.0", "yaml": "2.8.2", - "yargs": "18.0.0" + "yargs": "18.0.0", + "zod": "^4.3.6" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5793e37d4..ff6b0a16a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -697,6 +697,9 @@ importers: yargs: specifier: 18.0.0 version: 18.0.0 + zod: + specifier: ^4.3.6 + version: 4.3.6 packages: @@ -6175,6 +6178,9 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.3.6: + resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + zone.js@0.16.0: resolution: {integrity: sha512-LqLPpIQANebrlxY6jKcYKdgN5DTXyyHAKnnWWjE5pPfEQ4n7j5zn7mOEEpwNZVKGqx3kKKmvplEmoBrvpgROTA==} @@ -7603,7 +7609,7 @@ snapshots: eventsource: 3.0.7 eventsource-parser: 3.0.6 express: 5.2.1(supports-color@10.2.2) - express-rate-limit: 7.5.1(express@5.2.1(supports-color@10.2.2)) + express-rate-limit: 7.5.1(express@5.2.1) jose: 6.1.3 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -9270,7 +9276,7 @@ snapshots: exponential-backoff@3.1.3: optional: true - express-rate-limit@7.5.1(express@5.2.1(supports-color@10.2.2)): + express-rate-limit@7.5.1(express@5.2.1): dependencies: express: 5.2.1(supports-color@10.2.2) @@ -12416,4 +12422,6 @@ snapshots: zod@3.25.76: {} + zod@4.3.6: {} + zone.js@0.16.0: {}