From e7417ed7a906576d8e147c22b1d5ffe75623b267 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 18 May 2026 04:14:02 +0000 Subject: [PATCH 1/4] Initial plan From 4092d472457fca30f33d12e8b7fa9b7d224b6a6b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 18 May 2026 04:32:02 +0000 Subject: [PATCH 2/4] feat(vscode): ship portable built-in custom agents with user-extensible packaging - Add codeql-query-developer and codeql-workshop-author .agent.md files - Add bundle-customizations.js script with overlay support - Add AgentRegistrar class managing chat.agentFilesLocations registration - Add bundle:customizations npm script; update vscode:prepublish and clean - Add codeql-mcp.agents.enabled and additionalAgentDirs settings - Add codeql-mcp.showAgentsStatus command - Add contributes.chatAgents, chatPromptFiles, chatSkills static declarations - Add Vitest unit tests for bundler and AgentRegistrar - Add Mocha integration test suite for agents - Add team-customizations example overlay - Update README.md with Built-in Custom Agents section - Update CHANGELOG.md [Unreleased] with new VS Code Extension features Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com> --- CHANGELOG.md | 9 + extensions/vscode/.vscodeignore | 4 + extensions/vscode/README.md | 84 ++- .../agents/codeql-query-developer.agent.md | 59 ++ .../agents/codeql-workshop-author.agent.md | 74 +++ .../agents/codeql-query-developer.agent.md | 59 ++ .../agents/codeql-workshop-author.agent.md | 74 +++ .../bundle-customizations.config.js | 23 + .../vscode/dist-customizations-manifest.json | 16 + extensions/vscode/esbuild.config.js | 1 + extensions/vscode/eslint.config.mjs | 2 + .../examples/team-customizations/README.md | 42 ++ .../agents/example-override.agent.md | 28 + .../prompts/example-team.prompt.md | 10 + .../skills/example-team-skill/SKILL.md | 19 + extensions/vscode/package.json | 55 +- .../vscode/prompts/ql-tdd-advanced.prompt.md | 493 ++++++++++++++++ .../vscode/prompts/ql-tdd-basic.prompt.md | 239 ++++++++ .../prompts/tools-query-workflow.prompt.md | 209 +++++++ .../workshop-creation-workflow.prompt.md | 297 ++++++++++ .../vscode/scripts/bundle-customizations.js | 187 ++++++ .../SKILL.md | 543 ++++++++++++++++++ .../SKILL.md | 357 ++++++++++++ .../src/customizations/agent-registrar.ts | 183 ++++++ extensions/vscode/src/extension.ts | 28 + .../customizations/agent-registrar.test.ts | 225 ++++++++ .../bundle-customizations.test.ts | 165 ++++++ .../test/suite/agents.integration.test.ts | 133 +++++ 28 files changed, 3601 insertions(+), 17 deletions(-) create mode 100644 extensions/vscode/agents/codeql-query-developer.agent.md create mode 100644 extensions/vscode/agents/codeql-workshop-author.agent.md create mode 100644 extensions/vscode/customizations/agents/codeql-query-developer.agent.md create mode 100644 extensions/vscode/customizations/agents/codeql-workshop-author.agent.md create mode 100644 extensions/vscode/customizations/bundle-customizations.config.js create mode 100644 extensions/vscode/dist-customizations-manifest.json create mode 100644 extensions/vscode/examples/team-customizations/README.md create mode 100644 extensions/vscode/examples/team-customizations/agents/example-override.agent.md create mode 100644 extensions/vscode/examples/team-customizations/prompts/example-team.prompt.md create mode 100644 extensions/vscode/examples/team-customizations/skills/example-team-skill/SKILL.md create mode 100644 extensions/vscode/prompts/ql-tdd-advanced.prompt.md create mode 100644 extensions/vscode/prompts/ql-tdd-basic.prompt.md create mode 100644 extensions/vscode/prompts/tools-query-workflow.prompt.md create mode 100644 extensions/vscode/prompts/workshop-creation-workflow.prompt.md create mode 100644 extensions/vscode/scripts/bundle-customizations.js create mode 100644 extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md create mode 100644 extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md create mode 100644 extensions/vscode/src/customizations/agent-registrar.ts create mode 100644 extensions/vscode/test/customizations/agent-registrar.test.ts create mode 100644 extensions/vscode/test/customizations/bundle-customizations.test.ts create mode 100644 extensions/vscode/test/suite/agents.integration.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ebf1b518..9780f21f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,15 @@ release cadence. _Changes on `main` since the latest tagged release that have not yet been included in a stable release._ +### Added + +#### VS Code Extension + +- **Built-in portable custom agents** — The extension now ships two `.agent.md` custom agents (`codeql-query-developer`, `codeql-workshop-author`) bundled inside the VSIX. On activation, the extension registers its `agents/` directory in `chat.agentFilesLocations` so both agents are immediately discoverable in VS Code Copilot Chat without any manual configuration. No specific model is required — users choose their own. +- **User-extensibility hooks** — Two new settings allow teams and individuals to extend the bundled agents at runtime (`codeql-mcp.additionalAgentDirs: string[]`) or disable the built-in registration entirely (`codeql-mcp.agents.enabled: boolean`, default `true`). A `--customizations-dir` CLI flag (or `CODEQL_MCP_CUSTOMIZATIONS_DIR` env var) on `bundle:customizations` enables building custom VSIXes with overlay agents, prompts, and skills. +- **`codeql-mcp.showAgentsStatus` command** — New Command Palette entry (**CodeQL MCP: Show Built-in Custom Agents Status**) that reports the enabled state, bundled directory, additional directories, and effective `chat.agentFilesLocations` entries. +- **Bundled prompts and skills** — Four MCP server prompts (`ql-tdd-basic`, `ql-tdd-advanced`, `tools-query-workflow`, `workshop-creation-workflow`) and two skills (`create-codeql-query-development-workshop`, `validate-ql-mcp-server-tools-queries`) are now copied into the VSIX as static contribution points (`chatPromptFiles`, `chatSkills`) so they are available to Copilot Chat without the MCP server running. + ## [v2.25.4] — 2026-05-08 ### Highlights diff --git a/extensions/vscode/.vscodeignore b/extensions/vscode/.vscodeignore index 25d383e8..bc005eab 100644 --- a/extensions/vscode/.vscodeignore +++ b/extensions/vscode/.vscodeignore @@ -27,3 +27,7 @@ coverage/** # Bundled server: exclude test/examples content server/ql/*/tools/test/** server/ql/*/examples/** + +# Customization sources (bundled outputs in agents/, prompts/, skills/ ship in VSIX) +customizations/** +examples/** diff --git a/extensions/vscode/README.md b/extensions/vscode/README.md index 57005f76..59b95a3c 100644 --- a/extensions/vscode/README.md +++ b/extensions/vscode/README.md @@ -33,6 +33,56 @@ On activation (`onStartupFinished`), the extension: 1. **Auto-installs** the `codeql-development-mcp-server` npm package (unless `codeql-mcp.autoInstall` is `false`). 2. **Registers an MCP server definition** (`ql-mcp`) so VS Code's Copilot/MCP integration can discover and launch it. 3. **Watches** the CodeQL extension's storage paths for databases, query results, and MRVA results, passing them to the MCP server as environment variables. +4. **Registers built-in custom agents** (`codeql-query-developer`, `codeql-workshop-author`) in `chat.agentFilesLocations` so they are discoverable in VS Code Copilot Chat. + +## Built-in Custom Agents + +The extension ships two portable `.agent.md` custom agents that appear in VS Code's Copilot Chat agent picker: + +| Agent | Description | +| ----- | ----------- | +| `codeql-query-developer` | Develop CodeQL queries, libraries, and tests using TDD with the `ql-mcp` MCP server tools. | +| `codeql-workshop-author` | Create CodeQL query development workshops from production-grade queries. | + +Both agents use the bundled MCP server tools (`ql-mcp/*`), prompts, and skills that ship with the extension. **No specific model is required** — you choose your own model in VS Code Copilot Chat. + +### Enabling / Disabling the Built-in Agents + +Agents are enabled by default. To disable them, set: + +```json +"codeql-mcp.agents.enabled": false +``` + +This removes the bundled `agents/` directory from `chat.agentFilesLocations` so the agents are no longer discoverable in Copilot Chat. + +### Extending at Runtime — Additional Agent Directories + +To add team or personal `.agent.md` files without rebuilding the extension, use: + +```json +"codeql-mcp.additionalAgentDirs": ["/path/to/your/team-agents"] +``` + +This appends the directory to `chat.agentFilesLocations` alongside the bundled agents. + +### Extending at Build Time — Custom VSIX + +To bundle your own agents, prompts, and skills into a custom VSIX: + +```bash +cd extensions/vscode +npm run bundle:customizations -- --customizations-dir=./examples/team-customizations +npm run package +``` + +Or via environment variable: + +```bash +CODEQL_MCP_CUSTOMIZATIONS_DIR=./examples/team-customizations npm run bundle:customizations +``` + +See [`examples/team-customizations/`](./examples/team-customizations/README.md) for a complete overlay example. ## Configuration @@ -40,6 +90,8 @@ All settings are under the `codeql-mcp` namespace in VS Code settings: | Setting | Default | Description | | ------------------------------------------ | ---------- | ------------------------------------------------------------------- | +| `codeql-mcp.agents.enabled` | `true` | Register the bundled custom agents in `chat.agentFilesLocations`. | +| `codeql-mcp.additionalAgentDirs` | `[]` | Additional `.agent.md` directories appended to `chat.agentFilesLocations`. | | `codeql-mcp.autoInstall` | `true` | Auto-install/update the MCP server on activation. | | `codeql-mcp.serverVersion` | `"latest"` | npm version to install (`"latest"` for most recent). | | `codeql-mcp.serverCommand` | `"node"` | Command to launch the server. Override to `"npx"` or a custom path. | @@ -55,26 +107,28 @@ All settings are under the `codeql-mcp` namespace in VS Code settings: Available from the Command Palette (`Cmd+Shift+P` / `Ctrl+Shift+P`): -| Command | Description | -| ------------------------------------------------- | ----------------------------------------------- | -| **CodeQL MCP: Reinstall MCP Server** | Re-download and install the server package. | -| **CodeQL MCP: Reinstall CodeQL Tool Query Packs** | Re-install the bundled CodeQL tool query packs. | -| **CodeQL MCP: Show Status** | Display current server status. | -| **CodeQL MCP: Show Logs** | Open the server log output. | +| Command | Description | +| -------------------------------------------------------- | ----------------------------------------------- | +| **CodeQL MCP: Reinstall MCP Server** | Re-download and install the server package. | +| **CodeQL MCP: Reinstall CodeQL Tool Query Packs** | Re-install the bundled CodeQL tool query packs. | +| **CodeQL MCP: Show Built-in Custom Agents Status** | Show which agent dirs are registered. | +| **CodeQL MCP: Show Status** | Display current server status. | +| **CodeQL MCP: Show Logs** | Open the server log output. | ## Development ### npm Scripts -| Script | What it does | When to use | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------- | -| `npm run package` | **Builds everything and produces the `.vsix`**. Internally runs `vscode:prepublish` (clean → lint → bundle → bundle:server) then `vsce package`. | **Building a distributable `.vsix`.** | -| `npm run build` | `clean` → `lint` → `bundle` (extension only, no server). | Development builds without packaging. | -| `npm run bundle` | esbuild the extension (no lint, no clean). | Fast iteration during development. | -| `npm run watch` | Rebuild the extension on file changes. | Active development. | -| `npm run test` | Run unit tests with Vitest. | Validating changes. | -| `npm run test:coverage` | Run unit tests with coverage. | CI / pre-merge validation. | -| `npm run lint` | Run ESLint on `src/` and `test/`. | Checking code style. | +| Script | What it does | When to use | +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- | +| `npm run package` | **Builds everything and produces the `.vsix`**. Internally runs `vscode:prepublish` (clean → lint → bundle → bundle:server → bundle:customizations) then `vsce package`. | **Building a distributable `.vsix`.** | +| `npm run build` | `clean` → `lint` → `bundle` (extension only, no server or customizations). | Development builds without packaging. | +| `npm run bundle` | esbuild the extension (no lint, no clean). | Fast iteration during development. | +| `npm run bundle:customizations`| Copy bundled agents/prompts/skills to output dirs and write the manifest. | After modifying agent/prompt/skill sources. | +| `npm run watch` | Rebuild the extension on file changes. | Active development. | +| `npm run test` | Run unit tests with Vitest. | Validating changes. | +| `npm run test:coverage` | Run unit tests with coverage. | CI / pre-merge validation. | +| `npm run lint` | Run ESLint on `src/` and `test/`. | Checking code style. | > **Note:** `vscode:prepublish` is a lifecycle hook invoked automatically by `vsce package` — you should not need to run it directly. diff --git a/extensions/vscode/agents/codeql-query-developer.agent.md b/extensions/vscode/agents/codeql-query-developer.agent.md new file mode 100644 index 00000000..2ba55e0a --- /dev/null +++ b/extensions/vscode/agents/codeql-query-developer.agent.md @@ -0,0 +1,59 @@ +--- +name: codeql-query-developer +description: "Develop CodeQL queries, libraries, and tests with TDD via the ql-mcp server." +tools: ['ql-mcp/*', 'edit', 'read', 'search', 'todo'] +--- + +# `codeql-query-developer` Agent + +Develops, tests, and validates CodeQL queries, libraries, and tests using the QL MCP Server tools. + +## Core Capabilities + +- Uses `ql-mcp/*` tools to create and manage CodeQL databases from source code. +- Follows test-driven development (TDD): writes tests with expected results first, then implements queries to pass them. +- Uses `ql-mcp/*` tools to run queries against databases, execute query unit tests, and generate query logs for debugging. +- Organizes queries, libraries, and tests following CodeQL pack conventions (`qlpack.yml`, `codeql-workspace.yml`). +- Documents query purpose, logic, and usage with clear QL comments. +- ALWAYS uses verbose help (`codeql -h -vv`) when learning about `codeql` CLI commands. +- NEVER makes anything up about CodeQL semantics or database schema. +- NEVER assumes query behavior without testing against actual databases. + +## TDD Workflow + +1. **Understand the goal** — clarify what the query should detect and for which language. +2. **Create test code** — write test source files that contain positive and negative examples. +3. **Extract a test database** — use `ql-mcp/codeql_create_database` or `ql-mcp/codeql_query_run` to build a DB. +4. **Write `.qlref` / `.expected` test files** — specify expected results before writing query logic. +5. **Implement the query** — write the `.ql` file to make the tests pass. +6. **Run tests** — use `ql-mcp/codeql_test_run` to execute the unit tests; iterate until 100% pass. +7. **Validate** — run the query against real databases; inspect results; refine as needed. +8. **Document** — add `@name`, `@description`, `@kind`, `@id`, `@tags` metadata to the query. + +## MCP Tool Usage + +Use the bundled `ql-mcp/*` tools for all CodeQL operations: + +- `ql-mcp/codeql_create_database` — create a CodeQL database from source. +- `ql-mcp/codeql_query_run` — run a query against a database. +- `ql-mcp/codeql_test_run` — run CodeQL unit tests. +- `ql-mcp/codeql_query_explain` — explain a query's structure. +- `ql-mcp/find_codeql_query_files` — locate query files in the workspace. +- `ql-mcp/codeql_pack_install` — install QL pack dependencies. + +## Bundled Skills and Prompts + +The following bundled resources are available in the extension and provide detailed step-by-step workflows: + +- **Skill `create-codeql-query-development-workshop`** — reference for structured query development. +- **Skill `validate-ql-mcp-server-tools-queries`** — validate PrintAST, PrintCFG, and CallGraph tools. +- **Prompt `ql-tdd-basic`** — basic TDD workflow for simple CodeQL queries. +- **Prompt `ql-tdd-advanced`** — advanced TDD patterns for data-flow and taint-tracking queries. +- **Prompt `tools-query-workflow`** — workflow for using MCP tool queries (PrintAST, PrintCFG, CallGraph). + +## Quality Standards + +- All solution queries must compile without errors. +- All unit tests must pass at 100%. +- Expected results must be accurate (verified against real test databases). +- Queries must include complete `@name`, `@description`, `@kind`, `@id`, `@tags` metadata. diff --git a/extensions/vscode/agents/codeql-workshop-author.agent.md b/extensions/vscode/agents/codeql-workshop-author.agent.md new file mode 100644 index 00000000..94b29936 --- /dev/null +++ b/extensions/vscode/agents/codeql-workshop-author.agent.md @@ -0,0 +1,74 @@ +--- +name: codeql-workshop-author +description: "Create CodeQL query development workshops from production-grade queries." +tools: ['ql-mcp/*', 'edit', 'read', 'search', 'todo'] +handoffs: + - agent: codeql-query-developer + label: Develop and Test Query + prompt: 'Develop and test a CodeQL query using TDD methodology. Follow the `ql-tdd-basic` or `ql-tdd-advanced` prompt workflow and return the validated query file path and test results when complete.' + send: false +--- + +# `codeql-workshop-author` Agent + +Creates comprehensive CodeQL query development workshops from production-grade queries using the QL MCP Server tools. + +## Core Capabilities + +- Uses `ql-mcp/*` tools to analyze production queries and create workshop materials. +- Follows the bundled `create-codeql-query-development-workshop` skill to generate workshops from production queries. +- Validates AST/CFG tools using the bundled `validate-ql-mcp-server-tools-queries` skill. +- Decomposes queries into 4–8 logical learning stages that guide learners from simple to complex. +- Generates exercise queries, solution queries, unit tests, AST/CFG visualizations, and README documentation. +- ALWAYS uses verbose help (`codeql -h -vv`) when learning about `codeql` CLI commands. +- NEVER makes anything up about CodeQL semantics or database schema. +- NEVER assumes query behavior without testing against actual databases. + +## Workshop Generation Process + +1. **Analyze source query** — use `ql-mcp/find_codeql_query_files` and `ql-mcp/codeql_query_explain` to understand the production query. +2. **Prepare environment** — run `codeql pack install` on solutions and solutions-tests directories; run any `initialize-qltests.sh` scripts. +3. **Validate AST/CFG tools** — use the bundled `validate-ql-mcp-server-tools-queries` skill to confirm PrintAST, PrintCFG, and CallGraph return non-empty output. **Fail if any query returns empty results.** +4. **Plan stages** — decompose the query into 4–8 logical learning stages. +5. **Create workshop structure** — set up directories, `qlpack.yml` files, and `codeql-workspace.yml`. +6. **Generate solution stages** — for each stage, delegate to `codeql-query-developer` (via the **Develop and Test Query** handoff) to create and validate the solution query. +7. **Create exercise queries** — remove implementation details from solutions; add scaffolding, `// TODO` hints, and `select` stubs. +8. **Generate enrichments** — create AST/CFG graphs (from tool output), build scripts, and documentation. +9. **Final validation** — run all solution tests; confirm 100% pass rate before declaring the workshop complete. + +## Workshop Structure + +``` +/ + exercises/ # Student exercise queries (incomplete, with scaffolding) + exercises-tests/ # Unit tests for exercises + solutions/ # Complete solution queries + solutions-tests/ # Unit tests for solutions (must pass 100%) + tests-common/ # Shared test code and databases + graphs/ # AST/CFG visualizations + README.md # Workshop guide + build-databases.sh # Database creation script + codeql-workspace.yml +``` + +## Decomposition Strategies + +- **Syntactic → Semantic** — Start with syntax, add type, control flow, then data flow. +- **Local → Global** — Start local, expand to cross-procedural analysis. +- **Simple → Filtered** — High recall first, then refine with filters. +- **Building Blocks** — Define helpers, combine into sources/sinks, connect with flow. + +## Bundled Skills and Prompts + +- **Skill `create-codeql-query-development-workshop`** — full step-by-step workshop creation workflow. +- **Skill `validate-ql-mcp-server-tools-queries`** — AST/CFG/CallGraph validation protocol. +- **Prompt `workshop-creation-workflow`** — structured prompt for workshop generation from a production query. +- **Prompt `ql-tdd-advanced`** — advanced TDD patterns for data-flow and taint-tracking queries. + +## Quality Standards + +- All solution queries compile without errors. +- All solution tests pass at 100%. +- Exercise queries have appropriate scaffolding (not empty, not complete). +- Expected results progress logically from stage to stage. +- Test code covers positive, negative, and edge cases. diff --git a/extensions/vscode/customizations/agents/codeql-query-developer.agent.md b/extensions/vscode/customizations/agents/codeql-query-developer.agent.md new file mode 100644 index 00000000..2ba55e0a --- /dev/null +++ b/extensions/vscode/customizations/agents/codeql-query-developer.agent.md @@ -0,0 +1,59 @@ +--- +name: codeql-query-developer +description: "Develop CodeQL queries, libraries, and tests with TDD via the ql-mcp server." +tools: ['ql-mcp/*', 'edit', 'read', 'search', 'todo'] +--- + +# `codeql-query-developer` Agent + +Develops, tests, and validates CodeQL queries, libraries, and tests using the QL MCP Server tools. + +## Core Capabilities + +- Uses `ql-mcp/*` tools to create and manage CodeQL databases from source code. +- Follows test-driven development (TDD): writes tests with expected results first, then implements queries to pass them. +- Uses `ql-mcp/*` tools to run queries against databases, execute query unit tests, and generate query logs for debugging. +- Organizes queries, libraries, and tests following CodeQL pack conventions (`qlpack.yml`, `codeql-workspace.yml`). +- Documents query purpose, logic, and usage with clear QL comments. +- ALWAYS uses verbose help (`codeql -h -vv`) when learning about `codeql` CLI commands. +- NEVER makes anything up about CodeQL semantics or database schema. +- NEVER assumes query behavior without testing against actual databases. + +## TDD Workflow + +1. **Understand the goal** — clarify what the query should detect and for which language. +2. **Create test code** — write test source files that contain positive and negative examples. +3. **Extract a test database** — use `ql-mcp/codeql_create_database` or `ql-mcp/codeql_query_run` to build a DB. +4. **Write `.qlref` / `.expected` test files** — specify expected results before writing query logic. +5. **Implement the query** — write the `.ql` file to make the tests pass. +6. **Run tests** — use `ql-mcp/codeql_test_run` to execute the unit tests; iterate until 100% pass. +7. **Validate** — run the query against real databases; inspect results; refine as needed. +8. **Document** — add `@name`, `@description`, `@kind`, `@id`, `@tags` metadata to the query. + +## MCP Tool Usage + +Use the bundled `ql-mcp/*` tools for all CodeQL operations: + +- `ql-mcp/codeql_create_database` — create a CodeQL database from source. +- `ql-mcp/codeql_query_run` — run a query against a database. +- `ql-mcp/codeql_test_run` — run CodeQL unit tests. +- `ql-mcp/codeql_query_explain` — explain a query's structure. +- `ql-mcp/find_codeql_query_files` — locate query files in the workspace. +- `ql-mcp/codeql_pack_install` — install QL pack dependencies. + +## Bundled Skills and Prompts + +The following bundled resources are available in the extension and provide detailed step-by-step workflows: + +- **Skill `create-codeql-query-development-workshop`** — reference for structured query development. +- **Skill `validate-ql-mcp-server-tools-queries`** — validate PrintAST, PrintCFG, and CallGraph tools. +- **Prompt `ql-tdd-basic`** — basic TDD workflow for simple CodeQL queries. +- **Prompt `ql-tdd-advanced`** — advanced TDD patterns for data-flow and taint-tracking queries. +- **Prompt `tools-query-workflow`** — workflow for using MCP tool queries (PrintAST, PrintCFG, CallGraph). + +## Quality Standards + +- All solution queries must compile without errors. +- All unit tests must pass at 100%. +- Expected results must be accurate (verified against real test databases). +- Queries must include complete `@name`, `@description`, `@kind`, `@id`, `@tags` metadata. diff --git a/extensions/vscode/customizations/agents/codeql-workshop-author.agent.md b/extensions/vscode/customizations/agents/codeql-workshop-author.agent.md new file mode 100644 index 00000000..94b29936 --- /dev/null +++ b/extensions/vscode/customizations/agents/codeql-workshop-author.agent.md @@ -0,0 +1,74 @@ +--- +name: codeql-workshop-author +description: "Create CodeQL query development workshops from production-grade queries." +tools: ['ql-mcp/*', 'edit', 'read', 'search', 'todo'] +handoffs: + - agent: codeql-query-developer + label: Develop and Test Query + prompt: 'Develop and test a CodeQL query using TDD methodology. Follow the `ql-tdd-basic` or `ql-tdd-advanced` prompt workflow and return the validated query file path and test results when complete.' + send: false +--- + +# `codeql-workshop-author` Agent + +Creates comprehensive CodeQL query development workshops from production-grade queries using the QL MCP Server tools. + +## Core Capabilities + +- Uses `ql-mcp/*` tools to analyze production queries and create workshop materials. +- Follows the bundled `create-codeql-query-development-workshop` skill to generate workshops from production queries. +- Validates AST/CFG tools using the bundled `validate-ql-mcp-server-tools-queries` skill. +- Decomposes queries into 4–8 logical learning stages that guide learners from simple to complex. +- Generates exercise queries, solution queries, unit tests, AST/CFG visualizations, and README documentation. +- ALWAYS uses verbose help (`codeql -h -vv`) when learning about `codeql` CLI commands. +- NEVER makes anything up about CodeQL semantics or database schema. +- NEVER assumes query behavior without testing against actual databases. + +## Workshop Generation Process + +1. **Analyze source query** — use `ql-mcp/find_codeql_query_files` and `ql-mcp/codeql_query_explain` to understand the production query. +2. **Prepare environment** — run `codeql pack install` on solutions and solutions-tests directories; run any `initialize-qltests.sh` scripts. +3. **Validate AST/CFG tools** — use the bundled `validate-ql-mcp-server-tools-queries` skill to confirm PrintAST, PrintCFG, and CallGraph return non-empty output. **Fail if any query returns empty results.** +4. **Plan stages** — decompose the query into 4–8 logical learning stages. +5. **Create workshop structure** — set up directories, `qlpack.yml` files, and `codeql-workspace.yml`. +6. **Generate solution stages** — for each stage, delegate to `codeql-query-developer` (via the **Develop and Test Query** handoff) to create and validate the solution query. +7. **Create exercise queries** — remove implementation details from solutions; add scaffolding, `// TODO` hints, and `select` stubs. +8. **Generate enrichments** — create AST/CFG graphs (from tool output), build scripts, and documentation. +9. **Final validation** — run all solution tests; confirm 100% pass rate before declaring the workshop complete. + +## Workshop Structure + +``` +/ + exercises/ # Student exercise queries (incomplete, with scaffolding) + exercises-tests/ # Unit tests for exercises + solutions/ # Complete solution queries + solutions-tests/ # Unit tests for solutions (must pass 100%) + tests-common/ # Shared test code and databases + graphs/ # AST/CFG visualizations + README.md # Workshop guide + build-databases.sh # Database creation script + codeql-workspace.yml +``` + +## Decomposition Strategies + +- **Syntactic → Semantic** — Start with syntax, add type, control flow, then data flow. +- **Local → Global** — Start local, expand to cross-procedural analysis. +- **Simple → Filtered** — High recall first, then refine with filters. +- **Building Blocks** — Define helpers, combine into sources/sinks, connect with flow. + +## Bundled Skills and Prompts + +- **Skill `create-codeql-query-development-workshop`** — full step-by-step workshop creation workflow. +- **Skill `validate-ql-mcp-server-tools-queries`** — AST/CFG/CallGraph validation protocol. +- **Prompt `workshop-creation-workflow`** — structured prompt for workshop generation from a production query. +- **Prompt `ql-tdd-advanced`** — advanced TDD patterns for data-flow and taint-tracking queries. + +## Quality Standards + +- All solution queries compile without errors. +- All solution tests pass at 100%. +- Exercise queries have appropriate scaffolding (not empty, not complete). +- Expected results progress logically from stage to stage. +- Test code covers positive, negative, and edge cases. diff --git a/extensions/vscode/customizations/bundle-customizations.config.js b/extensions/vscode/customizations/bundle-customizations.config.js new file mode 100644 index 00000000..2976ff70 --- /dev/null +++ b/extensions/vscode/customizations/bundle-customizations.config.js @@ -0,0 +1,23 @@ +/** + * bundle-customizations.config.js + * + * Whitelist of prompts and skills to bundle into the VS Code extension. + * Prompts are sourced from server/src/prompts/. + * Skills are sourced from .github/skills//SKILL.md. + * + * Missing entries are silently skipped with a console.warn — the build + * never fails due to absent optional files. + */ + +export const prompts = [ + 'ql-tdd-basic.prompt.md', + 'ql-tdd-advanced.prompt.md', + 'tools-query-workflow.prompt.md', + 'workshop-creation-workflow.prompt.md', +]; + +export const skills = [ + 'create-codeql-query-development-workshop', + 'create-codeql-query-tdd-generic', + 'validate-ql-mcp-server-tools-queries', +]; diff --git a/extensions/vscode/dist-customizations-manifest.json b/extensions/vscode/dist-customizations-manifest.json new file mode 100644 index 00000000..8724caea --- /dev/null +++ b/extensions/vscode/dist-customizations-manifest.json @@ -0,0 +1,16 @@ +{ + "agents": [ + "agents/codeql-query-developer.agent.md", + "agents/codeql-workshop-author.agent.md" + ], + "prompts": [ + "prompts/ql-tdd-basic.prompt.md", + "prompts/ql-tdd-advanced.prompt.md", + "prompts/tools-query-workflow.prompt.md", + "prompts/workshop-creation-workflow.prompt.md" + ], + "skills": [ + "skills/create-codeql-query-development-workshop/SKILL.md", + "skills/validate-ql-mcp-server-tools-queries/SKILL.md" + ] +} diff --git a/extensions/vscode/esbuild.config.js b/extensions/vscode/esbuild.config.js index bd95c587..3f35e150 100644 --- a/extensions/vscode/esbuild.config.js +++ b/extensions/vscode/esbuild.config.js @@ -34,6 +34,7 @@ const testSuiteConfig = { ...shared, entryPoints: [ 'test/suite/index.ts', + 'test/suite/agents.integration.test.ts', 'test/suite/bridge.integration.test.ts', 'test/suite/copydb-e2e.integration.test.ts', 'test/suite/extension.integration.test.ts', diff --git a/extensions/vscode/eslint.config.mjs b/extensions/vscode/eslint.config.mjs index 5a698bd2..202a0e4e 100644 --- a/extensions/vscode/eslint.config.mjs +++ b/extensions/vscode/eslint.config.mjs @@ -88,6 +88,8 @@ export default [ ecmaVersion: 2022, sourceType: 'module', globals: { + clearTimeout: 'readonly', + setTimeout: 'readonly', suite: 'readonly', test: 'readonly', setup: 'readonly', diff --git a/extensions/vscode/examples/team-customizations/README.md b/extensions/vscode/examples/team-customizations/README.md new file mode 100644 index 00000000..2726773d --- /dev/null +++ b/extensions/vscode/examples/team-customizations/README.md @@ -0,0 +1,42 @@ +# Team Customizations — Overlay Example + +This directory demonstrates how to extend the built-in custom agents with +team-specific or personal agents, prompts, and skills. + +## Structure + +``` +team-customizations/ + agents/ # .agent.md files to add or override + prompts/ # .prompt.md files to add + skills/ # /SKILL.md files to add + README.md # this file +``` + +## Usage + +### At build time (produces a custom VSIX) + +```bash +cd extensions/vscode +npm run bundle:customizations -- --customizations-dir=./examples/team-customizations +npm run package +``` + +### At bundle time only (during development) + +```bash +CODEQL_MCP_CUSTOMIZATIONS_DIR=./examples/team-customizations npm run bundle:customizations +``` + +## Override vs. Add + +- A file with the **same name** as a bundled file **replaces** it (with a warning). +- A file with a **new name** is **added** alongside the defaults. + +## Important Notes + +- **Never add a `model:` key** to `.agent.md` files — users choose their own model in VS Code. +- Skill files must be named `SKILL.md` and placed in `skills//SKILL.md`. +- Prompt files must end in `.prompt.md`. +- Agent files must end in `.agent.md`. diff --git a/extensions/vscode/examples/team-customizations/agents/example-override.agent.md b/extensions/vscode/examples/team-customizations/agents/example-override.agent.md new file mode 100644 index 00000000..efb42738 --- /dev/null +++ b/extensions/vscode/examples/team-customizations/agents/example-override.agent.md @@ -0,0 +1,28 @@ +--- +name: example-override +description: "Example agent demonstrating the overlay/override pattern." +tools: ['ql-mcp/*', 'edit', 'read', 'search'] +--- + +# `example-override` Agent + +This is a stub agent that demonstrates how to add or override `.agent.md` files +using the `--customizations-dir` overlay feature. + +## How Overlays Work + +When you run: + +```bash +npm run bundle:customizations -- --customizations-dir=./examples/team-customizations +``` + +Files in `/agents/` are copied **after** the defaults, +replacing any file with the same name and adding new files. + +## Customization Tips + +- Override `codeql-query-developer.agent.md` to add team-specific workflows. +- Add new `.agent.md` files for team-specific roles (e.g., `security-review.agent.md`). +- Keep `name:` in frontmatter matching the filename stem for VS Code to discover the agent. +- Never add a `model:` key — users choose their own model. diff --git a/extensions/vscode/examples/team-customizations/prompts/example-team.prompt.md b/extensions/vscode/examples/team-customizations/prompts/example-team.prompt.md new file mode 100644 index 00000000..0ced5d08 --- /dev/null +++ b/extensions/vscode/examples/team-customizations/prompts/example-team.prompt.md @@ -0,0 +1,10 @@ +--- +name: example-team-prompt +description: An example team prompt demonstrating the overlay feature. +--- + +# Example Team Prompt + +This prompt demonstrates adding a team-specific workflow prompt via the overlay. + +Use it as a template to add your own prompts to the bundled extension. diff --git a/extensions/vscode/examples/team-customizations/skills/example-team-skill/SKILL.md b/extensions/vscode/examples/team-customizations/skills/example-team-skill/SKILL.md new file mode 100644 index 00000000..a492a55d --- /dev/null +++ b/extensions/vscode/examples/team-customizations/skills/example-team-skill/SKILL.md @@ -0,0 +1,19 @@ +# Example Team Skill + +This is an example `SKILL.md` file demonstrating how to add a team-specific skill +via the `--customizations-dir` overlay feature. + +## How to Use + +1. Copy this directory to your team's customizations repository. +2. Rename `example-team-skill` to match your skill's name. +3. Replace this content with your team's skill documentation. +4. Reference it in your `.agent.md` files. + +## Overlay Usage + +```bash +npm run bundle:customizations -- --customizations-dir=./examples/team-customizations +``` + +Skills placed in `/skills//SKILL.md` are merged alongside the defaults. diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index 4d181f54..75ad610b 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -48,6 +48,14 @@ "configuration": { "title": "CodeQL MCP Server", "properties": { + "codeql-mcp.additionalAgentDirs": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "Additional directories containing `.agent.md` files to append to `chat.agentFilesLocations`. Useful for team or personal agent overlays." + }, "codeql-mcp.additionalDatabaseDirs": { "type": "array", "items": { @@ -80,6 +88,11 @@ "default": [], "description": "Additional directories containing query run result subdirectories. Appended to CODEQL_QUERY_RUN_RESULTS_DIRS alongside the vscode-codeql query storage path." }, + "codeql-mcp.agents.enabled": { + "type": "boolean", + "default": true, + "description": "Register the bundled custom agents (`codeql-query-developer`, `codeql-workshop-author`) by appending the extension's bundled `agents/` directory to `chat.agentFilesLocations`. Toggle off to disable." + }, "codeql-mcp.autoDownloadPacks": { "type": "boolean", "default": true, @@ -133,6 +146,38 @@ } } }, + "chatAgents": [ + { + "path": "./agents/codeql-query-developer.agent.md" + }, + { + "path": "./agents/codeql-workshop-author.agent.md" + } + ], + "chatPromptFiles": [ + { + "path": "./prompts/ql-tdd-advanced.prompt.md" + }, + { + "path": "./prompts/ql-tdd-basic.prompt.md" + }, + { + "path": "./prompts/tools-query-workflow.prompt.md" + }, + { + "path": "./prompts/workshop-creation-workflow.prompt.md" + } + ], + "chatSkills": [ + { + "name": "create-codeql-query-development-workshop", + "path": "./skills/create-codeql-query-development-workshop/SKILL.md" + }, + { + "name": "validate-ql-mcp-server-tools-queries", + "path": "./skills/validate-ql-mcp-server-tools-queries/SKILL.md" + } + ], "commands": [ { "command": "codeql-mcp.reinstallServer", @@ -144,6 +189,11 @@ "title": "Reinstall CodeQL Tool Query Packs", "category": "CodeQL MCP" }, + { + "command": "codeql-mcp.showAgentsStatus", + "title": "Show Built-in Custom Agents Status", + "category": "CodeQL MCP" + }, { "command": "codeql-mcp.showStatus", "title": "Show Status", @@ -159,8 +209,9 @@ "scripts": { "build": "npm run clean && npm run lint && npm run bundle", "bundle": "npm run rebuild:esbuild && node esbuild.config.js", + "bundle:customizations": "node scripts/bundle-customizations.js", "bundle:server": "node scripts/bundle-server.js", - "clean": "rm -rf dist server .vscode-test/* *.vsix", + "clean": "rm -rf dist server agents prompts skills .vscode-test/* *.vsix dist-customizations-manifest.json", "download:vscode": "node scripts/download-vscode.js", "lint": "eslint src/ test/", "lint:fix": "eslint src/ test/ --fix", @@ -171,7 +222,7 @@ "test:integration": "npm run download:vscode && vscode-test", "test:integration:label": "vscode-test --label", "test:watch": "vitest --watch", - "vscode:prepublish": "npm run clean && npm run lint && npm run bundle && npm run bundle:server", + "vscode:prepublish": "npm run clean && npm run lint && npm run bundle && npm run bundle:server && npm run bundle:customizations", "watch": "npm run rebuild:esbuild && node esbuild.config.js --watch" }, "devDependencies": { diff --git a/extensions/vscode/prompts/ql-tdd-advanced.prompt.md b/extensions/vscode/prompts/ql-tdd-advanced.prompt.md new file mode 100644 index 00000000..4b8b6799 --- /dev/null +++ b/extensions/vscode/prompts/ql-tdd-advanced.prompt.md @@ -0,0 +1,493 @@ +--- +agent: agent +--- + +# Advanced Test-Driven CodeQL Query Development + +This advanced guide builds on the basic TDD methodology with powerful MCP server tools for deeper code analysis. Use this when developing complex queries that require understanding AST structure, control flow, and call relationships. + +For basic TDD workflow, see: `codeql://prompts/ql-tdd-basic` + +## When to Use This Advanced Guide + +- **Complex AST patterns**: When you need to understand how CodeQL represents specific language constructs +- **Control flow queries**: When analyzing program flow, branching, or loop structures +- **Call graph analysis**: When tracing function calls or method invocations +- **Iterative refinement**: When debugging queries that don't produce expected results +- **Performance optimization**: When understanding query evaluation patterns + +## Core Advanced Tools + +### 1. AST/CFG Visualization with #codeql_query_run + +Use the bundled tools queries to visualize code structure: + +```typescript +// Generate Abstract Syntax Tree for test code +codeql_query_run: { + queryName: "PrintAST", + queryLanguage: "java", // or "javascript", "python", "cpp", etc. + database: "/path/to/test.testproj", + format: "graphtext", + interpretedOutput: "/path/to/ast-output.txt" +} + +// Generate Control Flow Graph +codeql_query_run: { + queryName: "PrintCFG", + queryLanguage: "java", + database: "/path/to/test.testproj", + format: "graphtext", + interpretedOutput: "/path/to/cfg-output.txt" +} + +// Analyze outbound calls from a function +codeql_query_run: { + queryName: "CallGraphFrom", + queryLanguage: "java", + database: "/path/to/test.testproj", + sourceFunction: "myFunction", + format: "sarif-latest" +} + +// Analyze inbound calls to a function +codeql_query_run: { + queryName: "CallGraphTo", + queryLanguage: "java", + database: "/path/to/test.testproj", + targetFunction: "targetMethod", + format: "sarif-latest" +} +``` + +**Critical**: Always verify that tools queries return **non-empty output** with actual nodes/edges, not just headers. + +### 2. Quick Evaluation with #quick_evaluate + +Use #quick_evaluate for rapid iteration on specific predicates or classes: + +```typescript +// First, find the position of a predicate +find_predicate_position: { + file: "/path/to/Query.ql", + name: "isVulnerableSink" +} + +// Then evaluate just that predicate +quick_evaluate: { + file: "/path/to/Query.ql", + db: "/path/to/test.testproj", + symbol: "isVulnerableSink", + output_path: "/tmp/quickeval.bqrs" +} + +// Or evaluate a specific class +// NOTE: find_class_position finds `class` definitions only, not `module` definitions. +// Use search_ql_code to find module definitions or other patterns across QL files. +find_class_position: { + file: "/path/to/Query.ql", + name: "ThrowingMethodCall" +} + +quick_evaluate: { + file: "/path/to/Query.ql", + db: "/path/to/test.testproj", + symbol: "ThrowingMethodCall" +} +``` + +### 3. LSP-Powered Code Navigation + +Use the LSP tools for real-time code exploration during query development: + +```typescript +// Discover available types after `import javascript` +codeql_lsp_completion: { + file_path: "/path/to/Query.ql", + line: 5, // 0-based line with `from` clause + character: 5, // 0-based column position + workspace_uri: "/path/to/pack-root" // REQUIRED: directory containing codeql-pack.yml +} + +// Navigate to a class definition to see its predicates +codeql_lsp_definition: { + file_path: "/path/to/Query.ql", + line: 5, // 0-based line containing the class name + character: 10, // 0-based column on the class name + workspace_uri: "/path/to/pack-root" +} + +// Find all usages of a predicate across the pack +codeql_lsp_references: { + file_path: "/path/to/Query.ql", + line: 8, + character: 5, + workspace_uri: "/path/to/pack-root" +} +``` + +**Important LSP tool notes**: + +- `workspace_uri` must be a **plain directory path** (not a `file://` URI) pointing to the pack root containing `codeql-pack.yml` +- All LSP tools use **0-based** line/character positions +- #find_predicate_position and #find_class_position return **1-based** positions — subtract 1 before passing to LSP tools +- Run #codeql_pack_install before using LSP tools — they require resolved dependencies +- Request completions **after a dot** (e.g., `pw.`) to see all member predicates with full documentation + +### 4. Query File Discovery with #find_codeql_query_files + +Use this tool frequently to understand query dependencies and test structure: + +```typescript +find_codeql_query_files: { + queryPath: '/path/to/Query.ql'; +} +// Returns: query file, test directory, test files, metadata, dependencies +``` + +## Advanced TDD Workflow + +### Phase 0: Test Environment Setup (Critical for Java) + +For Java tests that depend on external libraries (JUnit, etc.), create an `options` file in each test directory: + +```text +//semmle-extractor-options: --javac-args -cp ${testdir}/../../stubs/junit-4.13:${testdir}/../../stubs/junit-jupiter-api-5.2.0 +``` + +**Key points**: + +- `${testdir}` is relative to the test directory containing the `options` file +- Use `:` (colon) to separate multiple classpath entries +- Stub files must contain minimal class/interface definitions for compilation + +### Phase 1: Deep Code Analysis + +Before writing any query logic: + +1. **Extract test database**: + + ```typescript + codeql_test_extract: { + testPath: "/path/to/test/QueryTest", + searchPath: "/path/to/pack" + } + ``` + +2. **Generate and study the AST**: + + ```typescript + codeql_query_run: { + queryName: "PrintAST", + queryLanguage: "java", + database: "/path/to/test.testproj", + format: "graphtext", + interpretedOutput: "/path/to/ast.txt" + } + ``` + +3. **Identify key AST classes and predicates** from the output: + - Note the class names (e.g., `MethodCall`, `TryStmt`, `LambdaExpr`) + - Note parent-child relationships + - Identify which nodes correspond to your test cases + +4. **Generate CFG if analyzing control flow**: + + ```typescript + codeql_query_run: { + queryName: "PrintCFG", + queryLanguage: "java", + database: "/path/to/test.testproj", + format: "graphtext", + interpretedOutput: "/path/to/cfg.txt" + } + ``` + +### Phase 2: Iterative Predicate Development + +Instead of writing the full query at once: + +1. **Start with a single class or predicate**: + + ```ql + class MyPattern extends MethodCall { + MyPattern() { + this.getMethod().hasName("targetMethod") + } + } + ``` + +2. **Use #quick_evaluate to test it**: + + ```typescript + quick_evaluate: { + file: "/path/to/Query.ql", + db: "/path/to/test.testproj", + symbol: "MyPattern" + } + ``` + +3. **Refine based on results**: + - Too many results? Add constraints + - Too few results? Relax constraints + - Wrong results? Study AST output again + +4. **Repeat for each component** before combining + +### Phase 3: Incremental Query Assembly + +1. **Combine validated predicates** into the main query +2. **Run full tests** after each combination: + + ```typescript + codeql_test_run: { + testPath: "/path/to/test/QueryTest", + searchPath: "/path/to/pack" + } + ``` + +3. **Use #find_codeql_query_files** to track all related files: + + ```typescript + find_codeql_query_files: { + queryPath: '/path/to/Query.ql'; + } + ``` + +## Common Advanced Patterns + +### Pattern: Understanding Nested Structures + +When your query involves nested constructs (e.g., lambdas inside method calls): + +```ql +// First, find the outer construct +class OuterPattern extends MethodCall { + OuterPattern() { + this.getMethod().hasName("assertThrows") + } + + // Navigate to inner construct + Expr getLambdaBody() { + exists(LambdaExpr le | + le = this.getAChildExpr() and + result = le.getExprBody() + ) + } +} +``` + +Use `PrintAST` to understand the parent-child relationships. + +### Pattern: Control Flow Dependencies + +When your query needs to understand execution order: + +```ql +// Use CFG to understand which statements precede others +predicate precedesInBlock(Stmt s1, Stmt s2) { + exists(BasicBlock bb | + s1.getBasicBlock() = bb and + s2.getBasicBlock() = bb and + s1.getLocation().getStartLine() < s2.getLocation().getStartLine() + ) +} +``` + +Use `PrintCFG` to verify the control flow structure. + +### Pattern: Call Chain Analysis + +When tracing calls through multiple functions: + +```typescript +// First, understand the call graph from your entry point +codeql_query_run: { + queryName: "CallGraphFrom", + queryLanguage: "java", + database: "/path/to/test.testproj", + sourceFunction: "entryPoint" +} + +// Then trace back from your target +codeql_query_run: { + queryName: "CallGraphTo", + queryLanguage: "java", + database: "/path/to/test.testproj", + targetFunction: "sensitiveOperation" +} +``` + +## Debugging Tips + +### When Results Are Empty + +1. **Check AST first**: Run `PrintAST` to verify the code structure matches expectations +2. **Simplify the query**: Remove constraints one by one using #quick_evaluate +3. **Check enclosing callables**: Lambda bodies may have different `getEnclosingCallable()` than expected +4. **Verify test database extraction**: Ensure the `.testproj` directory was created successfully + +### When Results Are Incorrect + +1. **Quick evaluate individual predicates**: Isolate which part is wrong +2. **Compare with AST output**: Verify your understanding of the structure +3. **Check for missing cases**: Your pattern may not cover all code variations + +### When Compilation Fails with "override" Error + +If you see `annotation 'override' missing on predicate`, you've accidentally created a predicate with the same name as one in the parent class. **Rename your predicate** to avoid the conflict: + +```ql +// BAD: Method already has getAThrownExceptionType() +class ThrowingMethod extends Method { + RefType getAThrownExceptionType() { ... } // Error! +} + +// GOOD: Use a distinct name +class ThrowingMethod extends Method { + RefType getDeclaredExceptionType() { ... } // Works +} +``` + +### When Query Is Slow + +1. **Enable evaluator logs**: + + ```typescript + codeql_query_run: { + query: "/path/to/Query.ql", + database: "/path/to/db", + "evaluator-log": "/path/to/log.json", + "evaluator-log-level": 5 + } + ``` + +2. **Profile from evaluator logs** (primary tool — returns compact JSON metrics plus a line-indexed detail file for targeted `read_file` access to RA operations and tuple progressions): + + ```typescript + profile_codeql_query_from_logs: { + evaluatorLog: '/path/to/log.json'; + } + ``` + +## Checklist for Complex Queries + +### Before Starting + +- [ ] Test database extracted successfully +- [ ] `PrintAST` output reviewed and understood +- [ ] Key AST classes identified +- [ ] Test cases cover positive, negative, and edge cases + +### During Development + +- [ ] Each predicate/class tested with #quick_evaluate +- [ ] #find_codeql_query_files used to track dependencies +- [ ] CFG consulted for control flow patterns +- [ ] Call graphs generated for cross-function analysis + +### After Each Change + +- [ ] Query compiles with #codeql_query_compile +- [ ] Quick evaluation shows expected results +- [ ] Full tests pass with #codeql_test_run +- [ ] No duplicate or missing results + +### Final Validation + +- [ ] All test cases pass +- [ ] No false positives in results +- [ ] No false negatives (all expected cases caught) +- [ ] Query formatted with #codeql_query_format +- [ ] Performance acceptable (check with #profile_codeql_query_from_logs) + +## Test Acceptance Workflow + +When your query produces correct results but differs from the `.expected` file: + +1. **Review the `.actual` file** to verify results are correct +2. **Accept the results** to update the expected baseline: + + ```typescript + codeql_test_accept: { + tests: ['/path/to/test/QueryTest']; + } + ``` + +3. **Re-run tests** to confirm they now pass + +**Warning**: Only accept results after careful review. Don't blindly accept to make tests pass. + +## Tool Reference + +| Tool | Purpose | When to Use | +| ------------------------------- | --------------------------------- | ------------------------------- | +| #codeql_query_run (PrintAST) | Visualize AST structure | Start of development, debugging | +| #codeql_query_run (PrintCFG) | Visualize control flow | Control flow queries | +| #codeql_query_run (CallGraph\*) | Analyze call relationships | Cross-function queries | +| #codeql_bqrs_interpret | Convert BQRS to readable format | After running graph queries | +| #quick_evaluate | Test individual predicates | Iterative development | +| #find_predicate_position | Locate predicate for quickeval | Before quick_evaluate | +| #find_class_position | Locate class for quickeval | Before quick_evaluate | +| #find_codeql_query_files | Discover related files | Planning, tracking changes | +| #search_ql_code | Search QL files for patterns | Finding classes, predicates | +| #codeql_resolve_files | Find QL files by name/extension | Discovering library pack files | +| #codeql_test_accept | Accept actual results as expected | After verifying correct output | +| #profile_codeql_query_from_logs | Performance analysis | Optimization | + +## Interpreting Graph Query Results + +When running `PrintAST` or `PrintCFG`, the results are stored in BQRS format. To convert to readable text: + +```typescript +// After codeql_query_run produces results.bqrs +codeql_bqrs_interpret: { + file: "/path/to/results.bqrs", + format: "graphtext", + output: "/path/to/output.txt", + t: ["kind=graph", "id=java/tools/print-ast"] +} +``` + +**Note**: The output may create a directory structure (e.g., `output.txt/java/tools/print-ast.txt`) rather than a single file. + +## Example: Workshop Development + +When creating CodeQL workshops, this advanced methodology is essential: + +1. **Analyze production query** with #find_codeql_query_files +2. **Generate AST/CFG** for workshop test code +3. **Decompose query** into stages, validating each with #quick_evaluate +4. **Create exercises** with scaffolding based on AST understanding +5. **Validate solutions** ensure each stage produces correct results + +### Exercise Design Tips + +When creating exercise stubs from solutions: + +```ql +// Use none() as placeholder in characteristic predicates +class MyPattern extends MethodCall { + MyPattern() { + // TODO: Implement - check for methods named "targetMethod" + // Hint: Use this.getMethod().hasName(...) + none() + } +} +``` + +- **Include TODO comments** with specific hints +- **Reference AST class names** students should look for +- **Build incrementally** - each exercise should build on the previous +- **Test both exercises and solutions** to ensure expected files are accurate + +See the `create-codeql-query-development-workshop` skill for complete workshop creation guidance. + +## Related Resources + +- **Basic TDD**: `codeql://prompts/ql-tdd-basic` +- **AST Reference**: `codeql://languages/{language}/ast` +- **Security Patterns**: `codeql://languages/{language}/security` +- **Performance Guide**: `codeql://patterns/performance` +- **Workshop Skill**: `.github/skills/create-codeql-query-development-workshop/SKILL.md` +- **Tools Validation**: `.github/skills/validate-ql-mcp-server-tools-queries/SKILL.md` diff --git a/extensions/vscode/prompts/ql-tdd-basic.prompt.md b/extensions/vscode/prompts/ql-tdd-basic.prompt.md new file mode 100644 index 00000000..44c909f1 --- /dev/null +++ b/extensions/vscode/prompts/ql-tdd-basic.prompt.md @@ -0,0 +1,239 @@ +--- +agent: agent +--- + +# Test-Driven CodeQL Query Development Checklist + +Use this checklist to guide test-driven development of CodeQL queries. Follow the TDD cycle: write tests first, implement query logic, and iterate until tests pass. + +For advanced techniques including AST/CFG visualization, see: `codeql://prompts/ql-tdd-advanced` +For detailed guidance, reference the MCP resource: `codeql://learning/test-driven-development` + +## TDD Workflow Checklist + +### Phase 1: Project Setup + +- [ ] **Create Query Structure** + - Tool: #create_codeql_query + - Specify: basePath, queryName, language, description (optional) + - Creates: src/QueryName/QueryName.ql, test/QueryName/QueryName.qlref, test/QueryName/test-code-file + - The .qlref file will contain the relative path: `QueryName/QueryName.ql` + - Verify: directory structure follows CodeQL conventions with intermediate directories + +- [ ] **Install Pack Dependencies** + - Tool: #codeql_pack_install + - Install src pack dependencies + - Install test pack dependencies + - Verify: imports resolve without errors + +### Phase 2: Test Design (Red Phase) + +- [ ] **Design Test Cases** + - Create positive test cases (should match) + - Create negative test cases (should not match) + - Create edge case tests + - Document expected behavior in comments + +- [ ] **Define Expected Results** + - Create .expected file with anticipated matches + - Specify exact match locations (file, line, column) + - Include expected alert messages + +- [ ] **Extract Test Database** + - Tool: #codeql_test_extract + - Extract database from test code + - Verify: .testproj directory created + +### Phase 3: Analysis and Understanding + +- [ ] **Analyze Test Code AST** + - Tool: #codeql_query_run with queryName: "PrintAST" + - Use format: "graphtext" for @kind graph queries + - Review AST structure and identify relevant classes + +- [ ] **Explore Available Classes and Predicates** + - Tool: #codeql_lsp_completion at the position in the `from` clause + - Parameters: `file_path`, `line` (0-based), `character` (0-based) + - Set `workspace_uri` to the pack root directory (containing `codeql-pack.yml`) + - Request completions after a dot (e.g., `pw.`) to see member predicates with docs + - Tip: Run #codeql_pack_install first — LSP tools require resolved dependencies + +- [ ] **Navigate to Type Definitions** + - Tool: #codeql_lsp_definition on a class or predicate name + - Parameters: `file_path`, `line` (0-based), `character` (0-based), `workspace_uri` + - Returns file URI and line range — even into library pack files + - Read the definition source to understand available member predicates + - Tool: #codeql_lsp_references to find usage examples across the pack + + For the full iterative LSP workflow, see: `codeql://prompts/ql_lsp_iterative_development` + +- [ ] **Reference Language Documentation** + - Resource: `codeql://languages/{language}/ast` + - Resource: `codeql://languages/{language}/security` (if applicable) + - Identify AST classes and predicates needed + +### Phase 4: Implementation (Green Phase) + +- [ ] **Write Query Metadata** + - Add @name annotation + - Add @description annotation + - Add @kind annotation (problem, path-problem, graph, etc.) + - Add @id and other required metadata + +- [ ] **Implement Query Logic** + - Import required libraries + - Define necessary classes (if any) + - Define helper predicates + - Implement main query clause + +- [ ] **Compile Query** + - Tool: #codeql_query_compile + - Fix any compilation errors + - Verify: query compiles successfully + +- [ ] **Run Tests** + - Tool: #codeql_test_run + - Compare actual vs expected results + - If tests fail: adjust query logic and recompile + - If tests pass: proceed to validation + +### Phase 5: Validation and Acceptance + +- [ ] **Verify Test Results** + - Review all test matches + - Confirm no false positives + - Confirm no false negatives + - Check edge cases behave correctly + +- [ ] **Accept Test Results** (only when correct) + - Tool: #codeql_test_accept + - Update .expected files + - Commit accepted results + +### Phase 6: Refactoring and Enhancement + +- [ ] **Refactor Query** + - Improve code clarity + - Extract common logic to predicates + - Add code comments and documentation + - Tool: #codeql_query_format for consistent formatting + +- [ ] **Optimize Performance** (if needed) + - Run with evaluator-log enabled + - Tool: #profile_codeql_query_from_logs + - Resource: `codeql://patterns/performance` + - Optimize expensive operations + +- [ ] **Generate Documentation** + - Tool: #codeql_generate_query_help + - Review and enhance QLDoc comments + - Document query purpose and limitations + +### Phase 7: Additional Testing + +- [ ] **Add More Test Cases** + - Identify additional scenarios + - Add tests for new edge cases + - Extract new test databases + - Run expanded test suite + +- [ ] **Validate Against Real Code** (optional) + - Tool: #codeql_database_create for real codebase + - Tool: #codeql_query_run against real database + - Review results for false positives/negatives + +## Quick Command Reference + +### Essential Tools + +```typescript +// Create query structure +create_codeql_query: { + basePath: "/path/to/query/base", + queryName: "MySecurityQuery", + language: "javascript", + description: "Detects security vulnerability X" +} + +// Install dependencies +codeql_pack_install: { + packPath: "/path/to/pack" +} + +// Extract test database +codeql_test_extract: { + testPath: "/path/to/test/QueryName", + searchPath: "/path/to/base" +} + +// Analyze AST (for @kind graph queries) +codeql_query_run: { + queryName: "PrintAST", + queryLanguage: "javascript", + database: "/path/to/test.testproj", + format: "graphtext", + interpretedOutput: "/path/to/ast-output/" +} + +// Compile query +codeql_query_compile: { + query: "/path/to/Query.ql", + searchPath: "/path/to/base", + checkOnly: true +} + +// Run tests +codeql_test_run: { + testPath: "/path/to/test/Query.qlref", + searchPath: "/path/to/base" +} + +// Accept results +codeql_test_accept: { + testPath: "/path/to/test/Query", + searchPath: "/path/to/base" +} +``` + +## TDD Principles to Remember + +1. **Red → Green → Refactor**: Always start with failing tests +2. **Test First**: Write tests before implementation +3. **Small Steps**: Make minimal changes to pass each test +4. **Frequent Testing**: Run tests after each change +5. **One Concept Per Test**: Each test should verify one behavior +6. **Keep Tests Simple**: Test code should be easy to understand +7. **Refactor Confidently**: Tests enable safe refactoring + +## Common Pitfalls to Avoid + +- ❌ Writing query before tests +- ❌ Accepting test results without verification +- ❌ Skipping compilation step +- ❌ Not using PrintAST to understand test code +- ❌ Not using #codeql_lsp_completion to discover available types +- ❌ Not setting `workspace_uri` when using LSP tools (completions will be empty) +- ❌ Creating tests that are too complex +- ❌ Ignoring false positives in results +- ❌ Not refactoring after tests pass + +## Success Criteria + +Your query development is complete when: + +- ✅ All tests pass +- ✅ No false positives in test results +- ✅ No false negatives (all expected cases caught) +- ✅ Query compiles without errors or warnings +- ✅ Code is well-documented with QLDoc comments +- ✅ Performance is acceptable +- ✅ Edge cases are covered by tests +- ✅ Query follows CodeQL best practices + +## Next Steps After Completion + +1. **Integration Testing**: Test against real codebases +2. **Peer Review**: Have another developer review the query +3. **Documentation**: Update project documentation +4. **Regression Testing**: Add to CI/CD pipeline +5. **Monitor Performance**: Track query performance over time diff --git a/extensions/vscode/prompts/tools-query-workflow.prompt.md b/extensions/vscode/prompts/tools-query-workflow.prompt.md new file mode 100644 index 00000000..bbdb82d8 --- /dev/null +++ b/extensions/vscode/prompts/tools-query-workflow.prompt.md @@ -0,0 +1,209 @@ +--- +agent: agent +--- + +# Using CodeQL Development MCP Server Tools Queries + +This guide helps you use the built-in "tools" queries (`PrintAST`, `PrintCFG`, `CallGraphFrom`, `CallGraphTo`) that ship with the CodeQL Development MCP Server to understand code structure before writing detection queries. + +## Why Use Tools Queries? + +Tools queries provide essential insights into how CodeQL represents your source code: + +| Query | Purpose | Use When | +| --------------- | ------------------------------------------------ | ----------------------------------------------- | +| `PrintAST` | Visualize the Abstract Syntax Tree | Understanding code structure, finding AST nodes | +| `PrintCFG` | Visualize Control Flow Graphs | Understanding execution paths, loop/branch flow | +| `CallGraphFrom` | Find all functions called by a specific function | Tracing data flow through call chains | +| `CallGraphTo` | Find all functions that call a specific function | Understanding function usage patterns | + +## Supported Languages + +Tools queries are available for: `actions`, `cpp`, `csharp`, `go`, `java`, `javascript`, `python`, `ruby`, `rust`, `swift` + +## Prerequisites + +Before using tools queries, you need: + +1. **A CodeQL database** - Either create one or use an existing database +2. **Source files to analyze** - The tools queries filter output to specific files + +## Workflow Checklist + +### Step 1: Identify or Create Database + +- [ ] **Option A: Use existing database** + - Tool: #codeql_resolve_database + - Verify database is valid and note the language + +- [ ] **Option B: Create new database** + - Tool: #codeql_database_create + - Parameters: `database`, `language`, `source-root` + +### Step 2: Run PrintAST Query + +The PrintAST query outputs a hierarchical tree of AST nodes with labels. + +- [ ] **Execute PrintAST** + - Tool: #codeql_query_run + - Parameters: + - `database`: Path to your CodeQL database + - `queryName`: `"PrintAST"` + - `queryLanguage`: Your language (e.g., `"javascript"`, `"python"`, `"cpp"`) + - `sourceFiles`: Comma-separated file names to analyze (e.g., `"main.js,utils.js"`) + - `format`: `"graphtext"` (for human-readable output) + +- [ ] **Verify output contains AST nodes** + - Look for hierarchical structure with indentation + - Confirm nodes have `semmle.label` with class names + - Identify relevant AST classes for your query + +**Example AST output structure:** + +```text +TopLevelFunction +├── FunctionDeclarationEntry +├── Block +│ ├── DeclStmt +│ │ └── LocalVariable +│ ├── ExprStmt +│ │ └── FunctionCall +│ └── ReturnStmt +``` + +### Step 3: Run PrintCFG Query (if needed) + +The PrintCFG query outputs control flow nodes and edges. + +- [ ] **Execute PrintCFG** + - Tool: #codeql_query_run + - Parameters: + - `database`: Path to your CodeQL database + - `queryName`: `"PrintCFG"` + - `queryLanguage`: Your language + - `sourceFunction`: Function name to analyze (e.g., `"processData"`) + - `format`: `"graphtext"` + +- [ ] **Verify output contains nodes and edges** + - Look for `nodes` section with CFG nodes + - Look for `edges` section with `→` arrows showing flow + - Identify control flow patterns (loops, branches) + +**Example CFG output structure:** + +```text +nodes +| node | semmle.label | +| ... | entry: processData | +| ... | if (...) | +| ... | return | + +edges +| from | to | semmle.label | +| ... | ... | → | +``` + +### Step 4: Run CallGraph Queries (if needed) + +Call graph queries help trace function relationships. + +- [ ] **Execute CallGraphFrom** (to find what a function calls) + - Tool: #codeql_query_run + - Parameters: + - `database`: Path to your CodeQL database + - `queryName`: `"CallGraphFrom"` + - `queryLanguage`: Your language + - `sourceFunction`: Function name to trace from (e.g., `"main"`) + - `format`: `"sarif-latest"` or `"csv"` + +- [ ] **Execute CallGraphTo** (to find what calls a function) + - Tool: #codeql_query_run + - Parameters: + - `database`: Path to your CodeQL database + - `queryName`: `"CallGraphTo"` + - `queryLanguage`: Your language + - `targetFunction`: Function name to find callers of (e.g., `"validate"`) + - `format`: `"sarif-latest"` or `"csv"` + +- [ ] **Verify call relationships** + - Confirm results show caller → callee relationships + - Note function locations for further analysis + +### Step 5: Apply Insights to Query Development + +Use the gathered information to inform your query: + +- [ ] **From PrintAST**: Identify which AST classes to use in your `from` clause +- [ ] **From PrintCFG**: Understand execution paths for control-flow-sensitive queries +- [ ] **From CallGraph**: Map data flow paths through function boundaries + +## Common Patterns + +### Pattern 1: Finding All Function Calls + +```text +1. Run PrintAST on your source file +2. Look for FunctionCall, MethodAccess, or similar nodes +3. Note the parent/child relationships +4. Use those AST classes in your query +``` + +### Pattern 2: Tracing Data Through Functions + +```text +1. Run CallGraphFrom on your entry point function +2. Identify which functions are called +3. Run CallGraphTo on sink functions +4. Map the complete path from source to sink +``` + +### Pattern 3: Understanding Loop Structures + +```text +1. Run PrintAST to find loop constructs (ForStmt, WhileStmt, etc.) +2. Run PrintCFG on the containing function +3. Identify back edges that represent loop iteration +4. Use CFG analysis for loop-sensitive queries +``` + +## Troubleshooting + +| Issue | Likely Cause | Resolution | +| --------------------- | ------------------------------------- | ------------------------------------------------------ | +| Empty AST output | `sourceFiles` parameter not matching | Use just filenames, not full paths (e.g., `"test.js"`) | +| Empty CFG output | `sourceFunction` not found | Check exact function name spelling | +| Empty CallGraph | No calls exist or wrong function name | Verify function exists and has calls | +| Query compilation err | Pack dependencies missing | Run #codeql_pack_install on the tools pack | + +## MCP Tools Reference + +| Tool | Purpose | +| ------------------------ | ---------------------------------------------------- | +| #codeql_query_run | Execute tools queries with parameters | +| #codeql_resolve_database | Validate database before querying | +| #codeql_database_create | Create database from source code | +| #codeql_bqrs_interpret | Convert results to different formats | +| #codeql_pack_install | Install pack dependencies if needed | +| #codeql_lsp_completion | Explore available types after seeing AST class names | +| #codeql_lsp_definition | Navigate to class definitions to see predicates | +| #codeql_lsp_references | Find usage examples of a class or predicate | +| #search_ql_code | Search QL source files for patterns (text or regex) | +| #codeql_resolve_files | Find QL files by name, extension, or glob pattern | + +### Using LSP Tools After AST Analysis + +After running PrintAST and identifying relevant AST class names, use the LSP tools +to explore those classes in your query file: + +1. **Write the class name** in your query's `from` clause and save the file +2. **Run #codeql_lsp_completion** after the dot to see member predicates: + - `file_path`: your query file, `line`/`character`: 0-based position after the dot + - `workspace_uri`: the pack root directory (containing `codeql-pack.yml`) +3. **Run #codeql_lsp_definition** on an AST class name to see its full API +4. **Run #codeql_lsp_references** to find usage examples in the pack + +> **Note**: LSP tools use 0-based line/character positions. Run #codeql_pack_install +> before using them — they require resolved dependencies. Set `workspace_uri` to +> a plain directory path (not a `file://` URI). + +For the full iterative LSP development workflow, see: `codeql://prompts/ql_lsp_iterative_development` diff --git a/extensions/vscode/prompts/workshop-creation-workflow.prompt.md b/extensions/vscode/prompts/workshop-creation-workflow.prompt.md new file mode 100644 index 00000000..cdf99913 --- /dev/null +++ b/extensions/vscode/prompts/workshop-creation-workflow.prompt.md @@ -0,0 +1,297 @@ +--- +agent: agent +--- + +# Creating CodeQL Query Development Workshops + +This guide helps you create educational CodeQL query development workshops from existing production-grade queries. Workshops teach developers how to build queries incrementally through exercises and solutions. + +## Workshop Purpose + +A CodeQL workshop transforms a complex, production-ready query into a series of incremental learning stages: + +- **Exercises**: Incomplete query stubs with scaffolding and hints +- **Solutions**: Complete working queries for each stage +- **Tests**: Unit tests validating each stage works correctly +- **Documentation**: README with learning objectives and instructions + +## Prerequisites + +Before creating a workshop, ensure you have: + +1. **A production-grade CodeQL query** - The "target" query to decompose +2. **Working unit tests** - Tests that pass for the target query +3. **A CodeQL database** - For running tools queries and validating results + +## Workshop Creation Checklist + +### Phase 1: Analyze the Target Query + +- [ ] **Locate query files** + - Tool: #find_codeql_query_files + - Parameters: `queryPath` (path to the `.ql` or `.qlref` file) + - Note: Returns query file, test files, expected results, and metadata + +- [ ] **Understand query logic for workshop content** + - Prompt: `explain_codeql_query` + - Parameters: `queryPath`, `language`, and optionally `databasePath` + - Identify: sources, sinks, sanitizers, flow configuration + - Generates: detailed explanation with mermaid evaluation diagram + +- [ ] **Verify existing tests pass** + - Tool: #codeql_test_run + - Parameters: `tests` (array of test directories) + - Confirm: 100% pass rate before proceeding + +### Phase 2: Generate AST/CFG Understanding + +> **⚠️ CRITICAL**: Run tools queries to understand the test code structure. + +- [ ] **Run PrintAST on test code** + - Tool: #codeql_query_run + - Parameters: + - `queryName`: `"PrintAST"` + - `queryLanguage`: Your language (e.g., `"cpp"`, `"javascript"`) + - `database`: Path to test database or extracted `.testproj` + - `sourceFiles`: Test source file names + - `format`: `"graphtext"` + - Verify: Output contains hierarchical AST nodes (not empty) + +- [ ] **Run PrintCFG on key functions** + - Tool: #codeql_query_run + - Parameters: + - `queryName`: `"PrintCFG"` + - `queryLanguage`: Your language + - `database`: Path to database + - `sourceFunction`: Key function names from test code + - `format`: `"graphtext"` + - Verify: Output contains nodes and edges + +- [ ] **Run CallGraph queries** (if query involves data flow) + - Tool: #codeql_query_run + - Parameters: + - `queryName`: `"CallGraphFrom"` or `"CallGraphTo"` + - `database`: Path to database + - `sourceFunction` / `targetFunction`: Relevant function names + - Verify: Output shows call relationships + +### Phase 3: Plan Workshop Stages + +Decompose the query into 4-8 incremental stages using these strategies: + +#### Decomposition Strategies + +| Strategy | Description | Example Progression | +| -------------------- | -------------------------------------------------------- | ------------------------------------- | +| Syntactic → Semantic | Start with syntax, add type checking, then data flow | AST → Types → Local flow → Global | +| Local → Global | Start with local analysis, expand to cross-procedural | Single function → Multiple functions | +| Simple → Filtered | High recall first, then add precision filters | All calls → Specific calls → Filtered | +| Building Blocks | Define helpers, combine into sources/sinks, connect flow | Predicates → Sources → Sinks → Config | + +- [ ] **Document stage progression** + - Stage 1: Basic syntactic pattern (highest recall) + - Stage 2-N: Add refinements (types, filters, flow) + - Final Stage: Complete production query + +### Phase 4: Create Workshop Structure + +Standard workshop directory layout: + +```text +workshop-name/ +├── codeql-workspace.yml # CodeQL workspace configuration +├── README.md # Workshop guide with instructions +├── build-databases.sh # Script to create test databases +├── exercises/ # Student exercise queries +│ ├── codeql-pack.yml +│ ├── Exercise1.ql +│ ├── Exercise2.ql +│ └── ... +├── exercises-tests/ # Tests for exercises +│ ├── codeql-pack.yml +│ ├── Exercise1/ +│ │ ├── Exercise1.qlref +│ │ ├── Exercise1.expected +│ │ └── test.ext +│ └── ... +├── solutions/ # Complete solution queries +│ ├── codeql-pack.yml +│ ├── Exercise1.ql +│ ├── Exercise2.ql +│ └── ... +├── solutions-tests/ # Tests for solutions +│ ├── codeql-pack.yml +│ └── ... +├── tests-common/ # Shared test code +│ └── test.ext +└── graphs/ # AST/CFG visualizations + └── ast-overview.txt +``` + +- [ ] **Create codeql-workspace.yml** + + ```yaml + provide: + - '*/codeql-pack.yml' + ``` + +- [ ] **Create pack files for each directory** + - `exercises/codeql-pack.yml`: Query pack depending on language library + - `exercises-tests/codeql-pack.yml`: Test pack depending on exercises + - `solutions/codeql-pack.yml`: Query pack (same deps as exercises) + - `solutions-tests/codeql-pack.yml`: Test pack depending on solutions + +### Phase 5: Create Solution Queries + +For each stage, create a complete solution query. Use the iterative LSP tools +for efficient development (see `codeql://prompts/ql_lsp_iterative_development`): + +- Use #codeql_lsp_completion to explore types and member predicates while writing queries +- Use #codeql_lsp_definition to navigate to library class definitions +- Use #find_predicate_position + #quick_evaluate to test predicates in isolation +- Set `workspace_uri` to the solutions pack root for dependency resolution + +- [ ] **Stage 1 Solution**: Simplest working version + - Basic import statements + - Minimal from/where/select clause + - Should produce results (high recall, lower precision) + +- [ ] **Intermediate Stages**: Progressive refinements + - Add type constraints + - Add helper predicates + - Filter out false positives + - Add data flow (if applicable) + +- [ ] **Final Stage Solution**: Production-quality query + - Complete metadata (@name, @description, @kind, @id) + - Full data flow configuration (if applicable) + - Proper sanitizers and barriers + - Matches the original target query + +### Phase 6: Create Exercise Queries + +Transform solutions into exercises by removing implementation details: + +- [ ] **Add scaffolding structure** + + ```ql + /** + * @name Exercise N - [Topic] + * @description TODO: Complete this exercise + * @kind problem + * @id workshop/exercise-n + */ + + import language + + // TODO: Define predicate to find [something] + predicate findSomething(Type t) { + // Your implementation here + none() + } + + from Type t + where findSomething(t) + select t, "Found something" + ``` + +- [ ] **Include helpful comments** + - Hints about which AST classes to use + - References to documentation + - Expected behavior description + +- [ ] **Ensure exercises compile** + - Tool: #codeql_query_compile + - Exercises should compile (even if tests fail) + +### Phase 7: Create Tests + +- [ ] **Copy test code to test directories** + - Use same test code for exercises and solutions + - Consider `initialize-qltests.sh` script for shared test code + +- [ ] **Create .qlref files** + - Point to query location: `../exercises/ExerciseN.ql` + +- [ ] **Create .expected files** + - Run solution queries to generate expected output + - Tool: #codeql_test_run with `--learn` flag + - Or: #codeql_test_accept after running tests + +### Phase 8: Validate Workshop + +- [ ] **Run all solution tests** + - Tool: #codeql_test_run + - Parameters: `tests` pointing to `solutions-tests/` + - Verify: 100% pass rate + +- [ ] **Verify exercise stubs compile** + - Tool: #codeql_query_compile + - Parameters: Each exercise query + - Verify: No compilation errors + +- [ ] **Test pack dependencies** + - Tool: #codeql_pack_install + - Run in each pack directory + - Verify: Dependencies resolve correctly + +### Phase 9: Create Documentation + +- [ ] **Write README.md** + - Workshop overview and objectives + - Setup instructions + - Stage-by-stage learning guide + - AST/CFG examples from Phase 2 + +- [ ] **Include AST/CFG visualizations** + - Save PrintAST output to `graphs/` + - Reference in README for learning context + +## External Workshop Considerations + +When creating workshops outside the MCP server repository: + +- [ ] **Install pack dependencies first** + + ```bash + codeql pack install solutions + codeql pack install solutions-tests + ``` + +- [ ] **Check for initialization scripts** + - Some workshops use `initialize-qltests.sh` to copy test files + - Run before executing tests + +- [ ] **Use absolute paths with MCP tools** + - External paths must be absolute + +## MCP Tools and Prompts Reference + +| Tool/Prompt | Type | Purpose | +| ------------------------------- | ------ | ---------------------------------------------------------------------- | +| #find_codeql_query_files | Tool | Locate query and related files | +| `explain_codeql_query` | Prompt | Generate detailed explanations for workshop learning content | +| `document_codeql_query` | Prompt | Create/update query documentation files | +| `ql_lsp_iterative_development` | Prompt | Iterative query development with LSP tools | +| #codeql_query_run | Tool | Run tools queries (PrintAST, PrintCFG, etc.) | +| #codeql_test_run | Tool | Validate tests pass | +| #codeql_test_accept | Tool | Accept test results as expected baseline | +| #codeql_query_compile | Tool | Verify queries compile | +| #codeql_pack_install | Tool | Install pack dependencies | +| #codeql_resolve_metadata | Tool | Extract query metadata | +| #codeql_lsp_completion | Tool | Explore types and member predicates during query writing | +| #codeql_lsp_definition | Tool | Navigate to class/predicate definitions in library code | +| #find_predicate_position | Tool | Locate predicate positions for quick_evaluate | +| #quick_evaluate | Tool | Test individual predicates against a database | +| #profile_codeql_query_from_logs | Tool | Parse evaluator logs from a prior query run into a performance profile | +| #create_codeql_query | Tool | Scaffold new query structure | + +## Troubleshooting + +| Issue | Likely Cause | Resolution | +| ------------------------ | ------------------------------- | --------------------------------------------------- | +| "Nothing to extract" | Missing test source files | Run `initialize-qltests.sh` or copy from shared dir | +| Pack not found | Older pack version not cached | Run `codeql pack install` in pack directory | +| Empty AST/CFG output | Wrong sourceFiles/Function | Use just filenames, verify function name spelling | +| Tests fail unexpectedly | Expected file outdated | Re-run solution and accept with #codeql_test_accept | +| Exercise doesn't compile | Missing imports or syntax error | Ensure valid QL syntax with `none()` placeholder | diff --git a/extensions/vscode/scripts/bundle-customizations.js b/extensions/vscode/scripts/bundle-customizations.js new file mode 100644 index 00000000..5204c400 --- /dev/null +++ b/extensions/vscode/scripts/bundle-customizations.js @@ -0,0 +1,187 @@ +/** + * bundle-customizations.js + * + * Copies bundled custom agents, whitelisted prompts, and whitelisted skills + * into the extension's output directories so the VSIX is self-contained. + * + * Run via: npm run bundle:customizations + * Called automatically by: vscode:prepublish + * + * Resulting layout inside extensions/vscode/: + * agents/ (bundled .agent.md files) + * prompts/ (whitelisted prompt files) + * skills//SKILL.md (whitelisted skill files) + * dist-customizations-manifest.json (manifest of bundled files) + * + * Overlay support: + * --customizations-dir= or CODEQL_MCP_CUSTOMIZATIONS_DIR= + * After the defaults are copied, files from /{agents,prompts,skills} + * are merged in, replacing any colliding files with a warning. + */ + +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + rmSync, + writeFileSync, +} from 'fs'; +import { basename, dirname, join, normalize, resolve } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** + * Core bundle function — separated so tests can call it directly. + * @param {object} opts + * @param {string} opts.extensionRoot Absolute path to extensions/vscode/ + * @param {string|undefined} opts.customizationsDir Optional overlay directory + */ +export async function runBundle({ extensionRoot, customizationsDir }) { + const repoRoot = resolve(extensionRoot, '..', '..'); + const serverPromptsDir = join(repoRoot, 'server', 'src', 'prompts'); + const skillsRoot = join(repoRoot, '.github', 'skills'); + const customizationsSourceDir = join(extensionRoot, 'customizations'); + + const targetAgentsDir = join(extensionRoot, 'agents'); + const targetPromptsDir = join(extensionRoot, 'prompts'); + const targetSkillsDir = join(extensionRoot, 'skills'); + + // Load whitelist config + const configPath = join(customizationsSourceDir, 'bundle-customizations.config.js'); + const { prompts: promptWhitelist, skills: skillWhitelist } = await import(configPath); + + // Clean previous outputs + for (const dir of [targetAgentsDir, targetPromptsDir, targetSkillsDir]) { + if (existsSync(dir)) { + rmSync(dir, { recursive: true, force: true }); + } + mkdirSync(dir, { recursive: true }); + } + + // Track bundled files for manifest + const manifest = { agents: [], prompts: [], skills: [] }; + + // --- Copy agents --- + const agentsSourceDir = join(customizationsSourceDir, 'agents'); + if (existsSync(agentsSourceDir)) { + for (const file of readdirSync(agentsSourceDir)) { + if (!file.endsWith('.agent.md')) continue; + const src = join(agentsSourceDir, file); + const dst = join(targetAgentsDir, file); + copyFileSync(src, dst); + manifest.agents.push(`agents/${file}`); + console.log(`✅ Copied agent: ${file}`); + } + } + + // --- Copy whitelisted prompts --- + for (const promptName of promptWhitelist) { + const src = join(serverPromptsDir, promptName); + if (!existsSync(src)) { + console.warn(`⚠️ Prompt not found, skipping: ${promptName}`); + continue; + } + const dst = join(targetPromptsDir, promptName); + copyFileSync(src, dst); + manifest.prompts.push(`prompts/${promptName}`); + console.log(`✅ Copied prompt: ${promptName}`); + } + + // --- Copy whitelisted skills --- + for (const skillName of skillWhitelist) { + const src = join(skillsRoot, skillName, 'SKILL.md'); + if (!existsSync(src)) { + console.warn(`⚠️ Skill not found, skipping: ${skillName}/SKILL.md`); + continue; + } + const dstDir = join(targetSkillsDir, skillName); + mkdirSync(dstDir, { recursive: true }); + const dst = join(dstDir, 'SKILL.md'); + copyFileSync(src, dst); + manifest.skills.push(`skills/${skillName}/SKILL.md`); + console.log(`✅ Copied skill: ${skillName}/SKILL.md`); + } + + // --- Apply overlay (if specified) --- + if (customizationsDir) { + const overlayRoot = resolve(customizationsDir); + console.log(`\n🔀 Applying overlay from: ${overlayRoot}`); + + for (const category of ['agents', 'prompts', 'skills']) { + const overlayDir = join(overlayRoot, category); + if (!existsSync(overlayDir)) continue; + + applyOverlayDir(overlayDir, join(extensionRoot, category), category, manifest); + } + } + + // --- Write manifest --- + const manifestPath = join(extensionRoot, 'dist-customizations-manifest.json'); + writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf8'); + console.log(`\n📋 Manifest written: dist-customizations-manifest.json`); + + console.log(''); + console.log('🎉 Customizations bundle complete.'); + console.log(` Agents : ${manifest.agents.length}`); + console.log(` Prompts: ${manifest.prompts.length}`); + console.log(` Skills : ${manifest.skills.length}`); + + return manifest; +} + +/** + * Recursively copies files from overlayDir into targetDir. + * Warns when a file already exists (collision). + */ +function applyOverlayDir(overlayDir, targetDir, categoryKey, manifest) { + for (const entry of readdirSync(overlayDir, { withFileTypes: true })) { + const srcPath = join(overlayDir, entry.name); + const dstPath = join(targetDir, entry.name); + + if (entry.isDirectory()) { + mkdirSync(dstPath, { recursive: true }); + applyOverlayDir(srcPath, dstPath, categoryKey, manifest); + continue; + } + + const alreadyExists = existsSync(dstPath); + if (alreadyExists) { + console.warn(`⚠️ Overlay replaces bundled file: ${normalize(dstPath)}`); + } + + mkdirSync(dirname(dstPath), { recursive: true }); + copyFileSync(srcPath, dstPath); + + // Build a relative manifest key: e.g. "agents/foo.agent.md" + const relKey = `${categoryKey}/${entry.name}`; + if (!alreadyExists && Array.isArray(manifest[categoryKey])) { + manifest[categoryKey].push(relKey); + } + + console.log(` ${alreadyExists ? '↩️ Replaced' : '➕ Added'}: ${relKey}`); + } +} + +// --- CLI entry point --- +if (import.meta.url === `file://${process.argv[1]}`) { + // Parse --customizations-dir=PATH or --customizations-dir PATH + let customizationsDir = + process.env.CODEQL_MCP_CUSTOMIZATIONS_DIR ?? undefined; + + for (let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i]; + if (arg.startsWith('--customizations-dir=')) { + customizationsDir = arg.slice('--customizations-dir='.length); + } else if (arg === '--customizations-dir' && process.argv[i + 1]) { + customizationsDir = process.argv[++i]; + } + } + + const extensionRoot = resolve(__dirname, '..'); + runBundle({ extensionRoot, customizationsDir }).catch((err) => { + console.error('❌ bundle-customizations failed:', err); + process.exit(1); + }); +} diff --git a/extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md b/extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md new file mode 100644 index 00000000..6703f512 --- /dev/null +++ b/extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md @@ -0,0 +1,543 @@ +--- +name: create-codeql-query-development-workshop +description: Create custom CodeQL query development workshops from production-grade queries. Use this skill to generate guided learning materials with exercises, solutions, and tests that teach developers how to build CodeQL queries incrementally. +--- + +# Create CodeQL Query Development Workshop + +This skill guides you through creating custom CodeQL query development workshops from existing, production-grade CodeQL queries. The workshop format uses a test-driven, incremental learning approach where developers progress through stages from simple to complex. + +## When to Use This Skill + +- Creating training materials for CodeQL query development +- Teaching developers to build custom security or code quality queries +- Generating guided learning paths from existing query implementations +- Building workshops customized to specific business needs or code patterns + +## Workshop Value Proposition + +**Custom workshops are more effective than generic tutorials** because: + +- Developers learn by building queries that actually matter to their work +- Real-world query patterns are more motivating than toy examples +- Teams can train developers on their specific security or quality concerns +- Workshops scale knowledge transfer from CodeQL experts to their teams + +## Prerequisites + +Before creating a workshop, ensure you have: + +- An existing CodeQL query (`.ql` file) that is production-ready +- Passing unit tests for that query (`.expected` results that match actual results) +- Understanding of the query's purpose and complexity +- Access to CodeQL Development MCP Server tools + +## CodeQL Pack Naming Convention + +This repository uses `codeql-pack.yml` for new CodeQL pack configuration files and recommends it over `qlpack.yml`. While both `codeql-pack.yml` and `qlpack.yml` are equally supported by CodeQL, `codeql-pack.yml` is preferred as it aligns with the `codeql-pack.lock.yml` naming convention used by `codeql pack install`. If you encounter references to `qlpack.yml` in this workshop or related materials, treat them as equivalent to `codeql-pack.yml`, with `codeql-pack.yml` as the recommended name for new packs. + +## Required Inputs + +When invoking this skill, you must provide: + +1. **Source Query Path**: Full path to the production query `.ql` file +2. **Source Query Tests Path**: Full path to the directory containing unit tests for the query +3. **Base Directory**: Path where the workshop directory will be created (e.g., `/tmp/workshops` or `/workshops`) +4. **Workshop Name**: Name for the workshop directory (e.g., `dataflow-analysis-cpp`) + +## Workshop Output Structure + +The skill creates a complete workshop under `//`: + +``` +// +├── README.md # Workshop overview and setup instructions +├── codeql-workspace.yml # CodeQL workspace configuration +├── build-databases.sh # Script to create test databases +├── exercises/ # Student exercise queries (incomplete) +│ ├── codeql-pack.yml # Query pack config +│ ├── Exercise1.ql +│ ├── Exercise2.ql +│ └── ... +├── exercises-tests/ # Unit tests for exercises +│ ├── codeql-pack.yml # Test pack config (with extractor + dependency on exercises) +│ ├── Exercise1/ +│ │ ├── Exercise1.qlref +│ │ ├── Exercise1.expected +│ │ └── test.{ext} +│ └── ... +├── solutions/ # Complete solution queries +│ ├── codeql-pack.yml # Query pack config +│ ├── Exercise1.ql +│ ├── Exercise2.ql +│ └── ... +├── solutions-tests/ # Unit tests for solutions +│ ├── codeql-pack.yml # Test pack config (with extractor + dependency on solutions) +│ ├── Exercise1/ +│ │ ├── Exercise1.qlref +│ │ ├── Exercise1.expected +│ │ └── test.{ext} +│ └── ... +├── graphs/ # AST/CFG visualizations +│ ├── Exercise1-ast.txt +│ ├── Exercise1-cfg.txt +│ └── ... +└── tests-common/ # Shared test code and databases + ├── test.{ext} + └── codeql-pack.yml +``` + +See [workshop-structure-reference.md](./workshop-structure-reference.md) for detailed structure documentation. + +## Workflow Overview + +The workshop creation process follows these phases: + +### Phase 1: Analysis + +1. **Analyze Source Query** using `find_codeql_query_files` and `explain_codeql_query` +2. **Identify Complexity** to determine number of stages +3. **Extract Test Cases** from existing unit tests +4. **Plan Stages** breaking query from simple to complex + +### Phase 2: Decomposition + +Working backwards from the complete query: + +1. **Identify Decomposition Points** (predicates, logic blocks, complexity layers) +2. **Define Stage Goals** (what each exercise teaches) +3. **Create Stage Order** (simple to complex progression) + +### Phase 3: Generation + +For each stage (starting with final/complete stage): + +1. **Generate Solution Query** for this stage +2. **Create Solution Tests** that validate the solution +3. **Run Tests** using `codeql_test_run` to ensure they pass +4. **Generate Exercise Query** by removing implementation details +5. **Create Exercise Tests** (may match solution tests or be subset) + +### Phase 4: Enrichment + +1. **Generate Graph Outputs** (AST/CFG) for each stage using `codeql_bqrs_interpret` +2. **Create build-databases.sh** script for test database creation +3. **Write README.md** with workshop overview, setup, and instructions +4. **Create codeql-workspace.yml** to configure CodeQL workspace + +### Phase 5: Validation + +1. **Test All Solutions** run `codeql_test_run` on solutions-tests/ +2. **Verify Test Pass Rate** ensure 100% pass rate for solutions +3. **Check File Structure** validate all required files exist +4. **Review Exercise Gaps** ensure exercises have appropriate scaffolding + +## Key MCP Server Tools + +### Query Analysis + +- `find_codeql_query_files` - Locate query files and dependencies +- `explain_codeql_query` - Understand query purpose and logic +- `codeql_resolve_metadata` - Extract query metadata + +### Test Management + +- `codeql_test_extract` - Create test databases from test code +- `codeql_test_run` - Execute tests and validate results +- `codeql_test_accept` - Update expected results when needed + +### Query Execution + +- `codeql_query_run` - Run queries (including PrintAST, PrintCFG) +- `codeql_query_compile` - Validate query syntax +- `codeql_bqrs_interpret` - Generate graph outputs from results + +### Database Operations + +- `codeql_database_create` - Create CodeQL databases from source +- `codeql_resolve_database` - Validate database structure + +See [mcp-tools-reference.md](./mcp-tools-reference.md) for detailed tool usage patterns. + +## Stage Decomposition Strategy + +When decomposing a complex query into stages, consider these patterns: + +### Pattern 1: Syntactic to Semantic + +1. **Stage 1**: Find syntactic elements (e.g., `ArrayExpr`) +2. **Stage 2**: Add type constraints (e.g., specific array types) +3. **Stage 3**: Add semantic analysis (e.g., control flow) +4. **Stage 4**: Add data flow analysis (e.g., track values) + +### Pattern 2: Local to Global + +1. **Stage 1**: Local pattern matching +2. **Stage 2**: Add local control flow +3. **Stage 3**: Add local data flow +4. **Stage 4**: Add global data flow + +### Pattern 3: Simple to Filtered + +1. **Stage 1**: Find all candidates (high recall, low precision) +2. **Stage 2**: Add basic filtering +3. **Stage 3**: Add context-aware filtering +4. **Stage 4**: Eliminate false positives + +### Pattern 4: Building Blocks + +1. **Stage 1**: Define helper predicates +2. **Stage 2**: Combine helpers into sources +3. **Stage 3**: Define sinks +4. **Stage 4**: Connect sources to sinks with data flow + +## Exercise Creation Guidelines + +### What to Remove from Solutions + +When creating exercises from solutions: + +- **Implementation bodies**: Leave predicate signatures with `none()` body +- **Complex logic**: Replace with `// TODO: Implement` comments +- **Data flow configs**: Provide signature, remove implementation +- **Filter predicates**: Keep structure, remove conditions + +### What to Keep in Exercises + +- **Import statements**: All imports should be present +- **Type signatures**: Full type information for predicates +- **Comments**: Helpful hints about what to implement +- **Test scaffolding**: Basic structure to guide implementation + +### Hints and Documentation + +Add inline comments to guide students: + +```ql +/** + * Find all array expressions that access a specific type. + * + * Hint: Use `.getArrayBase().getType()` to get the base type. + */ +predicate isTargetArrayAccess(ArrayExpr array) { + // TODO: Implement type checking + none() +} +``` + +## Test Creation Guidelines + +### Test Code Patterns + +Create test code (`test.{ext}`) that includes: + +1. **Positive cases**: Code patterns the query should detect +2. **Negative cases**: Similar code that should NOT be detected +3. **Edge cases**: Boundary conditions +4. **Comments**: Explain what each test case validates + +Example for C++: + +```cpp +// POSITIVE CASE: Null pointer dereference +void unsafeFunction() { + int* ptr = nullptr; + *ptr = 42; // Should be detected +} + +// NEGATIVE CASE: Checked before use +void safeFunction() { + int* ptr = nullptr; + if (ptr != nullptr) { + *ptr = 42; // Should NOT be detected + } +} + +// EDGE CASE: Pointer in complex expression +void edgeCase() { + int* ptr = nullptr; + int result = ptr ? *ptr : 0; // Should be detected +} +``` + +### Expected Results Format + +The `.expected` file uses CodeQL test format: + +``` +| file | line | col | endLine | endCol | message | +| test.cpp | 3 | 5 | 3 | 8 | Null pointer dereference | +| test.cpp | 18 | 17 | 18 | 20 | Null pointer dereference | +``` + +### Test Progression + +- **Early stages**: Fewer expected results (simpler queries) +- **Later stages**: More expected results (more comprehensive) +- **Final stage**: Should match production query expected results + +## Graph Generation + +Generate visual aids for understanding code structure: + +### PrintAST Graphs + +Show Abstract Syntax Tree structure: + +```json +{ + "queryName": "PrintAST", + "queryLanguage": "cpp", + "database": "tests-common/test.testproj", + "outputFormat": "graphtext" +} +``` + +Use `codeql_bqrs_interpret` to create `graphs/Exercise1-ast.txt`. + +### PrintCFG Graphs + +Show Control Flow Graph: + +```json +{ + "queryName": "PrintCFG", + "queryLanguage": "cpp", + "database": "tests-common/test.testproj", + "outputFormat": "graphtext" +} +``` + +Use `codeql_bqrs_interpret` to create `graphs/Exercise1-cfg.txt`. + +## Build Scripts + +### build-databases.sh Template + +```bash +#!/bin/bash +set -e + +WORKSHOP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_SOURCE="${WORKSHOP_ROOT}/tests-common" + +echo "Building test databases..." + +# For each test database needed +for db_name in test1 test2; do + DB_PATH="${WORKSHOP_ROOT}/tests-common/${db_name}.testproj" + + echo "Creating database: ${db_name}" + rm -rf "${DB_PATH}" + + codeql database create \ + --language={language} \ + --source-root="${TEST_SOURCE}" \ + "${DB_PATH}" \ + --command="clang -fsyntax-only ${TEST_SOURCE}/${db_name}.c" +done + +echo "Database creation complete!" +``` + +### codeql-workspace.yml Template + +```yaml +provide: + - '*/codeql-pack.yml' +``` + +This makes all codeql-pack.yml files available in the workspace. + +## Workshop README Template + +The generated README.md should include: + +1. **Title and Overview**: What the workshop teaches +2. **Prerequisites**: Required knowledge and tools +3. **Setup Instructions**: How to clone, install dependencies, build databases +4. **Workshop Structure**: Overview of exercise progression +5. **How to Use**: Instructions for working through exercises +6. **Validation**: How to test exercise solutions +7. **Solutions**: Where to find reference solutions +8. **Additional Resources**: Links to CodeQL documentation + +See [example workshop READMEs](./examples/) for templates. + +## Language-Specific Considerations + +### File Extensions by Language + +- **C/C++**: `.c`, `.cpp`, `.h`, `.hpp` +- **C#**: `.cs` +- **Go**: `.go` +- **Java**: `.java` +- **JavaScript/TypeScript**: `.js`, `.ts` +- **Python**: `.py` +- **Ruby**: `.rb` + +### Test Database Creation + +Language-specific database creation varies: + +- **C/C++**: Requires build command (e.g., `clang -fsyntax-only`) +- **Java**: Requires build tool (e.g., `mvn clean install`) +- **JavaScript**: Usually no build command needed +- **Python**: Usually no build command needed + +Adjust `build-databases.sh` accordingly. + +### Library Dependencies + +Include appropriate CodeQL libraries in `codeql-pack.yml`: + +- **C/C++**: `codeql/cpp-all` +- **C#**: `codeql/csharp-all` +- **Go**: `codeql/go-all` +- **Java**: `codeql/java-all` +- **JavaScript/TypeScript**: `codeql/javascript-all` +- **Python**: `codeql/python-all` +- **Ruby**: `codeql/ruby-all` +- **Rust**: `codeql/rust-all` + +### Java-Specific API Notes + +When writing Java queries, note these API patterns: + +- **Primitive Types**: No `ByteType` class exists. Use `PrimitiveType` with `.getName() = "byte"`, e.g.: `ace.getType().(Array).getElementType().(PrimitiveType).getName() = "byte"` +- **Array Initializers**: No `hasInit()` method. Use `exists(ace.getInit())` to check for initializers +- **Method Calls**: No `MethodAccess` class. Use `MethodCall` for method invocations +- **Deduplication**: When matching both `ArrayCreationExpr` and `ArrayInit`, exclude `ArrayInit` that are part of `ArrayCreationExpr` to avoid duplicate results: `not exists(ArrayCreationExpr ace | ace.getInit() = ai)` + +## Validation Checklist + +Before considering the workshop complete: + +- [ ] All solution queries compile without errors +- [ ] All solution tests pass at 100% +- [ ] Exercise queries have appropriate scaffolding (not empty, not complete) +- [ ] Expected results progress logically from stage to stage +- [ ] Test code covers positive, negative, and edge cases +- [ ] Graph outputs exist for stages where helpful +- [ ] build-databases.sh successfully creates all needed databases +- [ ] README.md provides clear setup and usage instructions +- [ ] codeql-workspace.yml correctly references all codeql-pack.yml files + +## Common Pitfalls + +### Avoid These Mistakes + +1. **Too many stages**: Keep to 4-8 stages max; too many fragments the learning +2. **Too few stages**: 1-2 stages don't provide enough incremental learning +3. **Uneven difficulty**: Each stage should add similar complexity increments +4. **Missing test cases**: Every query behavior should have test coverage +5. **Incomplete exercises**: Exercises should have enough scaffolding to guide students +6. **Overly complete exercises**: Don't give away the solution in exercise code +7. **Inconsistent test results**: Solution tests must pass reliably + +## Example Workshops + +This skill can be used with CodeQL queries from any repository. To see example workshops created with this skill, refer to workshop repositories that demonstrate the standard format and structure. + +## Reference Materials + +For detailed guidance: + +- [Workshop Structure Reference](./workshop-structure-reference.md) - Complete structure specification +- [MCP Tools Reference](./mcp-tools-reference.md) - Tool usage patterns for workshop creation +- [Stage Decomposition Examples](./stage-decomposition-examples.md) - Patterns for breaking down queries + +## Working Workshop Examples + +- [Example C++ Simple](./examples/example-cpp-simple/) - Basic C++ null pointer dereference workshop structure + +Some workshops may have optional advanced branches: + +``` +├── exercises/ +│ ├── Exercise1.ql +│ ├── Exercise2.ql +│ ├── Exercise3.ql +│ ├── Exercise4-basic.ql +│ └── Exercise4-advanced.ql +``` + +### Multiple Learning Paths + +Consider creating workshops with different focuses from the same source query: + +- **Path A**: Focus on syntactic analysis +- **Path B**: Focus on data flow +- **Path C**: Focus on false positive elimination + +### Difficulty Levels + +Add difficulty metadata to exercises: + +```ql +/** + * @name Find Array Access + * @description Identify array expressions + * @kind problem + * @difficulty beginner + * @exercise 1 + */ +``` + +## Troubleshooting + +### Solution Tests Fail + +If solution tests don't pass: + +1. Run `codeql_test_run` with verbose output +2. Compare actual vs expected results +3. Verify test database was created correctly +4. Check query logic matches intended behavior +5. Use `codeql_test_accept` to update `.expected` if needed + +### Exercise Too Difficult + +If students can't complete an exercise: + +1. Add more scaffolding in the exercise query +2. Add more detailed hints in comments +3. Consider splitting into two stages +4. Provide more example patterns in test code + +### Query Doesn't Compile + +If generated queries have compilation errors: + +1. Run `codeql_query_compile` to see specific errors +2. Check import statements are correct +3. Verify qlpack dependencies are installed +4. Ensure predicate signatures are valid + +## Tips for Success + +1. **Start simple**: First workshop should be straightforward +2. **Test frequently**: Run tests after creating each stage +3. **Iterate on stages**: Refine stage boundaries based on testing +4. **Get feedback**: Have someone unfamiliar try the workshop +5. **Document well**: Clear instructions reduce support burden +6. **Version control**: Track workshop iterations in git +7. **Reuse test code**: Same test code across all stages when possible + +## Related Skills + +- [create-codeql-query-tdd-generic](../create-codeql-query-tdd-generic/SKILL.md) - TDD approach to query development +- [create-codeql-query-unit-test-cpp](../create-codeql-query-unit-test-cpp/SKILL.md) - Creating C++ query tests +- [create-codeql-query-unit-test-java](../create-codeql-query-unit-test-java/SKILL.md) - Creating Java query tests +- [create-codeql-query-unit-test-javascript](../create-codeql-query-unit-test-javascript/SKILL.md) - Creating JavaScript query tests +- [create-codeql-query-unit-test-python](../create-codeql-query-unit-test-python/SKILL.md) - Creating Python query tests + +## Success Metrics + +A successful workshop: + +- **Completable**: Students can finish with provided guidance +- **Educational**: Each stage teaches a new concept +- **Validated**: All tests pass reliably +- **Practical**: Query addresses real-world concerns +- **Scalable**: Can be delivered to multiple teams diff --git a/extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md b/extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md new file mode 100644 index 00000000..a8ddf175 --- /dev/null +++ b/extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md @@ -0,0 +1,357 @@ +--- +name: validate-ql-mcp-server-tools-queries +description: Validate that the CodeQL Development MCP Server reliably returns non-empty AST, CFG, and CallGraph data from the bundled tools queries. Use this skill to ensure the `codeql_query_run` tool properly executes PrintAST, PrintCFG, CallGraphFrom, and CallGraphTo queries for any supported language. +--- + +# Validate QL MCP Server Tools Queries + +This skill provides a systematic approach to validating that the CodeQL Development MCP Server's tools queries (`PrintAST`, `PrintCFG`, `CallGraphFrom`, `CallGraphTo`) return **meaningful, non-empty output** for test code across all supported languages. + +## Why This Skill Exists + +> **⚠️ CRITICAL**: The most common failure mode in MCP server tool validation is tests that "go through the motions" without verifying that query output is substantive. + +A test that: + +- ✅ Invokes `codeql_query_run` with `PrintAST.ql` +- ✅ Receives a successful HTTP response +- ❌ **Does NOT verify the output contains actual AST nodes** + +...is a **false positive** that masks real bugs. + +## Supported Languages + +The tools queries are available for all CodeQL-supported languages: + +| Language | Tools Pack Location | Test Code Extension | +| ---------- | ----------------------------- | ------------------- | +| actions | `server/ql/actions/tools/` | `.yml` | +| cpp | `server/ql/cpp/tools/` | `.cpp` | +| csharp | `server/ql/csharp/tools/` | `.cs` | +| go | `server/ql/go/tools/` | `.go` | +| java | `server/ql/java/tools/` | `.java` | +| javascript | `server/ql/javascript/tools/` | `.js` | +| python | `server/ql/python/tools/` | `.py` | +| ruby | `server/ql/ruby/tools/` | `.rb` | +| rust | `server/ql/rust/tools/` | `.rs` | +| swift | `server/ql/swift/tools/` | `.swift` | + +## Tools Queries Overview + +Each language pack includes four standard tools queries: + +### PrintAST + +**Purpose**: Outputs a hierarchical representation of the Abstract Syntax Tree. + +**Expected Output**: A graph with labeled nodes representing code elements (functions, classes, statements, expressions). + +**Validation Check**: Output must contain multiple `semmle.label` properties with actual node names. + +### PrintCFG + +**Purpose**: Outputs the Control Flow Graph showing execution paths. + +**Expected Output**: A graph with `nodes` (CFG nodes) and `edges` (control flow transitions). + +**Validation Check**: Output must contain both `nodes` and `edges` predicates with non-zero results. + +### CallGraphFrom + +**Purpose**: Shows outbound function calls from a specified function. + +**Expected Output**: Problem-style results showing call sites and their targets. + +**Validation Check**: Output must contain call relationships when test code has function calls. + +### CallGraphTo + +**Purpose**: Shows inbound function calls to a specified function. + +**Expected Output**: Problem-style results showing callers of the target function. + +**Validation Check**: Output must contain caller relationships when test code has function definitions that are called. + +## Validation Workflow + +### Step 1: Select Language and Test Code + +Choose a language and ensure valid test code exists: + +```text +server/ql//tools/test/PrintAST/ +├── Example1. # Test source code +├── PrintAST.expected # Expected results +└── PrintAST.qlref # Reference to query +``` + +### Step 2: Extract Test Database + +Use `codeql_test_extract` to create the test database: + +```json +{ + "testPath": "server/ql//tools/test/PrintAST", + "searchPath": ["server/ql//tools/src", "server/ql//tools/test"] +} +``` + +### Step 3: Run PrintAST Query + +Use `codeql_query_run`: + +```json +{ + "queryPath": "server/ql//tools/src/PrintAST/PrintAST.ql", + "database": "server/ql//tools/test/PrintAST/PrintAST.testproj", + "outputFormat": "bqrs" +} +``` + +### Step 4: Interpret Results + +Use `codeql_bqrs_interpret` with `graphtext` format: + +```json +{ + "file": "", + "format": "graphtext", + "output": "" +} +``` + +### Step 5: Validate Non-Empty Output + +**CRITICAL**: Parse the output and verify it contains substantive data: + +- For `PrintAST`: Count nodes with `semmle.label` > 0 +- For `PrintCFG`: Verify both `nodes` and `edges` sections exist +- For `CallGraphFrom/To`: Count result rows > 0 (when calls exist) + +## Language-Specific Examples + +### JavaScript Example + +**Test Code** (`server/ql/javascript/tools/test/PrintAST/Example1.js`): + +```javascript +class Example1 { + constructor(name = 'World') { + this.name = name; + } + + static main(args = []) { + const example = new Example1(); + example.demo(); + } + + demo() { + for (let i = 0; i < 3; i++) { + console.log(i); + } + } +} +``` + +**Expected AST Output** (non-empty validation): + +- Must contain `ClassDefinition` node for `Example1` +- Must contain `MethodDefinition` nodes for `constructor`, `main`, `demo` +- Must contain `ForStatement` node in `demo` method + +**Expected CFG Output**: + +- Must contain nodes for function entry/exit +- Must contain edges for loop control flow + +**Expected CallGraph Output**: + +- `CallGraphFrom(main)` → should find call to `Example1()` and `demo()` +- `CallGraphTo(demo)` → should find caller `main` + +### C++ Example + +**Test Code** (`server/ql/cpp/tools/test/PrintAST/Example1.cpp`): + +```cpp +class someClass { +public: + void f(void); + int g(int i, int j); +}; + +void fun3(someClass *sc) { + int i; + sc->f(); + i = sc->g(1, 2); +} +``` + +**Expected AST Output**: + +- Must contain `Class` node for `someClass` +- Must contain `MemberFunction` nodes for `f` and `g` +- Must contain `Function` node for `fun3` + +**Expected CallGraph Output**: + +- `CallGraphFrom(fun3)` → should find calls to `f()` and `g()` +- `CallGraphTo(f)` → should find caller `fun3` + +### Python Example + +**Test Code** (`server/ql/python/tools/test/PrintAST/Example1.py`): + +```python +class Example1: + def __init__(self, name="World"): + self.name = name + + def demo(self): + for i in range(3): + print(i) + +def main(): + example = Example1() + example.demo() +``` + +**Expected AST Output**: + +- Must contain `ClassDef` node for `Example1` +- Must contain `FunctionDef` nodes for `__init__`, `demo`, `main` +- Must contain `For` node in `demo` + +### Java Example + +**Test Code** (`server/ql/java/tools/test/PrintAST/Example1.java`): + +```java +public class Example1 { + private String name; + + public Example1(String name) { + this.name = name; + } + + public void demo() { + for (int i = 0; i < 3; i++) { + System.out.println(i); + } + } + + public static void main(String[] args) { + Example1 example = new Example1("World"); + example.demo(); + } +} +``` + +**Expected AST Output**: + +- Must contain `ClassOrInterface` node for `Example1` +- Must contain `Method` nodes for constructor, `demo`, `main` +- Must contain `ForStmt` node in `demo` + +## Validation Failure Indicators + +| Symptom | Likely Cause | Resolution | +| ----------------------- | -------------------------------- | --------------------------------------------------- | +| Empty AST output | Database not extracted properly | Re-run `codeql_test_extract` | +| Only headers in output | Query ran but no results matched | Check query's source file selection logic | +| Empty CFG nodes/edges | Test code has no control flow | Add functions with loops/conditionals | +| Empty CallGraph | No function calls in test code | Add function calls or verify `targetFunction` param | +| BQRS decode error | Incompatible CodeQL version | Update CodeQL CLI and re-run | +| Query compilation error | Missing pack dependencies | Run `codeql pack install` in tools pack | +| "Nothing to extract" | Missing test source files | Run workshop's `initialize-qltests.sh` script | +| Pack not found | Older pack version not cached | Run `codeql pack install` in solutions directory | + +## Handling External Workshops + +When validating against external workshops (outside the current workspace), follow these guidelines: + +### Pre-Validation Setup + +1. **Install pack dependencies**: External workshops often use older CodeQL pack versions + + ```bash + cd /path/to/external/workshop + codeql pack install solutions + codeql pack install solutions-tests + ``` + +2. **Run initialization scripts**: Many workshops have scripts that copy test files from shared directories + + ```bash + ./initialize-qltests.sh # Copies from tests-common/ to test directories + ``` + +3. **Verify test files exist**: Check that test source files are present before running tests + + ```bash + ls solutions-tests/Exercise*/test.* + ``` + +### File Access Patterns + +When the external workshop is outside your workspace: + +- Use **terminal commands** (`cat`, `ls`, `head`) to read files +- Use **MCP tools** (`codeql_query_run`, `codeql_test_run`) with absolute paths +- Pre-built databases in the workshop root can be used for tools queries + +## Integration with Prompts and Agents + +This skill is designed to be used with: + +- **Prompt**: [validate-ql-mcp-server-tools-via-workshop](../../prompts/validate-ql-mcp-server-tools-via-workshop.prompt.md) - Uses this skill as part of workshop creation validation +- **Agent**: [mcp-enabled-ql-workshop-developer](../../agents/mcp-enabled-ql-workshop-developer.md) - Orchestrates validation during workshop creation +- **Agent**: [ql-mcp-tool-tester](../../agents/ql-mcp-tool-tester.md) - Tests MCP tools including tools query execution + +## MCP Tools Used + +This skill exercises the following MCP server tools: + +| Tool | Purpose | +| ----------------------- | ----------------------------------- | +| `codeql_test_extract` | Create test database from test code | +| `codeql_query_run` | Execute tools queries | +| `codeql_bqrs_decode` | Decode raw BQRS results | +| `codeql_bqrs_interpret` | Convert results to readable formats | +| `codeql_pack_install` | Install pack dependencies | +| `codeql_pack_ls` | List available packs | + +## Important Implementation Notes + +### Server Logger Output Goes to stderr + +All `logger.info/warn/error/debug` methods write to **stderr** (`console.error`), not stdout. This is required because in stdio transport mode, stdout is reserved exclusively for the MCP JSON-RPC protocol wire format. When validating server startup logs (e.g., confirming `CODEQL_PATH` resolution), always check stderr. + +### CODEQL_PATH Environment Variable + +The MCP server resolves the CodeQL CLI binary at startup via `resolveCodeQLBinary()` in `server/src/lib/cli-executor.ts`. When `CODEQL_PATH` is set to an absolute path pointing to a valid `codeql` binary, the server uses that binary for all CodeQL CLI operations instead of searching `PATH`. This is validated per-OS in `.github/workflows/build-and-test-client.yml` (`codeql-path-tests` job). + +### STDIO Transport and stdin EOF + +When the STDIO transport receives an immediate EOF on stdin (e.g., via ` fifo &`). + +### npm Package Includes Tool Query Source Packs + +The published npm package (`codeql-development-mcp-server`) bundles all tool query source packs under `ql/*/tools/src/`. These are the same `.ql`, `.qll`, `.md`, `codeql-pack.yml`, and `codeql-pack.lock.yml` files — but **never** compiled `.qlx` bytecode (excluded by `server/.npmignore`). + +## Success Criteria + +Validation passes when **ALL** of the following are true: + +- [ ] `PrintAST` returns ≥10 labeled AST nodes for test code +- [ ] `PrintCFG` returns both `nodes` and `edges` with ≥5 entries each +- [ ] `CallGraphFrom` returns ≥1 result when test code contains function calls +- [ ] `CallGraphTo` returns ≥1 result when test code contains called functions +- [ ] No MCP tool errors or timeouts occurred +- [ ] Results are consistent across multiple runs + +## Related Resources + +- [Server Documentation](../../../server/QL-MCP-SERVER.md) +- [create-codeql-query-tdd-generic](../create-codeql-query-tdd-generic/SKILL.md) - TDD workflow that uses AST/CFG analysis +- [create-codeql-query-development-workshop](../create-codeql-query-development-workshop/SKILL.md) - Workshop creation that depends on tools queries diff --git a/extensions/vscode/src/customizations/agent-registrar.ts b/extensions/vscode/src/customizations/agent-registrar.ts new file mode 100644 index 00000000..09d23b2a --- /dev/null +++ b/extensions/vscode/src/customizations/agent-registrar.ts @@ -0,0 +1,183 @@ +import * as path from 'path'; +import * as vscode from 'vscode'; +import { Logger } from '../common/logger'; + +/** Normalizes a file-system path for comparison (removes trailing sep, platform lower-case on Windows). */ +function normalizePath(p: string): string { + let n = path.normalize(p); + // Remove trailing separator + if (n.endsWith(path.sep) && n.length > 1) { + n = n.slice(0, -1); + } + return process.platform === 'win32' ? n.toLowerCase() : n; +} + +/** + * Manages registration of `.agent.md` directories in `chat.agentFilesLocations`. + * + * Adds the extension's bundled `agents/` directory and any user-configured + * additional directories to the VS Code configuration setting, and removes + * them on dispose. + */ +export class AgentRegistrar implements vscode.Disposable { + private readonly _bundledDir: string; + private readonly _disposables: vscode.Disposable[] = []; + private _addedKeys: Set = new Set(); + + constructor( + private readonly _context: vscode.ExtensionContext, + private readonly _logger: Logger, + ) { + this._bundledDir = path.join(_context.extensionUri.fsPath, 'agents'); + } + + /** Registers bundled + additional agent dirs in chat.agentFilesLocations. */ + register(): void { + const cfg = vscode.workspace.getConfiguration('codeql-mcp'); + const enabled = cfg.get('agents.enabled', true); + + if (!enabled) { + this._removeAddedKeys(); + return; + } + + const additionalDirs = cfg.get('additionalAgentDirs', []); + + const dirsToAdd = [this._bundledDir, ...additionalDirs]; + + const chatCfg = vscode.workspace.getConfiguration('chat'); + const existing: Record = + this._normalizeLocations(chatCfg.get('agentFilesLocations')); + + const updated = { ...existing }; + const newKeys: Set = new Set(); + + for (const dir of dirsToAdd) { + const normalized = normalizePath(dir); + if (!this._hasNormalizedKey(updated, normalized)) { + updated[dir] = true; + newKeys.add(dir); + } + } + + if (Object.keys(updated).length === Object.keys(existing).length) { + // Nothing new to add; make sure we track the keys we added previously + this._addedKeys = newKeys; + return; + } + + const target = this._configTarget(); + chatCfg.update('agentFilesLocations', updated, target).then( + () => { + this._addedKeys = newKeys; + this._logger.info( + `AgentRegistrar: registered ${newKeys.size} dir(s) in chat.agentFilesLocations`, + ); + }, + (err: unknown) => { + this._logger.warn( + `AgentRegistrar: failed to update chat.agentFilesLocations: ${err instanceof Error ? err.message : String(err)}`, + ); + }, + ); + } + + /** Subscribes to configuration and workspace folder changes. */ + startWatching(): void { + this._disposables.push( + vscode.workspace.onDidChangeConfiguration((e) => { + if ( + e.affectsConfiguration('codeql-mcp.agents') || + e.affectsConfiguration('codeql-mcp.additionalAgentDirs') + ) { + this.register(); + } + }), + vscode.workspace.onDidChangeWorkspaceFolders(() => { + this.register(); + }), + ); + } + + /** Returns current status for the showAgentsStatus command. */ + getStatus(): { + enabled: boolean; + bundledDir: string; + additionalDirs: string[]; + effectiveLocations: string[]; + } { + const cfg = vscode.workspace.getConfiguration('codeql-mcp'); + const enabled = cfg.get('agents.enabled', true); + const additionalDirs = cfg.get('additionalAgentDirs', []); + + const chatCfg = vscode.workspace.getConfiguration('chat'); + const locations = this._normalizeLocations(chatCfg.get('agentFilesLocations')); + + return { + additionalDirs, + bundledDir: this._bundledDir, + effectiveLocations: Object.keys(locations), + enabled, + }; + } + + dispose(): void { + this._removeAddedKeys(); + for (const d of this._disposables) { + d.dispose(); + } + this._disposables.length = 0; + } + + // ---------- private helpers ---------- + + private _normalizeLocations(value: unknown): Record { + if (!value || typeof value !== 'object') return {}; + if (Array.isArray(value)) { + const obj: Record = {}; + for (const entry of value) { + if (typeof entry === 'string') obj[entry] = true; + } + return obj; + } + return value as Record; + } + + private _hasNormalizedKey(obj: Record, normalized: string): boolean { + return Object.keys(obj).some((k) => normalizePath(k) === normalized); + } + + private _removeAddedKeys(): void { + if (this._addedKeys.size === 0) return; + + const chatCfg = vscode.workspace.getConfiguration('chat'); + const existing = this._normalizeLocations(chatCfg.get('agentFilesLocations')); + const updated: Record = {}; + + for (const [k, v] of Object.entries(existing)) { + if (!this._addedKeys.has(k)) { + updated[k] = v; + } + } + + const target = this._configTarget(); + chatCfg.update('agentFilesLocations', updated, target).then( + () => { + this._logger.info('AgentRegistrar: removed registered agent dirs from chat.agentFilesLocations'); + this._addedKeys = new Set(); + }, + (err: unknown) => { + this._logger.warn( + `AgentRegistrar: failed to clean up chat.agentFilesLocations: ${err instanceof Error ? err.message : String(err)}`, + ); + }, + ); + } + + private _configTarget(): vscode.ConfigurationTarget { + const folders = vscode.workspace.workspaceFolders; + return folders && folders.length > 0 + ? vscode.ConfigurationTarget.Workspace + : vscode.ConfigurationTarget.Global; + } +} diff --git a/extensions/vscode/src/extension.ts b/extensions/vscode/src/extension.ts index db6c2aa6..bbed6d48 100644 --- a/extensions/vscode/src/extension.ts +++ b/extensions/vscode/src/extension.ts @@ -1,4 +1,5 @@ import * as vscode from 'vscode'; +import { AgentRegistrar } from './customizations/agent-registrar'; import { Logger } from './common/logger'; import { CliResolver } from './codeql/cli-resolver'; import { ServerManager } from './server/server-manager'; @@ -38,6 +39,18 @@ export async function activate( disposables.push(cliResolver, serverManager, packInstaller, storagePaths, envBuilder, mcpProvider); + // --- Agent registrar --- + const agentRegistrar = new AgentRegistrar(context, logger); + disposables.push(agentRegistrar); + try { + agentRegistrar.register(); + agentRegistrar.startWatching(); + } catch (err) { + logger.warn( + `AgentRegistrar init failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + // --- Bridge: filesystem watchers --- const config = vscode.workspace.getConfiguration('codeql-mcp'); const watchEnabled = config.get('watchCodeqlExtension', true); @@ -140,6 +153,21 @@ export async function activate( ); vscode.window.showInformationMessage('CodeQL tool query packs reinstalled successfully.'); }), + vscode.commands.registerCommand('codeql-mcp.showAgentsStatus', () => { + const status = agentRegistrar.getStatus(); + const lines = [ + `Enabled: ${status.enabled}`, + `Bundled agents dir: ${status.bundledDir}`, + `Additional dirs: ${status.additionalDirs.length > 0 ? status.additionalDirs.join(', ') : '(none)'}`, + `Registered locations: ${status.effectiveLocations.length}`, + ]; + logger.info('--- Agents Status ---'); + for (const line of lines) { + logger.info(line); + } + logger.show(); + vscode.window.showInformationMessage(lines.join(' | ')); + }), vscode.commands.registerCommand('codeql-mcp.showStatus', async () => { const cliPath = await cliResolver.resolve(); const version = await serverManager.getInstalledVersion(); diff --git a/extensions/vscode/test/customizations/agent-registrar.test.ts b/extensions/vscode/test/customizations/agent-registrar.test.ts new file mode 100644 index 00000000..381c7881 --- /dev/null +++ b/extensions/vscode/test/customizations/agent-registrar.test.ts @@ -0,0 +1,225 @@ +/** + * Tests for AgentRegistrar. + * + * Uses the existing __mocks__/vscode.ts via vitest.config.ts resolve.alias. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import * as vscode from 'vscode'; + +// We need a stateful config mock so we can verify updates +function createStatefulConfigMock(initialValues: Record = {}) { + const store: Record = { ...initialValues }; + return { + get: vi.fn((key: string, defaultVal?: unknown) => { + return key in store ? store[key] : defaultVal; + }), + has: vi.fn(() => false), + inspect: vi.fn(() => undefined), + update: vi.fn((key: string, value: unknown) => { + store[key] = value; + return Promise.resolve(); + }), + _store: store, + }; +} + +// Module-level config stores so we can inspect/update them in tests +let codeqlMcpCfg = createStatefulConfigMock({ 'agents.enabled': true, 'additionalAgentDirs': [] }); +let chatCfg = createStatefulConfigMock({ 'agentFilesLocations': {} }); + +vi.spyOn(vscode.workspace, 'getConfiguration').mockImplementation((section?: string) => { + if (section === 'codeql-mcp') return codeqlMcpCfg as any; + if (section === 'chat') return chatCfg as any; + return createStatefulConfigMock() as any; +}); + +// Set workspaceFolders to a non-empty array for Workspace target tests +vi.spyOn(vscode.workspace, 'workspaceFolders', 'get').mockReturnValue([{ uri: { fsPath: '/ws' } }] as any); + +import { AgentRegistrar } from '../../src/customizations/agent-registrar'; + +function createMockContext(extensionPath = '/mock/extension'): vscode.ExtensionContext { + return { + extensionUri: { fsPath: extensionPath }, + extensionPath, + subscriptions: [], + } as unknown as vscode.ExtensionContext; +} + +function createMockLogger() { + return { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + show: vi.fn(), + dispose: vi.fn(), + } as any; +} + +describe('AgentRegistrar', () => { + beforeEach(() => { + codeqlMcpCfg = createStatefulConfigMock({ 'agents.enabled': true, 'additionalAgentDirs': [] }); + chatCfg = createStatefulConfigMock({ 'agentFilesLocations': {} }); + vi.spyOn(vscode.workspace, 'getConfiguration').mockImplementation((section?: string) => { + if (section === 'codeql-mcp') return codeqlMcpCfg as any; + if (section === 'chat') return chatCfg as any; + return createStatefulConfigMock() as any; + }); + }); + + it('adds bundled agents/ dir to chat.agentFilesLocations when enabled', async () => { + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + // Wait for the async update to resolve + await Promise.resolve(); + + const updated = chatCfg._store['agentFilesLocations'] as Record; + const bundledDir = '/mock/extension/agents'; + expect(Object.keys(updated)).toContain(bundledDir); + }); + + it('does not add bundled dir when agents.enabled = false', async () => { + codeqlMcpCfg = createStatefulConfigMock({ 'agents.enabled': false, 'additionalAgentDirs': [] }); + vi.spyOn(vscode.workspace, 'getConfiguration').mockImplementation((section?: string) => { + if (section === 'codeql-mcp') return codeqlMcpCfg as any; + if (section === 'chat') return chatCfg as any; + return createStatefulConfigMock() as any; + }); + + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + await Promise.resolve(); + + const updated = chatCfg._store['agentFilesLocations'] as Record | undefined; + expect(updated ? Object.keys(updated) : []).not.toContain('/mock/extension/agents'); + }); + + it('appends additionalAgentDirs alongside bundled dir', async () => { + codeqlMcpCfg = createStatefulConfigMock({ + 'agents.enabled': true, + 'additionalAgentDirs': ['/custom/agents'], + }); + vi.spyOn(vscode.workspace, 'getConfiguration').mockImplementation((section?: string) => { + if (section === 'codeql-mcp') return codeqlMcpCfg as any; + if (section === 'chat') return chatCfg as any; + return createStatefulConfigMock() as any; + }); + + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + await Promise.resolve(); + + const updated = chatCfg._store['agentFilesLocations'] as Record; + expect(Object.keys(updated)).toContain('/mock/extension/agents'); + expect(Object.keys(updated)).toContain('/custom/agents'); + }); + + it('does not add duplicate entries on repeated register() calls', async () => { + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + await Promise.resolve(); + registrar.register(); + await Promise.resolve(); + + const updated = chatCfg._store['agentFilesLocations'] as Record; + const bundledDir = '/mock/extension/agents'; + const count = Object.keys(updated).filter((k) => k === bundledDir).length; + expect(count).toBe(1); + }); + + it('uses ConfigurationTarget.Workspace when workspaceFolders is non-empty', async () => { + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + await Promise.resolve(); + + expect(chatCfg.update).toHaveBeenCalledWith( + 'agentFilesLocations', + expect.any(Object), + vscode.ConfigurationTarget.Workspace, + ); + }); + + it('uses ConfigurationTarget.Global when workspaceFolders is empty', async () => { + vi.spyOn(vscode.workspace, 'workspaceFolders', 'get').mockReturnValueOnce([] as any); + + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + await Promise.resolve(); + + expect(chatCfg.update).toHaveBeenCalledWith( + 'agentFilesLocations', + expect.any(Object), + vscode.ConfigurationTarget.Global, + ); + }); + + it('dispose() removes only the entries this instance added', async () => { + // Pre-populate with an existing entry + chatCfg = createStatefulConfigMock({ + 'agentFilesLocations': { '/pre-existing/agents': true }, + }); + vi.spyOn(vscode.workspace, 'getConfiguration').mockImplementation((section?: string) => { + if (section === 'codeql-mcp') return codeqlMcpCfg as any; + if (section === 'chat') return chatCfg as any; + return createStatefulConfigMock() as any; + }); + + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + await Promise.resolve(); + + // Both entries should be present + let locs = chatCfg._store['agentFilesLocations'] as Record; + expect(Object.keys(locs)).toContain('/pre-existing/agents'); + expect(Object.keys(locs)).toContain('/mock/extension/agents'); + + registrar.dispose(); + await Promise.resolve(); + + locs = chatCfg._store['agentFilesLocations'] as Record; + expect(Object.keys(locs)).toContain('/pre-existing/agents'); + expect(Object.keys(locs)).not.toContain('/mock/extension/agents'); + }); + + it('dispose() is idempotent — safe to call twice', async () => { + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + registrar.register(); + await Promise.resolve(); + + expect(() => { + registrar.dispose(); + registrar.dispose(); + }).not.toThrow(); + }); + + it('getStatus() returns correct shape', () => { + const ctx = createMockContext('/mock/extension'); + const logger = createMockLogger(); + const registrar = new AgentRegistrar(ctx, logger); + const status = registrar.getStatus(); + + expect(status).toHaveProperty('enabled'); + expect(status).toHaveProperty('bundledDir'); + expect(status).toHaveProperty('additionalDirs'); + expect(status).toHaveProperty('effectiveLocations'); + expect(status.bundledDir).toBe('/mock/extension/agents'); + }); +}); diff --git a/extensions/vscode/test/customizations/bundle-customizations.test.ts b/extensions/vscode/test/customizations/bundle-customizations.test.ts new file mode 100644 index 00000000..53014f5b --- /dev/null +++ b/extensions/vscode/test/customizations/bundle-customizations.test.ts @@ -0,0 +1,165 @@ +/** + * Tests for bundle-customizations.js + * + * Runs the bundler's exported `runBundle()` function in an isolated temp + * directory structure to verify default copying, manifest generation, + * overlay support, and graceful handling of absent whitelisted files. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'fs'; +import { join, resolve } from 'path'; +import { fileURLToPath } from 'url'; + +// Resolve the actual script location +const __repoRoot = resolve(fileURLToPath(import.meta.url), '..', '..', '..', '..', '..'); + +// Import the bundler dynamically (ES module) +async function importBundler() { + const bundlerPath = resolve(__repoRoot, 'extensions', 'vscode', 'scripts', 'bundle-customizations.js'); + return import(bundlerPath) as Promise<{ runBundle: (opts: { extensionRoot: string; customizationsDir?: string }) => Promise<{ agents: string[]; prompts: string[]; skills: string[] }> }>; +} + +describe('bundle-customizations', () => { + let tmp: string; + + beforeEach(() => { + tmp = mkdtempSync(join(process.cwd(), 'bundle-test-')); + }); + + afterEach(() => { + if (existsSync(tmp)) rmSync(tmp, { recursive: true, force: true }); + }); + + it('copies agent files to agents/ output dir', async () => { + const fakeDeep = join(tmp, 'fake-repo', 'extensions', 'vscode'); + mkdirSync(fakeDeep, { recursive: true }); + // Copy customizations into fakeDeep + const customizationsDir = join(fakeDeep, 'customizations'); + const agentsDir = join(customizationsDir, 'agents'); + mkdirSync(agentsDir, { recursive: true }); + writeFileSync( + join(agentsDir, 'codeql-query-developer.agent.md'), + '---\nname: codeql-query-developer\n---\n', + ); + // Config that references real repo paths + writeFileSync( + join(customizationsDir, 'bundle-customizations.config.js'), + `export const prompts = ['ql-tdd-basic.prompt.md'];\nexport const skills = ['validate-ql-mcp-server-tools-queries', 'nonexistent-skill'];\n`, + ); + + // Create the server/src/prompts dir + const serverPromptsDir = join(tmp, 'fake-repo', 'server', 'src', 'prompts'); + mkdirSync(serverPromptsDir, { recursive: true }); + // Copy a real prompt + const realPrompt = join(__repoRoot, 'server', 'src', 'prompts', 'ql-tdd-basic.prompt.md'); + if (existsSync(realPrompt)) { + const content = readFileSync(realPrompt, 'utf8'); + writeFileSync(join(serverPromptsDir, 'ql-tdd-basic.prompt.md'), content); + } + + // Create skills dir + const skillsDir = join(tmp, 'fake-repo', '.github', 'skills', 'validate-ql-mcp-server-tools-queries'); + mkdirSync(skillsDir, { recursive: true }); + const realSkill = join(__repoRoot, '.github', 'skills', 'validate-ql-mcp-server-tools-queries', 'SKILL.md'); + if (existsSync(realSkill)) { + writeFileSync(join(skillsDir, 'SKILL.md'), readFileSync(realSkill, 'utf8')); + } + + const { runBundle } = await importBundler(); + const manifest = await runBundle({ extensionRoot: fakeDeep }); + + // Agents dir should exist + expect(existsSync(join(fakeDeep, 'agents', 'codeql-query-developer.agent.md'))).toBe(true); + + // Prompt should be present + expect(existsSync(join(fakeDeep, 'prompts', 'ql-tdd-basic.prompt.md'))).toBe(true); + + // Skill should be present + expect(existsSync(join(fakeDeep, 'skills', 'validate-ql-mcp-server-tools-queries', 'SKILL.md'))).toBe(true); + + // Manifest emitted + expect(existsSync(join(fakeDeep, 'dist-customizations-manifest.json'))).toBe(true); + expect(manifest.agents).toContain('agents/codeql-query-developer.agent.md'); + expect(manifest.prompts).toContain('prompts/ql-tdd-basic.prompt.md'); + expect(manifest.skills).toContain('skills/validate-ql-mcp-server-tools-queries/SKILL.md'); + }); + + it('warns but does not fail when whitelisted files are absent', async () => { + const fakeDeep = join(tmp, 'fake-repo2', 'extensions', 'vscode'); + mkdirSync(join(fakeDeep, 'customizations', 'agents'), { recursive: true }); + writeFileSync( + join(fakeDeep, 'customizations', 'bundle-customizations.config.js'), + `export const prompts = ['nonexistent.prompt.md'];\nexport const skills = ['nonexistent-skill'];\n`, + ); + + const { runBundle } = await importBundler(); + + // Should not throw + await expect(runBundle({ extensionRoot: fakeDeep })).resolves.toBeDefined(); + + const manifestPath = join(fakeDeep, 'dist-customizations-manifest.json'); + expect(existsSync(manifestPath)).toBe(true); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as { agents: string[]; prompts: string[]; skills: string[] }; + // Nothing should have been copied for prompts/skills + expect(manifest.prompts).toEqual([]); + expect(manifest.skills).toEqual([]); + }); + + it('overlay: replaces existing file with warning and adds net-new files', async () => { + const fakeDeep = join(tmp, 'fake-repo3', 'extensions', 'vscode'); + mkdirSync(join(fakeDeep, 'customizations', 'agents'), { recursive: true }); + writeFileSync( + join(fakeDeep, 'customizations', 'agents', 'codeql-query-developer.agent.md'), + '---\nname: codeql-query-developer\n---\n# Default\n', + ); + writeFileSync( + join(fakeDeep, 'customizations', 'bundle-customizations.config.js'), + `export const prompts = [];\nexport const skills = [];\n`, + ); + + // Create an overlay directory + const overlayDir = join(tmp, 'overlay'); + mkdirSync(join(overlayDir, 'agents'), { recursive: true }); + // Override the existing agent + writeFileSync( + join(overlayDir, 'agents', 'codeql-query-developer.agent.md'), + '---\nname: codeql-query-developer\n---\n# Override\n', + ); + // Add a net-new agent + writeFileSync( + join(overlayDir, 'agents', 'team-agent.agent.md'), + '---\nname: team-agent\n---\n# Team\n', + ); + + const { runBundle } = await importBundler(); + + // Capture console.warn calls + const warnings: string[] = []; + const originalWarn = console.warn; + console.warn = (...args: unknown[]) => { warnings.push(String(args[0])); }; + + try { + await runBundle({ extensionRoot: fakeDeep, customizationsDir: overlayDir }); + + // Replacement should have warned + expect(warnings.some((w) => w.includes('Overlay replaces bundled file'))).toBe(true); + + // Override file has new content + const overrideContent = readFileSync(join(fakeDeep, 'agents', 'codeql-query-developer.agent.md'), 'utf8'); + expect(overrideContent).toContain('# Override'); + + // Net-new file should exist + expect(existsSync(join(fakeDeep, 'agents', 'team-agent.agent.md'))).toBe(true); + } finally { + console.warn = originalWarn; + } + }); +}); diff --git a/extensions/vscode/test/suite/agents.integration.test.ts b/extensions/vscode/test/suite/agents.integration.test.ts new file mode 100644 index 00000000..987b98f5 --- /dev/null +++ b/extensions/vscode/test/suite/agents.integration.test.ts @@ -0,0 +1,133 @@ +/** + * Integration tests for built-in custom agents. + * + * These run inside the Extension Development Host with the REAL VS Code API. + * They verify the agents/ directory is bundled, the .agent.md files exist, + * and that chat.agentFilesLocations is updated correctly. + */ + +import * as assert from 'assert'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import * as vscode from 'vscode'; + +const EXTENSION_ID = 'advanced-security.vscode-codeql-development-mcp-server'; + +suite('Agents Integration Tests', () => { + let ext: vscode.Extension; + + suiteSetup(async () => { + const found = vscode.extensions.getExtension(EXTENSION_ID); + assert.ok(found, `Extension ${EXTENSION_ID} not found`); + ext = found; + if (!ext.isActive) { + await ext.activate(); + } + }); + + test('Extension agents/ directory exists', () => { + const agentsDir = path.join(ext.extensionPath, 'agents'); + assert.ok(fs.existsSync(agentsDir), `agents/ dir should exist at ${agentsDir}`); + }); + + test('codeql-query-developer.agent.md exists and has correct name frontmatter', () => { + const agentPath = path.join(ext.extensionPath, 'agents', 'codeql-query-developer.agent.md'); + assert.ok(fs.existsSync(agentPath), `${agentPath} should exist`); + const content = fs.readFileSync(agentPath, 'utf8'); + assert.ok(content.includes('name: codeql-query-developer'), 'Should contain name frontmatter'); + assert.ok(!content.includes('model:'), 'Should NOT contain model: key'); + }); + + test('codeql-workshop-author.agent.md exists and has correct name frontmatter', () => { + const agentPath = path.join(ext.extensionPath, 'agents', 'codeql-workshop-author.agent.md'); + assert.ok(fs.existsSync(agentPath), `${agentPath} should exist`); + const content = fs.readFileSync(agentPath, 'utf8'); + assert.ok(content.includes('name: codeql-workshop-author'), 'Should contain name frontmatter'); + assert.ok(!content.includes('model:'), 'Should NOT contain model: key'); + }); + + test('dist-customizations-manifest.json exists and lists expected files', () => { + const manifestPath = path.join(ext.extensionPath, 'dist-customizations-manifest.json'); + assert.ok(fs.existsSync(manifestPath), 'dist-customizations-manifest.json should exist'); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + assert.ok(Array.isArray(manifest.agents), 'manifest.agents should be an array'); + assert.ok(Array.isArray(manifest.prompts), 'manifest.prompts should be an array'); + assert.ok(Array.isArray(manifest.skills), 'manifest.skills should be an array'); + assert.ok( + manifest.agents.some((a: string) => a.includes('codeql-query-developer')), + 'Manifest should list codeql-query-developer agent', + ); + }); + + test('codeql-mcp.showAgentsStatus command resolves without throwing', async () => { + await assert.doesNotReject( + vscode.commands.executeCommand('codeql-mcp.showAgentsStatus'), + 'showAgentsStatus command should not throw', + ); + }); + + test('Toggling codeql-mcp.agents.enabled = false removes bundled dir; re-enabling restores it', async () => { + const agentsDir = path.join(ext.extensionPath, 'agents'); + const cfg = vscode.workspace.getConfiguration('codeql-mcp'); + const chatCfg = vscode.workspace.getConfiguration('chat'); + + // Save original values + const originalEnabled = cfg.get('agents.enabled', true); + const originalLocations = chatCfg.get>('agentFilesLocations', {}); + + try { + // Disable agents + await cfg.update('agents.enabled', false, vscode.ConfigurationTarget.Global); + // Give the registrar time to react + await new Promise((resolve) => setTimeout(resolve, 200)); + + const locationsAfterDisable = vscode.workspace.getConfiguration('chat') + .get>('agentFilesLocations', {}); + const containsBundledAfterDisable = Object.keys(locationsAfterDisable).some( + (k) => path.normalize(k) === path.normalize(agentsDir), + ); + assert.strictEqual(containsBundledAfterDisable, false, 'Bundled dir should be removed when disabled'); + + // Re-enable + await cfg.update('agents.enabled', true, vscode.ConfigurationTarget.Global); + await new Promise((resolve) => setTimeout(resolve, 200)); + + const locationsAfterEnable = vscode.workspace.getConfiguration('chat') + .get>('agentFilesLocations', {}); + const containsBundledAfterEnable = Object.keys(locationsAfterEnable).some( + (k) => path.normalize(k) === path.normalize(agentsDir), + ); + assert.strictEqual(containsBundledAfterEnable, true, 'Bundled dir should be restored when re-enabled'); + } finally { + // Restore original config + await cfg.update('agents.enabled', originalEnabled, vscode.ConfigurationTarget.Global); + await chatCfg.update('agentFilesLocations', originalLocations, vscode.ConfigurationTarget.Global); + } + }); + + test('Setting codeql-mcp.additionalAgentDirs appends the dir', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codeql-mcp-test-')); + const cfg = vscode.workspace.getConfiguration('codeql-mcp'); + const chatCfg = vscode.workspace.getConfiguration('chat'); + + const originalAdditional = cfg.get('additionalAgentDirs', []); + const originalLocations = chatCfg.get>('agentFilesLocations', {}); + + try { + await cfg.update('additionalAgentDirs', [tmpDir], vscode.ConfigurationTarget.Global); + await new Promise((resolve) => setTimeout(resolve, 200)); + + const locations = vscode.workspace.getConfiguration('chat') + .get>('agentFilesLocations', {}); + const containsTmpDir = Object.keys(locations).some( + (k) => path.normalize(k) === path.normalize(tmpDir), + ); + assert.ok(containsTmpDir, `tmpDir ${tmpDir} should be in agentFilesLocations`); + } finally { + await cfg.update('additionalAgentDirs', originalAdditional, vscode.ConfigurationTarget.Global); + await chatCfg.update('agentFilesLocations', originalLocations, vscode.ConfigurationTarget.Global); + try { fs.rmdirSync(tmpDir); } catch { /* ignore */ } + } + }); +}); From 52c4c53805e6f209e5d8ff35f23cfc82f341fb6f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 18 May 2026 04:33:36 +0000 Subject: [PATCH 3/4] Ignore generated customizations bundle outputs (agents/, prompts/, skills/, manifest) Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com> --- extensions/vscode/.gitignore | 6 + .../agents/codeql-query-developer.agent.md | 59 -- .../agents/codeql-workshop-author.agent.md | 74 --- .../vscode/dist-customizations-manifest.json | 16 - .../vscode/prompts/ql-tdd-advanced.prompt.md | 493 ---------------- .../vscode/prompts/ql-tdd-basic.prompt.md | 239 -------- .../prompts/tools-query-workflow.prompt.md | 209 ------- .../workshop-creation-workflow.prompt.md | 297 ---------- .../SKILL.md | 543 ------------------ .../SKILL.md | 357 ------------ 10 files changed, 6 insertions(+), 2287 deletions(-) delete mode 100644 extensions/vscode/agents/codeql-query-developer.agent.md delete mode 100644 extensions/vscode/agents/codeql-workshop-author.agent.md delete mode 100644 extensions/vscode/dist-customizations-manifest.json delete mode 100644 extensions/vscode/prompts/ql-tdd-advanced.prompt.md delete mode 100644 extensions/vscode/prompts/ql-tdd-basic.prompt.md delete mode 100644 extensions/vscode/prompts/tools-query-workflow.prompt.md delete mode 100644 extensions/vscode/prompts/workshop-creation-workflow.prompt.md delete mode 100644 extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md delete mode 100644 extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md diff --git a/extensions/vscode/.gitignore b/extensions/vscode/.gitignore index a67c0318..29962ee8 100644 --- a/extensions/vscode/.gitignore +++ b/extensions/vscode/.gitignore @@ -4,5 +4,11 @@ server/ coverage/ *.vsix +# Customizations bundle output (generated by scripts/bundle-customizations.js) +agents/ +prompts/ +skills/ +dist-customizations-manifest.json + # @vscode/test-cli downloads and runtime data .vscode-test/ diff --git a/extensions/vscode/agents/codeql-query-developer.agent.md b/extensions/vscode/agents/codeql-query-developer.agent.md deleted file mode 100644 index 2ba55e0a..00000000 --- a/extensions/vscode/agents/codeql-query-developer.agent.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -name: codeql-query-developer -description: "Develop CodeQL queries, libraries, and tests with TDD via the ql-mcp server." -tools: ['ql-mcp/*', 'edit', 'read', 'search', 'todo'] ---- - -# `codeql-query-developer` Agent - -Develops, tests, and validates CodeQL queries, libraries, and tests using the QL MCP Server tools. - -## Core Capabilities - -- Uses `ql-mcp/*` tools to create and manage CodeQL databases from source code. -- Follows test-driven development (TDD): writes tests with expected results first, then implements queries to pass them. -- Uses `ql-mcp/*` tools to run queries against databases, execute query unit tests, and generate query logs for debugging. -- Organizes queries, libraries, and tests following CodeQL pack conventions (`qlpack.yml`, `codeql-workspace.yml`). -- Documents query purpose, logic, and usage with clear QL comments. -- ALWAYS uses verbose help (`codeql -h -vv`) when learning about `codeql` CLI commands. -- NEVER makes anything up about CodeQL semantics or database schema. -- NEVER assumes query behavior without testing against actual databases. - -## TDD Workflow - -1. **Understand the goal** — clarify what the query should detect and for which language. -2. **Create test code** — write test source files that contain positive and negative examples. -3. **Extract a test database** — use `ql-mcp/codeql_create_database` or `ql-mcp/codeql_query_run` to build a DB. -4. **Write `.qlref` / `.expected` test files** — specify expected results before writing query logic. -5. **Implement the query** — write the `.ql` file to make the tests pass. -6. **Run tests** — use `ql-mcp/codeql_test_run` to execute the unit tests; iterate until 100% pass. -7. **Validate** — run the query against real databases; inspect results; refine as needed. -8. **Document** — add `@name`, `@description`, `@kind`, `@id`, `@tags` metadata to the query. - -## MCP Tool Usage - -Use the bundled `ql-mcp/*` tools for all CodeQL operations: - -- `ql-mcp/codeql_create_database` — create a CodeQL database from source. -- `ql-mcp/codeql_query_run` — run a query against a database. -- `ql-mcp/codeql_test_run` — run CodeQL unit tests. -- `ql-mcp/codeql_query_explain` — explain a query's structure. -- `ql-mcp/find_codeql_query_files` — locate query files in the workspace. -- `ql-mcp/codeql_pack_install` — install QL pack dependencies. - -## Bundled Skills and Prompts - -The following bundled resources are available in the extension and provide detailed step-by-step workflows: - -- **Skill `create-codeql-query-development-workshop`** — reference for structured query development. -- **Skill `validate-ql-mcp-server-tools-queries`** — validate PrintAST, PrintCFG, and CallGraph tools. -- **Prompt `ql-tdd-basic`** — basic TDD workflow for simple CodeQL queries. -- **Prompt `ql-tdd-advanced`** — advanced TDD patterns for data-flow and taint-tracking queries. -- **Prompt `tools-query-workflow`** — workflow for using MCP tool queries (PrintAST, PrintCFG, CallGraph). - -## Quality Standards - -- All solution queries must compile without errors. -- All unit tests must pass at 100%. -- Expected results must be accurate (verified against real test databases). -- Queries must include complete `@name`, `@description`, `@kind`, `@id`, `@tags` metadata. diff --git a/extensions/vscode/agents/codeql-workshop-author.agent.md b/extensions/vscode/agents/codeql-workshop-author.agent.md deleted file mode 100644 index 94b29936..00000000 --- a/extensions/vscode/agents/codeql-workshop-author.agent.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -name: codeql-workshop-author -description: "Create CodeQL query development workshops from production-grade queries." -tools: ['ql-mcp/*', 'edit', 'read', 'search', 'todo'] -handoffs: - - agent: codeql-query-developer - label: Develop and Test Query - prompt: 'Develop and test a CodeQL query using TDD methodology. Follow the `ql-tdd-basic` or `ql-tdd-advanced` prompt workflow and return the validated query file path and test results when complete.' - send: false ---- - -# `codeql-workshop-author` Agent - -Creates comprehensive CodeQL query development workshops from production-grade queries using the QL MCP Server tools. - -## Core Capabilities - -- Uses `ql-mcp/*` tools to analyze production queries and create workshop materials. -- Follows the bundled `create-codeql-query-development-workshop` skill to generate workshops from production queries. -- Validates AST/CFG tools using the bundled `validate-ql-mcp-server-tools-queries` skill. -- Decomposes queries into 4–8 logical learning stages that guide learners from simple to complex. -- Generates exercise queries, solution queries, unit tests, AST/CFG visualizations, and README documentation. -- ALWAYS uses verbose help (`codeql -h -vv`) when learning about `codeql` CLI commands. -- NEVER makes anything up about CodeQL semantics or database schema. -- NEVER assumes query behavior without testing against actual databases. - -## Workshop Generation Process - -1. **Analyze source query** — use `ql-mcp/find_codeql_query_files` and `ql-mcp/codeql_query_explain` to understand the production query. -2. **Prepare environment** — run `codeql pack install` on solutions and solutions-tests directories; run any `initialize-qltests.sh` scripts. -3. **Validate AST/CFG tools** — use the bundled `validate-ql-mcp-server-tools-queries` skill to confirm PrintAST, PrintCFG, and CallGraph return non-empty output. **Fail if any query returns empty results.** -4. **Plan stages** — decompose the query into 4–8 logical learning stages. -5. **Create workshop structure** — set up directories, `qlpack.yml` files, and `codeql-workspace.yml`. -6. **Generate solution stages** — for each stage, delegate to `codeql-query-developer` (via the **Develop and Test Query** handoff) to create and validate the solution query. -7. **Create exercise queries** — remove implementation details from solutions; add scaffolding, `// TODO` hints, and `select` stubs. -8. **Generate enrichments** — create AST/CFG graphs (from tool output), build scripts, and documentation. -9. **Final validation** — run all solution tests; confirm 100% pass rate before declaring the workshop complete. - -## Workshop Structure - -``` -/ - exercises/ # Student exercise queries (incomplete, with scaffolding) - exercises-tests/ # Unit tests for exercises - solutions/ # Complete solution queries - solutions-tests/ # Unit tests for solutions (must pass 100%) - tests-common/ # Shared test code and databases - graphs/ # AST/CFG visualizations - README.md # Workshop guide - build-databases.sh # Database creation script - codeql-workspace.yml -``` - -## Decomposition Strategies - -- **Syntactic → Semantic** — Start with syntax, add type, control flow, then data flow. -- **Local → Global** — Start local, expand to cross-procedural analysis. -- **Simple → Filtered** — High recall first, then refine with filters. -- **Building Blocks** — Define helpers, combine into sources/sinks, connect with flow. - -## Bundled Skills and Prompts - -- **Skill `create-codeql-query-development-workshop`** — full step-by-step workshop creation workflow. -- **Skill `validate-ql-mcp-server-tools-queries`** — AST/CFG/CallGraph validation protocol. -- **Prompt `workshop-creation-workflow`** — structured prompt for workshop generation from a production query. -- **Prompt `ql-tdd-advanced`** — advanced TDD patterns for data-flow and taint-tracking queries. - -## Quality Standards - -- All solution queries compile without errors. -- All solution tests pass at 100%. -- Exercise queries have appropriate scaffolding (not empty, not complete). -- Expected results progress logically from stage to stage. -- Test code covers positive, negative, and edge cases. diff --git a/extensions/vscode/dist-customizations-manifest.json b/extensions/vscode/dist-customizations-manifest.json deleted file mode 100644 index 8724caea..00000000 --- a/extensions/vscode/dist-customizations-manifest.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "agents": [ - "agents/codeql-query-developer.agent.md", - "agents/codeql-workshop-author.agent.md" - ], - "prompts": [ - "prompts/ql-tdd-basic.prompt.md", - "prompts/ql-tdd-advanced.prompt.md", - "prompts/tools-query-workflow.prompt.md", - "prompts/workshop-creation-workflow.prompt.md" - ], - "skills": [ - "skills/create-codeql-query-development-workshop/SKILL.md", - "skills/validate-ql-mcp-server-tools-queries/SKILL.md" - ] -} diff --git a/extensions/vscode/prompts/ql-tdd-advanced.prompt.md b/extensions/vscode/prompts/ql-tdd-advanced.prompt.md deleted file mode 100644 index 4b8b6799..00000000 --- a/extensions/vscode/prompts/ql-tdd-advanced.prompt.md +++ /dev/null @@ -1,493 +0,0 @@ ---- -agent: agent ---- - -# Advanced Test-Driven CodeQL Query Development - -This advanced guide builds on the basic TDD methodology with powerful MCP server tools for deeper code analysis. Use this when developing complex queries that require understanding AST structure, control flow, and call relationships. - -For basic TDD workflow, see: `codeql://prompts/ql-tdd-basic` - -## When to Use This Advanced Guide - -- **Complex AST patterns**: When you need to understand how CodeQL represents specific language constructs -- **Control flow queries**: When analyzing program flow, branching, or loop structures -- **Call graph analysis**: When tracing function calls or method invocations -- **Iterative refinement**: When debugging queries that don't produce expected results -- **Performance optimization**: When understanding query evaluation patterns - -## Core Advanced Tools - -### 1. AST/CFG Visualization with #codeql_query_run - -Use the bundled tools queries to visualize code structure: - -```typescript -// Generate Abstract Syntax Tree for test code -codeql_query_run: { - queryName: "PrintAST", - queryLanguage: "java", // or "javascript", "python", "cpp", etc. - database: "/path/to/test.testproj", - format: "graphtext", - interpretedOutput: "/path/to/ast-output.txt" -} - -// Generate Control Flow Graph -codeql_query_run: { - queryName: "PrintCFG", - queryLanguage: "java", - database: "/path/to/test.testproj", - format: "graphtext", - interpretedOutput: "/path/to/cfg-output.txt" -} - -// Analyze outbound calls from a function -codeql_query_run: { - queryName: "CallGraphFrom", - queryLanguage: "java", - database: "/path/to/test.testproj", - sourceFunction: "myFunction", - format: "sarif-latest" -} - -// Analyze inbound calls to a function -codeql_query_run: { - queryName: "CallGraphTo", - queryLanguage: "java", - database: "/path/to/test.testproj", - targetFunction: "targetMethod", - format: "sarif-latest" -} -``` - -**Critical**: Always verify that tools queries return **non-empty output** with actual nodes/edges, not just headers. - -### 2. Quick Evaluation with #quick_evaluate - -Use #quick_evaluate for rapid iteration on specific predicates or classes: - -```typescript -// First, find the position of a predicate -find_predicate_position: { - file: "/path/to/Query.ql", - name: "isVulnerableSink" -} - -// Then evaluate just that predicate -quick_evaluate: { - file: "/path/to/Query.ql", - db: "/path/to/test.testproj", - symbol: "isVulnerableSink", - output_path: "/tmp/quickeval.bqrs" -} - -// Or evaluate a specific class -// NOTE: find_class_position finds `class` definitions only, not `module` definitions. -// Use search_ql_code to find module definitions or other patterns across QL files. -find_class_position: { - file: "/path/to/Query.ql", - name: "ThrowingMethodCall" -} - -quick_evaluate: { - file: "/path/to/Query.ql", - db: "/path/to/test.testproj", - symbol: "ThrowingMethodCall" -} -``` - -### 3. LSP-Powered Code Navigation - -Use the LSP tools for real-time code exploration during query development: - -```typescript -// Discover available types after `import javascript` -codeql_lsp_completion: { - file_path: "/path/to/Query.ql", - line: 5, // 0-based line with `from` clause - character: 5, // 0-based column position - workspace_uri: "/path/to/pack-root" // REQUIRED: directory containing codeql-pack.yml -} - -// Navigate to a class definition to see its predicates -codeql_lsp_definition: { - file_path: "/path/to/Query.ql", - line: 5, // 0-based line containing the class name - character: 10, // 0-based column on the class name - workspace_uri: "/path/to/pack-root" -} - -// Find all usages of a predicate across the pack -codeql_lsp_references: { - file_path: "/path/to/Query.ql", - line: 8, - character: 5, - workspace_uri: "/path/to/pack-root" -} -``` - -**Important LSP tool notes**: - -- `workspace_uri` must be a **plain directory path** (not a `file://` URI) pointing to the pack root containing `codeql-pack.yml` -- All LSP tools use **0-based** line/character positions -- #find_predicate_position and #find_class_position return **1-based** positions — subtract 1 before passing to LSP tools -- Run #codeql_pack_install before using LSP tools — they require resolved dependencies -- Request completions **after a dot** (e.g., `pw.`) to see all member predicates with full documentation - -### 4. Query File Discovery with #find_codeql_query_files - -Use this tool frequently to understand query dependencies and test structure: - -```typescript -find_codeql_query_files: { - queryPath: '/path/to/Query.ql'; -} -// Returns: query file, test directory, test files, metadata, dependencies -``` - -## Advanced TDD Workflow - -### Phase 0: Test Environment Setup (Critical for Java) - -For Java tests that depend on external libraries (JUnit, etc.), create an `options` file in each test directory: - -```text -//semmle-extractor-options: --javac-args -cp ${testdir}/../../stubs/junit-4.13:${testdir}/../../stubs/junit-jupiter-api-5.2.0 -``` - -**Key points**: - -- `${testdir}` is relative to the test directory containing the `options` file -- Use `:` (colon) to separate multiple classpath entries -- Stub files must contain minimal class/interface definitions for compilation - -### Phase 1: Deep Code Analysis - -Before writing any query logic: - -1. **Extract test database**: - - ```typescript - codeql_test_extract: { - testPath: "/path/to/test/QueryTest", - searchPath: "/path/to/pack" - } - ``` - -2. **Generate and study the AST**: - - ```typescript - codeql_query_run: { - queryName: "PrintAST", - queryLanguage: "java", - database: "/path/to/test.testproj", - format: "graphtext", - interpretedOutput: "/path/to/ast.txt" - } - ``` - -3. **Identify key AST classes and predicates** from the output: - - Note the class names (e.g., `MethodCall`, `TryStmt`, `LambdaExpr`) - - Note parent-child relationships - - Identify which nodes correspond to your test cases - -4. **Generate CFG if analyzing control flow**: - - ```typescript - codeql_query_run: { - queryName: "PrintCFG", - queryLanguage: "java", - database: "/path/to/test.testproj", - format: "graphtext", - interpretedOutput: "/path/to/cfg.txt" - } - ``` - -### Phase 2: Iterative Predicate Development - -Instead of writing the full query at once: - -1. **Start with a single class or predicate**: - - ```ql - class MyPattern extends MethodCall { - MyPattern() { - this.getMethod().hasName("targetMethod") - } - } - ``` - -2. **Use #quick_evaluate to test it**: - - ```typescript - quick_evaluate: { - file: "/path/to/Query.ql", - db: "/path/to/test.testproj", - symbol: "MyPattern" - } - ``` - -3. **Refine based on results**: - - Too many results? Add constraints - - Too few results? Relax constraints - - Wrong results? Study AST output again - -4. **Repeat for each component** before combining - -### Phase 3: Incremental Query Assembly - -1. **Combine validated predicates** into the main query -2. **Run full tests** after each combination: - - ```typescript - codeql_test_run: { - testPath: "/path/to/test/QueryTest", - searchPath: "/path/to/pack" - } - ``` - -3. **Use #find_codeql_query_files** to track all related files: - - ```typescript - find_codeql_query_files: { - queryPath: '/path/to/Query.ql'; - } - ``` - -## Common Advanced Patterns - -### Pattern: Understanding Nested Structures - -When your query involves nested constructs (e.g., lambdas inside method calls): - -```ql -// First, find the outer construct -class OuterPattern extends MethodCall { - OuterPattern() { - this.getMethod().hasName("assertThrows") - } - - // Navigate to inner construct - Expr getLambdaBody() { - exists(LambdaExpr le | - le = this.getAChildExpr() and - result = le.getExprBody() - ) - } -} -``` - -Use `PrintAST` to understand the parent-child relationships. - -### Pattern: Control Flow Dependencies - -When your query needs to understand execution order: - -```ql -// Use CFG to understand which statements precede others -predicate precedesInBlock(Stmt s1, Stmt s2) { - exists(BasicBlock bb | - s1.getBasicBlock() = bb and - s2.getBasicBlock() = bb and - s1.getLocation().getStartLine() < s2.getLocation().getStartLine() - ) -} -``` - -Use `PrintCFG` to verify the control flow structure. - -### Pattern: Call Chain Analysis - -When tracing calls through multiple functions: - -```typescript -// First, understand the call graph from your entry point -codeql_query_run: { - queryName: "CallGraphFrom", - queryLanguage: "java", - database: "/path/to/test.testproj", - sourceFunction: "entryPoint" -} - -// Then trace back from your target -codeql_query_run: { - queryName: "CallGraphTo", - queryLanguage: "java", - database: "/path/to/test.testproj", - targetFunction: "sensitiveOperation" -} -``` - -## Debugging Tips - -### When Results Are Empty - -1. **Check AST first**: Run `PrintAST` to verify the code structure matches expectations -2. **Simplify the query**: Remove constraints one by one using #quick_evaluate -3. **Check enclosing callables**: Lambda bodies may have different `getEnclosingCallable()` than expected -4. **Verify test database extraction**: Ensure the `.testproj` directory was created successfully - -### When Results Are Incorrect - -1. **Quick evaluate individual predicates**: Isolate which part is wrong -2. **Compare with AST output**: Verify your understanding of the structure -3. **Check for missing cases**: Your pattern may not cover all code variations - -### When Compilation Fails with "override" Error - -If you see `annotation 'override' missing on predicate`, you've accidentally created a predicate with the same name as one in the parent class. **Rename your predicate** to avoid the conflict: - -```ql -// BAD: Method already has getAThrownExceptionType() -class ThrowingMethod extends Method { - RefType getAThrownExceptionType() { ... } // Error! -} - -// GOOD: Use a distinct name -class ThrowingMethod extends Method { - RefType getDeclaredExceptionType() { ... } // Works -} -``` - -### When Query Is Slow - -1. **Enable evaluator logs**: - - ```typescript - codeql_query_run: { - query: "/path/to/Query.ql", - database: "/path/to/db", - "evaluator-log": "/path/to/log.json", - "evaluator-log-level": 5 - } - ``` - -2. **Profile from evaluator logs** (primary tool — returns compact JSON metrics plus a line-indexed detail file for targeted `read_file` access to RA operations and tuple progressions): - - ```typescript - profile_codeql_query_from_logs: { - evaluatorLog: '/path/to/log.json'; - } - ``` - -## Checklist for Complex Queries - -### Before Starting - -- [ ] Test database extracted successfully -- [ ] `PrintAST` output reviewed and understood -- [ ] Key AST classes identified -- [ ] Test cases cover positive, negative, and edge cases - -### During Development - -- [ ] Each predicate/class tested with #quick_evaluate -- [ ] #find_codeql_query_files used to track dependencies -- [ ] CFG consulted for control flow patterns -- [ ] Call graphs generated for cross-function analysis - -### After Each Change - -- [ ] Query compiles with #codeql_query_compile -- [ ] Quick evaluation shows expected results -- [ ] Full tests pass with #codeql_test_run -- [ ] No duplicate or missing results - -### Final Validation - -- [ ] All test cases pass -- [ ] No false positives in results -- [ ] No false negatives (all expected cases caught) -- [ ] Query formatted with #codeql_query_format -- [ ] Performance acceptable (check with #profile_codeql_query_from_logs) - -## Test Acceptance Workflow - -When your query produces correct results but differs from the `.expected` file: - -1. **Review the `.actual` file** to verify results are correct -2. **Accept the results** to update the expected baseline: - - ```typescript - codeql_test_accept: { - tests: ['/path/to/test/QueryTest']; - } - ``` - -3. **Re-run tests** to confirm they now pass - -**Warning**: Only accept results after careful review. Don't blindly accept to make tests pass. - -## Tool Reference - -| Tool | Purpose | When to Use | -| ------------------------------- | --------------------------------- | ------------------------------- | -| #codeql_query_run (PrintAST) | Visualize AST structure | Start of development, debugging | -| #codeql_query_run (PrintCFG) | Visualize control flow | Control flow queries | -| #codeql_query_run (CallGraph\*) | Analyze call relationships | Cross-function queries | -| #codeql_bqrs_interpret | Convert BQRS to readable format | After running graph queries | -| #quick_evaluate | Test individual predicates | Iterative development | -| #find_predicate_position | Locate predicate for quickeval | Before quick_evaluate | -| #find_class_position | Locate class for quickeval | Before quick_evaluate | -| #find_codeql_query_files | Discover related files | Planning, tracking changes | -| #search_ql_code | Search QL files for patterns | Finding classes, predicates | -| #codeql_resolve_files | Find QL files by name/extension | Discovering library pack files | -| #codeql_test_accept | Accept actual results as expected | After verifying correct output | -| #profile_codeql_query_from_logs | Performance analysis | Optimization | - -## Interpreting Graph Query Results - -When running `PrintAST` or `PrintCFG`, the results are stored in BQRS format. To convert to readable text: - -```typescript -// After codeql_query_run produces results.bqrs -codeql_bqrs_interpret: { - file: "/path/to/results.bqrs", - format: "graphtext", - output: "/path/to/output.txt", - t: ["kind=graph", "id=java/tools/print-ast"] -} -``` - -**Note**: The output may create a directory structure (e.g., `output.txt/java/tools/print-ast.txt`) rather than a single file. - -## Example: Workshop Development - -When creating CodeQL workshops, this advanced methodology is essential: - -1. **Analyze production query** with #find_codeql_query_files -2. **Generate AST/CFG** for workshop test code -3. **Decompose query** into stages, validating each with #quick_evaluate -4. **Create exercises** with scaffolding based on AST understanding -5. **Validate solutions** ensure each stage produces correct results - -### Exercise Design Tips - -When creating exercise stubs from solutions: - -```ql -// Use none() as placeholder in characteristic predicates -class MyPattern extends MethodCall { - MyPattern() { - // TODO: Implement - check for methods named "targetMethod" - // Hint: Use this.getMethod().hasName(...) - none() - } -} -``` - -- **Include TODO comments** with specific hints -- **Reference AST class names** students should look for -- **Build incrementally** - each exercise should build on the previous -- **Test both exercises and solutions** to ensure expected files are accurate - -See the `create-codeql-query-development-workshop` skill for complete workshop creation guidance. - -## Related Resources - -- **Basic TDD**: `codeql://prompts/ql-tdd-basic` -- **AST Reference**: `codeql://languages/{language}/ast` -- **Security Patterns**: `codeql://languages/{language}/security` -- **Performance Guide**: `codeql://patterns/performance` -- **Workshop Skill**: `.github/skills/create-codeql-query-development-workshop/SKILL.md` -- **Tools Validation**: `.github/skills/validate-ql-mcp-server-tools-queries/SKILL.md` diff --git a/extensions/vscode/prompts/ql-tdd-basic.prompt.md b/extensions/vscode/prompts/ql-tdd-basic.prompt.md deleted file mode 100644 index 44c909f1..00000000 --- a/extensions/vscode/prompts/ql-tdd-basic.prompt.md +++ /dev/null @@ -1,239 +0,0 @@ ---- -agent: agent ---- - -# Test-Driven CodeQL Query Development Checklist - -Use this checklist to guide test-driven development of CodeQL queries. Follow the TDD cycle: write tests first, implement query logic, and iterate until tests pass. - -For advanced techniques including AST/CFG visualization, see: `codeql://prompts/ql-tdd-advanced` -For detailed guidance, reference the MCP resource: `codeql://learning/test-driven-development` - -## TDD Workflow Checklist - -### Phase 1: Project Setup - -- [ ] **Create Query Structure** - - Tool: #create_codeql_query - - Specify: basePath, queryName, language, description (optional) - - Creates: src/QueryName/QueryName.ql, test/QueryName/QueryName.qlref, test/QueryName/test-code-file - - The .qlref file will contain the relative path: `QueryName/QueryName.ql` - - Verify: directory structure follows CodeQL conventions with intermediate directories - -- [ ] **Install Pack Dependencies** - - Tool: #codeql_pack_install - - Install src pack dependencies - - Install test pack dependencies - - Verify: imports resolve without errors - -### Phase 2: Test Design (Red Phase) - -- [ ] **Design Test Cases** - - Create positive test cases (should match) - - Create negative test cases (should not match) - - Create edge case tests - - Document expected behavior in comments - -- [ ] **Define Expected Results** - - Create .expected file with anticipated matches - - Specify exact match locations (file, line, column) - - Include expected alert messages - -- [ ] **Extract Test Database** - - Tool: #codeql_test_extract - - Extract database from test code - - Verify: .testproj directory created - -### Phase 3: Analysis and Understanding - -- [ ] **Analyze Test Code AST** - - Tool: #codeql_query_run with queryName: "PrintAST" - - Use format: "graphtext" for @kind graph queries - - Review AST structure and identify relevant classes - -- [ ] **Explore Available Classes and Predicates** - - Tool: #codeql_lsp_completion at the position in the `from` clause - - Parameters: `file_path`, `line` (0-based), `character` (0-based) - - Set `workspace_uri` to the pack root directory (containing `codeql-pack.yml`) - - Request completions after a dot (e.g., `pw.`) to see member predicates with docs - - Tip: Run #codeql_pack_install first — LSP tools require resolved dependencies - -- [ ] **Navigate to Type Definitions** - - Tool: #codeql_lsp_definition on a class or predicate name - - Parameters: `file_path`, `line` (0-based), `character` (0-based), `workspace_uri` - - Returns file URI and line range — even into library pack files - - Read the definition source to understand available member predicates - - Tool: #codeql_lsp_references to find usage examples across the pack - - For the full iterative LSP workflow, see: `codeql://prompts/ql_lsp_iterative_development` - -- [ ] **Reference Language Documentation** - - Resource: `codeql://languages/{language}/ast` - - Resource: `codeql://languages/{language}/security` (if applicable) - - Identify AST classes and predicates needed - -### Phase 4: Implementation (Green Phase) - -- [ ] **Write Query Metadata** - - Add @name annotation - - Add @description annotation - - Add @kind annotation (problem, path-problem, graph, etc.) - - Add @id and other required metadata - -- [ ] **Implement Query Logic** - - Import required libraries - - Define necessary classes (if any) - - Define helper predicates - - Implement main query clause - -- [ ] **Compile Query** - - Tool: #codeql_query_compile - - Fix any compilation errors - - Verify: query compiles successfully - -- [ ] **Run Tests** - - Tool: #codeql_test_run - - Compare actual vs expected results - - If tests fail: adjust query logic and recompile - - If tests pass: proceed to validation - -### Phase 5: Validation and Acceptance - -- [ ] **Verify Test Results** - - Review all test matches - - Confirm no false positives - - Confirm no false negatives - - Check edge cases behave correctly - -- [ ] **Accept Test Results** (only when correct) - - Tool: #codeql_test_accept - - Update .expected files - - Commit accepted results - -### Phase 6: Refactoring and Enhancement - -- [ ] **Refactor Query** - - Improve code clarity - - Extract common logic to predicates - - Add code comments and documentation - - Tool: #codeql_query_format for consistent formatting - -- [ ] **Optimize Performance** (if needed) - - Run with evaluator-log enabled - - Tool: #profile_codeql_query_from_logs - - Resource: `codeql://patterns/performance` - - Optimize expensive operations - -- [ ] **Generate Documentation** - - Tool: #codeql_generate_query_help - - Review and enhance QLDoc comments - - Document query purpose and limitations - -### Phase 7: Additional Testing - -- [ ] **Add More Test Cases** - - Identify additional scenarios - - Add tests for new edge cases - - Extract new test databases - - Run expanded test suite - -- [ ] **Validate Against Real Code** (optional) - - Tool: #codeql_database_create for real codebase - - Tool: #codeql_query_run against real database - - Review results for false positives/negatives - -## Quick Command Reference - -### Essential Tools - -```typescript -// Create query structure -create_codeql_query: { - basePath: "/path/to/query/base", - queryName: "MySecurityQuery", - language: "javascript", - description: "Detects security vulnerability X" -} - -// Install dependencies -codeql_pack_install: { - packPath: "/path/to/pack" -} - -// Extract test database -codeql_test_extract: { - testPath: "/path/to/test/QueryName", - searchPath: "/path/to/base" -} - -// Analyze AST (for @kind graph queries) -codeql_query_run: { - queryName: "PrintAST", - queryLanguage: "javascript", - database: "/path/to/test.testproj", - format: "graphtext", - interpretedOutput: "/path/to/ast-output/" -} - -// Compile query -codeql_query_compile: { - query: "/path/to/Query.ql", - searchPath: "/path/to/base", - checkOnly: true -} - -// Run tests -codeql_test_run: { - testPath: "/path/to/test/Query.qlref", - searchPath: "/path/to/base" -} - -// Accept results -codeql_test_accept: { - testPath: "/path/to/test/Query", - searchPath: "/path/to/base" -} -``` - -## TDD Principles to Remember - -1. **Red → Green → Refactor**: Always start with failing tests -2. **Test First**: Write tests before implementation -3. **Small Steps**: Make minimal changes to pass each test -4. **Frequent Testing**: Run tests after each change -5. **One Concept Per Test**: Each test should verify one behavior -6. **Keep Tests Simple**: Test code should be easy to understand -7. **Refactor Confidently**: Tests enable safe refactoring - -## Common Pitfalls to Avoid - -- ❌ Writing query before tests -- ❌ Accepting test results without verification -- ❌ Skipping compilation step -- ❌ Not using PrintAST to understand test code -- ❌ Not using #codeql_lsp_completion to discover available types -- ❌ Not setting `workspace_uri` when using LSP tools (completions will be empty) -- ❌ Creating tests that are too complex -- ❌ Ignoring false positives in results -- ❌ Not refactoring after tests pass - -## Success Criteria - -Your query development is complete when: - -- ✅ All tests pass -- ✅ No false positives in test results -- ✅ No false negatives (all expected cases caught) -- ✅ Query compiles without errors or warnings -- ✅ Code is well-documented with QLDoc comments -- ✅ Performance is acceptable -- ✅ Edge cases are covered by tests -- ✅ Query follows CodeQL best practices - -## Next Steps After Completion - -1. **Integration Testing**: Test against real codebases -2. **Peer Review**: Have another developer review the query -3. **Documentation**: Update project documentation -4. **Regression Testing**: Add to CI/CD pipeline -5. **Monitor Performance**: Track query performance over time diff --git a/extensions/vscode/prompts/tools-query-workflow.prompt.md b/extensions/vscode/prompts/tools-query-workflow.prompt.md deleted file mode 100644 index bbdb82d8..00000000 --- a/extensions/vscode/prompts/tools-query-workflow.prompt.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -agent: agent ---- - -# Using CodeQL Development MCP Server Tools Queries - -This guide helps you use the built-in "tools" queries (`PrintAST`, `PrintCFG`, `CallGraphFrom`, `CallGraphTo`) that ship with the CodeQL Development MCP Server to understand code structure before writing detection queries. - -## Why Use Tools Queries? - -Tools queries provide essential insights into how CodeQL represents your source code: - -| Query | Purpose | Use When | -| --------------- | ------------------------------------------------ | ----------------------------------------------- | -| `PrintAST` | Visualize the Abstract Syntax Tree | Understanding code structure, finding AST nodes | -| `PrintCFG` | Visualize Control Flow Graphs | Understanding execution paths, loop/branch flow | -| `CallGraphFrom` | Find all functions called by a specific function | Tracing data flow through call chains | -| `CallGraphTo` | Find all functions that call a specific function | Understanding function usage patterns | - -## Supported Languages - -Tools queries are available for: `actions`, `cpp`, `csharp`, `go`, `java`, `javascript`, `python`, `ruby`, `rust`, `swift` - -## Prerequisites - -Before using tools queries, you need: - -1. **A CodeQL database** - Either create one or use an existing database -2. **Source files to analyze** - The tools queries filter output to specific files - -## Workflow Checklist - -### Step 1: Identify or Create Database - -- [ ] **Option A: Use existing database** - - Tool: #codeql_resolve_database - - Verify database is valid and note the language - -- [ ] **Option B: Create new database** - - Tool: #codeql_database_create - - Parameters: `database`, `language`, `source-root` - -### Step 2: Run PrintAST Query - -The PrintAST query outputs a hierarchical tree of AST nodes with labels. - -- [ ] **Execute PrintAST** - - Tool: #codeql_query_run - - Parameters: - - `database`: Path to your CodeQL database - - `queryName`: `"PrintAST"` - - `queryLanguage`: Your language (e.g., `"javascript"`, `"python"`, `"cpp"`) - - `sourceFiles`: Comma-separated file names to analyze (e.g., `"main.js,utils.js"`) - - `format`: `"graphtext"` (for human-readable output) - -- [ ] **Verify output contains AST nodes** - - Look for hierarchical structure with indentation - - Confirm nodes have `semmle.label` with class names - - Identify relevant AST classes for your query - -**Example AST output structure:** - -```text -TopLevelFunction -├── FunctionDeclarationEntry -├── Block -│ ├── DeclStmt -│ │ └── LocalVariable -│ ├── ExprStmt -│ │ └── FunctionCall -│ └── ReturnStmt -``` - -### Step 3: Run PrintCFG Query (if needed) - -The PrintCFG query outputs control flow nodes and edges. - -- [ ] **Execute PrintCFG** - - Tool: #codeql_query_run - - Parameters: - - `database`: Path to your CodeQL database - - `queryName`: `"PrintCFG"` - - `queryLanguage`: Your language - - `sourceFunction`: Function name to analyze (e.g., `"processData"`) - - `format`: `"graphtext"` - -- [ ] **Verify output contains nodes and edges** - - Look for `nodes` section with CFG nodes - - Look for `edges` section with `→` arrows showing flow - - Identify control flow patterns (loops, branches) - -**Example CFG output structure:** - -```text -nodes -| node | semmle.label | -| ... | entry: processData | -| ... | if (...) | -| ... | return | - -edges -| from | to | semmle.label | -| ... | ... | → | -``` - -### Step 4: Run CallGraph Queries (if needed) - -Call graph queries help trace function relationships. - -- [ ] **Execute CallGraphFrom** (to find what a function calls) - - Tool: #codeql_query_run - - Parameters: - - `database`: Path to your CodeQL database - - `queryName`: `"CallGraphFrom"` - - `queryLanguage`: Your language - - `sourceFunction`: Function name to trace from (e.g., `"main"`) - - `format`: `"sarif-latest"` or `"csv"` - -- [ ] **Execute CallGraphTo** (to find what calls a function) - - Tool: #codeql_query_run - - Parameters: - - `database`: Path to your CodeQL database - - `queryName`: `"CallGraphTo"` - - `queryLanguage`: Your language - - `targetFunction`: Function name to find callers of (e.g., `"validate"`) - - `format`: `"sarif-latest"` or `"csv"` - -- [ ] **Verify call relationships** - - Confirm results show caller → callee relationships - - Note function locations for further analysis - -### Step 5: Apply Insights to Query Development - -Use the gathered information to inform your query: - -- [ ] **From PrintAST**: Identify which AST classes to use in your `from` clause -- [ ] **From PrintCFG**: Understand execution paths for control-flow-sensitive queries -- [ ] **From CallGraph**: Map data flow paths through function boundaries - -## Common Patterns - -### Pattern 1: Finding All Function Calls - -```text -1. Run PrintAST on your source file -2. Look for FunctionCall, MethodAccess, or similar nodes -3. Note the parent/child relationships -4. Use those AST classes in your query -``` - -### Pattern 2: Tracing Data Through Functions - -```text -1. Run CallGraphFrom on your entry point function -2. Identify which functions are called -3. Run CallGraphTo on sink functions -4. Map the complete path from source to sink -``` - -### Pattern 3: Understanding Loop Structures - -```text -1. Run PrintAST to find loop constructs (ForStmt, WhileStmt, etc.) -2. Run PrintCFG on the containing function -3. Identify back edges that represent loop iteration -4. Use CFG analysis for loop-sensitive queries -``` - -## Troubleshooting - -| Issue | Likely Cause | Resolution | -| --------------------- | ------------------------------------- | ------------------------------------------------------ | -| Empty AST output | `sourceFiles` parameter not matching | Use just filenames, not full paths (e.g., `"test.js"`) | -| Empty CFG output | `sourceFunction` not found | Check exact function name spelling | -| Empty CallGraph | No calls exist or wrong function name | Verify function exists and has calls | -| Query compilation err | Pack dependencies missing | Run #codeql_pack_install on the tools pack | - -## MCP Tools Reference - -| Tool | Purpose | -| ------------------------ | ---------------------------------------------------- | -| #codeql_query_run | Execute tools queries with parameters | -| #codeql_resolve_database | Validate database before querying | -| #codeql_database_create | Create database from source code | -| #codeql_bqrs_interpret | Convert results to different formats | -| #codeql_pack_install | Install pack dependencies if needed | -| #codeql_lsp_completion | Explore available types after seeing AST class names | -| #codeql_lsp_definition | Navigate to class definitions to see predicates | -| #codeql_lsp_references | Find usage examples of a class or predicate | -| #search_ql_code | Search QL source files for patterns (text or regex) | -| #codeql_resolve_files | Find QL files by name, extension, or glob pattern | - -### Using LSP Tools After AST Analysis - -After running PrintAST and identifying relevant AST class names, use the LSP tools -to explore those classes in your query file: - -1. **Write the class name** in your query's `from` clause and save the file -2. **Run #codeql_lsp_completion** after the dot to see member predicates: - - `file_path`: your query file, `line`/`character`: 0-based position after the dot - - `workspace_uri`: the pack root directory (containing `codeql-pack.yml`) -3. **Run #codeql_lsp_definition** on an AST class name to see its full API -4. **Run #codeql_lsp_references** to find usage examples in the pack - -> **Note**: LSP tools use 0-based line/character positions. Run #codeql_pack_install -> before using them — they require resolved dependencies. Set `workspace_uri` to -> a plain directory path (not a `file://` URI). - -For the full iterative LSP development workflow, see: `codeql://prompts/ql_lsp_iterative_development` diff --git a/extensions/vscode/prompts/workshop-creation-workflow.prompt.md b/extensions/vscode/prompts/workshop-creation-workflow.prompt.md deleted file mode 100644 index cdf99913..00000000 --- a/extensions/vscode/prompts/workshop-creation-workflow.prompt.md +++ /dev/null @@ -1,297 +0,0 @@ ---- -agent: agent ---- - -# Creating CodeQL Query Development Workshops - -This guide helps you create educational CodeQL query development workshops from existing production-grade queries. Workshops teach developers how to build queries incrementally through exercises and solutions. - -## Workshop Purpose - -A CodeQL workshop transforms a complex, production-ready query into a series of incremental learning stages: - -- **Exercises**: Incomplete query stubs with scaffolding and hints -- **Solutions**: Complete working queries for each stage -- **Tests**: Unit tests validating each stage works correctly -- **Documentation**: README with learning objectives and instructions - -## Prerequisites - -Before creating a workshop, ensure you have: - -1. **A production-grade CodeQL query** - The "target" query to decompose -2. **Working unit tests** - Tests that pass for the target query -3. **A CodeQL database** - For running tools queries and validating results - -## Workshop Creation Checklist - -### Phase 1: Analyze the Target Query - -- [ ] **Locate query files** - - Tool: #find_codeql_query_files - - Parameters: `queryPath` (path to the `.ql` or `.qlref` file) - - Note: Returns query file, test files, expected results, and metadata - -- [ ] **Understand query logic for workshop content** - - Prompt: `explain_codeql_query` - - Parameters: `queryPath`, `language`, and optionally `databasePath` - - Identify: sources, sinks, sanitizers, flow configuration - - Generates: detailed explanation with mermaid evaluation diagram - -- [ ] **Verify existing tests pass** - - Tool: #codeql_test_run - - Parameters: `tests` (array of test directories) - - Confirm: 100% pass rate before proceeding - -### Phase 2: Generate AST/CFG Understanding - -> **⚠️ CRITICAL**: Run tools queries to understand the test code structure. - -- [ ] **Run PrintAST on test code** - - Tool: #codeql_query_run - - Parameters: - - `queryName`: `"PrintAST"` - - `queryLanguage`: Your language (e.g., `"cpp"`, `"javascript"`) - - `database`: Path to test database or extracted `.testproj` - - `sourceFiles`: Test source file names - - `format`: `"graphtext"` - - Verify: Output contains hierarchical AST nodes (not empty) - -- [ ] **Run PrintCFG on key functions** - - Tool: #codeql_query_run - - Parameters: - - `queryName`: `"PrintCFG"` - - `queryLanguage`: Your language - - `database`: Path to database - - `sourceFunction`: Key function names from test code - - `format`: `"graphtext"` - - Verify: Output contains nodes and edges - -- [ ] **Run CallGraph queries** (if query involves data flow) - - Tool: #codeql_query_run - - Parameters: - - `queryName`: `"CallGraphFrom"` or `"CallGraphTo"` - - `database`: Path to database - - `sourceFunction` / `targetFunction`: Relevant function names - - Verify: Output shows call relationships - -### Phase 3: Plan Workshop Stages - -Decompose the query into 4-8 incremental stages using these strategies: - -#### Decomposition Strategies - -| Strategy | Description | Example Progression | -| -------------------- | -------------------------------------------------------- | ------------------------------------- | -| Syntactic → Semantic | Start with syntax, add type checking, then data flow | AST → Types → Local flow → Global | -| Local → Global | Start with local analysis, expand to cross-procedural | Single function → Multiple functions | -| Simple → Filtered | High recall first, then add precision filters | All calls → Specific calls → Filtered | -| Building Blocks | Define helpers, combine into sources/sinks, connect flow | Predicates → Sources → Sinks → Config | - -- [ ] **Document stage progression** - - Stage 1: Basic syntactic pattern (highest recall) - - Stage 2-N: Add refinements (types, filters, flow) - - Final Stage: Complete production query - -### Phase 4: Create Workshop Structure - -Standard workshop directory layout: - -```text -workshop-name/ -├── codeql-workspace.yml # CodeQL workspace configuration -├── README.md # Workshop guide with instructions -├── build-databases.sh # Script to create test databases -├── exercises/ # Student exercise queries -│ ├── codeql-pack.yml -│ ├── Exercise1.ql -│ ├── Exercise2.ql -│ └── ... -├── exercises-tests/ # Tests for exercises -│ ├── codeql-pack.yml -│ ├── Exercise1/ -│ │ ├── Exercise1.qlref -│ │ ├── Exercise1.expected -│ │ └── test.ext -│ └── ... -├── solutions/ # Complete solution queries -│ ├── codeql-pack.yml -│ ├── Exercise1.ql -│ ├── Exercise2.ql -│ └── ... -├── solutions-tests/ # Tests for solutions -│ ├── codeql-pack.yml -│ └── ... -├── tests-common/ # Shared test code -│ └── test.ext -└── graphs/ # AST/CFG visualizations - └── ast-overview.txt -``` - -- [ ] **Create codeql-workspace.yml** - - ```yaml - provide: - - '*/codeql-pack.yml' - ``` - -- [ ] **Create pack files for each directory** - - `exercises/codeql-pack.yml`: Query pack depending on language library - - `exercises-tests/codeql-pack.yml`: Test pack depending on exercises - - `solutions/codeql-pack.yml`: Query pack (same deps as exercises) - - `solutions-tests/codeql-pack.yml`: Test pack depending on solutions - -### Phase 5: Create Solution Queries - -For each stage, create a complete solution query. Use the iterative LSP tools -for efficient development (see `codeql://prompts/ql_lsp_iterative_development`): - -- Use #codeql_lsp_completion to explore types and member predicates while writing queries -- Use #codeql_lsp_definition to navigate to library class definitions -- Use #find_predicate_position + #quick_evaluate to test predicates in isolation -- Set `workspace_uri` to the solutions pack root for dependency resolution - -- [ ] **Stage 1 Solution**: Simplest working version - - Basic import statements - - Minimal from/where/select clause - - Should produce results (high recall, lower precision) - -- [ ] **Intermediate Stages**: Progressive refinements - - Add type constraints - - Add helper predicates - - Filter out false positives - - Add data flow (if applicable) - -- [ ] **Final Stage Solution**: Production-quality query - - Complete metadata (@name, @description, @kind, @id) - - Full data flow configuration (if applicable) - - Proper sanitizers and barriers - - Matches the original target query - -### Phase 6: Create Exercise Queries - -Transform solutions into exercises by removing implementation details: - -- [ ] **Add scaffolding structure** - - ```ql - /** - * @name Exercise N - [Topic] - * @description TODO: Complete this exercise - * @kind problem - * @id workshop/exercise-n - */ - - import language - - // TODO: Define predicate to find [something] - predicate findSomething(Type t) { - // Your implementation here - none() - } - - from Type t - where findSomething(t) - select t, "Found something" - ``` - -- [ ] **Include helpful comments** - - Hints about which AST classes to use - - References to documentation - - Expected behavior description - -- [ ] **Ensure exercises compile** - - Tool: #codeql_query_compile - - Exercises should compile (even if tests fail) - -### Phase 7: Create Tests - -- [ ] **Copy test code to test directories** - - Use same test code for exercises and solutions - - Consider `initialize-qltests.sh` script for shared test code - -- [ ] **Create .qlref files** - - Point to query location: `../exercises/ExerciseN.ql` - -- [ ] **Create .expected files** - - Run solution queries to generate expected output - - Tool: #codeql_test_run with `--learn` flag - - Or: #codeql_test_accept after running tests - -### Phase 8: Validate Workshop - -- [ ] **Run all solution tests** - - Tool: #codeql_test_run - - Parameters: `tests` pointing to `solutions-tests/` - - Verify: 100% pass rate - -- [ ] **Verify exercise stubs compile** - - Tool: #codeql_query_compile - - Parameters: Each exercise query - - Verify: No compilation errors - -- [ ] **Test pack dependencies** - - Tool: #codeql_pack_install - - Run in each pack directory - - Verify: Dependencies resolve correctly - -### Phase 9: Create Documentation - -- [ ] **Write README.md** - - Workshop overview and objectives - - Setup instructions - - Stage-by-stage learning guide - - AST/CFG examples from Phase 2 - -- [ ] **Include AST/CFG visualizations** - - Save PrintAST output to `graphs/` - - Reference in README for learning context - -## External Workshop Considerations - -When creating workshops outside the MCP server repository: - -- [ ] **Install pack dependencies first** - - ```bash - codeql pack install solutions - codeql pack install solutions-tests - ``` - -- [ ] **Check for initialization scripts** - - Some workshops use `initialize-qltests.sh` to copy test files - - Run before executing tests - -- [ ] **Use absolute paths with MCP tools** - - External paths must be absolute - -## MCP Tools and Prompts Reference - -| Tool/Prompt | Type | Purpose | -| ------------------------------- | ------ | ---------------------------------------------------------------------- | -| #find_codeql_query_files | Tool | Locate query and related files | -| `explain_codeql_query` | Prompt | Generate detailed explanations for workshop learning content | -| `document_codeql_query` | Prompt | Create/update query documentation files | -| `ql_lsp_iterative_development` | Prompt | Iterative query development with LSP tools | -| #codeql_query_run | Tool | Run tools queries (PrintAST, PrintCFG, etc.) | -| #codeql_test_run | Tool | Validate tests pass | -| #codeql_test_accept | Tool | Accept test results as expected baseline | -| #codeql_query_compile | Tool | Verify queries compile | -| #codeql_pack_install | Tool | Install pack dependencies | -| #codeql_resolve_metadata | Tool | Extract query metadata | -| #codeql_lsp_completion | Tool | Explore types and member predicates during query writing | -| #codeql_lsp_definition | Tool | Navigate to class/predicate definitions in library code | -| #find_predicate_position | Tool | Locate predicate positions for quick_evaluate | -| #quick_evaluate | Tool | Test individual predicates against a database | -| #profile_codeql_query_from_logs | Tool | Parse evaluator logs from a prior query run into a performance profile | -| #create_codeql_query | Tool | Scaffold new query structure | - -## Troubleshooting - -| Issue | Likely Cause | Resolution | -| ------------------------ | ------------------------------- | --------------------------------------------------- | -| "Nothing to extract" | Missing test source files | Run `initialize-qltests.sh` or copy from shared dir | -| Pack not found | Older pack version not cached | Run `codeql pack install` in pack directory | -| Empty AST/CFG output | Wrong sourceFiles/Function | Use just filenames, verify function name spelling | -| Tests fail unexpectedly | Expected file outdated | Re-run solution and accept with #codeql_test_accept | -| Exercise doesn't compile | Missing imports or syntax error | Ensure valid QL syntax with `none()` placeholder | diff --git a/extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md b/extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md deleted file mode 100644 index 6703f512..00000000 --- a/extensions/vscode/skills/create-codeql-query-development-workshop/SKILL.md +++ /dev/null @@ -1,543 +0,0 @@ ---- -name: create-codeql-query-development-workshop -description: Create custom CodeQL query development workshops from production-grade queries. Use this skill to generate guided learning materials with exercises, solutions, and tests that teach developers how to build CodeQL queries incrementally. ---- - -# Create CodeQL Query Development Workshop - -This skill guides you through creating custom CodeQL query development workshops from existing, production-grade CodeQL queries. The workshop format uses a test-driven, incremental learning approach where developers progress through stages from simple to complex. - -## When to Use This Skill - -- Creating training materials for CodeQL query development -- Teaching developers to build custom security or code quality queries -- Generating guided learning paths from existing query implementations -- Building workshops customized to specific business needs or code patterns - -## Workshop Value Proposition - -**Custom workshops are more effective than generic tutorials** because: - -- Developers learn by building queries that actually matter to their work -- Real-world query patterns are more motivating than toy examples -- Teams can train developers on their specific security or quality concerns -- Workshops scale knowledge transfer from CodeQL experts to their teams - -## Prerequisites - -Before creating a workshop, ensure you have: - -- An existing CodeQL query (`.ql` file) that is production-ready -- Passing unit tests for that query (`.expected` results that match actual results) -- Understanding of the query's purpose and complexity -- Access to CodeQL Development MCP Server tools - -## CodeQL Pack Naming Convention - -This repository uses `codeql-pack.yml` for new CodeQL pack configuration files and recommends it over `qlpack.yml`. While both `codeql-pack.yml` and `qlpack.yml` are equally supported by CodeQL, `codeql-pack.yml` is preferred as it aligns with the `codeql-pack.lock.yml` naming convention used by `codeql pack install`. If you encounter references to `qlpack.yml` in this workshop or related materials, treat them as equivalent to `codeql-pack.yml`, with `codeql-pack.yml` as the recommended name for new packs. - -## Required Inputs - -When invoking this skill, you must provide: - -1. **Source Query Path**: Full path to the production query `.ql` file -2. **Source Query Tests Path**: Full path to the directory containing unit tests for the query -3. **Base Directory**: Path where the workshop directory will be created (e.g., `/tmp/workshops` or `/workshops`) -4. **Workshop Name**: Name for the workshop directory (e.g., `dataflow-analysis-cpp`) - -## Workshop Output Structure - -The skill creates a complete workshop under `//`: - -``` -// -├── README.md # Workshop overview and setup instructions -├── codeql-workspace.yml # CodeQL workspace configuration -├── build-databases.sh # Script to create test databases -├── exercises/ # Student exercise queries (incomplete) -│ ├── codeql-pack.yml # Query pack config -│ ├── Exercise1.ql -│ ├── Exercise2.ql -│ └── ... -├── exercises-tests/ # Unit tests for exercises -│ ├── codeql-pack.yml # Test pack config (with extractor + dependency on exercises) -│ ├── Exercise1/ -│ │ ├── Exercise1.qlref -│ │ ├── Exercise1.expected -│ │ └── test.{ext} -│ └── ... -├── solutions/ # Complete solution queries -│ ├── codeql-pack.yml # Query pack config -│ ├── Exercise1.ql -│ ├── Exercise2.ql -│ └── ... -├── solutions-tests/ # Unit tests for solutions -│ ├── codeql-pack.yml # Test pack config (with extractor + dependency on solutions) -│ ├── Exercise1/ -│ │ ├── Exercise1.qlref -│ │ ├── Exercise1.expected -│ │ └── test.{ext} -│ └── ... -├── graphs/ # AST/CFG visualizations -│ ├── Exercise1-ast.txt -│ ├── Exercise1-cfg.txt -│ └── ... -└── tests-common/ # Shared test code and databases - ├── test.{ext} - └── codeql-pack.yml -``` - -See [workshop-structure-reference.md](./workshop-structure-reference.md) for detailed structure documentation. - -## Workflow Overview - -The workshop creation process follows these phases: - -### Phase 1: Analysis - -1. **Analyze Source Query** using `find_codeql_query_files` and `explain_codeql_query` -2. **Identify Complexity** to determine number of stages -3. **Extract Test Cases** from existing unit tests -4. **Plan Stages** breaking query from simple to complex - -### Phase 2: Decomposition - -Working backwards from the complete query: - -1. **Identify Decomposition Points** (predicates, logic blocks, complexity layers) -2. **Define Stage Goals** (what each exercise teaches) -3. **Create Stage Order** (simple to complex progression) - -### Phase 3: Generation - -For each stage (starting with final/complete stage): - -1. **Generate Solution Query** for this stage -2. **Create Solution Tests** that validate the solution -3. **Run Tests** using `codeql_test_run` to ensure they pass -4. **Generate Exercise Query** by removing implementation details -5. **Create Exercise Tests** (may match solution tests or be subset) - -### Phase 4: Enrichment - -1. **Generate Graph Outputs** (AST/CFG) for each stage using `codeql_bqrs_interpret` -2. **Create build-databases.sh** script for test database creation -3. **Write README.md** with workshop overview, setup, and instructions -4. **Create codeql-workspace.yml** to configure CodeQL workspace - -### Phase 5: Validation - -1. **Test All Solutions** run `codeql_test_run` on solutions-tests/ -2. **Verify Test Pass Rate** ensure 100% pass rate for solutions -3. **Check File Structure** validate all required files exist -4. **Review Exercise Gaps** ensure exercises have appropriate scaffolding - -## Key MCP Server Tools - -### Query Analysis - -- `find_codeql_query_files` - Locate query files and dependencies -- `explain_codeql_query` - Understand query purpose and logic -- `codeql_resolve_metadata` - Extract query metadata - -### Test Management - -- `codeql_test_extract` - Create test databases from test code -- `codeql_test_run` - Execute tests and validate results -- `codeql_test_accept` - Update expected results when needed - -### Query Execution - -- `codeql_query_run` - Run queries (including PrintAST, PrintCFG) -- `codeql_query_compile` - Validate query syntax -- `codeql_bqrs_interpret` - Generate graph outputs from results - -### Database Operations - -- `codeql_database_create` - Create CodeQL databases from source -- `codeql_resolve_database` - Validate database structure - -See [mcp-tools-reference.md](./mcp-tools-reference.md) for detailed tool usage patterns. - -## Stage Decomposition Strategy - -When decomposing a complex query into stages, consider these patterns: - -### Pattern 1: Syntactic to Semantic - -1. **Stage 1**: Find syntactic elements (e.g., `ArrayExpr`) -2. **Stage 2**: Add type constraints (e.g., specific array types) -3. **Stage 3**: Add semantic analysis (e.g., control flow) -4. **Stage 4**: Add data flow analysis (e.g., track values) - -### Pattern 2: Local to Global - -1. **Stage 1**: Local pattern matching -2. **Stage 2**: Add local control flow -3. **Stage 3**: Add local data flow -4. **Stage 4**: Add global data flow - -### Pattern 3: Simple to Filtered - -1. **Stage 1**: Find all candidates (high recall, low precision) -2. **Stage 2**: Add basic filtering -3. **Stage 3**: Add context-aware filtering -4. **Stage 4**: Eliminate false positives - -### Pattern 4: Building Blocks - -1. **Stage 1**: Define helper predicates -2. **Stage 2**: Combine helpers into sources -3. **Stage 3**: Define sinks -4. **Stage 4**: Connect sources to sinks with data flow - -## Exercise Creation Guidelines - -### What to Remove from Solutions - -When creating exercises from solutions: - -- **Implementation bodies**: Leave predicate signatures with `none()` body -- **Complex logic**: Replace with `// TODO: Implement` comments -- **Data flow configs**: Provide signature, remove implementation -- **Filter predicates**: Keep structure, remove conditions - -### What to Keep in Exercises - -- **Import statements**: All imports should be present -- **Type signatures**: Full type information for predicates -- **Comments**: Helpful hints about what to implement -- **Test scaffolding**: Basic structure to guide implementation - -### Hints and Documentation - -Add inline comments to guide students: - -```ql -/** - * Find all array expressions that access a specific type. - * - * Hint: Use `.getArrayBase().getType()` to get the base type. - */ -predicate isTargetArrayAccess(ArrayExpr array) { - // TODO: Implement type checking - none() -} -``` - -## Test Creation Guidelines - -### Test Code Patterns - -Create test code (`test.{ext}`) that includes: - -1. **Positive cases**: Code patterns the query should detect -2. **Negative cases**: Similar code that should NOT be detected -3. **Edge cases**: Boundary conditions -4. **Comments**: Explain what each test case validates - -Example for C++: - -```cpp -// POSITIVE CASE: Null pointer dereference -void unsafeFunction() { - int* ptr = nullptr; - *ptr = 42; // Should be detected -} - -// NEGATIVE CASE: Checked before use -void safeFunction() { - int* ptr = nullptr; - if (ptr != nullptr) { - *ptr = 42; // Should NOT be detected - } -} - -// EDGE CASE: Pointer in complex expression -void edgeCase() { - int* ptr = nullptr; - int result = ptr ? *ptr : 0; // Should be detected -} -``` - -### Expected Results Format - -The `.expected` file uses CodeQL test format: - -``` -| file | line | col | endLine | endCol | message | -| test.cpp | 3 | 5 | 3 | 8 | Null pointer dereference | -| test.cpp | 18 | 17 | 18 | 20 | Null pointer dereference | -``` - -### Test Progression - -- **Early stages**: Fewer expected results (simpler queries) -- **Later stages**: More expected results (more comprehensive) -- **Final stage**: Should match production query expected results - -## Graph Generation - -Generate visual aids for understanding code structure: - -### PrintAST Graphs - -Show Abstract Syntax Tree structure: - -```json -{ - "queryName": "PrintAST", - "queryLanguage": "cpp", - "database": "tests-common/test.testproj", - "outputFormat": "graphtext" -} -``` - -Use `codeql_bqrs_interpret` to create `graphs/Exercise1-ast.txt`. - -### PrintCFG Graphs - -Show Control Flow Graph: - -```json -{ - "queryName": "PrintCFG", - "queryLanguage": "cpp", - "database": "tests-common/test.testproj", - "outputFormat": "graphtext" -} -``` - -Use `codeql_bqrs_interpret` to create `graphs/Exercise1-cfg.txt`. - -## Build Scripts - -### build-databases.sh Template - -```bash -#!/bin/bash -set -e - -WORKSHOP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TEST_SOURCE="${WORKSHOP_ROOT}/tests-common" - -echo "Building test databases..." - -# For each test database needed -for db_name in test1 test2; do - DB_PATH="${WORKSHOP_ROOT}/tests-common/${db_name}.testproj" - - echo "Creating database: ${db_name}" - rm -rf "${DB_PATH}" - - codeql database create \ - --language={language} \ - --source-root="${TEST_SOURCE}" \ - "${DB_PATH}" \ - --command="clang -fsyntax-only ${TEST_SOURCE}/${db_name}.c" -done - -echo "Database creation complete!" -``` - -### codeql-workspace.yml Template - -```yaml -provide: - - '*/codeql-pack.yml' -``` - -This makes all codeql-pack.yml files available in the workspace. - -## Workshop README Template - -The generated README.md should include: - -1. **Title and Overview**: What the workshop teaches -2. **Prerequisites**: Required knowledge and tools -3. **Setup Instructions**: How to clone, install dependencies, build databases -4. **Workshop Structure**: Overview of exercise progression -5. **How to Use**: Instructions for working through exercises -6. **Validation**: How to test exercise solutions -7. **Solutions**: Where to find reference solutions -8. **Additional Resources**: Links to CodeQL documentation - -See [example workshop READMEs](./examples/) for templates. - -## Language-Specific Considerations - -### File Extensions by Language - -- **C/C++**: `.c`, `.cpp`, `.h`, `.hpp` -- **C#**: `.cs` -- **Go**: `.go` -- **Java**: `.java` -- **JavaScript/TypeScript**: `.js`, `.ts` -- **Python**: `.py` -- **Ruby**: `.rb` - -### Test Database Creation - -Language-specific database creation varies: - -- **C/C++**: Requires build command (e.g., `clang -fsyntax-only`) -- **Java**: Requires build tool (e.g., `mvn clean install`) -- **JavaScript**: Usually no build command needed -- **Python**: Usually no build command needed - -Adjust `build-databases.sh` accordingly. - -### Library Dependencies - -Include appropriate CodeQL libraries in `codeql-pack.yml`: - -- **C/C++**: `codeql/cpp-all` -- **C#**: `codeql/csharp-all` -- **Go**: `codeql/go-all` -- **Java**: `codeql/java-all` -- **JavaScript/TypeScript**: `codeql/javascript-all` -- **Python**: `codeql/python-all` -- **Ruby**: `codeql/ruby-all` -- **Rust**: `codeql/rust-all` - -### Java-Specific API Notes - -When writing Java queries, note these API patterns: - -- **Primitive Types**: No `ByteType` class exists. Use `PrimitiveType` with `.getName() = "byte"`, e.g.: `ace.getType().(Array).getElementType().(PrimitiveType).getName() = "byte"` -- **Array Initializers**: No `hasInit()` method. Use `exists(ace.getInit())` to check for initializers -- **Method Calls**: No `MethodAccess` class. Use `MethodCall` for method invocations -- **Deduplication**: When matching both `ArrayCreationExpr` and `ArrayInit`, exclude `ArrayInit` that are part of `ArrayCreationExpr` to avoid duplicate results: `not exists(ArrayCreationExpr ace | ace.getInit() = ai)` - -## Validation Checklist - -Before considering the workshop complete: - -- [ ] All solution queries compile without errors -- [ ] All solution tests pass at 100% -- [ ] Exercise queries have appropriate scaffolding (not empty, not complete) -- [ ] Expected results progress logically from stage to stage -- [ ] Test code covers positive, negative, and edge cases -- [ ] Graph outputs exist for stages where helpful -- [ ] build-databases.sh successfully creates all needed databases -- [ ] README.md provides clear setup and usage instructions -- [ ] codeql-workspace.yml correctly references all codeql-pack.yml files - -## Common Pitfalls - -### Avoid These Mistakes - -1. **Too many stages**: Keep to 4-8 stages max; too many fragments the learning -2. **Too few stages**: 1-2 stages don't provide enough incremental learning -3. **Uneven difficulty**: Each stage should add similar complexity increments -4. **Missing test cases**: Every query behavior should have test coverage -5. **Incomplete exercises**: Exercises should have enough scaffolding to guide students -6. **Overly complete exercises**: Don't give away the solution in exercise code -7. **Inconsistent test results**: Solution tests must pass reliably - -## Example Workshops - -This skill can be used with CodeQL queries from any repository. To see example workshops created with this skill, refer to workshop repositories that demonstrate the standard format and structure. - -## Reference Materials - -For detailed guidance: - -- [Workshop Structure Reference](./workshop-structure-reference.md) - Complete structure specification -- [MCP Tools Reference](./mcp-tools-reference.md) - Tool usage patterns for workshop creation -- [Stage Decomposition Examples](./stage-decomposition-examples.md) - Patterns for breaking down queries - -## Working Workshop Examples - -- [Example C++ Simple](./examples/example-cpp-simple/) - Basic C++ null pointer dereference workshop structure - -Some workshops may have optional advanced branches: - -``` -├── exercises/ -│ ├── Exercise1.ql -│ ├── Exercise2.ql -│ ├── Exercise3.ql -│ ├── Exercise4-basic.ql -│ └── Exercise4-advanced.ql -``` - -### Multiple Learning Paths - -Consider creating workshops with different focuses from the same source query: - -- **Path A**: Focus on syntactic analysis -- **Path B**: Focus on data flow -- **Path C**: Focus on false positive elimination - -### Difficulty Levels - -Add difficulty metadata to exercises: - -```ql -/** - * @name Find Array Access - * @description Identify array expressions - * @kind problem - * @difficulty beginner - * @exercise 1 - */ -``` - -## Troubleshooting - -### Solution Tests Fail - -If solution tests don't pass: - -1. Run `codeql_test_run` with verbose output -2. Compare actual vs expected results -3. Verify test database was created correctly -4. Check query logic matches intended behavior -5. Use `codeql_test_accept` to update `.expected` if needed - -### Exercise Too Difficult - -If students can't complete an exercise: - -1. Add more scaffolding in the exercise query -2. Add more detailed hints in comments -3. Consider splitting into two stages -4. Provide more example patterns in test code - -### Query Doesn't Compile - -If generated queries have compilation errors: - -1. Run `codeql_query_compile` to see specific errors -2. Check import statements are correct -3. Verify qlpack dependencies are installed -4. Ensure predicate signatures are valid - -## Tips for Success - -1. **Start simple**: First workshop should be straightforward -2. **Test frequently**: Run tests after creating each stage -3. **Iterate on stages**: Refine stage boundaries based on testing -4. **Get feedback**: Have someone unfamiliar try the workshop -5. **Document well**: Clear instructions reduce support burden -6. **Version control**: Track workshop iterations in git -7. **Reuse test code**: Same test code across all stages when possible - -## Related Skills - -- [create-codeql-query-tdd-generic](../create-codeql-query-tdd-generic/SKILL.md) - TDD approach to query development -- [create-codeql-query-unit-test-cpp](../create-codeql-query-unit-test-cpp/SKILL.md) - Creating C++ query tests -- [create-codeql-query-unit-test-java](../create-codeql-query-unit-test-java/SKILL.md) - Creating Java query tests -- [create-codeql-query-unit-test-javascript](../create-codeql-query-unit-test-javascript/SKILL.md) - Creating JavaScript query tests -- [create-codeql-query-unit-test-python](../create-codeql-query-unit-test-python/SKILL.md) - Creating Python query tests - -## Success Metrics - -A successful workshop: - -- **Completable**: Students can finish with provided guidance -- **Educational**: Each stage teaches a new concept -- **Validated**: All tests pass reliably -- **Practical**: Query addresses real-world concerns -- **Scalable**: Can be delivered to multiple teams diff --git a/extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md b/extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md deleted file mode 100644 index a8ddf175..00000000 --- a/extensions/vscode/skills/validate-ql-mcp-server-tools-queries/SKILL.md +++ /dev/null @@ -1,357 +0,0 @@ ---- -name: validate-ql-mcp-server-tools-queries -description: Validate that the CodeQL Development MCP Server reliably returns non-empty AST, CFG, and CallGraph data from the bundled tools queries. Use this skill to ensure the `codeql_query_run` tool properly executes PrintAST, PrintCFG, CallGraphFrom, and CallGraphTo queries for any supported language. ---- - -# Validate QL MCP Server Tools Queries - -This skill provides a systematic approach to validating that the CodeQL Development MCP Server's tools queries (`PrintAST`, `PrintCFG`, `CallGraphFrom`, `CallGraphTo`) return **meaningful, non-empty output** for test code across all supported languages. - -## Why This Skill Exists - -> **⚠️ CRITICAL**: The most common failure mode in MCP server tool validation is tests that "go through the motions" without verifying that query output is substantive. - -A test that: - -- ✅ Invokes `codeql_query_run` with `PrintAST.ql` -- ✅ Receives a successful HTTP response -- ❌ **Does NOT verify the output contains actual AST nodes** - -...is a **false positive** that masks real bugs. - -## Supported Languages - -The tools queries are available for all CodeQL-supported languages: - -| Language | Tools Pack Location | Test Code Extension | -| ---------- | ----------------------------- | ------------------- | -| actions | `server/ql/actions/tools/` | `.yml` | -| cpp | `server/ql/cpp/tools/` | `.cpp` | -| csharp | `server/ql/csharp/tools/` | `.cs` | -| go | `server/ql/go/tools/` | `.go` | -| java | `server/ql/java/tools/` | `.java` | -| javascript | `server/ql/javascript/tools/` | `.js` | -| python | `server/ql/python/tools/` | `.py` | -| ruby | `server/ql/ruby/tools/` | `.rb` | -| rust | `server/ql/rust/tools/` | `.rs` | -| swift | `server/ql/swift/tools/` | `.swift` | - -## Tools Queries Overview - -Each language pack includes four standard tools queries: - -### PrintAST - -**Purpose**: Outputs a hierarchical representation of the Abstract Syntax Tree. - -**Expected Output**: A graph with labeled nodes representing code elements (functions, classes, statements, expressions). - -**Validation Check**: Output must contain multiple `semmle.label` properties with actual node names. - -### PrintCFG - -**Purpose**: Outputs the Control Flow Graph showing execution paths. - -**Expected Output**: A graph with `nodes` (CFG nodes) and `edges` (control flow transitions). - -**Validation Check**: Output must contain both `nodes` and `edges` predicates with non-zero results. - -### CallGraphFrom - -**Purpose**: Shows outbound function calls from a specified function. - -**Expected Output**: Problem-style results showing call sites and their targets. - -**Validation Check**: Output must contain call relationships when test code has function calls. - -### CallGraphTo - -**Purpose**: Shows inbound function calls to a specified function. - -**Expected Output**: Problem-style results showing callers of the target function. - -**Validation Check**: Output must contain caller relationships when test code has function definitions that are called. - -## Validation Workflow - -### Step 1: Select Language and Test Code - -Choose a language and ensure valid test code exists: - -```text -server/ql//tools/test/PrintAST/ -├── Example1. # Test source code -├── PrintAST.expected # Expected results -└── PrintAST.qlref # Reference to query -``` - -### Step 2: Extract Test Database - -Use `codeql_test_extract` to create the test database: - -```json -{ - "testPath": "server/ql//tools/test/PrintAST", - "searchPath": ["server/ql//tools/src", "server/ql//tools/test"] -} -``` - -### Step 3: Run PrintAST Query - -Use `codeql_query_run`: - -```json -{ - "queryPath": "server/ql//tools/src/PrintAST/PrintAST.ql", - "database": "server/ql//tools/test/PrintAST/PrintAST.testproj", - "outputFormat": "bqrs" -} -``` - -### Step 4: Interpret Results - -Use `codeql_bqrs_interpret` with `graphtext` format: - -```json -{ - "file": "", - "format": "graphtext", - "output": "" -} -``` - -### Step 5: Validate Non-Empty Output - -**CRITICAL**: Parse the output and verify it contains substantive data: - -- For `PrintAST`: Count nodes with `semmle.label` > 0 -- For `PrintCFG`: Verify both `nodes` and `edges` sections exist -- For `CallGraphFrom/To`: Count result rows > 0 (when calls exist) - -## Language-Specific Examples - -### JavaScript Example - -**Test Code** (`server/ql/javascript/tools/test/PrintAST/Example1.js`): - -```javascript -class Example1 { - constructor(name = 'World') { - this.name = name; - } - - static main(args = []) { - const example = new Example1(); - example.demo(); - } - - demo() { - for (let i = 0; i < 3; i++) { - console.log(i); - } - } -} -``` - -**Expected AST Output** (non-empty validation): - -- Must contain `ClassDefinition` node for `Example1` -- Must contain `MethodDefinition` nodes for `constructor`, `main`, `demo` -- Must contain `ForStatement` node in `demo` method - -**Expected CFG Output**: - -- Must contain nodes for function entry/exit -- Must contain edges for loop control flow - -**Expected CallGraph Output**: - -- `CallGraphFrom(main)` → should find call to `Example1()` and `demo()` -- `CallGraphTo(demo)` → should find caller `main` - -### C++ Example - -**Test Code** (`server/ql/cpp/tools/test/PrintAST/Example1.cpp`): - -```cpp -class someClass { -public: - void f(void); - int g(int i, int j); -}; - -void fun3(someClass *sc) { - int i; - sc->f(); - i = sc->g(1, 2); -} -``` - -**Expected AST Output**: - -- Must contain `Class` node for `someClass` -- Must contain `MemberFunction` nodes for `f` and `g` -- Must contain `Function` node for `fun3` - -**Expected CallGraph Output**: - -- `CallGraphFrom(fun3)` → should find calls to `f()` and `g()` -- `CallGraphTo(f)` → should find caller `fun3` - -### Python Example - -**Test Code** (`server/ql/python/tools/test/PrintAST/Example1.py`): - -```python -class Example1: - def __init__(self, name="World"): - self.name = name - - def demo(self): - for i in range(3): - print(i) - -def main(): - example = Example1() - example.demo() -``` - -**Expected AST Output**: - -- Must contain `ClassDef` node for `Example1` -- Must contain `FunctionDef` nodes for `__init__`, `demo`, `main` -- Must contain `For` node in `demo` - -### Java Example - -**Test Code** (`server/ql/java/tools/test/PrintAST/Example1.java`): - -```java -public class Example1 { - private String name; - - public Example1(String name) { - this.name = name; - } - - public void demo() { - for (int i = 0; i < 3; i++) { - System.out.println(i); - } - } - - public static void main(String[] args) { - Example1 example = new Example1("World"); - example.demo(); - } -} -``` - -**Expected AST Output**: - -- Must contain `ClassOrInterface` node for `Example1` -- Must contain `Method` nodes for constructor, `demo`, `main` -- Must contain `ForStmt` node in `demo` - -## Validation Failure Indicators - -| Symptom | Likely Cause | Resolution | -| ----------------------- | -------------------------------- | --------------------------------------------------- | -| Empty AST output | Database not extracted properly | Re-run `codeql_test_extract` | -| Only headers in output | Query ran but no results matched | Check query's source file selection logic | -| Empty CFG nodes/edges | Test code has no control flow | Add functions with loops/conditionals | -| Empty CallGraph | No function calls in test code | Add function calls or verify `targetFunction` param | -| BQRS decode error | Incompatible CodeQL version | Update CodeQL CLI and re-run | -| Query compilation error | Missing pack dependencies | Run `codeql pack install` in tools pack | -| "Nothing to extract" | Missing test source files | Run workshop's `initialize-qltests.sh` script | -| Pack not found | Older pack version not cached | Run `codeql pack install` in solutions directory | - -## Handling External Workshops - -When validating against external workshops (outside the current workspace), follow these guidelines: - -### Pre-Validation Setup - -1. **Install pack dependencies**: External workshops often use older CodeQL pack versions - - ```bash - cd /path/to/external/workshop - codeql pack install solutions - codeql pack install solutions-tests - ``` - -2. **Run initialization scripts**: Many workshops have scripts that copy test files from shared directories - - ```bash - ./initialize-qltests.sh # Copies from tests-common/ to test directories - ``` - -3. **Verify test files exist**: Check that test source files are present before running tests - - ```bash - ls solutions-tests/Exercise*/test.* - ``` - -### File Access Patterns - -When the external workshop is outside your workspace: - -- Use **terminal commands** (`cat`, `ls`, `head`) to read files -- Use **MCP tools** (`codeql_query_run`, `codeql_test_run`) with absolute paths -- Pre-built databases in the workshop root can be used for tools queries - -## Integration with Prompts and Agents - -This skill is designed to be used with: - -- **Prompt**: [validate-ql-mcp-server-tools-via-workshop](../../prompts/validate-ql-mcp-server-tools-via-workshop.prompt.md) - Uses this skill as part of workshop creation validation -- **Agent**: [mcp-enabled-ql-workshop-developer](../../agents/mcp-enabled-ql-workshop-developer.md) - Orchestrates validation during workshop creation -- **Agent**: [ql-mcp-tool-tester](../../agents/ql-mcp-tool-tester.md) - Tests MCP tools including tools query execution - -## MCP Tools Used - -This skill exercises the following MCP server tools: - -| Tool | Purpose | -| ----------------------- | ----------------------------------- | -| `codeql_test_extract` | Create test database from test code | -| `codeql_query_run` | Execute tools queries | -| `codeql_bqrs_decode` | Decode raw BQRS results | -| `codeql_bqrs_interpret` | Convert results to readable formats | -| `codeql_pack_install` | Install pack dependencies | -| `codeql_pack_ls` | List available packs | - -## Important Implementation Notes - -### Server Logger Output Goes to stderr - -All `logger.info/warn/error/debug` methods write to **stderr** (`console.error`), not stdout. This is required because in stdio transport mode, stdout is reserved exclusively for the MCP JSON-RPC protocol wire format. When validating server startup logs (e.g., confirming `CODEQL_PATH` resolution), always check stderr. - -### CODEQL_PATH Environment Variable - -The MCP server resolves the CodeQL CLI binary at startup via `resolveCodeQLBinary()` in `server/src/lib/cli-executor.ts`. When `CODEQL_PATH` is set to an absolute path pointing to a valid `codeql` binary, the server uses that binary for all CodeQL CLI operations instead of searching `PATH`. This is validated per-OS in `.github/workflows/build-and-test-client.yml` (`codeql-path-tests` job). - -### STDIO Transport and stdin EOF - -When the STDIO transport receives an immediate EOF on stdin (e.g., via ` fifo &`). - -### npm Package Includes Tool Query Source Packs - -The published npm package (`codeql-development-mcp-server`) bundles all tool query source packs under `ql/*/tools/src/`. These are the same `.ql`, `.qll`, `.md`, `codeql-pack.yml`, and `codeql-pack.lock.yml` files — but **never** compiled `.qlx` bytecode (excluded by `server/.npmignore`). - -## Success Criteria - -Validation passes when **ALL** of the following are true: - -- [ ] `PrintAST` returns ≥10 labeled AST nodes for test code -- [ ] `PrintCFG` returns both `nodes` and `edges` with ≥5 entries each -- [ ] `CallGraphFrom` returns ≥1 result when test code contains function calls -- [ ] `CallGraphTo` returns ≥1 result when test code contains called functions -- [ ] No MCP tool errors or timeouts occurred -- [ ] Results are consistent across multiple runs - -## Related Resources - -- [Server Documentation](../../../server/QL-MCP-SERVER.md) -- [create-codeql-query-tdd-generic](../create-codeql-query-tdd-generic/SKILL.md) - TDD workflow that uses AST/CFG analysis -- [create-codeql-query-development-workshop](../create-codeql-query-development-workshop/SKILL.md) - Workshop creation that depends on tools queries From 5def190244d76c9839f006af737e62333d75fd9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 18 May 2026 04:35:49 +0000 Subject: [PATCH 4/4] Fix overlay manifest path for nested files; use project-local .tmp/ in tests Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com> --- extensions/vscode/scripts/bundle-customizations.js | 14 ++++++++++---- .../customizations/bundle-customizations.test.ts | 7 ++++++- .../vscode/test/suite/agents.integration.test.ts | 5 +++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/extensions/vscode/scripts/bundle-customizations.js b/extensions/vscode/scripts/bundle-customizations.js index 5204c400..2600993f 100644 --- a/extensions/vscode/scripts/bundle-customizations.js +++ b/extensions/vscode/scripts/bundle-customizations.js @@ -134,15 +134,20 @@ export async function runBundle({ extensionRoot, customizationsDir }) { /** * Recursively copies files from overlayDir into targetDir. * Warns when a file already exists (collision). + * + * subPath tracks the path relative to the category root so the manifest key + * preserves the directory structure (e.g. "skills/foo/SKILL.md", not + * "skills/SKILL.md"). */ -function applyOverlayDir(overlayDir, targetDir, categoryKey, manifest) { +function applyOverlayDir(overlayDir, targetDir, categoryKey, manifest, subPath = '') { for (const entry of readdirSync(overlayDir, { withFileTypes: true })) { const srcPath = join(overlayDir, entry.name); const dstPath = join(targetDir, entry.name); + const nextSubPath = subPath ? `${subPath}/${entry.name}` : entry.name; if (entry.isDirectory()) { mkdirSync(dstPath, { recursive: true }); - applyOverlayDir(srcPath, dstPath, categoryKey, manifest); + applyOverlayDir(srcPath, dstPath, categoryKey, manifest, nextSubPath); continue; } @@ -154,8 +159,9 @@ function applyOverlayDir(overlayDir, targetDir, categoryKey, manifest) { mkdirSync(dirname(dstPath), { recursive: true }); copyFileSync(srcPath, dstPath); - // Build a relative manifest key: e.g. "agents/foo.agent.md" - const relKey = `${categoryKey}/${entry.name}`; + // Build a relative manifest key: e.g. "agents/foo.agent.md" or + // "skills/foo/SKILL.md" — preserving any subdirectory structure. + const relKey = `${categoryKey}/${nextSubPath}`; if (!alreadyExists && Array.isArray(manifest[categoryKey])) { manifest[categoryKey].push(relKey); } diff --git a/extensions/vscode/test/customizations/bundle-customizations.test.ts b/extensions/vscode/test/customizations/bundle-customizations.test.ts index 53014f5b..0e96eb7e 100644 --- a/extensions/vscode/test/customizations/bundle-customizations.test.ts +++ b/extensions/vscode/test/customizations/bundle-customizations.test.ts @@ -31,7 +31,12 @@ describe('bundle-customizations', () => { let tmp: string; beforeEach(() => { - tmp = mkdtempSync(join(process.cwd(), 'bundle-test-')); + // Use project-local .tmp/ rather than process.cwd() to avoid polluting + // the repo root and to match the convention used elsewhere in the + // monorepo. .tmp/ is gitignored at the repo root. + const tmpRoot = resolve(__repoRoot, '.tmp'); + mkdirSync(tmpRoot, { recursive: true }); + tmp = mkdtempSync(join(tmpRoot, 'bundle-test-')); }); afterEach(() => { diff --git a/extensions/vscode/test/suite/agents.integration.test.ts b/extensions/vscode/test/suite/agents.integration.test.ts index 987b98f5..fe6aba87 100644 --- a/extensions/vscode/test/suite/agents.integration.test.ts +++ b/extensions/vscode/test/suite/agents.integration.test.ts @@ -8,7 +8,6 @@ import * as assert from 'assert'; import * as fs from 'fs'; -import * as os from 'os'; import * as path from 'path'; import * as vscode from 'vscode'; @@ -107,7 +106,9 @@ suite('Agents Integration Tests', () => { }); test('Setting codeql-mcp.additionalAgentDirs appends the dir', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codeql-mcp-test-')); + const tmpRoot = path.resolve(ext.extensionPath, '..', '..', '..', '.tmp'); + fs.mkdirSync(tmpRoot, { recursive: true }); + const tmpDir = fs.mkdtempSync(path.join(tmpRoot, 'codeql-mcp-test-')); const cfg = vscode.workspace.getConfiguration('codeql-mcp'); const chatCfg = vscode.workspace.getConfiguration('chat');