diff --git a/.agents/.gitignore b/.agents/.gitignore new file mode 100644 index 00000000000000..7c94641724aabe --- /dev/null +++ b/.agents/.gitignore @@ -0,0 +1,3 @@ +# Auto-generated by dotagents. Do not edit. +# Managed skills (installed by dotagents) +/skills/brand-guidelines/ diff --git a/AGENTS.md b/AGENTS.md index cad25dbeea42aa..821bae23713d00 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,11 +75,25 @@ When writing requirements in `develop-docs/`: ``` +## Content Authoring + +- **ALWAYS** run `/brand-guidelines` to audit any user-facing content before committing. See `.agents/skills/brand-guidelines/SKILL.md` +- Use `docs-review` skill for Sentry style and voice review. See `.claude/skills/docs-review/SKILL.md` +- Use `technical-docs` skill when writing or reviewing SDK documentation. See `.claude/skills/technical-docs/SKILL.md` + +## LLM-Friendly MD Exports + +- Every page at `docs.sentry.io/` has a `.md` export at `docs.sentry.io/.md` +- `scripts/generate-md-exports.mjs` generates these as a post-build step +- Frontmatter metadata (title, description, URL) is emitted as a YAML frontmatter block in MD exports — pages missing descriptions lose LLM relevance signal +- MDX override templates live in `md-overrides/` +- Architecture spec: `specs/llm-friendly-docs.md` + ## Plan Mode - Make the plan extremely concise. Sacrifice grammar for the sake of concision. - At the end of each plan, give me a list of unresolved questions to answer, if any. -## Pull Request generation +## Pull Request generation Use .github/PULL_REQUEST_TEMPLATE.md and add Co-Authored-By: Claude diff --git a/Makefile b/Makefile index 3d20e1636a2060..54c185968a2326 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ develop: setup-git [ -f .env.development ] || cp .env.example .env.development pnpm install + npx @sentry/dotagents install setup-git: ifneq (, $(shell which pre-commit)) diff --git a/agents.lock b/agents.lock new file mode 100644 index 00000000000000..833496d70d012c --- /dev/null +++ b/agents.lock @@ -0,0 +1,10 @@ +# Auto-generated by dotagents. Do not edit. +version = 1 + +[skills.brand-guidelines] +source = "getsentry/skills" +resolved_url = "https://github.com/getsentry/skills.git" +resolved_path = ".agents/skills/brand-guidelines" +commit = "5bf1b9870096afd55e62069514c7e77d4e62b5f0" +integrity = "sha256-OI4bxji4am79cV/3TNDVUfsnt6L2HBysG7epE4HFkVI=" + diff --git a/agents.toml b/agents.toml new file mode 100644 index 00000000000000..c4b3f97cc84d2e --- /dev/null +++ b/agents.toml @@ -0,0 +1,8 @@ +version = 1 +# Check skills into git so collaborators get them without running 'dotagents install'. +# Set to true (or remove) to gitignore managed skills instead. +gitignore = true + +[[skills]] +name = "brand-guidelines" +source = "getsentry/skills" diff --git a/docs/contributing/environment.mdx b/docs/contributing/environment.mdx index 8c9dbcd8e59e8f..c3cf1a7cef931d 100644 --- a/docs/contributing/environment.mdx +++ b/docs/contributing/environment.mdx @@ -22,6 +22,8 @@ Next, navigate into the cloned repo and run the following to install dependencie make ``` +This also installs agent skills via `@sentry/dotagents` for AI-assisted development workflows. + Now, run the development webserver (depending on the docs you want to run): ```bash diff --git a/docs/contributing/pages/llm-support.mdx b/docs/contributing/pages/llm-support.mdx index 6aeab6476ccbdf..d480e4b3fc8f0d 100644 --- a/docs/contributing/pages/llm-support.mdx +++ b/docs/contributing/pages/llm-support.mdx @@ -29,7 +29,8 @@ Our existing documentation principles naturally support LLM consumption: When writing documentation, keep in mind: -- **Frontmatter descriptions** appear in section listings and help LLMs understand page purpose +- **Frontmatter descriptions are included in MD exports** as a YAML frontmatter block (with title, description, and canonical URL). Pages without a `description` in their frontmatter lose this LLM relevance signal — LLM agents read the first few lines to decide if a page is useful, so a good description helps them prioritize correctly. - **Code examples** should be complete and runnable - LLMs often copy them directly - **Avoid ambiguous references** like "click the button above" that require visual context - Pages with `noindex: true` are excluded from search and LLM discovery +- **Hub pages** (pages that only contain ``) rely on their frontmatter `description` for context in MD exports — make sure they have one diff --git a/docs/platforms/android/index.mdx b/docs/platforms/android/index.mdx index ded14c1bd01fcb..addfbc8d3011a0 100644 --- a/docs/platforms/android/index.mdx +++ b/docs/platforms/android/index.mdx @@ -1,5 +1,6 @@ --- title: Android +description: "Learn how to set up Sentry's Android SDK for error monitoring and performance tracking." caseStyle: camelCase supportLevel: production sdk: sentry.java.android diff --git a/docs/platforms/apple/index.mdx b/docs/platforms/apple/index.mdx index 87e696cd5bc851..57f27863a61c96 100644 --- a/docs/platforms/apple/index.mdx +++ b/docs/platforms/apple/index.mdx @@ -1,5 +1,6 @@ --- title: Apple +description: "Learn how to set up Sentry's Apple SDK for error monitoring and performance tracking." sdk: sentry.cocoa caseStyle: camelCase supportLevel: production diff --git a/docs/platforms/godot/index.mdx b/docs/platforms/godot/index.mdx index f2ccb5b1a4c8fd..d5478fa2a2150b 100644 --- a/docs/platforms/godot/index.mdx +++ b/docs/platforms/godot/index.mdx @@ -1,5 +1,6 @@ --- title: Godot Engine +description: "Learn how to set up Sentry's Godot Engine SDK for error monitoring and performance tracking." caseStyle: snake_case supportLevel: production sdk: sentry.godot diff --git a/docs/platforms/kotlin/index.mdx b/docs/platforms/kotlin/index.mdx index b4118c394dc07d..f4362da74e7e6e 100644 --- a/docs/platforms/kotlin/index.mdx +++ b/docs/platforms/kotlin/index.mdx @@ -1,5 +1,6 @@ --- title: Kotlin +description: "Learn how to set up Sentry's Kotlin SDK for error monitoring and performance tracking." caseStyle: camelCase supportLevel: production categories: diff --git a/docs/platforms/nintendo-switch/index.mdx b/docs/platforms/nintendo-switch/index.mdx index 99e92a848ad3d8..cc81e89922bb3d 100644 --- a/docs/platforms/nintendo-switch/index.mdx +++ b/docs/platforms/nintendo-switch/index.mdx @@ -1,5 +1,6 @@ --- title: Nintendo Switch +description: "Learn how to set up Sentry's Nintendo Switch SDK for error monitoring and performance tracking." caseStyle: snake_case supportLevel: production sdk: sentry.nintendo-switch diff --git a/docs/platforms/playstation/index.mdx b/docs/platforms/playstation/index.mdx index 50e4eadb257c86..1e768ef49d2130 100644 --- a/docs/platforms/playstation/index.mdx +++ b/docs/platforms/playstation/index.mdx @@ -1,5 +1,6 @@ --- title: PlayStation +description: "Learn how to set up Sentry's PlayStation SDK for error monitoring and performance tracking." caseStyle: snake_case supportLevel: production sdk: sentry.playstation diff --git a/docs/platforms/powershell/index.mdx b/docs/platforms/powershell/index.mdx index 6e7df0d05760bc..146c61e5fffebd 100644 --- a/docs/platforms/powershell/index.mdx +++ b/docs/platforms/powershell/index.mdx @@ -1,5 +1,6 @@ --- title: PowerShell +description: "Learn how to set up Sentry's PowerShell SDK for error monitoring and performance tracking." caseStyle: PascalCase supportLevel: production sdk: sentry.dotnet.powershell diff --git a/docs/platforms/rust/index.mdx b/docs/platforms/rust/index.mdx index 116ad0d3b95a88..48266832689914 100644 --- a/docs/platforms/rust/index.mdx +++ b/docs/platforms/rust/index.mdx @@ -1,5 +1,6 @@ --- title: Rust +description: "Learn how to set up Sentry's Rust SDK for error monitoring and performance tracking." sdk: sentry.rust caseStyle: snake_case supportLevel: production diff --git a/docs/platforms/unity/index.mdx b/docs/platforms/unity/index.mdx index fdf7385f89f0a0..331a8d8f6bfb14 100644 --- a/docs/platforms/unity/index.mdx +++ b/docs/platforms/unity/index.mdx @@ -1,5 +1,6 @@ --- title: Unity +description: "Learn how to set up Sentry's Unity SDK for error monitoring and performance tracking." caseStyle: PascalCase supportLevel: production sdk: sentry.dotnet.unity diff --git a/docs/platforms/unreal/index.mdx b/docs/platforms/unreal/index.mdx index 31a2c13181a990..f4e435acf4cf6b 100644 --- a/docs/platforms/unreal/index.mdx +++ b/docs/platforms/unreal/index.mdx @@ -1,5 +1,6 @@ --- title: Unreal Engine +description: "Learn how to set up Sentry's Unreal Engine SDK for error monitoring and performance tracking." caseStyle: PascalCase supportLevel: production sdk: sentry.unreal diff --git a/docs/platforms/xbox/index.mdx b/docs/platforms/xbox/index.mdx index f7216782be22cf..63bdf069fd3b0c 100644 --- a/docs/platforms/xbox/index.mdx +++ b/docs/platforms/xbox/index.mdx @@ -1,5 +1,6 @@ --- title: Xbox +description: "Learn how to set up Sentry's Xbox SDK for error monitoring and performance tracking." caseStyle: snake_case supportLevel: production sdk: sentry.xbox diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 42400b9cd00be4..3a4ed4c3a0b28f 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -294,6 +294,60 @@ function buildFallbackChildSection(parentPath, children) { return childSection; } +/** + * Walks the doctree to build a map of relativePath → {title, description, url} + * for YAML frontmatter generation. + */ +function buildFrontmatterMap(docTree) { + const map = new Map(); + if (!docTree) { + return map; + } + + function walk(node) { + if (node.path) { + const relativePath = node.path + '.md'; + map.set(relativePath, { + title: getTitle(node), + description: node.frontmatter?.description || '', + url: `${DOCS_ORIGIN}/${node.path}/`, + }); + } + if (node.children) { + for (const child of node.children) { + walk(child); + } + } + } + + if (docTree.children) { + for (const child of docTree.children) { + walk(child); + } + } + + return map; +} + +/** + * Formats a YAML frontmatter block for a markdown file. + * Only includes fields that have non-empty values. + */ +function formatYamlFrontmatter({title, description, url}) { + let yaml = '---\n'; + if (title) { + yaml += `title: "${title.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, ' ')}"\n`; + } + if (description) { + yaml += `description: "${description.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, ' ')}"\n`; + } + if (url) { + yaml += `url: ${url}\n`; + } + yaml += '---\n\n'; + return yaml; +} + // --- MDX template rendering for full-page overrides --- function buildMdxComponents(docTree, createElement) { @@ -533,6 +587,9 @@ async function createWork() { console.warn(' Falling back to slug-based navigation'); } + // Build frontmatter map for YAML metadata in markdown output + const frontmatterMap = buildFrontmatterMap(docTree); + // Render MDX template overrides (full-page content replacements) const mdxOverrides = await renderMdxOverrides(root, docTree); @@ -602,11 +659,24 @@ async function createWork() { const relativePath = path.relative(OUTPUT_DIR, targetPath); // Use MDX override HTML if available, otherwise use Next.js build HTML const mdxOverride = mdxOverrides.get(relativePath); + let taskFrontmatter = null; + if (mdxOverride) { + const fm = mdxOverride.frontmatter; + const urlPath = relativePath.replace(/\.md$/, '').replace(/\/index$|^index$/, ''); + taskFrontmatter = { + title: fm.title || '', + description: fm.description || '', + url: `${DOCS_ORIGIN}/${urlPath}${urlPath ? '/' : ''}`, + }; + } else { + taskFrontmatter = frontmatterMap.get(relativePath) || null; + } workerTasks[workerIdx].push({ sourcePath: mdxOverride ? mdxOverride.htmlPath : sourcePath, targetPath, relativePath, r2Hash: existingFilesOnR2 ? existingFilesOnR2.get(relativePath) : null, + frontmatter: taskFrontmatter, }); workerIdx = (workerIdx + 1) % numWorkers; numFiles++; @@ -712,8 +782,11 @@ async function createWork() { // Append child listings to parent index files. // Skip pages whose MDX override sets append_sections: false. + const hasR2 = !!(accessKeyId && secretAccessKey && existingFilesOnR2); let updatedCount = 0; - const r2Uploads = []; + // Store the latest content per key for R2 upload after child section append. + const r2Uploads = new Map(); + for (const [parentPath, children] of pathsByParent) { const overrideFm = mdxOverrides.get(parentPath)?.frontmatter; if (overrideFm?.append_sections === false) { @@ -749,30 +822,30 @@ async function createWork() { const updatedContent = existingContent + childSection; await writeFile(parentFile, updatedContent, {encoding: 'utf8'}); updatedCount++; - - // Collect R2 uploads needed (will be uploaded in parallel below) - if (accessKeyId && secretAccessKey && existingFilesOnR2) { - const fileHash = md5(updatedContent); - const existingHash = existingFilesOnR2.get(parentPath); - if (existingHash !== fileHash) { - r2Uploads.push({parentPath, updatedContent}); - } + if (hasR2) { + r2Uploads.set(parentPath, updatedContent); } } } + console.log(`📑 Added child page listings to ${updatedCount} section index files`); - // Upload all modified section index files to R2 in parallel - if (r2Uploads.length > 0) { - const limit = pLimit(50); - const s3Client = getS3Client(); - await Promise.all( - r2Uploads.map(({parentPath, updatedContent}) => - limit(() => uploadToCFR2(s3Client, parentPath, updatedContent)) - ) - ); - console.log(`📤 Uploaded ${r2Uploads.length} section index files to R2`); + // Upload modified files to R2, skipping those whose hash already matches + if (r2Uploads.size > 0) { + const toUpload = []; + for (const [key, data] of r2Uploads) { + if (existingFilesOnR2.get(key) !== md5(data)) { + toUpload.push([key, data]); + } + } + if (toUpload.length > 0) { + const limit = pLimit(50); + const s3Client = getS3Client(); + await Promise.all( + toUpload.map(([key, data]) => limit(() => uploadToCFR2(s3Client, key, data))) + ); + console.log(`📤 Uploaded ${toUpload.length} modified files to R2`); + } } - console.log(`📑 Added child page listings to ${updatedCount} section index files`); // Clean up unused cache files to prevent unbounded growth if (!noCache) { @@ -880,7 +953,7 @@ function extractContentForCacheKey(html) { return title + '\0' + canonical + '\0' + normalizedMain; } -async function genMDFromHTML(source, target, {cacheDir, noCache, usedCacheFiles}) { +async function genMDFromHTML(source, {cacheDir, noCache, usedCacheFiles}) { const rawHTML = await readFile(source, {encoding: 'utf8'}); // Strip build-specific HTML elements for faster parsing. // See stripUnstableElements() for details on what's removed and why. @@ -895,7 +968,6 @@ async function genMDFromHTML(source, target, {cacheDir, noCache, usedCacheFiles} const data = await text( compose(createReadStream(cacheFile), createBrotliDecompress()) ); - await writeFile(target, data, {encoding: 'utf8'}); // Track that we used this cache file if (usedCacheFiles) { @@ -976,28 +1048,18 @@ async function genMDFromHTML(source, target, {cacheDir, noCache, usedCacheFiles} .use(remarkStringify) .process(strippedHTML) ); - const reader = Readable.from(data); - - await Promise.all([ - pipeline( - reader, - createWriteStream(target, { - encoding: 'utf8', - }) - ), - pipeline( - reader, - createBrotliCompress({ - chunkSize: 32 * 1024, - params: { - [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT, - [zlibConstants.BROTLI_PARAM_QUALITY]: CACHE_COMPRESS_LEVEL, - [zlibConstants.BROTLI_PARAM_SIZE_HINT]: data.length, - }, - }), - createWriteStream(cacheFile) - ).catch(err => console.warn('Error writing cache file:', err)), - ]); + await pipeline( + Readable.from(data), + createBrotliCompress({ + chunkSize: 32 * 1024, + params: { + [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT, + [zlibConstants.BROTLI_PARAM_QUALITY]: CACHE_COMPRESS_LEVEL, + [zlibConstants.BROTLI_PARAM_SIZE_HINT]: data.length, + }, + }), + createWriteStream(cacheFile) + ).catch(err => console.warn('Error writing cache file:', err)); // Track that we created this cache file if (usedCacheFiles) { @@ -1013,14 +1075,15 @@ async function processTaskList({id, tasks, cacheDir, noCache, usedCacheFiles}) { usedCacheFiles = new Set(); } - const s3Client = getS3Client(); + const hasR2 = tasks.some(t => t.r2Hash !== null); + const s3Client = hasR2 ? getS3Client() : null; const failedTasks = []; let cacheMisses = []; let r2CacheMisses = []; console.log(`🤖 Worker[${id}]: Starting to process ${tasks.length} files...`); - for (const {sourcePath, targetPath, relativePath, r2Hash} of tasks) { + for (const {sourcePath, targetPath, relativePath, r2Hash, frontmatter} of tasks) { try { - const {data, cacheHit} = await genMDFromHTML(sourcePath, targetPath, { + const {data, cacheHit} = await genMDFromHTML(sourcePath, { cacheDir, noCache, usedCacheFiles, @@ -1029,12 +1092,16 @@ async function processTaskList({id, tasks, cacheDir, noCache, usedCacheFiles}) { cacheMisses.push(relativePath); } - if (r2Hash !== null) { - const fileHash = md5(data); + // Prepend YAML frontmatter and write to target + const output = frontmatter ? formatYamlFrontmatter(frontmatter) + data : data; + await writeFile(targetPath, output, {encoding: 'utf8'}); + + if (r2Hash !== null && s3Client) { + const fileHash = md5(output); if (r2Hash !== fileHash) { r2CacheMisses.push(relativePath); - await uploadToCFR2(s3Client, relativePath, data); + await uploadToCFR2(s3Client, relativePath, output); } } } catch (error) { @@ -1052,8 +1119,10 @@ async function processTaskList({id, tasks, cacheDir, noCache, usedCacheFiles}) { ); } const cacheHits = success - cacheMisses.length; + const missRate = + success > 0 ? ((cacheMisses.length / success) * 100).toFixed(1) : '0.0'; console.log( - `📈 Worker[${id}]: Cache stats: ${cacheHits} hits, ${cacheMisses.length} misses (${((cacheMisses.length / success) * 100).toFixed(1)}% miss rate)` + `📈 Worker[${id}]: Cache stats: ${cacheHits} hits, ${cacheMisses.length} misses (${missRate}% miss rate)` ); if (cacheMisses.length / tasks.length > 0.1) { diff --git a/specs/llm-friendly-docs.md b/specs/llm-friendly-docs.md index 6cab1d29dd07da..8c4eb8223cf7d0 100644 --- a/specs/llm-friendly-docs.md +++ b/specs/llm-friendly-docs.md @@ -28,6 +28,7 @@ Markdown exports are not just raw dumps of HTML content. They are adapted for LL | **Links** | Relative HTML paths | Absolute `.md` URLs (e.g., `https://docs.sentry.io/platforms/javascript.md`) | | **Images** | Relative paths | Absolute URLs | | **Page structure** | Header, sidebar, main content, footer | Title + main content + navigation sections | +| **Description** | HTML meta tag only | YAML frontmatter block with title, description, and canonical URL | ## Page Customization Architecture @@ -140,6 +141,33 @@ Pages without a matching override get a generic "Pages in this section" listing - Sorted by `sidebar_order`, then alphabetically by title - Hidden/draft/versioned pages filtered out +## YAML Frontmatter + +Every markdown export includes a YAML frontmatter block with metadata from the doctree. This gives LLM agents a relevance signal before the content begins and provides the canonical URL for navigation. + +**How it works:** + +1. Before workers start, `buildFrontmatterMap(docTree)` walks the doctree and creates a map of `relativePath → {title, description, url}` +2. Each task carries its frontmatter metadata (from the map, or from MDX override frontmatter) +3. After `genMDFromHTML` returns cached/converted markdown, `processTaskList` prepends the YAML frontmatter +4. The combined output (frontmatter + markdown) is written to the target file and used for R2 hash comparison + +**Result:** + +```markdown +--- +title: "Sentry for Python" +description: "Sentry's Python SDK enables automatic reporting of errors and performance data." +url: https://docs.sentry.io/platforms/python/ +--- + +# Python | Sentry for Python + +## Prerequisites +``` + +Pages without a doctree entry (e.g., error pages) get no frontmatter. The cache stores raw markdown without frontmatter, so metadata changes don't cause unnecessary cache invalidation. + ## Current Override Registry ### MDX Template Overrides @@ -178,10 +206,10 @@ pnpm generate-md-exports → public/md-exports/**/*.md (+ R2 sync) During `generate-md-exports`: -1. Load doctree +1. Load doctree, build frontmatter map 2. Render MDX templates from `md-overrides/` → HTML in `.next/cache/md-override-html/` -3. Discover HTML files, swapping source path for MDX override pages -4. Workers convert HTML → Markdown (parallel, cached, with R2 sync) +3. Discover HTML files, swapping source path for MDX override pages, attaching frontmatter to tasks +4. Workers convert HTML → Markdown, prepend YAML frontmatter, write to target (parallel, cached, with R2 sync) 5. Append navigation sections to parent pages using `pageOverrides` ## Adding a New Override