feat(docs): improve discoverability of API reference docs

docs/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "build": "yarn clean && yarn preprocess && yarn spellcheck && yarn preprocess:move && yarn validate:redirects && yarn validate:api-ref-links && docusaurus build && node scripts/append_api_docs_to_llms.js",
+    "build": "yarn clean && yarn preprocess && yarn spellcheck && yarn preprocess:move && yarn validate:redirects && yarn validate:api-ref-links && docusaurus build && node scripts/augment_sitemap.js && node scripts/append_api_docs_to_llms.js",
     "validate:redirects": "./scripts/validate_redirect_targets.sh",
     "validate:api-ref-links": "./scripts/validate_api_ref_links.sh",
     "clean": "./scripts/clean.sh",

docs/scripts/append_api_docs_to_llms.js

@@ -38,10 +38,19 @@ if (defaultType && fs.existsSync(path.join(STATIC_DIR, `aztec-nr-api/${defaultTy
     name: "Aztec.nr API Reference",
     dir: `aztec-nr-api/${defaultType}`,
     description: `Auto-generated API documentation for Aztec.nr (${defaultVersion})`,
+    format: "html",
   });
 } else if (!defaultType) {
   console.warn("Warning: No default version found for API docs");
 }
+if (defaultType && fs.existsSync(path.join(STATIC_DIR, `typescript-api/${defaultType}`))) {
+  API_DIRS.push({
+    name: "TypeScript API Reference",
+    dir: `typescript-api/${defaultType}`,
+    description: `Auto-generated TypeScript API documentation for Aztec packages (${defaultVersion})`,
+    format: "markdown",
+  });
+}
 
 /**
  * Extract text content from HTML, stripping tags and normalizing whitespace.
@@ -94,9 +103,9 @@ function htmlToText(html) {
 }
 
 /**
- * Recursively find all HTML files in a directory.
+ * Recursively find all files with a given extension in a directory.
  */
-function findHtmlFiles(dir, files = []) {
+function findFiles(dir, ext, files = []) {
   if (!fs.existsSync(dir)) {
     return files;
   }
@@ -106,15 +115,30 @@ function findHtmlFiles(dir, files = []) {
   for (const entry of entries) {
     const fullPath = path.join(dir, entry.name);
     if (entry.isDirectory()) {
-      findHtmlFiles(fullPath, files);
-    } else if (entry.name.endsWith(".html")) {
+      findFiles(fullPath, ext, files);
+    } else if (entry.name.endsWith(ext)) {
       files.push(fullPath);
     }
   }
 
   return files;
 }
 
+/**
+ * Recursively find all HTML files in a directory.
+ */
+function findHtmlFiles(dir) {
+  return findFiles(dir, ".html");
+}
+
+/**
+ * Recursively find all markdown files in a directory.
+ * Note: `llm-summary.txt` is naturally excluded since it does not end in `.md`.
+ */
+function findMarkdownFiles(dir) {
+  return findFiles(dir, ".md");
+}
+
 /**
  * Get the relative URL path for a file.
  */
@@ -182,10 +206,14 @@ function main() {
       continue;
     }
 
-    const htmlFiles = sortByImportance(findHtmlFiles(dirPath));
-    console.log(`Found ${htmlFiles.length} HTML files in ${apiDir.dir}`);
+    const isMarkdown = apiDir.format === "markdown";
+    const files = isMarkdown
+      ? findMarkdownFiles(dirPath)
+      : sortByImportance(findHtmlFiles(dirPath));
+    const ext = isMarkdown ? ".md" : ".html";
+    console.log(`Found ${files.length} ${isMarkdown ? "markdown" : "HTML"} files in ${apiDir.dir}`);
 
-    if (htmlFiles.length === 0) {
+    if (files.length === 0) {
       continue;
     }
 
@@ -195,30 +223,43 @@ function main() {
     fullContentSection += `## ${apiDir.name}\n\n`;
     fullContentSection += `${apiDir.description}\n\n`;
 
-    // Process only index files for links to avoid overwhelming the llms.txt
-    const indexFiles = htmlFiles.filter(
-      (f) => f.endsWith("index.html") || f.includes("/fn.") || f.includes("/struct.") || f.includes("/trait.")
-    );
-
-    // Add links for key files
-    for (const file of indexFiles.slice(0, 100)) {
-      // Limit to 100 links per section
-      const urlPath = getUrlPath(file, STATIC_DIR);
-      const fileName = path.basename(file, ".html");
-      linksSection += `- [${fileName}](${urlPath})\n`;
-    }
+    if (isMarkdown) {
+      // For markdown API docs, add a link per file and include llm-summary.txt if present
+      const summaryPath = path.join(dirPath, "llm-summary.txt");
+      if (fs.existsSync(summaryPath)) {
+        linksSection += fs.readFileSync(summaryPath, "utf-8") + "\n\n";
+      }
+      for (const file of files) {
+        const urlPath = getUrlPath(file, STATIC_DIR);
+        const fileName = path.basename(file, ext);
+        linksSection += `- [${fileName}](${urlPath})\n`;
+      }
+    } else {
+      // For HTML API docs, process only index files for links
+      const indexFiles = files.filter(
+        (f) => f.endsWith("index.html") || f.includes("/fn.") || f.includes("/struct.") || f.includes("/trait.")
+      );
+
+      // Add links for key files
+      for (const file of indexFiles.slice(0, 100)) {
+        // Limit to 100 links per section
+        const urlPath = getUrlPath(file, STATIC_DIR);
+        const fileName = path.basename(file, ext);
+        linksSection += `- [${fileName}](${urlPath})\n`;
+      }
 
-    if (indexFiles.length > 100) {
-      linksSection += `- ... and ${indexFiles.length - 100} more files\n`;
+      if (indexFiles.length > 100) {
+        linksSection += `- ... and ${indexFiles.length - 100} more files\n`;
+      }
     }
 
     linksSection += "\n";
 
     // Add full content for all files
-    for (const file of htmlFiles) {
+    for (const file of files) {
       try {
-        const html = fs.readFileSync(file, "utf-8");
-        const text = htmlToText(html);
+        const raw = fs.readFileSync(file, "utf-8");
+        const text = isMarkdown ? raw.trim() : htmlToText(raw);
 
         if (text.length > 100) {
           // Only include if there's meaningful content

docs/scripts/augment_sitemap.js

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+/**
+ * Post-build script to add static API documentation URLs to the sitemap.
+ *
+ * Docusaurus only includes its managed routes in sitemap.xml. This script
+ * appends entries for the auto-generated API docs in static/ that are
+ * copied to build/ but not indexed by the sitemap plugin.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const BUILD_DIR = path.join(__dirname, "..", "build");
+// Override with SITE_URL env var if the canonical URL ever changes.
+// Kept in sync with `url` in docusaurus.config.js.
+const SITE_URL = process.env.SITE_URL || "https://docs.aztec.network";
+
+// Load version config to determine which version subdirectory to index.
+let developerVersionConfig;
+try {
+  developerVersionConfig = require("../developer_version_config.json");
+} catch {
+  developerVersionConfig = null;
+}
+
+const defaultType = developerVersionConfig?.mainnet
+  ? "mainnet"
+  : developerVersionConfig?.testnet
+    ? "testnet"
+    : null;
+
+if (!defaultType) {
+  console.warn("Warning: No default version found — skipping sitemap augmentation");
+  process.exit(0);
+}
+
+/**
+ * Recursively find all files with a given extension.
+ */
+function findFiles(dir, ext) {
+  const results = [];
+  if (!fs.existsSync(dir)) return results;
+
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...findFiles(fullPath, ext));
+    } else if (entry.name.endsWith(ext)) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+
+function main() {
+  const sitemapPath = path.join(BUILD_DIR, "sitemap.xml");
+
+  if (!fs.existsSync(sitemapPath)) {
+    console.error("Error: build/sitemap.xml not found. Run the build first.");
+    process.exit(1);
+  }
+
+  let sitemap = fs.readFileSync(sitemapPath, "utf-8");
+
+  // Aztec.nr API HTML files (skip raw markdown — those aren't browsable pages)
+  const nrApiDir = path.join(BUILD_DIR, `aztec-nr-api/${defaultType}`);
+  // Exclude Noir stdlib (duplicated at noir-lang.org), the all.html mega-index,
+  // and per-constant global.*.html pages. Keeps the sitemap aligned with the
+  // Typesense stop_urls list so both discovery paths surface the same content.
+  const EXCLUDE_RE = new RegExp(
+    `aztec-nr-api/${defaultType}/(std/|all\\.html$|.*/global\\.[^/]+\\.html$)`
+  );
+  const htmlFiles = findFiles(nrApiDir, ".html").filter(
+    (f) => !EXCLUDE_RE.test(f.replace(/\\/g, "/"))
+  );
+
+  if (htmlFiles.length === 0) {
+    console.log("No static API docs found to add to sitemap");
+    return;
+  }
+
+  // Build XML entries
+  const entries = htmlFiles
+    .map((file) => {
+      const relativePath = path.relative(BUILD_DIR, file).replace(/\\/g, "/");
+      return `<url><loc>${SITE_URL}/${relativePath}</loc><changefreq>monthly</changefreq><priority>0.3</priority></url>`;
+    })
+    .join("");
+
+  // Insert before closing </urlset>
+  if (!sitemap.includes("</urlset>")) {
+    console.error("Error: build/sitemap.xml missing </urlset> closing tag — aborting.");
+    process.exit(1);
+  }
+  sitemap = sitemap.replace("</urlset>", entries + "</urlset>");
+
+  fs.writeFileSync(sitemapPath, sitemap);
+  console.log(`Added ${htmlFiles.length} Aztec.nr API doc URLs to sitemap.xml`);
+}
+
+main();

docs/typesense.config.json

@@ -1,26 +1,52 @@
 {
   "index_name": "aztec-docs",
   "start_urls": [
-      "https://docs.aztec.network/"
+      {
+        "url": "https://docs.aztec.network/",
+        "page_rank": 10
+      },
+      {
+        "url": "https://docs.aztec.network/aztec-nr-api/mainnet/",
+        "selectors_key": "api-nr",
+        "page_rank": 5
+      }
     ],
     "sitemap_urls": [
       "https://docs.aztec.network/sitemap.xml"
     ],
+    "stop_urls": [
+      "https://docs.aztec.network/aztec-nr-api/mainnet/std/",
+      "https://docs.aztec.network/aztec-nr-api/mainnet/all.html",
+      "aztec-nr-api/.*/global\\.[^/]+\\.html$"
+    ],
   "sitemap_alternate_links": true,
   "selectors": {
-    "lvl0": {
-      "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]",
-      "type": "xpath",
-      "global": true,
-      "default_value": "Documentation"
+    "default": {
+      "lvl0": {
+        "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]",
+        "type": "xpath",
+        "global": true,
+        "default_value": "Documentation"
+      },
+      "lvl1": "header h1",
+      "lvl2": "header h2",
+      "lvl3": "header h3",
+      "lvl4": "header h4",
+      "lvl5": "header h5",
+      "lvl6": "header h6",
+      "text": "article p, article li, article td:last-child"
     },
-    "lvl1": "header h1",
-    "lvl2": "header h2",
-    "lvl3": "header h3",
-    "lvl4": "header h4",
-    "lvl5": "header h5",
-    "lvl6": "header h6",
-    "text": "article p, article li, article td:last-child"
+    "api-nr": {
+      "lvl0": {
+        "selector": "nav.sidebar h1 a",
+        "default_value": "Aztec.nr API Reference"
+      },
+      "lvl1": "main h1",
+      "lvl2": "main h2",
+      "lvl3": "main h3",
+      "lvl4": "main h4",
+      "text": "main .comments p, main .item-description, main li, main pre code"
+    }
   },
   "strip_chars": " .,;:#",
   "custom_settings": {