Fixing issues with github actions when generating docs and toolkit changes detectiong

.github/workflows/generate-toolkit-docs.yml

@@ -10,6 +10,9 @@ on:
   repository_dispatch:
     types: [porter_deploy_succeeded]
   workflow_dispatch:
+  # 11:00 UTC = 3 AM PST / 4 AM PDT — late enough that DST drift doesn't matter.
+  schedule:
+    - cron: "0 11 * * *"
 
 permissions:
   contents: write
@@ -27,8 +30,6 @@ jobs:
 
       - name: Install pnpm
         uses: pnpm/action-setup@v4
-        with:
-          version: 9
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -45,11 +46,14 @@ jobs:
 
       - name: Generate toolkit docs
         run: |
-          pnpm start generate \
+          pnpm dlx tsx src/cli/index.ts generate \
             --all \
             --skip-unchanged \
-            --engine-api-url "$ENGINE_API_URL" \
-            --engine-api-key "$ENGINE_API_KEY" \
+            --require-complete \
+            --verbose \
+            --api-source tool-metadata \
+            --tool-metadata-url "$ENGINE_API_URL" \
+            --tool-metadata-key "$ENGINE_API_KEY" \
             --llm-provider openai \
             --llm-model "$OPENAI_MODEL" \
             --llm-api-key "$OPENAI_API_KEY" \
@@ -62,7 +66,7 @@ jobs:
           OPENAI_MODEL: ${{ secrets.OPENAI_MODEL || 'gpt-4o-mini' }}
 
       - name: Sync toolkit sidebar navigation
-        run: pnpm dlx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --prune --verbose
+        run: pnpm dlx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --remove-empty-sections=false --verbose
 
       - name: Check for changes
         id: check-changes
@@ -76,10 +80,12 @@ jobs:
       - name: Create pull request
         if: steps.check-changes.outputs.has_changes == 'true'
         uses: peter-evans/create-pull-request@v7
+        env:
+          HUSKY: 0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: "chore: update toolkit docs after Porter deploy"
-          title: "Update toolkit docs after Porter deploy"
+          commit-message: "[AUTO] Adding MCP Servers docs update"
+          title: "[AUTO] Adding MCP Servers docs update"
           body: |
             This PR was generated after a Porter deploy succeeded.
 

.gitignore

@@ -24,6 +24,7 @@ styles/write-good/
 
 # Toolkit overview input files
 toolkit-docs-generator/overview-input/
+toolkit-docs-generator-verification/logs/
 
 # Generated toolkit markdown (built at build time, not committed)
 public/toolkit-markdown/

.husky/pre-commit

@@ -132,7 +132,7 @@ if git diff --cached --name-only | grep -q "next.config.ts"; then
 fi
 
 # --- Lint Staged (formatting) ---
-pnpm dlx lint-staged
+pnpm exec lint-staged
 
 # --- Stash + Format ---
 # Skip this block during merge/rebase: git stash --keep-index destroys
@@ -150,17 +150,43 @@ STAGED_HASH=$(git diff --cached | sha256sum | cut -d' ' -f1)
 STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACMR)
 PARTIALLY_STAGED=$(git diff --name-only)
 
+# If a file is both staged and unstaged, stash/pop can produce conflicts.
+# In that case rely on lint-staged only, which already ran above.
+if [ -n "$PARTIALLY_STAGED" ] && [ -n "$STAGED_FILES" ]; then
+  for file in $PARTIALLY_STAGED; do
+    if [ -f "$file" ] && echo "$STAGED_FILES" | grep -qxF "$file"; then
+      echo "⏭️  Skipping stash+format (partially staged files detected)"
+      exit 0
+    fi
+  done
+fi
+
 # Stash unstaged changes to preserve working directory
 # --keep-index keeps staged changes in working tree
-git stash push --quiet --keep-index --message "pre-commit-stash" || true
-STASHED=$?
+STASH_CREATED=false
+STASH_MESSAGE="pre-commit-stash-$$-$(date +%s)"
+if ! git diff --quiet; then
+  git stash push --quiet --keep-index --message "$STASH_MESSAGE"
+  TOP_STASH_SUBJECT="$(git stash list -1 --format='%s' || true)"
+  case "$TOP_STASH_SUBJECT" in
+    *"$STASH_MESSAGE")
+      STASH_CREATED=true
+      ;;
+  esac
+fi
 
 # Run formatter on the staged files
-pnpm dlx ultracite fix
-FORMAT_EXIT_CODE=$?
+if [ -n "$STAGED_FILES" ]; then
+  for file in $STAGED_FILES; do
+    if [ -f "$file" ]; then
+      pnpm exec ultracite fix "$file"
+    fi
+  done
+fi
+FORMAT_EXIT_CODE=0
 
 # Restore working directory state
-if [ $STASHED -eq 0 ]; then
+if [ "$STASH_CREATED" = true ]; then
   # Re-stage the formatted files
   if [ -n "$STAGED_FILES" ]; then
     echo "$STAGED_FILES" | while IFS= read -r file; do
@@ -171,17 +197,10 @@ if [ $STASHED -eq 0 ]; then
   fi
 
   # Restore unstaged changes
-  git stash pop --quiet || true
-
-  # Restore partial staging if files were partially staged
-  if [ -n "$PARTIALLY_STAGED" ]; then
-    for file in $PARTIALLY_STAGED; do
-      if [ -f "$file" ] && echo "$STAGED_FILES" | grep -q "^$file$"; then
-        # File was partially staged - need to unstage the unstaged parts
-        git restore --staged "$file" 2>/dev/null || true
-        git add -p "$file" < /dev/null 2>/dev/null || git add "$file"
-      fi
-    done
+  if ! git stash pop --quiet; then
+    echo "❌ Failed to restore stashed changes during pre-commit."
+    echo "   Resolve conflicts, then re-stage files and commit again."
+    exit 1
   fi
 else
   # No stash was created, just re-add the formatted files

app/_components/toolkit-docs/components/toolkit-page.tsx

@@ -583,6 +583,15 @@ export function ToolkitPage({ data }: ToolkitPageProps) {
         <h1 className="mb-6 font-bold text-4xl text-foreground tracking-tight">
           {data.label}
         </h1>
+        <ToolkitHeader
+          auth={data.auth}
+          description={data.description}
+          id={data.id}
+          label={data.label}
+          metadata={metadata}
+          toolStats={toolStats}
+          version={data.version}
+        />
         <DocumentationChunkRenderer
           chunks={documentationChunks}
           location="header"
@@ -593,15 +602,6 @@ export function ToolkitPage({ data }: ToolkitPageProps) {
           location="description"
           position="before"
         />
-        <ToolkitHeader
-          auth={data.auth}
-          description={data.description}
-          id={data.id}
-          label={data.label}
-          metadata={metadata}
-          toolStats={toolStats}
-          version={data.version}
-        />
         <DocumentationChunkRenderer
           chunks={documentationChunks}
           location="description"

app/_lib/toolkit-static-params.ts

@@ -1,6 +1,7 @@
 import { readdir, readFile } from "node:fs/promises";
+import { createRequire } from "node:module";
 import { join } from "node:path";
-import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system";
+import { pathToFileURL } from "node:url";
 import { readToolkitData, readToolkitIndex } from "./toolkit-data";
 import { getToolkitSlug, normalizeToolkitId } from "./toolkit-slug";
 
@@ -31,6 +32,45 @@ export type ToolkitRouteEntry = {
   category: IntegrationCategory;
 };
 
+const require = createRequire(import.meta.url);
+let cachedDesignSystemToolkits: ToolkitCatalogEntry[] | null = null;
+
+const isToolkitCatalogEntry = (
+  value: unknown
+): value is ToolkitCatalogEntry => {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+  const entry = value as Record<string, unknown>;
+  return typeof entry.id === "string";
+};
+
+const loadDesignSystemToolkits = async (): Promise<ToolkitCatalogEntry[]> => {
+  if (cachedDesignSystemToolkits) {
+    return cachedDesignSystemToolkits;
+  }
+
+  try {
+    const designSystemEntry = require.resolve("@arcadeai/design-system");
+    const designSystem = (await import(
+      pathToFileURL(designSystemEntry).href
+    )) as {
+      TOOLKITS?: unknown;
+    };
+    const toolkits = Array.isArray(designSystem.TOOLKITS)
+      ? designSystem.TOOLKITS
+      : [];
+
+    cachedDesignSystemToolkits = toolkits.flatMap((toolkit) =>
+      isToolkitCatalogEntry(toolkit) ? [toolkit] : []
+    );
+  } catch {
+    cachedDesignSystemToolkits = [];
+  }
+
+  return cachedDesignSystemToolkits;
+};
+
 function normalizeCategory(
   value: string | null | undefined
 ): IntegrationCategory {
@@ -98,42 +138,35 @@ const listToolkitRoutesFromDataDir = async (options?: {
   return [...unique.values()];
 };
 
-const shouldReadToolkitData = (entry?: ToolkitCatalogEntry): boolean => {
-  if (!entry) {
-    return true;
-  }
-
-  // Read toolkit data if we need a docsLink or hidden flag.
-  return (
-    typeof entry.docsLink === "undefined" ||
-    typeof entry.isHidden === "undefined"
-  );
-};
-
 const resolveToolkitRoute = async (
   toolkit: { id: string; category?: string },
   catalogByNormalizedId: Map<string, ToolkitCatalogEntry>,
   dataDir?: string
 ): Promise<ToolkitRouteEntry | null> => {
   const normalizedId = normalizeToolkitId(toolkit.id);
   const catalogEntry = catalogByNormalizedId.get(normalizedId);
-  const data = shouldReadToolkitData(catalogEntry)
-    ? await readToolkitData(toolkit.id, dataDir ? { dataDir } : undefined)
-    : null;
+  // Always read the JSON file — it is the source of truth for category, docsLink,
+  // and isHidden. The design system catalog is only used as a fallback for
+  // visibility when the JSON file is absent.
+  const data = await readToolkitData(
+    toolkit.id,
+    dataDir ? { dataDir } : undefined
+  );
 
-  const isHidden = catalogEntry?.isHidden ?? data?.metadata?.isHidden ?? false;
+  const isHidden = data?.metadata?.isHidden ?? catalogEntry?.isHidden ?? false;
   if (isHidden) {
     return null;
   }
 
   const slug = getToolkitSlug({
-    id: catalogEntry?.id ?? toolkit.id,
-    docsLink: catalogEntry?.docsLink ?? data?.metadata?.docsLink,
+    id: toolkit.id,
+    docsLink: data?.metadata?.docsLink ?? catalogEntry?.docsLink,
   });
-  // Prefer the full JSON data file's category over the index.json summary,
-  // because the index may have stale/incorrect categories.
+  // JSON file is the source of truth for category. The generator is responsible
+  // for writing the correct value; the design system catalog and index.json are
+  // only used as a last resort when the JSON is missing.
   const category = normalizeCategory(
-    catalogEntry?.category ?? data?.metadata?.category ?? toolkit.category
+    data?.metadata?.category ?? catalogEntry?.category ?? toolkit.category
   );
   return { toolkitId: slug, category };
 };
@@ -150,7 +183,8 @@ export async function listToolkitRoutes(options?: {
     return await listToolkitRoutesFromDataDir(options);
   }
 
-  const toolkitsCatalog = options?.toolkitsCatalog ?? DESIGN_SYSTEM_TOOLKITS;
+  const toolkitsCatalog =
+    options?.toolkitsCatalog ?? (await loadDesignSystemToolkits());
   const catalogByNormalizedId = new Map(
     toolkitsCatalog.map((toolkit) => [normalizeToolkitId(toolkit.id), toolkit])
   );

biome.jsonc

@@ -132,6 +132,8 @@
       "!build",
       "!node_modules",
       "!public",
+      "!toolkit-docs-generator/data/toolkits",
+      "!toolkit-docs-generator/data/toolkits",
       "!scripts",
       "!agents",
       "!.vscode",

next.config.ts

@@ -1,8 +1,21 @@
+import { createRequire } from "node:module";
 import type { NextConfig } from "next";
 import nextra from "nextra";
 import { withLlmsTxt } from "./lib/next-plugin-llmstxt";
 import { remarkGlossary } from "./lib/remark-glossary";
 
+const require = createRequire(import.meta.url);
+const designSystemMainPath = require.resolve("@arcadeai/design-system");
+const designSystemUtilsPath = require.resolve(
+  "@arcadeai/design-system/lib/utils"
+);
+const designSystemVariablesPath = require.resolve(
+  "@arcadeai/design-system/assets/variables.css"
+);
+const designSystemArcadeLogoPath = require.resolve(
+  "@arcadeai/design-system/assets/images/arcade-logo"
+);
+
 // Set up Nextra with its configuration
 const withNextra = nextra({
   defaultShowCopyCode: true,
@@ -844,6 +857,23 @@ const nextConfig: NextConfig = withLlmsTxt({
         },
       ];
     },
+    webpack: (config) => {
+      config.resolve ??= {};
+      config.resolve.alias ??= {};
+
+      if (typeof config.resolve.alias === "object") {
+        config.resolve.alias["@arcadeai/design-system$"] = designSystemMainPath;
+        config.resolve.alias["@arcadeai/design-system/lib/utils"] =
+          designSystemUtilsPath;
+        config.resolve.alias["@arcadeai/design-system/assets/variables.css"] =
+          designSystemVariablesPath;
+        config.resolve.alias[
+          "@arcadeai/design-system/assets/images/arcade-logo"
+        ] = designSystemArcadeLogoPath;
+      }
+
+      return config;
+    },
     headers: async () => [
       {
         source: "/(.*)",

package.json

@@ -7,8 +7,8 @@
     "dev": "next dev --webpack",
     "build": "pnpm run toolkit-markdown && next build --webpack && pnpm run custompagefind",
     "start": "next start",
-    "lint": "pnpm dlx ultracite check",
-    "format": "pnpm dlx ultracite fix",
+    "lint": "pnpm exec ultracite check",
+    "format": "pnpm exec ultracite fix",
     "prepare": "husky install",
     "toolkit-markdown": "pnpm dlx tsx toolkit-docs-generator/scripts/generate-toolkit-markdown.ts",
     "postbuild": "if [ \"$SKIP_POSTBUILD\" != \"true\" ]; then pnpm run generate:markdown && pnpm run custompagefind; fi",
@@ -42,7 +42,7 @@
   },
   "homepage": "https://arcade.dev/",
   "dependencies": {
-    "@arcadeai/design-system": "^3.26.1",
+    "@arcadeai/design-system": "^3.27.7",
     "@mdx-js/mdx": "^3.1.1",
     "@mdx-js/react": "^3.1.1",
     "@next/third-parties": "16.0.1",
@@ -112,7 +112,7 @@
   },
   "lint-staged": {
     "!(examples)/**/*.{js,jsx,ts,tsx,json,jsonc,css,scss,md,mdx}": [
-      "pnpm dlx ultracite fix "
+      "pnpm exec ultracite fix"
     ]
   },
   "packageManager": "pnpm@10.11.0",