diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml
index 185d89286..2c5dcd73f 100644
--- a/.github/workflows/generate-toolkit-docs.yml
+++ b/.github/workflows/generate-toolkit-docs.yml
@@ -10,6 +10,9 @@ on:
repository_dispatch:
types: [porter_deploy_succeeded]
workflow_dispatch:
+ # 11:00 UTC = 3 AM PST / 4 AM PDT — late enough that DST drift doesn't matter.
+ schedule:
+ - cron: "0 11 * * *"
permissions:
contents: write
@@ -27,8 +30,6 @@ jobs:
- name: Install pnpm
uses: pnpm/action-setup@v4
- with:
- version: 9
- name: Setup Node.js
uses: actions/setup-node@v4
@@ -45,14 +46,19 @@ jobs:
- name: Generate toolkit docs
run: |
- pnpm start generate \
+ pnpm dlx tsx src/cli/index.ts generate \
--all \
--skip-unchanged \
- --engine-api-url "$ENGINE_API_URL" \
- --engine-api-key "$ENGINE_API_KEY" \
+ --require-complete \
+ --verbose \
+ --api-source tool-metadata \
+ --tool-metadata-url "$ENGINE_API_URL" \
+ --tool-metadata-key "$ENGINE_API_KEY" \
--llm-provider openai \
--llm-model "$OPENAI_MODEL" \
--llm-api-key "$OPENAI_API_KEY" \
+ --toolkit-concurrency 8 \
+ --llm-concurrency 15 \
--output data/toolkits
working-directory: toolkit-docs-generator
env:
@@ -62,7 +68,7 @@ jobs:
OPENAI_MODEL: ${{ secrets.OPENAI_MODEL || 'gpt-4o-mini' }}
- name: Sync toolkit sidebar navigation
- run: pnpm dlx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --prune --verbose
+ run: pnpm dlx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --remove-empty-sections=false --verbose
- name: Check for changes
id: check-changes
@@ -76,10 +82,12 @@ jobs:
- name: Create pull request
if: steps.check-changes.outputs.has_changes == 'true'
uses: peter-evans/create-pull-request@v7
+ env:
+ HUSKY: 0
with:
token: ${{ secrets.GITHUB_TOKEN }}
- commit-message: "chore: update toolkit docs after Porter deploy"
- title: "Update toolkit docs after Porter deploy"
+ commit-message: "[AUTO] Adding MCP Servers docs update"
+ title: "[AUTO] Adding MCP Servers docs update"
body: |
This PR was generated after a Porter deploy succeeded.
diff --git a/.gitignore b/.gitignore
index 3e4c055ee..5814a04cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@ styles/write-good/
# Toolkit overview input files
toolkit-docs-generator/overview-input/
+toolkit-docs-generator-verification/logs/
# Generated toolkit markdown (built at build time, not committed)
public/toolkit-markdown/
diff --git a/.husky/pre-commit b/.husky/pre-commit
index 76674e5f5..4ffa0437b 100755
--- a/.husky/pre-commit
+++ b/.husky/pre-commit
@@ -132,7 +132,7 @@ if git diff --cached --name-only | grep -q "next.config.ts"; then
fi
# --- Lint Staged (formatting) ---
-pnpm dlx lint-staged
+pnpm exec lint-staged
# --- Stash + Format ---
# Skip this block during merge/rebase: git stash --keep-index destroys
@@ -150,17 +150,43 @@ STAGED_HASH=$(git diff --cached | sha256sum | cut -d' ' -f1)
STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACMR)
PARTIALLY_STAGED=$(git diff --name-only)
+# If a file is both staged and unstaged, stash/pop can produce conflicts.
+# In that case rely on lint-staged only, which already ran above.
+if [ -n "$PARTIALLY_STAGED" ] && [ -n "$STAGED_FILES" ]; then
+ for file in $PARTIALLY_STAGED; do
+ if [ -f "$file" ] && echo "$STAGED_FILES" | grep -qxF "$file"; then
+ echo "⏭️ Skipping stash+format (partially staged files detected)"
+ exit 0
+ fi
+ done
+fi
+
# Stash unstaged changes to preserve working directory
# --keep-index keeps staged changes in working tree
-git stash push --quiet --keep-index --message "pre-commit-stash" || true
-STASHED=$?
+STASH_CREATED=false
+STASH_MESSAGE="pre-commit-stash-$$-$(date +%s)"
+if ! git diff --quiet; then
+ git stash push --quiet --keep-index --message "$STASH_MESSAGE"
+ TOP_STASH_SUBJECT="$(git stash list -1 --format='%s' || true)"
+ case "$TOP_STASH_SUBJECT" in
+ *"$STASH_MESSAGE")
+ STASH_CREATED=true
+ ;;
+ esac
+fi
# Run formatter on the staged files
-pnpm dlx ultracite fix
-FORMAT_EXIT_CODE=$?
+if [ -n "$STAGED_FILES" ]; then
+ for file in $STAGED_FILES; do
+ if [ -f "$file" ]; then
+ pnpm exec ultracite fix "$file"
+ fi
+ done
+fi
+FORMAT_EXIT_CODE=0
# Restore working directory state
-if [ $STASHED -eq 0 ]; then
+if [ "$STASH_CREATED" = true ]; then
# Re-stage the formatted files
if [ -n "$STAGED_FILES" ]; then
echo "$STAGED_FILES" | while IFS= read -r file; do
@@ -171,17 +197,10 @@ if [ $STASHED -eq 0 ]; then
fi
# Restore unstaged changes
- git stash pop --quiet || true
-
- # Restore partial staging if files were partially staged
- if [ -n "$PARTIALLY_STAGED" ]; then
- for file in $PARTIALLY_STAGED; do
- if [ -f "$file" ] && echo "$STAGED_FILES" | grep -q "^$file$"; then
- # File was partially staged - need to unstage the unstaged parts
- git restore --staged "$file" 2>/dev/null || true
- git add -p "$file" < /dev/null 2>/dev/null || git add "$file"
- fi
- done
+ if ! git stash pop --quiet; then
+ echo "❌ Failed to restore stashed changes during pre-commit."
+ echo " Resolve conflicts, then re-stage files and commit again."
+ exit 1
fi
else
# No stash was created, just re-add the formatted files
diff --git a/app/_components/toolkit-docs/components/toolkit-page.tsx b/app/_components/toolkit-docs/components/toolkit-page.tsx
index 6c1d9abe3..a9e969019 100644
--- a/app/_components/toolkit-docs/components/toolkit-page.tsx
+++ b/app/_components/toolkit-docs/components/toolkit-page.tsx
@@ -583,6 +583,15 @@ export function ToolkitPage({ data }: ToolkitPageProps) {
{data.label}
+
-
{
+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
+ return false;
+ }
+ const entry = value as Record;
+ return typeof entry.id === "string";
+};
+
+const loadDesignSystemToolkits = async (): Promise => {
+ if (cachedDesignSystemToolkits) {
+ return cachedDesignSystemToolkits;
+ }
+
+ try {
+ const designSystemEntry = require.resolve("@arcadeai/design-system");
+ const designSystem = (await import(
+ pathToFileURL(designSystemEntry).href
+ )) as {
+ TOOLKITS?: unknown;
+ };
+ const toolkits = Array.isArray(designSystem.TOOLKITS)
+ ? designSystem.TOOLKITS
+ : [];
+
+ cachedDesignSystemToolkits = toolkits.flatMap((toolkit) =>
+ isToolkitCatalogEntry(toolkit) ? [toolkit] : []
+ );
+ } catch {
+ cachedDesignSystemToolkits = [];
+ }
+
+ return cachedDesignSystemToolkits;
+};
+
function normalizeCategory(
value: string | null | undefined
): IntegrationCategory {
@@ -98,18 +138,6 @@ const listToolkitRoutesFromDataDir = async (options?: {
return [...unique.values()];
};
-const shouldReadToolkitData = (entry?: ToolkitCatalogEntry): boolean => {
- if (!entry) {
- return true;
- }
-
- // Read toolkit data if we need a docsLink or hidden flag.
- return (
- typeof entry.docsLink === "undefined" ||
- typeof entry.isHidden === "undefined"
- );
-};
-
const resolveToolkitRoute = async (
toolkit: { id: string; category?: string },
catalogByNormalizedId: Map,
@@ -117,23 +145,28 @@ const resolveToolkitRoute = async (
): Promise => {
const normalizedId = normalizeToolkitId(toolkit.id);
const catalogEntry = catalogByNormalizedId.get(normalizedId);
- const data = shouldReadToolkitData(catalogEntry)
- ? await readToolkitData(toolkit.id, dataDir ? { dataDir } : undefined)
- : null;
+ // Always read the JSON file — it is the source of truth for category, docsLink,
+ // and isHidden. The design system catalog is only used as a fallback for
+ // visibility when the JSON file is absent.
+ const data = await readToolkitData(
+ toolkit.id,
+ dataDir ? { dataDir } : undefined
+ );
- const isHidden = catalogEntry?.isHidden ?? data?.metadata?.isHidden ?? false;
+ const isHidden = data?.metadata?.isHidden ?? catalogEntry?.isHidden ?? false;
if (isHidden) {
return null;
}
const slug = getToolkitSlug({
- id: catalogEntry?.id ?? toolkit.id,
- docsLink: catalogEntry?.docsLink ?? data?.metadata?.docsLink,
+ id: toolkit.id,
+ docsLink: data?.metadata?.docsLink ?? catalogEntry?.docsLink,
});
- // Prefer the full JSON data file's category over the index.json summary,
- // because the index may have stale/incorrect categories.
+ // JSON file is the source of truth for category. The generator is responsible
+ // for writing the correct value; the design system catalog and index.json are
+ // only used as a last resort when the JSON is missing.
const category = normalizeCategory(
- catalogEntry?.category ?? data?.metadata?.category ?? toolkit.category
+ data?.metadata?.category ?? catalogEntry?.category ?? toolkit.category
);
return { toolkitId: slug, category };
};
@@ -150,7 +183,8 @@ export async function listToolkitRoutes(options?: {
return await listToolkitRoutesFromDataDir(options);
}
- const toolkitsCatalog = options?.toolkitsCatalog ?? DESIGN_SYSTEM_TOOLKITS;
+ const toolkitsCatalog =
+ options?.toolkitsCatalog ?? (await loadDesignSystemToolkits());
const catalogByNormalizedId = new Map(
toolkitsCatalog.map((toolkit) => [normalizeToolkitId(toolkit.id), toolkit])
);
diff --git a/app/en/resources/integrations/_meta.tsx b/app/en/resources/integrations/_meta.tsx
index 32201d550..1cf921397 100644
--- a/app/en/resources/integrations/_meta.tsx
+++ b/app/en/resources/integrations/_meta.tsx
@@ -36,6 +36,9 @@ const meta: MetaRecord = {
sales: {
title: "Sales",
},
+ databases: {
+ title: "Databases",
+ },
"customer-support": {
title: "Customer Support",
},
diff --git a/app/en/resources/integrations/databases/_meta.tsx b/app/en/resources/integrations/databases/_meta.tsx
new file mode 100644
index 000000000..d0b9a9cc9
--- /dev/null
+++ b/app/en/resources/integrations/databases/_meta.tsx
@@ -0,0 +1,10 @@
+import type { MetaRecord } from "nextra";
+
+const meta: MetaRecord = {
+ "weaviate-api": {
+ title: "Weaviate API",
+ href: "/en/resources/integrations/databases/weaviate-api",
+ },
+};
+
+export default meta;
diff --git a/app/en/resources/integrations/development/_meta.tsx b/app/en/resources/integrations/development/_meta.tsx
index 8722736fb..70a1a6050 100644
--- a/app/en/resources/integrations/development/_meta.tsx
+++ b/app/en/resources/integrations/development/_meta.tsx
@@ -101,10 +101,6 @@ const meta: MetaRecord = {
title: "Vercel API",
href: "/en/resources/integrations/development/vercel-api",
},
- "weaviate-api": {
- title: "Weaviate API",
- href: "/en/resources/integrations/development/weaviate-api",
- },
};
export default meta;
diff --git a/biome.jsonc b/biome.jsonc
index 7ba3126d8..46f808c2a 100644
--- a/biome.jsonc
+++ b/biome.jsonc
@@ -132,6 +132,7 @@
"!build",
"!node_modules",
"!public",
+ "!toolkit-docs-generator/data/toolkits",
"!scripts",
"!agents",
"!.vscode",
diff --git a/package.json b/package.json
index 538702cad..6a056f0b9 100644
--- a/package.json
+++ b/package.json
@@ -7,8 +7,8 @@
"dev": "next dev --webpack",
"build": "pnpm run toolkit-markdown && next build --webpack && pnpm run custompagefind",
"start": "next start",
- "lint": "pnpm dlx ultracite check",
- "format": "pnpm dlx ultracite fix",
+ "lint": "pnpm exec ultracite check",
+ "format": "pnpm exec ultracite fix",
"prepare": "husky install",
"toolkit-markdown": "pnpm dlx tsx toolkit-docs-generator/scripts/generate-toolkit-markdown.ts",
"postbuild": "if [ \"$SKIP_POSTBUILD\" != \"true\" ]; then pnpm run generate:markdown && pnpm run custompagefind; fi",
@@ -112,7 +112,7 @@
},
"lint-staged": {
"!(examples)/**/*.{js,jsx,ts,tsx,json,jsonc,css,scss,md,mdx}": [
- "pnpm dlx ultracite fix "
+ "pnpm exec ultracite fix"
]
},
"packageManager": "pnpm@10.11.0",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1615a6836..52e0b4156 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -2164,21 +2164,12 @@ packages:
peerDependencies:
tailwindcss: '>=3.0.0 || insiders || >=4.0.0-alpha.20 || >=4.0.0-beta.1'
- '@tanstack/react-virtual@3.13.13':
- resolution: {integrity: sha512-4o6oPMDvQv+9gMi8rE6gWmsOjtUZUYIJHv7EB+GblyYdi8U6OqLl8rhHWIUZSL1dUU2dPwTdTgybCKf9EjIrQg==}
- peerDependencies:
- react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
- react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
-
'@tanstack/react-virtual@3.13.18':
resolution: {integrity: sha512-dZkhyfahpvlaV0rIKnvQiVoWPyURppl6w4m9IwMDpuIjcJ1sD9YGWrt0wISvgU7ewACXx2Ct46WPgI6qAD4v6A==}
peerDependencies:
react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0
- '@tanstack/virtual-core@3.13.13':
- resolution: {integrity: sha512-uQFoSdKKf5S8k51W5t7b2qpfkyIbdHMzAn+AMQvHPxKUPeo1SsGaA4JRISQT87jm28b7z8OEqPcg1IOZagQHcA==}
-
'@tanstack/virtual-core@3.13.18':
resolution: {integrity: sha512-Mx86Hqu1k39icq2Zusq+Ey2J6dDWTjDvEv43PJtRCoEYTLyfaPnxIQ6iy7YAOK0NV/qOEmZQ/uCufrppZxTgcg==}
@@ -5212,7 +5203,7 @@ snapshots:
'@floating-ui/react': 0.26.28(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
'@react-aria/focus': 3.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
'@react-aria/interactions': 3.25.6(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
- '@tanstack/react-virtual': 3.13.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
+ '@tanstack/react-virtual': 3.13.18(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
react: 19.2.3
react-dom: 19.2.3(react@19.2.3)
use-sync-external-store: 1.6.0(react@19.2.3)
@@ -7110,20 +7101,12 @@ snapshots:
postcss-selector-parser: 6.0.10
tailwindcss: 4.1.16
- '@tanstack/react-virtual@3.13.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)':
- dependencies:
- '@tanstack/virtual-core': 3.13.13
- react: 19.2.3
- react-dom: 19.2.3(react@19.2.3)
-
'@tanstack/react-virtual@3.13.18(react-dom@19.2.3(react@19.2.3))(react@19.2.3)':
dependencies:
'@tanstack/virtual-core': 3.13.18
react: 19.2.3
react-dom: 19.2.3(react@19.2.3)
- '@tanstack/virtual-core@3.13.13': {}
-
'@tanstack/virtual-core@3.13.18': {}
'@theguild/remark-mermaid@0.3.0(react@19.2.3)':
diff --git a/tests/external-url-check.test.ts b/tests/external-url-check.test.ts
index 44c2e4ec6..635ed75e3 100644
--- a/tests/external-url-check.test.ts
+++ b/tests/external-url-check.test.ts
@@ -5,6 +5,19 @@ import { expect, test } from "vitest";
const TIMEOUT = 120_000;
const CONCURRENCY = 10;
const REQUEST_TIMEOUT = 8000;
+const RETRY_ATTEMPTS = 2;
+const RETRY_DELAY_MS = 400;
+const TRANSIENT_ERROR_PATTERNS: readonly RegExp[] = [
+ /aborted/i,
+ /timeout/i,
+ /timed out/i,
+ /network/i,
+ /fetch failed/i,
+ /socket hang up/i,
+ /econnreset/i,
+ /eai_again/i,
+ /enotfound/i,
+];
const SKIP_PATTERNS: RegExp[] = [
// Placeholder / example domains
@@ -67,44 +80,68 @@ function extractExternalUrls(content: string): string[] {
return urls;
}
-async function checkUrl(
- url: string
-): Promise<{ ok: boolean; status?: number; error?: string }> {
+function sleep(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isTransientError(message: string | undefined): boolean {
+ if (!message) {
+ return false;
+ }
+
+ return TRANSIENT_ERROR_PATTERNS.some((pattern) => pattern.test(message));
+}
+
+async function fetchWithTimeout(
+ url: string,
+ method: "HEAD" | "GET"
+): Promise {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT);
-
try {
- let res = await fetch(url, {
- method: "HEAD",
+ return await fetch(url, {
+ method,
signal: controller.signal,
redirect: "follow",
});
+ } finally {
+ clearTimeout(timer);
+ }
+}
- if (res.status === 405 || res.status === 403) {
- const controller2 = new AbortController();
- const timer2 = setTimeout(() => controller2.abort(), REQUEST_TIMEOUT);
- try {
- res = await fetch(url, {
- method: "GET",
- signal: controller2.signal,
- redirect: "follow",
- });
- } finally {
- clearTimeout(timer2);
+async function checkUrl(
+ url: string
+): Promise<{ ok: boolean; status?: number; error?: string }> {
+ for (let attempt = 0; attempt <= RETRY_ATTEMPTS; attempt += 1) {
+ try {
+ let res = await fetchWithTimeout(url, "HEAD");
+
+ if (res.status === 405 || res.status === 403) {
+ res = await fetchWithTimeout(url, "GET");
}
- }
- if (res.status === 429 || (res.status >= 200 && res.status < 400)) {
- return { ok: true };
- }
+ if (res.status === 429 || (res.status >= 200 && res.status < 400)) {
+ return { ok: true };
+ }
- return { ok: false, status: res.status };
- } catch (err) {
- const message = err instanceof Error ? err.message : String(err);
- return { ok: false, error: message };
- } finally {
- clearTimeout(timer);
+ // Retry transient upstream failures.
+ if (res.status >= 500 && attempt < RETRY_ATTEMPTS) {
+ await sleep(RETRY_DELAY_MS * (attempt + 1));
+ continue;
+ }
+
+ return { ok: false, status: res.status };
+ } catch (err) {
+ const message = err instanceof Error ? err.message : String(err);
+ if (isTransientError(message) && attempt < RETRY_ATTEMPTS) {
+ await sleep(RETRY_DELAY_MS * (attempt + 1));
+ continue;
+ }
+ return { ok: false, error: message };
+ }
}
+
+ return { ok: false, error: "Unknown link check failure" };
}
function pLimit(concurrency: number) {
diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md
index 7b9e05eed..9311eb56a 100644
--- a/toolkit-docs-generator/README.md
+++ b/toolkit-docs-generator/README.md
@@ -77,13 +77,15 @@ This step does not change JSON output. It only updates navigation files.
## Local usage
+Run these commands from the `toolkit-docs-generator` directory.
+
Generate a single toolkit:
```bash
-pnpm start generate \
+pnpm dlx tsx src/cli/index.ts generate \
--providers "Github:1.0.0" \
- --engine-api-url "$ENGINE_API_URL" \
- --engine-api-key "$ENGINE_API_KEY" \
+ --tool-metadata-url "$ENGINE_API_URL" \
+ --tool-metadata-key "$ENGINE_API_KEY" \
--llm-provider openai \
--llm-model gpt-4.1-mini \
--llm-api-key "$OPENAI_API_KEY" \
@@ -93,11 +95,11 @@ pnpm start generate \
Generate all toolkits:
```bash
-pnpm start generate \
+pnpm dlx tsx src/cli/index.ts generate \
--all \
--skip-unchanged \
- --engine-api-url "$ENGINE_API_URL" \
- --engine-api-key "$ENGINE_API_KEY" \
+ --tool-metadata-url "$ENGINE_API_URL" \
+ --tool-metadata-key "$ENGINE_API_KEY" \
--llm-provider openai \
--llm-model gpt-4.1-mini \
--llm-api-key "$OPENAI_API_KEY" \
@@ -107,13 +109,12 @@ pnpm start generate \
Generate without LLM output:
```bash
-pnpm start generate \
+pnpm dlx tsx src/cli/index.ts generate \
--providers "Asana:0.1.3" \
- --engine-api-url "$ENGINE_API_URL" \
- --engine-api-key "$ENGINE_API_KEY" \
+ --tool-metadata-url "$ENGINE_API_URL" \
+ --tool-metadata-key "$ENGINE_API_KEY" \
--skip-examples \
--skip-summary \
- --skip-overview \
--output data/toolkits
```
@@ -132,11 +133,11 @@ pnpm dlx tsx .github/scripts/sync-toolkit-sidebar.ts
(only with the explicit flag), or `mock`
- `--previous-output` compare against a previous output directory
- `--custom-sections` load curated docs sections
-- `--skip-examples`, `--skip-summary`, `--skip-overview` disable LLM steps
+- `--skip-examples`, `--skip-summary` disable LLM steps
- `--no-verify-output` skip output verification
## Troubleshooting
- **Nothing regenerated**: `--skip-unchanged` exits early when tool definitions did not change.
- **Missing metadata**: the generator falls back to the metadata JSON file when design system metadata is unavailable.
-- **Verify output fails**: run `pnpm start verify-output --output data/toolkits` and fix the reported mismatch.
+- **Verify output fails**: run `pnpm dlx tsx src/cli/index.ts verify-output --output data/toolkits` and fix the reported mismatch.
diff --git a/toolkit-docs-generator/data/toolkits/index.json b/toolkit-docs-generator/data/toolkits/index.json
index af48d1e93..5aea862b4 100644
--- a/toolkit-docs-generator/data/toolkits/index.json
+++ b/toolkit-docs-generator/data/toolkits/index.json
@@ -785,7 +785,7 @@
"id": "WeaviateApi",
"label": "Weaviate API",
"version": "2.0.0",
- "category": "development",
+ "category": "databases",
"toolCount": 81,
"authType": "none",
"type": "arcade_starter"
diff --git a/toolkit-docs-generator/data/toolkits/weaviateapi.json b/toolkit-docs-generator/data/toolkits/weaviateapi.json
index 75b2bef3a..0fdf5d920 100644
--- a/toolkit-docs-generator/data/toolkits/weaviateapi.json
+++ b/toolkit-docs-generator/data/toolkits/weaviateapi.json
@@ -4,12 +4,12 @@
"version": "2.0.0",
"description": "Tools that enable LLMs to interact directly with the weaviate API.",
"metadata": {
- "category": "development",
+ "category": "databases",
"iconUrl": "https://design-system.arcade.dev/icons/weaviate.svg",
"isBYOC": false,
"isPro": false,
"type": "arcade_starter",
- "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/weaviate-api",
+ "docsLink": "https://docs.arcade.dev/en/mcp-servers/databases/weaviate-api",
"isComingSoon": false,
"isHidden": false
},
diff --git a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts
index a8e517eda..abbaabf95 100644
--- a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts
+++ b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts
@@ -13,6 +13,8 @@
* npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts
* npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --dry-run
* npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --verbose
+ * npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --remove-empty-sections
+ * npx tsx toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts --remove-empty-sections=false
*/
import {
@@ -23,27 +25,42 @@ import {
rmSync,
writeFileSync,
} from "node:fs";
+import { createRequire } from "node:module";
import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// Import design system toolkits
-let TOOLKITS: Array<{
+type DesignSystemToolkit = {
id: string;
label: string;
category?: string;
isHidden?: boolean;
-}> = [];
+};
+
+let TOOLKITS: DesignSystemToolkit[] = [];
+const require = createRequire(import.meta.url);
try {
- const designSystem = await import("@arcadeai/design-system");
+ const designSystemEntry = require.resolve("@arcadeai/design-system");
+ const designSystem = (await import(
+ pathToFileURL(designSystemEntry).href
+ )) as {
+ TOOLKITS?: DesignSystemToolkit[];
+ };
TOOLKITS = designSystem.TOOLKITS || [];
} catch {
- console.warn(
- "Warning: @arcadeai/design-system not found, using fallback category detection"
- );
+ if (!process.env.VITEST) {
+ console.warn(
+ "Warning: @arcadeai/design-system not found, using fallback category detection"
+ );
+ }
+}
+
+export function setToolkitsForTesting(toolkits: DesignSystemToolkit[]): void {
+ TOOLKITS = toolkits;
}
// Get project root (two levels up from toolkit-docs-generator/scripts)
@@ -140,6 +157,36 @@ export type SyncResult = {
errors: string[];
};
+type SyncOptions = {
+ dryRun?: boolean;
+ verbose?: boolean;
+ /** Preferred flag name (default: false) */
+ removeEmptySections?: boolean;
+ /** Backward-compatible alias for removeEmptySections */
+ prune?: boolean;
+};
+
+export const resolveRemoveEmptySections = (options: SyncOptions): boolean =>
+ options.removeEmptySections ?? options.prune ?? false;
+
+export const parseBooleanCliFlag = (
+ args: readonly string[],
+ flagName: string
+): boolean | undefined => {
+ const valueArg = args.find((arg) => arg.startsWith(`${flagName}=`));
+ if (valueArg) {
+ const rawValue = valueArg.slice(flagName.length + 1).toLowerCase();
+ if (rawValue === "true") {
+ return true;
+ }
+ if (rawValue === "false") {
+ return false;
+ }
+ }
+
+ return args.includes(flagName) ? true : undefined;
+};
+
/**
* Get list of toolkit JSON files (excluding index.json)
*/
@@ -274,8 +321,10 @@ function resolveToolkitInfo(
return null;
}
+ // Keep sidebar routes aligned with static params: toolkit JSON is source of
+ // truth for category, with design system as fallback when JSON is missing.
const category =
- designSystemToolkit?.category ?? jsonData?.metadata?.category ?? "others";
+ jsonData?.metadata?.category ?? designSystemToolkit?.category ?? "others";
const labelFromDesignSystem = designSystemToolkit?.label ?? null;
const labelFromJson = jsonData?.label ?? jsonData?.name ?? null;
const typeFromJson = jsonData?.metadata?.type ?? null;
@@ -464,10 +513,9 @@ export default meta;
* Sync toolkit sidebar with available JSON files
*/
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Main orchestration function requires multiple steps
-export function syncToolkitSidebar(
- options: { dryRun?: boolean; verbose?: boolean; prune?: boolean } = {}
-): SyncResult {
+export function syncToolkitSidebar(options: SyncOptions = {}): SyncResult {
const { dryRun = false, verbose = false } = options;
+ const removeEmptySections = resolveRemoveEmptySections(options);
const result: SyncResult = {
categoriesUpdated: [],
@@ -548,20 +596,21 @@ export function syncToolkitSidebar(
}
}
- // Remove category directories that no longer have any toolkits.
- // This handles the case where toolkits move between categories (e.g. from
- // "others" to "development") and the old category becomes empty.
- // The --prune flag is accepted for backward compatibility but cleanup
- // always runs to prevent stale sidebar entries and orphaned routes.
- for (const existingDir of existingDirs) {
- if (!activeCategories.includes(existingDir)) {
- const categoryDir = join(CONFIG.integrationsDir, existingDir);
- log(`Removing empty category: ${existingDir}`);
- if (!dryRun) {
- rmSync(categoryDir, { recursive: true });
+ // Remove category directories only when explicitly enabled.
+ // This prevents automatic route removals during regular sync runs.
+ if (removeEmptySections) {
+ for (const existingDir of existingDirs) {
+ if (!activeCategories.includes(existingDir)) {
+ const categoryDir = join(CONFIG.integrationsDir, existingDir);
+ log(`Removing empty category: ${existingDir}`);
+ if (!dryRun) {
+ rmSync(categoryDir, { recursive: true });
+ }
+ result.categoriesRemoved.push(existingDir);
}
- result.categoriesRemoved.push(existingDir);
}
+ } else {
+ log("Skipping empty category removal (--remove-empty-sections=false)");
}
// Update main _meta.tsx
@@ -636,13 +685,16 @@ if (isMainModule) {
const args = process.argv.slice(2);
const dryRun = args.includes("--dry-run");
const verbose = args.includes("--verbose") || args.includes("-v");
- const prune = args.includes("--prune");
+ const removeEmptySections =
+ parseBooleanCliFlag(args, "--remove-empty-sections") ??
+ parseBooleanCliFlag(args, "--prune") ??
+ false;
if (dryRun) {
console.log("Running in dry-run mode (no changes will be made)\n");
}
- const result = syncToolkitSidebar({ dryRun, verbose, prune });
+ const result = syncToolkitSidebar({ dryRun, verbose, removeEmptySections });
printResults(result);
if (result.errors.length > 0) {
diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts
index 19dfa7768..a391fec51 100644
--- a/toolkit-docs-generator/src/cli/index.ts
+++ b/toolkit-docs-generator/src/cli/index.ts
@@ -13,7 +13,7 @@
import chalk from "chalk";
import { Command } from "commander";
-import { access, mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
+import { readdir, readFile, rm } from "fs/promises";
import ora from "ora";
import { join, resolve } from "path";
import {
@@ -23,6 +23,7 @@ import {
getChangedToolkitIds,
hasChanges,
} from "../diff/index.js";
+import { parsePreviousToolkitForDiff } from "../diff/previous-output.js";
import {
createJsonGenerator,
type VerificationProgress,
@@ -33,7 +34,6 @@ import {
type LlmClient,
type LlmProvider,
LlmToolExampleGenerator,
- LlmToolkitOverviewGenerator,
LlmToolkitSummaryGenerator,
} from "../llm/index.js";
import type { MergeResult } from "../merger/data-merger.js";
@@ -43,14 +43,13 @@ import { createDesignSystemMetadataSource } from "../sources/design-system-metad
import { createEmptyCustomSectionsSource } from "../sources/in-memory.js";
import { createMockMetadataSource } from "../sources/mock-metadata.js";
import { createDesignSystemProviderIdResolver } from "../sources/oauth-provider-resolver.js";
-import { createOverviewInstructionsFileSource } from "../sources/overview-instructions-file.js";
import {
createArcadeToolkitDataSource,
createEngineToolkitDataSource,
createMockToolkitDataSource,
type IToolkitDataSource,
+ type ToolkitData,
} from "../sources/toolkit-data-source.js";
-import { normalizeId } from "../utils/fp.js";
import {
createProgressTracker,
formatToolkitComplete,
@@ -72,7 +71,6 @@ type ApiSource = "list-tools" | "tool-metadata" | "mock";
import {
type MergedToolkit,
- MergedToolkitSchema,
type ProviderVersion,
ProviderVersionSchema,
} from "../types/index.js";
@@ -106,40 +104,6 @@ const parseProviders = (input: string): ProviderVersion[] => {
});
};
-const parseToolkitList = (input: string): string[] =>
- input
- .split(",")
- .map((value) => value.trim())
- .filter(Boolean);
-
-const buildOverviewTemplate = (toolkitId: string, label?: string) => ({
- toolkitId,
- label: label ?? toolkitId,
- sources: [],
- instructions:
- "Write an overview section for this toolkit. Start with a short paragraph, then add a **Capabilities** list. Mention auth type/scopes and secrets if relevant.",
-});
-
-const buildOverviewFilePath = (dir: string, toolkitId: string): string => {
- const fileName = `${normalizeId(toolkitId)}.json`;
- return resolve(dir, fileName);
-};
-
-const resolveProviderIds = async (
- providers: ProviderVersion[],
- metadataSource: unknown
-): Promise => {
- const source = metadataSource as {
- getAllToolkitsMetadata?: () => Promise<
- readonly { id: string; label: string }[]
- >;
- };
- if (!source.getAllToolkitsMetadata) return providers;
-
- const all = await source.getAllToolkitsMetadata();
- return resolveProviderIdsFromMetadata(providers, all);
-};
-
/**
* Get the default fixture paths (for mock mode)
*/
@@ -167,20 +131,19 @@ const getDefaultVerificationDir = (): string => {
const getDefaultLogDir = (): string =>
resolve(getDefaultVerificationDir(), "logs");
-const getDefaultOverviewDir = (): string => {
- const cwd = process.cwd();
- if (cwd.endsWith("toolkit-docs-generator")) {
- return resolve(cwd, "overview-input");
- }
- return resolve(cwd, "toolkit-docs-generator", "overview-input");
-};
-
const buildLogPaths = (logDir: string) => ({
runLogPath: join(logDir, "runs.log"),
changeLogPath: join(logDir, "changes.log"),
failedToolsPath: join(logDir, "failed-tools.json"),
});
+const getToolkitIdsWithoutMetadata = (
+ toolkitsData: ReadonlyMap
+): string[] =>
+ Array.from(toolkitsData.entries())
+ .filter(([, toolkitData]) => toolkitData.metadata === null)
+ .map(([toolkitId]) => toolkitId);
+
const createMetadataSource = async (options: {
metadataFile: string;
useMetadataFile: boolean;
@@ -230,101 +193,33 @@ const createMetadataSource = async (options: {
return createMockMetadataSource(options.metadataFile);
};
-type OverviewInitOptions = {
- toolkits?: string;
- overviewDir: string;
- metadataFile?: string;
- force: boolean;
-};
-
type MetadataSource = Awaited>;
-const getOverviewInitToolkits = (options: OverviewInitOptions): string[] => {
- if (!options.toolkits) {
- throw new Error('Missing required option "--toolkits".');
- }
-
- const toolkits = parseToolkitList(options.toolkits);
- if (toolkits.length === 0) {
- throw new Error("No toolkit IDs provided.");
- }
-
- return toolkits;
-};
-
-const resolveOverviewInitMetadataSource = async (
- options: OverviewInitOptions
-): Promise => {
- const metadataFile =
- options.metadataFile ?? join(getDefaultMockDataDir(), "metadata.json");
- return createMetadataSource({
- metadataFile,
- useMetadataFile: Boolean(options.metadataFile),
- verbose: false,
- });
-};
-
-const shouldSkipOverviewFile = async (
- filePath: string,
- force: boolean
-): Promise => {
- if (force) {
- return false;
- }
-
- try {
- await access(filePath);
- return true;
- } catch {
- return false;
- }
-};
-
-const getOverviewTemplateLabel = async (
- metadataSource: MetadataSource,
- toolkitId: string
-): Promise => {
- if ("getToolkitMetadata" in metadataSource) {
- const metadata = await metadataSource.getToolkitMetadata(toolkitId);
- return metadata?.label ?? undefined;
- }
-
- return;
+const resolveProviderIds = async (
+ providers: ProviderVersion[],
+ metadataSource: MetadataSource
+): Promise => {
+ const all = await metadataSource.getAllToolkitsMetadata();
+ return resolveProviderIdsFromMetadata(providers, all);
};
-const writeOverviewTemplate = async (
- filePath: string,
- template: ReturnType
-): Promise => {
- await writeFile(filePath, `${JSON.stringify(template, null, 2)}\n`, "utf-8");
-};
+const filterProvidersByMetadataPresence = async (
+ providers: readonly ProviderVersion[],
+ metadataSource: MetadataSource
+): Promise<{ included: ProviderVersion[]; excluded: string[] }> => {
+ const included: ProviderVersion[] = [];
+ const excluded: string[] = [];
-const createOverviewFiles = async (options: {
- toolkits: readonly string[];
- overviewDir: string;
- metadataSource: MetadataSource;
- force: boolean;
-}): Promise<{ created: number; skipped: number }> => {
- let created = 0;
- let skipped = 0;
-
- for (const toolkitId of options.toolkits) {
- const filePath = buildOverviewFilePath(options.overviewDir, toolkitId);
- if (await shouldSkipOverviewFile(filePath, options.force)) {
- skipped += 1;
- continue;
+ for (const provider of providers) {
+ const metadata = await metadataSource.getToolkitMetadata(provider.provider);
+ if (metadata) {
+ included.push(provider);
+ } else {
+ excluded.push(provider.provider);
}
-
- const label = await getOverviewTemplateLabel(
- options.metadataSource,
- toolkitId
- );
- const template = buildOverviewTemplate(toolkitId, label);
- await writeOverviewTemplate(filePath, template);
- created += 1;
}
- return { created, skipped };
+ return { included, excluded };
};
const buildChangeLogDetails = (
@@ -339,7 +234,8 @@ const buildChangeLogDetails = (
(change) =>
change.changeType === "modified" &&
change.toolChanges.length === 0 &&
- change.versionChanged
+ change.versionChanged &&
+ !change.metadataChanged
)
.map((change) => change.toolkitId);
@@ -610,49 +506,169 @@ const createToolkitDataSourceForApi = (
const normalizeToolkitKey = (toolkitId: string): string =>
toolkitId.toLowerCase();
+interface PreviousToolkitLoadStats {
+ scannedFiles: number;
+ loadedFiles: number;
+ fallbackFiles: number;
+ missingFiles: number;
+ failedFiles: string[];
+}
+
+interface PreviousToolkitLoadResult {
+ toolkits: ReadonlyMap;
+ stats: PreviousToolkitLoadStats;
+}
+
+const createPreviousToolkitLoadStats = (): PreviousToolkitLoadStats => ({
+ scannedFiles: 0,
+ loadedFiles: 0,
+ fallbackFiles: 0,
+ missingFiles: 0,
+ failedFiles: [],
+});
+
+const getFileStem = (filePath: string): string => {
+ const fileName = filePath.split("/").pop() ?? "unknown.json";
+ return fileName.replace(/\.json$/i, "");
+};
+
+interface PreviousToolkitLoadOutcome {
+ toolkit: MergedToolkit | null;
+ missing: boolean;
+ usedFallback: boolean;
+ reason?: string;
+}
+
const loadPreviousToolkit = async (
filePath: string
-): Promise => {
+): Promise => {
try {
const content = await readFile(filePath, "utf-8");
const parsed = JSON.parse(content) as unknown;
- const result = MergedToolkitSchema.safeParse(parsed);
- if (!result.success) {
- return null;
+ const parsedToolkit = parsePreviousToolkitForDiff(
+ parsed,
+ getFileStem(filePath)
+ );
+ return {
+ toolkit: parsedToolkit.toolkit,
+ missing: false,
+ usedFallback: parsedToolkit.usedFallback,
+ ...(parsedToolkit.reason ? { reason: parsedToolkit.reason } : {}),
+ };
+ } catch (error) {
+ const errorWithCode =
+ error && typeof error === "object" && "code" in error
+ ? (error as { code?: string })
+ : undefined;
+ if (errorWithCode?.code === "ENOENT") {
+ return {
+ toolkit: null,
+ missing: true,
+ usedFallback: false,
+ };
}
- return result.data;
- } catch {
- return null;
+
+ return {
+ toolkit: null,
+ missing: false,
+ usedFallback: false,
+ reason: error instanceof Error ? error.message : String(error),
+ };
}
};
+const formatPreviousToolkitLoadStats = (
+ stats: PreviousToolkitLoadStats
+): string =>
+ [
+ `${stats.scannedFiles} scanned`,
+ `${stats.loadedFiles} loaded`,
+ `${stats.fallbackFiles} fallback`,
+ `${stats.missingFiles} missing`,
+ `${stats.failedFiles.length} failed`,
+ ].join(", ");
+
+const addLoadedPreviousToolkit = (
+ previousToolkits: Map,
+ stats: PreviousToolkitLoadStats,
+ loaded: PreviousToolkitLoadOutcome
+): boolean => {
+ if (!loaded.toolkit) {
+ return false;
+ }
+ previousToolkits.set(normalizeToolkitKey(loaded.toolkit.id), loaded.toolkit);
+ stats.loadedFiles += 1;
+ if (loaded.usedFallback) {
+ stats.fallbackFiles += 1;
+ }
+ return true;
+};
+
+const getPreviousToolkitFailureReason = (
+ loaded: PreviousToolkitLoadOutcome
+): string => loaded.reason ?? "unable to parse previous output";
+
+const logFallbackToolkitLoad = (
+ itemLabel: string,
+ loaded: PreviousToolkitLoadOutcome,
+ verbose: boolean
+): void => {
+ if (!(verbose && loaded.usedFallback)) {
+ return;
+ }
+ console.log(
+ chalk.dim(
+ `Loaded ${itemLabel} with fallback parser (${loaded.reason ?? "schema mismatch"}).`
+ )
+ );
+};
+
const loadPreviousToolkitsForProviders = async (
dir: string,
providers: ProviderVersion[],
verbose: boolean
-): Promise> => {
+): Promise => {
const previousToolkits = new Map();
+ const stats = createPreviousToolkitLoadStats();
for (const provider of providers) {
- const filePath = join(dir, `${provider.provider.toLowerCase()}.json`);
- const toolkit = await loadPreviousToolkit(filePath);
- if (toolkit) {
- previousToolkits.set(normalizeToolkitKey(toolkit.id), toolkit);
- } else if (verbose) {
+ stats.scannedFiles += 1;
+ const providerName = provider.provider;
+ const filePath = join(dir, `${providerName.toLowerCase()}.json`);
+ const loaded = await loadPreviousToolkit(filePath);
+
+ if (addLoadedPreviousToolkit(previousToolkits, stats, loaded)) {
+ logFallbackToolkitLoad(providerName, loaded, verbose);
+ continue;
+ }
+
+ if (loaded.missing) {
+ stats.missingFiles += 1;
+ if (verbose) {
+ console.log(chalk.dim(`No previous output found for ${providerName}.`));
+ }
+ continue;
+ }
+
+ stats.failedFiles.push(
+ `${providerName}: ${getPreviousToolkitFailureReason(loaded)}`
+ );
+ if (verbose) {
console.log(
- chalk.dim(`No previous output found for ${provider.provider}.`)
+ chalk.dim(`Failed to parse previous output for ${providerName}.`)
);
}
}
- return previousToolkits;
+ return { toolkits: previousToolkits, stats };
};
const loadPreviousToolkitsFromDir = async (
dir: string,
verbose: boolean
-): Promise> => {
+): Promise => {
const previousToolkits = new Map();
+ const stats = createPreviousToolkitLoadStats();
try {
const entries = await readdir(dir);
@@ -661,11 +677,22 @@ const loadPreviousToolkitsFromDir = async (
);
for (const fileName of jsonFiles) {
+ stats.scannedFiles += 1;
const filePath = join(dir, fileName);
- const toolkit = await loadPreviousToolkit(filePath);
- if (toolkit) {
- previousToolkits.set(normalizeToolkitKey(toolkit.id), toolkit);
+ const loaded = await loadPreviousToolkit(filePath);
+ if (addLoadedPreviousToolkit(previousToolkits, stats, loaded)) {
+ logFallbackToolkitLoad(fileName, loaded, verbose);
+ continue;
+ }
+
+ if (loaded.missing) {
+ stats.missingFiles += 1;
+ continue;
}
+
+ stats.failedFiles.push(
+ `${fileName}: ${getPreviousToolkitFailureReason(loaded)}`
+ );
}
} catch (error) {
if (verbose) {
@@ -673,7 +700,7 @@ const loadPreviousToolkitsFromDir = async (
}
}
- return previousToolkits;
+ return { toolkits: previousToolkits, stats };
};
const processProviders = async (
@@ -709,46 +736,6 @@ program
.description("Generate documentation JSON for Arcade toolkits")
.version("1.0.0");
-program
- .command("overview-init")
- .description("Create overview instruction file(s) for toolkits")
- .option(
- "--toolkits ",
- 'Comma-separated toolkit IDs (e.g. "Github,Slack")'
- )
- .option(
- "--overview-dir ",
- "Directory for overview instruction files",
- getDefaultOverviewDir()
- )
- .option("--metadata-file ", "Path to metadata JSON file")
- .option("--force", "Overwrite existing overview files", false)
- .action(async (options: OverviewInitOptions) => {
- const spinner = ora("Preparing overview files...").start();
- try {
- const toolkits = getOverviewInitToolkits(options);
- const overviewDir = resolve(options.overviewDir);
- await mkdir(overviewDir, { recursive: true });
-
- const metadataSource = await resolveOverviewInitMetadataSource(options);
- const { created, skipped } = await createOverviewFiles({
- toolkits,
- overviewDir,
- metadataSource,
- force: options.force,
- });
-
- spinner.succeed(
- `Overview files created: ${created}${skipped > 0 ? ` (skipped ${skipped})` : ""}`
- );
- } catch (error) {
- spinner.fail(
- `Error: ${error instanceof Error ? error.message : String(error)}`
- );
- process.exit(1);
- }
- });
-
program
.command("generate")
.description("Generate documentation for specific providers or all toolkits")
@@ -765,15 +752,6 @@ program
.option("--log-dir ", "Directory for run logs", getDefaultLogDir())
.option("--mock-data-dir ", "Path to mock data directory")
.option("--metadata-file ", "Path to metadata JSON file")
- .option(
- "--overview-dir ",
- "Directory with toolkit overview instruction files",
- getDefaultOverviewDir()
- )
- .option(
- "--overview-file ",
- "Path to a single overview instruction file"
- )
.option(
"--api-source ",
'API source: "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata), or "mock" (default: auto-detect)'
@@ -825,12 +803,12 @@ program
)
.option(
"--llm-concurrency ",
- "Max concurrent LLM calls per toolkit (default: 5)",
+ "Max concurrent LLM calls per toolkit (default: 10)",
(value) => Number.parseInt(value, 10)
)
.option(
"--toolkit-concurrency ",
- "Max concurrent toolkit processing (default: 3)",
+ "Max concurrent toolkit processing (default: 5)",
(value) => Number.parseInt(value, 10)
)
.option(
@@ -839,7 +817,6 @@ program
)
.option("--skip-examples", "Skip LLM example generation", false)
.option("--skip-summary", "Skip LLM summary generation", false)
- .option("--skip-overview", "Skip LLM overview generation", false)
.option("--no-verify-output", "Skip output verification")
.option("--custom-sections ", "Path to custom sections JSON")
.option(
@@ -857,6 +834,11 @@ program
"Only regenerate toolkits with changed tool definitions",
false
)
+ .option(
+ "--require-complete",
+ "Require complete metadata. Only include toolkits with data in Engine and Design System.",
+ false
+ )
.option("--verbose", "Enable verbose logging", false)
.action(
async (options: {
@@ -890,19 +872,18 @@ program
indexFromOutput: boolean;
skipExamples: boolean;
skipSummary: boolean;
- skipOverview: boolean;
- overviewDir: string;
- overviewFile?: string;
verifyOutput: boolean;
customSections?: string;
resume: boolean;
incremental: boolean;
skipUnchanged: boolean;
+ requireComplete: boolean;
verbose: boolean;
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: legacy CLI flow
}) => {
const spinner = ora("Parsing input...").start();
const logPaths = buildLogPaths(resolve(options.logDir));
+ const requireComplete = options.requireComplete;
try {
const runAll = options.all;
@@ -983,6 +964,36 @@ program
providers = await resolveProviderIds(providers, metadataSource);
}
+ if (requireComplete && providers && providers.length > 0) {
+ const { included, excluded } =
+ await filterProvidersByMetadataPresence(providers, metadataSource);
+ providers = included;
+
+ if (excluded.length > 0) {
+ const label = excluded.length === 1 ? "provider" : "providers";
+ console.log(
+ chalk.yellow(
+ `Skipping ${excluded.length} ${label} not found in metadata source: ${excluded.join(", ")}`
+ )
+ );
+ }
+
+ if (providers.length === 0) {
+ spinner.succeed(
+ "No providers left after metadata filtering. Nothing to generate."
+ );
+ await appendLogEntry(logPaths.runLogPath, {
+ title: "generate (metadata-filter no-op)",
+ details: [
+ `output=${resolve(options.output)}`,
+ "mode=providers",
+ "reason=no providers matched metadata source",
+ ],
+ });
+ process.exit(0);
+ }
+ }
+
// Create toolkit data source based on API source
const apiSource = resolveApiSource(options);
const toolkitDataSource = createToolkitDataSourceForApi(
@@ -996,8 +1007,7 @@ program
const needsExamples = !options.skipExamples;
const needsSummary = !options.skipSummary;
- const needsOverview = !options.skipOverview;
- const needsLlm = needsExamples || needsSummary || needsOverview;
+ const needsLlm = needsExamples || needsSummary;
const llmConfig = needsLlm
? resolveLlmConfig(options, options.verbose)
@@ -1018,21 +1028,11 @@ program
}
toolkitSummaryGenerator = new LlmToolkitSummaryGenerator(llmConfig);
}
- let overviewGenerator: LlmToolkitOverviewGenerator | undefined;
- if (needsOverview) {
- if (llmConfig) {
- overviewGenerator = new LlmToolkitOverviewGenerator(llmConfig);
- } else {
- spinner.warn(
- "Overview generation skipped (missing LLM configuration)."
- );
- }
- }
const previousOutputDir = options.forceRegenerate
? undefined
: (options.previousOutput ??
(options.overwriteOutput ? undefined : options.output));
- const previousToolkits = previousOutputDir
+ const previousToolkitLoad = previousOutputDir
? runAll
? await loadPreviousToolkitsFromDir(
previousOutputDir,
@@ -1044,20 +1044,33 @@ program
options.verbose
)
: undefined;
+ const previousToolkits = previousToolkitLoad?.toolkits;
+ if (previousToolkitLoad) {
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ `Loaded previous output: ${formatPreviousToolkitLoadStats(previousToolkitLoad.stats)}`
+ )
+ );
+ }
+ if (previousToolkitLoad.stats.failedFiles.length > 0) {
+ console.log(
+ chalk.yellow(
+ `⚠ Skipped ${previousToolkitLoad.stats.failedFiles.length} previous toolkit file(s) during diffing.`
+ )
+ );
+ if (options.verbose) {
+ for (const failure of previousToolkitLoad.stats.failedFiles) {
+ console.log(chalk.dim(` - ${failure}`));
+ }
+ }
+ }
+ }
// Custom sections source
const customSectionsSource = options.customSections
? createCustomSectionsFileSource(options.customSections)
: createEmptyCustomSectionsSource();
- const overviewInstructionsSource = options.overviewFile
- ? createOverviewInstructionsFileSource({
- filePath: resolve(options.overviewFile),
- allowMissing: false,
- })
- : createOverviewInstructionsFileSource({
- dirPath: resolve(options.overviewDir ?? getDefaultOverviewDir()),
- allowMissing: true,
- });
// Build provider ID resolver from design system OAuth catalogue
const resolveProviderId =
@@ -1097,23 +1110,55 @@ program
spinner.start("Detecting changes in tool definitions...");
// Fetch all current tools
+ const fetchStartedAt = Date.now();
const currentToolkitsData =
await toolkitDataSource.fetchAllToolkitsData();
+ const fetchDurationMs = Date.now() - fetchStartedAt;
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ ` Fetched ${currentToolkitsData.size} toolkit(s) from ${apiSource} in ${fetchDurationMs}ms`
+ )
+ );
+ }
- // Build map of toolkit ID -> tools for comparison
- const currentToolkitTools = new Map<
- string,
- readonly import("../types/index.js").ToolDefinition[]
- >();
+ const metadataExcludedToolkitIds = requireComplete
+ ? getToolkitIdsWithoutMetadata(currentToolkitsData)
+ : [];
+ const metadataExcludedToolkitIdSet = new Set(
+ metadataExcludedToolkitIds.map((id) => id.toLowerCase())
+ );
+ if (options.verbose && metadataExcludedToolkitIds.length > 0) {
+ console.log(
+ chalk.dim(
+ ` Excluding ${metadataExcludedToolkitIds.length} toolkit(s) without metadata before change detection`
+ )
+ );
+ }
+
+ // Build map of toolkit ID -> current toolkit data for comparison
+ const currentToolkitDataForDiff = new Map();
for (const [id, data] of currentToolkitsData) {
- currentToolkitTools.set(id, data.tools);
+ if (metadataExcludedToolkitIdSet.has(id.toLowerCase())) {
+ continue;
+ }
+ currentToolkitDataForDiff.set(id, data);
}
// Detect changes
+ const compareStartedAt = Date.now();
const detectedChanges = detectChanges(
- currentToolkitTools,
+ currentToolkitDataForDiff,
previousToolkits ?? new Map()
);
+ const compareDurationMs = Date.now() - compareStartedAt;
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ ` Compared ${currentToolkitDataForDiff.size} current toolkit(s) against ${previousToolkits?.size ?? 0} previous toolkit(s) in ${compareDurationMs}ms`
+ )
+ );
+ }
if (!hasChanges(detectedChanges)) {
spinner.succeed(
@@ -1140,15 +1185,24 @@ program
const changedIds = getChangedToolkitIds(detectedChanges);
changedToolkitIds = new Set(changedIds.map((id) => id.toLowerCase()));
changeResult = detectedChanges;
+ const changedPreview =
+ changedIds.length <= 20
+ ? changedIds.join(", ")
+ : `${changedIds.slice(0, 20).join(", ")} ... +${changedIds.length - 20} more`;
spinner.succeed(
- `Detected ${changedIds.length} changed toolkit(s): ${changedIds.join(", ")}`
+ `Detected ${changedIds.length} changed toolkit(s): ${changedPreview}`
);
if (options.verbose) {
console.log(
chalk.dim(` Summary: ${formatChangeSummary(detectedChanges)}`)
);
+ if (changedIds.length > 20) {
+ console.log(
+ chalk.dim(` Full changed list: ${changedIds.join(", ")}`)
+ );
+ }
}
}
@@ -1191,9 +1245,6 @@ program
const merger = createDataMerger({
toolkitDataSource,
customSectionsSource,
- overviewInstructionsSource,
- ...(overviewGenerator ? { overviewGenerator } : {}),
- skipOverview: options.skipOverview || !overviewGenerator,
...(toolExampleGenerator ? { toolExampleGenerator } : {}),
...(toolkitSummaryGenerator ? { toolkitSummaryGenerator } : {}),
...(previousToolkits ? { previousToolkits } : {}),
@@ -1205,6 +1256,7 @@ program
: {}),
...(runAll ? { onToolkitProgress } : {}),
...(skipToolkitIds.size > 0 ? { skipToolkitIds } : {}),
+ requireCompleteData: requireComplete,
...(onToolkitComplete ? { onToolkitComplete } : {}),
...(resolveProviderId ? { resolveProviderId } : {}),
});
@@ -1214,8 +1266,33 @@ program
if (runAll) {
// First, get the toolkit count to set up progress tracking
spinner.start("Fetching toolkit list...");
+ const fetchToolkitListStartedAt = Date.now();
const toolkitList = await toolkitDataSource.fetchAllToolkitsData();
+ const fetchToolkitListDurationMs =
+ Date.now() - fetchToolkitListStartedAt;
const totalToolkits = toolkitList.size;
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ ` Toolkit list fetched in ${fetchToolkitListDurationMs}ms (${totalToolkits} toolkit(s))`
+ )
+ );
+ }
+
+ if (requireComplete) {
+ const metadataExcludedToolkitIds =
+ getToolkitIdsWithoutMetadata(toolkitList);
+ for (const toolkitId of metadataExcludedToolkitIds) {
+ skipToolkitIds.add(toolkitId.toLowerCase());
+ }
+ if (options.verbose && metadataExcludedToolkitIds.length > 0) {
+ console.log(
+ chalk.dim(
+ ` Excluding ${metadataExcludedToolkitIds.length} toolkit(s) without metadata`
+ )
+ );
+ }
+ }
// If --skip-unchanged, only process changed toolkits
// Add unchanged toolkits to skipToolkitIds
@@ -1231,17 +1308,26 @@ program
const toProcess = totalToolkits - skipToolkitIds.size;
if (toProcess === 0) {
- spinner.succeed(`All ${totalToolkits} toolkit(s) already complete`);
+ spinner.succeed("No toolkits to process after applying filters");
allResults = [];
} else {
- const skipReason = changedToolkitIds
- ? `(${skipToolkitIds.size} unchanged)`
- : skipToolkitIds.size > 0
- ? `(${skipToolkitIds.size} skipped)`
- : "";
+ const skipReason =
+ skipToolkitIds.size > 0 ? `(${skipToolkitIds.size} skipped)` : "";
spinner.succeed(
`Found ${totalToolkits} toolkit(s), ${toProcess} to process ${skipReason}`.trim()
);
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ ` Starting merge with toolkitConcurrency=${options.toolkitConcurrency ?? 5}, llmConcurrency=${options.llmConcurrency ?? 10}`
+ )
+ );
+ console.log(
+ chalk.dim(
+ ` LLM steps: examples=${needsExamples}, summary=${needsSummary}`
+ )
+ );
+ }
// Set up progress tracker
const progressTracker = createProgressTracker({
@@ -1274,9 +1360,6 @@ program
const mergerWithProgress = createDataMerger({
toolkitDataSource,
customSectionsSource,
- overviewInstructionsSource,
- ...(overviewGenerator ? { overviewGenerator } : {}),
- skipOverview: options.skipOverview || !overviewGenerator,
...(toolExampleGenerator ? { toolExampleGenerator } : {}),
...(toolkitSummaryGenerator ? { toolkitSummaryGenerator } : {}),
...(previousToolkits ? { previousToolkits } : {}),
@@ -1288,6 +1371,7 @@ program
: {}),
onToolkitProgress,
...(skipToolkitIds.size > 0 ? { skipToolkitIds } : {}),
+ requireCompleteData: requireComplete,
...(onToolkitComplete ? { onToolkitComplete } : {}),
...(resolveProviderId ? { resolveProviderId } : {}),
});
@@ -1389,9 +1473,11 @@ program
};
spinner.text = `${phaseLabels[progress.phase]} ${progress.current}/${progress.total}: ${progress.fileName ?? ""}`;
};
+ const allowLegacyFallback = !options.overwriteOutput;
const verification = await verifyOutputDir(
resolve(options.output),
- onVerifyProgress
+ onVerifyProgress,
+ { allowLegacyFallback }
);
if (!verification.valid) {
spinner.fail("Output verification failed.");
@@ -1424,6 +1510,7 @@ program
`apiSource=${apiSource}`,
`mode=${runAll ? "all" : "providers"}`,
`skipUnchanged=${options.skipUnchanged}`,
+ `requireComplete=${requireComplete}`,
`filesWritten=${filesWritten.length}`,
`warnings=${warningCount}`,
`writeErrors=${writeErrors.length}`,
@@ -1487,15 +1574,6 @@ program
.option("-o, --output ", "Output directory", getDefaultOutputDir())
.option("--mock-data-dir ", "Path to mock data directory")
.option("--metadata-file ", "Path to metadata JSON file")
- .option(
- "--overview-dir ",
- "Directory with toolkit overview instruction files",
- getDefaultOverviewDir()
- )
- .option(
- "--overview-file ",
- "Path to a single overview instruction file"
- )
.option(
"--api-source ",
'API source: "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata), or "mock" (default: auto-detect)'
@@ -1547,12 +1625,12 @@ program
)
.option(
"--llm-concurrency ",
- "Max concurrent LLM calls per toolkit (default: 5)",
+ "Max concurrent LLM calls per toolkit (default: 10)",
(value) => Number.parseInt(value, 10)
)
.option(
"--toolkit-concurrency ",
- "Max concurrent toolkit processing (default: 3)",
+ "Max concurrent toolkit processing (default: 5)",
(value) => Number.parseInt(value, 10)
)
.option(
@@ -1573,6 +1651,11 @@ program
"Write each toolkit immediately after processing (default when --resume)",
false
)
+ .option(
+ "--require-complete",
+ "Require complete metadata. Only include toolkits with data in Engine and Design System.",
+ false
+ )
.option("--verbose", "Enable verbose logging", false)
.action(
async (options: {
@@ -1602,17 +1685,16 @@ program
indexFromOutput: boolean;
skipExamples: boolean;
skipSummary: boolean;
- skipOverview: boolean;
- overviewDir: string;
- overviewFile?: string;
verifyOutput: boolean;
customSections?: string;
resume: boolean;
incremental: boolean;
+ requireComplete: boolean;
verbose: boolean;
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: legacy CLI flow
}) => {
const spinner = ora("Initializing...").start();
+ const requireComplete = options.requireComplete;
try {
const mockDataDir = options.mockDataDir ?? getDefaultMockDataDir();
@@ -1637,8 +1719,7 @@ program
const needsExamples = !options.skipExamples;
const needsSummary = !options.skipSummary;
- const needsOverview = !options.skipOverview;
- const needsLlm = needsExamples || needsSummary || needsOverview;
+ const needsLlm = needsExamples || needsSummary;
const llmConfig = needsLlm
? resolveLlmConfig(options, options.verbose)
@@ -1659,39 +1740,42 @@ program
}
toolkitSummaryGenerator = new LlmToolkitSummaryGenerator(llmConfig);
}
- let overviewGenerator: LlmToolkitOverviewGenerator | undefined;
- if (needsOverview) {
- if (llmConfig) {
- overviewGenerator = new LlmToolkitOverviewGenerator(llmConfig);
- } else {
- spinner.warn(
- "Overview generation skipped (missing LLM configuration)."
- );
- }
- }
const previousOutputDir = options.forceRegenerate
? undefined
: (options.previousOutput ??
(options.overwriteOutput ? undefined : options.output));
- const previousToolkits = previousOutputDir
+ const previousToolkitLoad = previousOutputDir
? await loadPreviousToolkitsFromDir(
previousOutputDir,
options.verbose
)
: undefined;
+ const previousToolkits = previousToolkitLoad?.toolkits;
+ if (previousToolkitLoad) {
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ `Loaded previous output: ${formatPreviousToolkitLoadStats(previousToolkitLoad.stats)}`
+ )
+ );
+ }
+ if (previousToolkitLoad.stats.failedFiles.length > 0) {
+ console.log(
+ chalk.yellow(
+ `⚠ Skipped ${previousToolkitLoad.stats.failedFiles.length} previous toolkit file(s) during diffing.`
+ )
+ );
+ if (options.verbose) {
+ for (const failure of previousToolkitLoad.stats.failedFiles) {
+ console.log(chalk.dim(` - ${failure}`));
+ }
+ }
+ }
+ }
const customSectionsSource = options.customSections
? createCustomSectionsFileSource(options.customSections)
: createEmptyCustomSectionsSource();
- const overviewInstructionsSource = options.overviewFile
- ? createOverviewInstructionsFileSource({
- filePath: resolve(options.overviewFile),
- allowMissing: false,
- })
- : createOverviewInstructionsFileSource({
- dirPath: resolve(options.overviewDir ?? getDefaultOverviewDir()),
- allowMissing: true,
- });
// Build provider ID resolver from design system OAuth catalogue
const resolveProviderId =
@@ -1752,16 +1836,54 @@ program
// First, get the toolkit count to set up progress tracking
spinner.start("Fetching toolkit list...");
+ const fetchToolkitListStartedAt = Date.now();
const toolkitList = await toolkitDataSource.fetchAllToolkitsData();
+ const fetchToolkitListDurationMs =
+ Date.now() - fetchToolkitListStartedAt;
const totalToolkits = toolkitList.size;
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ ` Toolkit list fetched in ${fetchToolkitListDurationMs}ms (${totalToolkits} toolkit(s))`
+ )
+ );
+ }
+
+ if (requireComplete) {
+ const metadataExcludedToolkitIds =
+ getToolkitIdsWithoutMetadata(toolkitList);
+ for (const toolkitId of metadataExcludedToolkitIds) {
+ skipToolkitIds.add(toolkitId.toLowerCase());
+ }
+ if (options.verbose && metadataExcludedToolkitIds.length > 0) {
+ console.log(
+ chalk.dim(
+ ` Excluding ${metadataExcludedToolkitIds.length} toolkit(s) without metadata`
+ )
+ );
+ }
+ }
+
const toProcess = totalToolkits - skipToolkitIds.size;
if (toProcess === 0) {
- spinner.succeed(`All ${totalToolkits} toolkit(s) already complete`);
+ spinner.succeed("No toolkits to process after applying filters");
} else {
spinner.succeed(
`Found ${totalToolkits} toolkit(s), ${toProcess} to process${skipToolkitIds.size > 0 ? ` (${skipToolkitIds.size} skipped)` : ""}`
);
+ if (options.verbose) {
+ console.log(
+ chalk.dim(
+ ` Starting merge with toolkitConcurrency=${options.toolkitConcurrency ?? 5}, llmConcurrency=${options.llmConcurrency ?? 10}`
+ )
+ );
+ console.log(
+ chalk.dim(
+ ` LLM steps: examples=${needsExamples}, summary=${needsSummary}`
+ )
+ );
+ }
// Set up progress tracker
const progressTracker = createProgressTracker({
@@ -1794,9 +1916,6 @@ program
const merger = createDataMerger({
toolkitDataSource,
customSectionsSource,
- overviewInstructionsSource,
- ...(overviewGenerator ? { overviewGenerator } : {}),
- skipOverview: options.skipOverview || !overviewGenerator,
...(toolExampleGenerator ? { toolExampleGenerator } : {}),
...(toolkitSummaryGenerator ? { toolkitSummaryGenerator } : {}),
...(previousToolkits ? { previousToolkits } : {}),
@@ -1808,6 +1927,7 @@ program
: {}),
onToolkitProgress,
...(skipToolkitIds.size > 0 ? { skipToolkitIds } : {}),
+ requireCompleteData: requireComplete,
...(onToolkitComplete ? { onToolkitComplete } : {}),
...(resolveProviderId ? { resolveProviderId } : {}),
});
@@ -1879,9 +1999,11 @@ program
};
spinner.text = `${phaseLabels[progress.phase]} ${progress.current}/${progress.total}: ${progress.fileName ?? ""}`;
};
+ const allowLegacyFallback = !options.overwriteOutput;
const verification = await verifyOutputDir(
resolve(options.output),
- onVerifyProgress
+ onVerifyProgress,
+ { allowLegacyFallback }
);
if (!verification.valid) {
spinner.fail("Output verification failed.");
@@ -2081,39 +2203,81 @@ program
// Fetch current data from API
spinner.text = "Fetching current tool definitions from API...";
+ const fetchStartedAt = Date.now();
const currentToolkitsData =
await toolkitDataSource.fetchAllToolkitsData();
+ const fetchDurationMs = Date.now() - fetchStartedAt;
- // Build map of toolkit ID -> tools
- const currentToolkitTools = new Map<
- string,
- readonly import("../types/index.js").ToolDefinition[]
- >();
+ // Build map of toolkit ID -> current toolkit data
+ const currentToolkitDataForDiff = new Map();
for (const [id, data] of currentToolkitsData) {
- currentToolkitTools.set(id, data.tools);
+ currentToolkitDataForDiff.set(id, data);
}
// Load previous output
spinner.text = "Loading previous output...";
- const previousToolkits = await loadPreviousToolkitsFromDir(
+ const loadPreviousStartedAt = Date.now();
+ const previousToolkitLoad = await loadPreviousToolkitsFromDir(
resolve(options.output),
- false
+ options.verbose
);
+ const loadPreviousDurationMs = Date.now() - loadPreviousStartedAt;
+ const previousToolkits = previousToolkitLoad.toolkits;
// Detect changes
spinner.text = "Comparing tool definitions...";
+ const compareStartedAt = Date.now();
const changeResult = detectChanges(
- currentToolkitTools,
+ currentToolkitDataForDiff,
previousToolkits
);
+ const compareDurationMs = Date.now() - compareStartedAt;
spinner.stop();
+ if (options.verbose) {
+ console.log(chalk.dim("\nDiagnostics:"));
+ console.log(
+ chalk.dim(
+ ` Fetched ${currentToolkitDataForDiff.size} current toolkit(s) in ${fetchDurationMs}ms`
+ )
+ );
+ console.log(
+ chalk.dim(
+ ` Loaded previous output in ${loadPreviousDurationMs}ms (${formatPreviousToolkitLoadStats(previousToolkitLoad.stats)})`
+ )
+ );
+ console.log(
+ chalk.dim(` Compared signatures in ${compareDurationMs}ms`)
+ );
+ if (previousToolkitLoad.stats.failedFiles.length > 0) {
+ console.log(
+ chalk.yellow(
+ ` ⚠ ${previousToolkitLoad.stats.failedFiles.length} previous file(s) could not be parsed and were excluded from diffing.`
+ )
+ );
+ for (const failure of previousToolkitLoad.stats.failedFiles) {
+ console.log(chalk.dim(` - ${failure}`));
+ }
+ }
+ console.log();
+ } else if (previousToolkitLoad.stats.failedFiles.length > 0) {
+ console.log(
+ chalk.yellow(
+ `⚠ Excluded ${previousToolkitLoad.stats.failedFiles.length} previous toolkit file(s) from diffing. Run with --verbose for details.`
+ )
+ );
+ }
+
await appendLogEntry(logPaths.runLogPath, {
title: "check-changes",
details: [
`output=${resolve(options.output)}`,
`apiSource=${apiSource}`,
+ `fetchDurationMs=${fetchDurationMs}`,
+ `loadPreviousDurationMs=${loadPreviousDurationMs}`,
+ `compareDurationMs=${compareDurationMs}`,
+ `previousLoadStats=${formatPreviousToolkitLoadStats(previousToolkitLoad.stats)}`,
...buildChangeLogDetails(changeResult),
],
});
@@ -2124,7 +2288,25 @@ program
// Output results
if (options.json) {
- console.log(JSON.stringify(changeResult, null, 2));
+ console.log(
+ JSON.stringify(
+ {
+ ...changeResult,
+ diagnostics: {
+ currentToolkitCount: currentToolkitDataForDiff.size,
+ previousToolkitCount: previousToolkits.size,
+ previousLoad: previousToolkitLoad.stats,
+ timingMs: {
+ fetch: fetchDurationMs,
+ loadPrevious: loadPreviousDurationMs,
+ compare: compareDurationMs,
+ },
+ },
+ },
+ null,
+ 2
+ )
+ );
} else {
console.log(chalk.bold("\n📋 Change Detection Results\n"));
diff --git a/toolkit-docs-generator/src/diff/previous-output.ts b/toolkit-docs-generator/src/diff/previous-output.ts
new file mode 100644
index 000000000..13b7dc2c8
--- /dev/null
+++ b/toolkit-docs-generator/src/diff/previous-output.ts
@@ -0,0 +1,344 @@
+import {
+ type MergedToolkit,
+ MergedToolkitAuthSchema,
+ MergedToolkitMetadataSchema,
+ MergedToolkitSchema,
+ type ToolDefinition,
+ ToolDefinitionSchema,
+} from "../types/index.js";
+
+const DEFAULT_PREVIOUS_TOOLKIT_METADATA = {
+ category: "development" as const,
+ iconUrl: "",
+ isBYOC: false,
+ isPro: false,
+ type: "arcade" as const,
+ docsLink: "",
+ isComingSoon: false,
+ isHidden: false,
+};
+
+const toRecord = (value: unknown): Record | null =>
+ typeof value === "object" && value !== null && !Array.isArray(value)
+ ? (value as Record)
+ : null;
+
+const toStringOrNull = (value: unknown): string | null =>
+ typeof value === "string" ? value : null;
+
+const toStringArray = (value: unknown): string[] =>
+ Array.isArray(value)
+ ? value.flatMap((item) => (typeof item === "string" ? [item] : []))
+ : [];
+
+const deriveQualifiedName = (
+ raw: Record,
+ fallbackToolkitId: string,
+ fallbackIndex: number
+): string => {
+ const qualifiedName = raw.qualifiedName;
+ if (typeof qualifiedName === "string" && qualifiedName.length > 0) {
+ return qualifiedName;
+ }
+
+ const fullyQualifiedName = raw.fullyQualifiedName;
+ if (typeof fullyQualifiedName === "string" && fullyQualifiedName.length > 0) {
+ return (
+ fullyQualifiedName.split("@")[0] ??
+ `${fallbackToolkitId}.Tool${fallbackIndex}`
+ );
+ }
+
+ const name = raw.name;
+ if (typeof name === "string" && name.length > 0) {
+ return `${fallbackToolkitId}.${name}`;
+ }
+
+ return `${fallbackToolkitId}.Tool${fallbackIndex}`;
+};
+
+const deriveVersion = (
+ raw: Record,
+ fallbackVersion: string
+): string => {
+ const fullyQualifiedName = raw.fullyQualifiedName;
+ if (
+ typeof fullyQualifiedName !== "string" ||
+ fullyQualifiedName.length === 0
+ ) {
+ return fallbackVersion;
+ }
+ const version = fullyQualifiedName.split("@")[1];
+ return version && version.length > 0 ? version : fallbackVersion;
+};
+
+const normalizeParameter = (
+ value: unknown
+): ToolDefinition["parameters"][number] | null => {
+ const record = toRecord(value);
+ if (!record) return null;
+
+ const name = record.name;
+ if (typeof name !== "string" || name.length === 0) {
+ return null;
+ }
+
+ const enumValues = Array.isArray(record.enum)
+ ? toStringArray(record.enum)
+ : null;
+
+ return {
+ name,
+ type: typeof record.type === "string" ? record.type : "string",
+ ...(typeof record.innerType === "string"
+ ? { innerType: record.innerType }
+ : {}),
+ required: Boolean(record.required),
+ description: toStringOrNull(record.description),
+ enum: enumValues,
+ inferrable:
+ typeof record.inferrable === "boolean" ? record.inferrable : true,
+ };
+};
+
+const normalizeAuth = (value: unknown): ToolDefinition["auth"] => {
+ const record = toRecord(value);
+ if (!record) return null;
+
+ const providerType =
+ typeof record.providerType === "string"
+ ? record.providerType
+ : typeof record.provider_type === "string"
+ ? record.provider_type
+ : null;
+
+ if (!providerType) {
+ return null;
+ }
+
+ const providerId =
+ typeof record.providerId === "string"
+ ? record.providerId
+ : typeof record.provider_id === "string"
+ ? record.provider_id
+ : null;
+
+ return {
+ providerId,
+ providerType,
+ scopes: toStringArray(record.scopes),
+ };
+};
+
+const normalizeOutput = (value: unknown): ToolDefinition["output"] => {
+ if (value == null) return null;
+ const record = toRecord(value);
+ if (!record) return null;
+
+ const outputType =
+ typeof record.type === "string"
+ ? record.type
+ : (() => {
+ const valueSchema = toRecord(record.value_schema);
+ return valueSchema && typeof valueSchema.val_type === "string"
+ ? valueSchema.val_type
+ : "unknown";
+ })();
+
+ return {
+ type: outputType,
+ description: toStringOrNull(record.description),
+ };
+};
+
+const normalizeTool = (
+ value: unknown,
+ toolkitId: string,
+ toolkitVersion: string,
+ fallbackIndex: number
+): ToolDefinition | null => {
+ const record = toRecord(value);
+ if (!record) return null;
+
+ const qualifiedName = deriveQualifiedName(record, toolkitId, fallbackIndex);
+ const toolName =
+ typeof record.name === "string" && record.name.length > 0
+ ? record.name
+ : (qualifiedName.split(".").at(-1) ?? `Tool${fallbackIndex}`);
+ const toolVersion = deriveVersion(record, toolkitVersion);
+
+ const candidate: ToolDefinition = {
+ name: toolName,
+ qualifiedName,
+ fullyQualifiedName:
+ typeof record.fullyQualifiedName === "string" &&
+ record.fullyQualifiedName.length > 0
+ ? record.fullyQualifiedName
+ : `${qualifiedName}@${toolVersion}`,
+ description: toStringOrNull(record.description),
+ toolkitDescription: toStringOrNull(record.toolkitDescription),
+ parameters: Array.isArray(record.parameters)
+ ? record.parameters.flatMap((parameter) => {
+ const normalized = normalizeParameter(parameter);
+ return normalized ? [normalized] : [];
+ })
+ : [],
+ auth: normalizeAuth(record.auth),
+ secrets: toStringArray(record.secrets),
+ output: normalizeOutput(record.output),
+ };
+
+ const parsed = ToolDefinitionSchema.safeParse(candidate);
+ return parsed.success ? parsed.data : null;
+};
+
+const getVersionFromTool = (
+ tool: ToolDefinition | undefined
+): string | undefined => {
+ if (!tool) return;
+ const version = tool.fullyQualifiedName.split("@")[1];
+ return version && version.length > 0 ? version : undefined;
+};
+
+const getNonEmptyString = (value: unknown): string | undefined =>
+ typeof value === "string" && value.length > 0 ? value : undefined;
+
+const getFallbackReason = (error: {
+ issues: ReadonlyArray<{ path: readonly PropertyKey[]; message: string }>;
+ message: string;
+}): string => {
+ const firstIssue = error.issues[0];
+ return firstIssue
+ ? `${firstIssue.path.map(String).join(".")}: ${firstIssue.message}`
+ : error.message;
+};
+
+type FallbackToolsResult = {
+ tools: ToolDefinition[];
+ droppedToolCount: number;
+ droppedParameterCount: number;
+};
+
+const normalizeFallbackTools = (
+ record: Record,
+ toolkitId: string,
+ declaredVersion: string
+): FallbackToolsResult => {
+ const rawTools = Array.isArray(record.tools) ? record.tools : [];
+ let droppedToolCount = 0;
+ let droppedParameterCount = 0;
+
+ const tools = rawTools.flatMap((rawTool, index) => {
+ const rawRecord = toRecord(rawTool);
+ const rawParamCount =
+ rawRecord && Array.isArray(rawRecord.parameters)
+ ? rawRecord.parameters.length
+ : 0;
+
+ const normalized = normalizeTool(
+ rawTool,
+ toolkitId,
+ declaredVersion,
+ index + 1
+ );
+ if (!normalized) {
+ droppedToolCount += 1;
+ return [];
+ }
+
+ droppedParameterCount += rawParamCount - normalized.parameters.length;
+ return [normalized];
+ });
+
+ return { tools, droppedToolCount, droppedParameterCount };
+};
+
+type FallbackToolkitResult = {
+ toolkit: MergedToolkit;
+ droppedToolCount: number;
+ droppedParameterCount: number;
+};
+
+const buildFallbackToolkit = (
+ record: Record,
+ fallbackId: string
+): FallbackToolkitResult => {
+ const toolkitId = getNonEmptyString(record.id) ?? fallbackId;
+ const label = getNonEmptyString(record.label) ?? toolkitId;
+ const description = toStringOrNull(record.description);
+ const declaredVersion = getNonEmptyString(record.version) ?? "0.0.0";
+
+ const { tools, droppedToolCount, droppedParameterCount } =
+ normalizeFallbackTools(record, toolkitId, declaredVersion);
+ const version = getVersionFromTool(tools[0]) ?? declaredVersion;
+
+ const metadataResult = MergedToolkitMetadataSchema.safeParse(record.metadata);
+ const authResult = MergedToolkitAuthSchema.safeParse(record.auth);
+ const summary =
+ typeof record.summary === "string" ? record.summary : undefined;
+ const generatedAt =
+ typeof record.generatedAt === "string" ? record.generatedAt : undefined;
+
+ return {
+ toolkit: {
+ id: toolkitId,
+ label,
+ version,
+ description,
+ ...(summary ? { summary } : {}),
+ metadata: metadataResult.success
+ ? metadataResult.data
+ : DEFAULT_PREVIOUS_TOOLKIT_METADATA,
+ auth: authResult.success ? authResult.data : null,
+ tools: tools.map((tool) => ({
+ ...tool,
+ secretsInfo: [],
+ documentationChunks: [],
+ })),
+ documentationChunks: [],
+ customImports: [],
+ subPages: [],
+ ...(generatedAt ? { generatedAt } : {}),
+ },
+ droppedToolCount,
+ droppedParameterCount,
+ };
+};
+
+export type PreviousToolkitParseResult = {
+ toolkit: MergedToolkit | null;
+ usedFallback: boolean;
+ reason?: string;
+ droppedToolCount?: number;
+ droppedParameterCount?: number;
+};
+
+export const parsePreviousToolkitForDiff = (
+ payload: unknown,
+ fallbackId: string
+): PreviousToolkitParseResult => {
+ const strictResult = MergedToolkitSchema.safeParse(payload);
+ if (strictResult.success) {
+ return { toolkit: strictResult.data, usedFallback: false };
+ }
+
+ const record = toRecord(payload);
+ if (!record) {
+ return {
+ toolkit: null,
+ usedFallback: true,
+ reason: "Previous output file is not a JSON object",
+ };
+ }
+
+ const fallbackReason = getFallbackReason(strictResult.error);
+ const { toolkit, droppedToolCount, droppedParameterCount } =
+ buildFallbackToolkit(record, fallbackId);
+ return {
+ toolkit,
+ usedFallback: true,
+ reason: fallbackReason,
+ droppedToolCount,
+ droppedParameterCount,
+ };
+};
diff --git a/toolkit-docs-generator/src/diff/toolkit-diff.ts b/toolkit-docs-generator/src/diff/toolkit-diff.ts
index d0993d666..7b3c55199 100644
--- a/toolkit-docs-generator/src/diff/toolkit-diff.ts
+++ b/toolkit-docs-generator/src/diff/toolkit-diff.ts
@@ -5,8 +5,16 @@
* Used to determine which toolkits need regeneration.
*/
-import { buildToolSignature, extractVersion } from "../merger/data-merger.js";
-import type { MergedToolkit, ToolDefinition } from "../types/index.js";
+import {
+ buildToolSignatureInput,
+ extractVersion,
+ stableStringify,
+} from "../merger/data-merger.js";
+import type {
+ MergedToolkit,
+ ToolDefinition,
+ ToolkitMetadata,
+} from "../types/index.js";
// ============================================================================
// Types
@@ -37,6 +45,8 @@ export interface ToolkitChange {
readonly previousVersion: string;
/** Whether the toolkit version changed */
readonly versionChanged: boolean;
+ /** Whether relevant metadata fields changed */
+ readonly metadataChanged: boolean;
/** Current tool count (0 if removed) */
readonly currentToolCount: number;
/** Previous tool count (0 if new) */
@@ -69,6 +79,26 @@ export interface ChangeSummary {
readonly modifiedTools: number;
}
+export interface CurrentToolkitData {
+ /** Tool definitions fetched from the API */
+ readonly tools: readonly ToolDefinition[];
+ /** Toolkit metadata fetched from Design System */
+ readonly metadata: ToolkitMetadata | null;
+}
+
+export type CurrentToolkitDiffInput =
+ | readonly ToolDefinition[]
+ | CurrentToolkitData;
+
+const isCurrentToolkitData = (
+ value: CurrentToolkitDiffInput
+): value is CurrentToolkitData =>
+ !Array.isArray(value) &&
+ typeof value === "object" &&
+ value !== null &&
+ "tools" in value &&
+ "metadata" in value;
+
// ============================================================================
// Signature Building for Tool Definitions
// ============================================================================
@@ -77,8 +107,55 @@ export interface ChangeSummary {
* Build a signature for a ToolDefinition (from API)
* This is different from MergedTool signatures as it uses the raw API format
*/
+const normalizeOutputTypeForDiff = (value: string): string =>
+ value === "unknown" ? "string" : value;
+
+const normalizeToolSignatureInputForDiff = (
+ tool: ToolDefinition | MergedToolkit["tools"][number]
+): Record => {
+ const signatureInput = buildToolSignatureInput(tool);
+ const parameters = signatureInput.parameters.map((parameter) => ({
+ ...parameter,
+ // Parameter descriptions are not stable across /v1/tools and /v1/tool_metadata.
+ description: null,
+ // tool_metadata may emit [] while list-tools uses null for "no enum values".
+ enum: parameter.enum && parameter.enum.length > 0 ? parameter.enum : null,
+ }));
+ const auth = signatureInput.auth
+ ? {
+ ...signatureInput.auth,
+ // OAuth provider IDs can differ by endpoint shape; scopes/type are stable.
+ providerId:
+ signatureInput.auth.providerType === "oauth2"
+ ? null
+ : signatureInput.auth.providerId,
+ }
+ : null;
+ const output = signatureInput.output
+ ? {
+ ...signatureInput.output,
+ type: normalizeOutputTypeForDiff(signatureInput.output.type),
+ // Output descriptions are not stable across endpoints.
+ description: null,
+ }
+ : null;
+
+ return {
+ ...signatureInput,
+ // Tool descriptions are documentation metadata and vary by source.
+ description: null,
+ parameters,
+ auth,
+ output,
+ };
+};
+
+const buildDiffToolSignature = (
+ tool: ToolDefinition | MergedToolkit["tools"][number]
+): string => stableStringify(normalizeToolSignatureInputForDiff(tool));
+
export const buildToolDefinitionSignature = (tool: ToolDefinition): string =>
- buildToolSignature(tool);
+ buildDiffToolSignature(tool);
/**
* Build a map of tool signatures for quick lookup
@@ -101,7 +178,7 @@ const buildMergedToolSignatureMap = (
): ReadonlyMap => {
const map = new Map();
for (const tool of toolkit.tools) {
- map.set(tool.qualifiedName, buildToolSignature(tool));
+ map.set(tool.qualifiedName, buildDiffToolSignature(tool));
}
return map;
};
@@ -114,6 +191,65 @@ const getToolkitVersion = (tools: readonly ToolDefinition[]): string => {
return firstTool ? extractVersion(firstTool.fullyQualifiedName) : "0.0.0";
};
+type ComparableMetadataSnapshot = {
+ label: string;
+ category: string;
+ isHidden: boolean;
+ type: string;
+};
+
+const buildCurrentMetadataSnapshot = (
+ metadata: ToolkitMetadata | null
+): ComparableMetadataSnapshot | null => {
+ if (!metadata) {
+ return null;
+ }
+
+ return {
+ label: metadata.label,
+ category: metadata.category,
+ isHidden: metadata.isHidden,
+ type: metadata.type,
+ };
+};
+
+const buildPreviousMetadataSnapshot = (
+ toolkit: MergedToolkit | undefined
+): ComparableMetadataSnapshot | null => {
+ if (!toolkit) {
+ return null;
+ }
+
+ return {
+ label: toolkit.label,
+ category: toolkit.metadata.category,
+ isHidden: toolkit.metadata.isHidden,
+ type: toolkit.metadata.type,
+ };
+};
+
+const hasRelevantMetadataChanges = (
+ currentMetadata: ToolkitMetadata | null,
+ previousToolkit: MergedToolkit | undefined
+): boolean => {
+ const current = buildCurrentMetadataSnapshot(currentMetadata);
+ const previous = buildPreviousMetadataSnapshot(previousToolkit);
+ // When either side lacks metadata (new toolkit or metadata not yet available),
+ // treat as "no metadata change" — the toolkit is already flagged as added/removed
+ // by the tool-level diff, so a missing metadata snapshot should not independently
+ // trigger regeneration.
+ if (!(current && previous)) {
+ return false;
+ }
+
+ return (
+ current.label !== previous.label ||
+ current.category !== previous.category ||
+ current.isHidden !== previous.isHidden ||
+ current.type !== previous.type
+ );
+};
+
// ============================================================================
// Change Detection Functions
// ============================================================================
@@ -190,13 +326,18 @@ export const compareTools = (
export const compareToolkit = (
toolkitId: string,
currentTools: readonly ToolDefinition[],
- previousToolkit: MergedToolkit | undefined
+ previousToolkit: MergedToolkit | undefined,
+ currentMetadata: ToolkitMetadata | null = null
): ToolkitChange => {
const toolChanges = compareTools(currentTools, previousToolkit);
const currentVersion = getToolkitVersion(currentTools);
const previousVersion = previousToolkit?.version ?? "0.0.0";
const versionChanged =
Boolean(previousToolkit) && currentVersion !== previousVersion;
+ const metadataChanged = hasRelevantMetadataChanges(
+ currentMetadata,
+ previousToolkit
+ );
// Determine overall change type
let changeType: ToolkitChangeType;
@@ -205,7 +346,7 @@ export const compareToolkit = (
} else if (currentTools.length === 0 && previousToolkit.tools.length > 0) {
// This shouldn't happen normally, but handle it
changeType = "removed";
- } else if (toolChanges.length === 0 && !versionChanged) {
+ } else if (toolChanges.length === 0 && !versionChanged && !metadataChanged) {
changeType = "unchanged";
} else {
changeType = "modified";
@@ -218,11 +359,21 @@ export const compareToolkit = (
currentVersion,
previousVersion,
versionChanged,
+ metadataChanged,
currentToolCount: currentTools.length,
previousToolCount: previousToolkit?.tools.length ?? 0,
};
};
+const normalizeCurrentToolkitData = (
+ value: CurrentToolkitDiffInput
+): CurrentToolkitData => {
+ if (isCurrentToolkitData(value)) {
+ return { tools: value.tools, metadata: value.metadata };
+ }
+ return { tools: value, metadata: null };
+};
+
/**
* Compare all toolkits to detect changes
*
@@ -230,7 +381,7 @@ export const compareToolkit = (
* @param previousToolkits - Map of toolkit ID to previous merged output
*/
export const detectChanges = (
- currentToolkitTools: ReadonlyMap,
+ currentToolkitData: ReadonlyMap,
previousToolkits: ReadonlyMap
): ChangeDetectionResult => {
const toolkitChanges: ToolkitChange[] = [];
@@ -248,7 +399,8 @@ export const detectChanges = (
const seenPreviousIds = new Set();
// Check current toolkits
- for (const [toolkitId, tools] of currentToolkitTools) {
+ for (const [toolkitId, current] of currentToolkitData) {
+ const normalizedCurrent = normalizeCurrentToolkitData(current);
const normalizedId = normalizeKey(toolkitId);
const previousToolkit = previousByNormalizedId.get(normalizedId);
@@ -256,7 +408,12 @@ export const detectChanges = (
seenPreviousIds.add(normalizedId);
}
- const change = compareToolkit(toolkitId, tools, previousToolkit);
+ const change = compareToolkit(
+ toolkitId,
+ normalizedCurrent.tools,
+ previousToolkit,
+ normalizedCurrent.metadata
+ );
toolkitChanges.push(change);
}
@@ -275,6 +432,7 @@ export const detectChanges = (
currentVersion: "0.0.0",
previousVersion: toolkit.version,
versionChanged: false,
+ metadataChanged: false,
currentToolCount: 0,
previousToolCount: toolkit.tools.length,
});
@@ -324,7 +482,11 @@ const calculateSummary = (
break;
case "modified":
modifiedToolkits++;
- if (change.toolChanges.length === 0 && change.versionChanged) {
+ if (
+ change.toolChanges.length === 0 &&
+ change.versionChanged &&
+ !change.metadataChanged
+ ) {
versionOnlyToolkits++;
}
for (const toolChange of change.toolChanges) {
@@ -441,7 +603,14 @@ const shouldListToolChanges = (toolkitChange: ToolkitChange): boolean =>
const shouldNoteVersionOnlyUpdate = (toolkitChange: ToolkitChange): boolean =>
toolkitChange.changeType === "modified" &&
toolkitChange.toolChanges.length === 0 &&
- toolkitChange.versionChanged;
+ toolkitChange.versionChanged &&
+ !toolkitChange.metadataChanged;
+
+const shouldNoteMetadataOnlyUpdate = (toolkitChange: ToolkitChange): boolean =>
+ toolkitChange.changeType === "modified" &&
+ toolkitChange.toolChanges.length === 0 &&
+ !toolkitChange.versionChanged &&
+ toolkitChange.metadataChanged;
const formatToolkitLine = (toolkitChange: ToolkitChange): string => {
const changeLabel = TOOLKIT_CHANGE_LABELS[toolkitChange.changeType];
@@ -459,6 +628,9 @@ const appendToolChanges = (
if (shouldNoteVersionOnlyUpdate(toolkitChange)) {
lines.push(" ~ version update only");
}
+ if (shouldNoteMetadataOnlyUpdate(toolkitChange)) {
+ lines.push(" ~ metadata update only");
+ }
for (const toolChange of toolkitChange.toolChanges) {
const toolLabel = TOOL_CHANGE_LABELS[toolChange.changeType];
lines.push(`${toolLabel}${toolChange.toolName}`);
diff --git a/toolkit-docs-generator/src/generator/json-generator.ts b/toolkit-docs-generator/src/generator/json-generator.ts
index 7e9539cd5..f51865e74 100644
--- a/toolkit-docs-generator/src/generator/json-generator.ts
+++ b/toolkit-docs-generator/src/generator/json-generator.ts
@@ -5,6 +5,7 @@
*/
import { mkdir, readFile, stat, writeFile } from "fs/promises";
import { dirname, join } from "path";
+import { parsePreviousToolkitForDiff } from "../diff/previous-output.js";
import type {
MergedToolkit,
ToolkitIndex,
@@ -108,7 +109,9 @@ export class JsonGenerator {
async getCompletedToolkitIds(): Promise> {
const completedIds = new Set();
try {
- const result = await readToolkitsFromDir(this.outputDir);
+ const result = await readToolkitsFromDir(this.outputDir, undefined, {
+ allowLegacyFallback: true,
+ });
for (const toolkit of result.toolkits) {
completedIds.add(toolkit.id.toLowerCase());
}
@@ -131,7 +134,8 @@ export class JsonGenerator {
if (result.success) {
return result.data;
}
- return null;
+ const fallback = parsePreviousToolkitForDiff(parsed, toolkitId);
+ return fallback.toolkit;
} catch {
return null;
}
@@ -210,7 +214,9 @@ export class JsonGenerator {
private async getToolkitsFromOutputDir(
errors: string[]
): Promise {
- const readResult = await readToolkitsFromDir(this.outputDir);
+ const readResult = await readToolkitsFromDir(this.outputDir, undefined, {
+ allowLegacyFallback: true,
+ });
if (readResult.errors.length > 0) {
errors.push(...readResult.errors);
}
diff --git a/toolkit-docs-generator/src/generator/output-verifier.ts b/toolkit-docs-generator/src/generator/output-verifier.ts
index e6768070f..b28763c17 100644
--- a/toolkit-docs-generator/src/generator/output-verifier.ts
+++ b/toolkit-docs-generator/src/generator/output-verifier.ts
@@ -1,5 +1,6 @@
import { readdir, readFile } from "fs/promises";
import { basename, join } from "path";
+import { parsePreviousToolkitForDiff } from "../diff/previous-output.js";
import type { MergedToolkit, ToolkitIndex } from "../types/index.js";
import { MergedToolkitSchema, ToolkitIndexSchema } from "../types/index.js";
@@ -12,6 +13,7 @@ export interface OutputVerificationResult {
export interface ToolkitReadResult {
toolkits: MergedToolkit[];
errors: string[];
+ warnings: string[];
}
export interface VerificationProgress {
@@ -25,6 +27,14 @@ export type VerificationProgressCallback = (
progress: VerificationProgress
) => void;
+export interface ReadToolkitsOptions {
+ allowLegacyFallback?: boolean;
+}
+
+export interface VerifyOutputOptions {
+ allowLegacyFallback?: boolean;
+}
+
const isToolkitFile = (fileName: string): boolean =>
fileName.endsWith(".json") && fileName !== "index.json";
@@ -34,10 +44,12 @@ const normalizeToolkitKey = (toolkitId: string): string =>
type ReadToolkitFileResult = {
toolkit: MergedToolkit | null;
error?: string;
+ warning?: string;
};
const readToolkitFile = async (
- filePath: string
+ filePath: string,
+ allowLegacyFallback: boolean
): Promise => {
const fileName = basename(filePath);
let content: string;
@@ -63,22 +75,38 @@ const readToolkitFile = async (
}
const result = MergedToolkitSchema.safeParse(parsed);
- if (!result.success) {
- return {
- toolkit: null,
- error: `Invalid toolkit schema in ${fileName}: ${result.error.message}`,
- };
+ if (result.success) {
+ return { toolkit: result.data };
}
- return { toolkit: result.data };
+ if (allowLegacyFallback) {
+ const fallback = parsePreviousToolkitForDiff(
+ parsed,
+ basename(fileName, ".json")
+ );
+ if (fallback.toolkit) {
+ return {
+ toolkit: fallback.toolkit,
+ warning: `Loaded ${fileName} with fallback parser (${fallback.reason ?? "schema mismatch"}).`,
+ };
+ }
+ }
+
+ return {
+ toolkit: null,
+ error: `Invalid toolkit schema in ${fileName}: ${result.error.message}`,
+ };
};
export const readToolkitsFromDir = async (
dir: string,
- onProgress?: VerificationProgressCallback
+ onProgress?: VerificationProgressCallback,
+ options: ReadToolkitsOptions = {}
): Promise => {
const toolkits: MergedToolkit[] = [];
const errors: string[] = [];
+ const warnings: string[] = [];
+ const allowLegacyFallback = options.allowLegacyFallback ?? false;
let entries: string[] = [];
try {
@@ -87,6 +115,7 @@ export const readToolkitsFromDir = async (
return {
toolkits: [],
errors: [`Failed to read output directory: ${error}`],
+ warnings: [],
};
}
@@ -105,15 +134,18 @@ export const readToolkitsFromDir = async (
});
const filePath = join(dir, fileName);
- const result = await readToolkitFile(filePath);
+ const result = await readToolkitFile(filePath, allowLegacyFallback);
if (!result.toolkit) {
errors.push(result.error ?? `Invalid toolkit JSON: ${fileName}`);
continue;
}
+ if (result.warning) {
+ warnings.push(result.warning);
+ }
toolkits.push(result.toolkit);
}
- return { toolkits, errors };
+ return { toolkits, errors, warnings };
};
const readIndexFile = async (dir: string): Promise => {
@@ -271,16 +303,22 @@ const validateFileNames = async (
export const verifyOutputDir = async (
dir: string,
- onProgress?: VerificationProgressCallback
+ onProgress?: VerificationProgressCallback,
+ options: VerifyOutputOptions = {}
): Promise => {
const errors: string[] = [];
const warnings: string[] = [];
-
- const { toolkits, errors: toolkitErrors } = await readToolkitsFromDir(
- dir,
- onProgress
- );
+ const allowLegacyFallback = options.allowLegacyFallback ?? false;
+
+ const {
+ toolkits,
+ errors: toolkitErrors,
+ warnings: toolkitWarnings,
+ } = await readToolkitsFromDir(dir, onProgress, {
+ allowLegacyFallback,
+ });
errors.push(...toolkitErrors);
+ warnings.push(...toolkitWarnings);
if (toolkits.length === 0) {
errors.push("No toolkit JSON files found in output directory.");
diff --git a/toolkit-docs-generator/src/llm/index.ts b/toolkit-docs-generator/src/llm/index.ts
index 40c80d3cc..162d4b879 100644
--- a/toolkit-docs-generator/src/llm/index.ts
+++ b/toolkit-docs-generator/src/llm/index.ts
@@ -1,4 +1,3 @@
export * from "./client.js";
export * from "./tool-example-generator.js";
-export * from "./toolkit-overview-generator.js";
export * from "./toolkit-summary-generator.js";
diff --git a/toolkit-docs-generator/src/llm/toolkit-overview-generator.ts b/toolkit-docs-generator/src/llm/toolkit-overview-generator.ts
deleted file mode 100644
index 4e330ce22..000000000
--- a/toolkit-docs-generator/src/llm/toolkit-overview-generator.ts
+++ /dev/null
@@ -1,229 +0,0 @@
-import type {
- OverviewGenerationInput,
- OverviewGenerationResult,
- ToolkitOverviewGenerator,
-} from "../overview/types.js";
-import type { DocumentationChunk, SecretType } from "../types/index.js";
-import type { LlmClient } from "./client.js";
-
-export interface LlmToolkitOverviewGeneratorConfig {
- readonly client: LlmClient;
- readonly model: string;
- readonly temperature?: number;
- readonly maxTokens?: number;
- readonly systemPrompt?: string;
-}
-
-const defaultSystemPrompt =
- "Return only valid JSON. No markdown fences, no extra text.";
-
-const collectSecrets = (tools: OverviewGenerationInput["toolkit"]["tools"]) => {
- const secretNames = new Set();
- const secretTypes = new Set();
-
- for (const tool of tools) {
- for (const name of tool.secrets) {
- secretNames.add(name);
- }
- for (const secret of tool.secretsInfo ?? []) {
- secretTypes.add(secret.type);
- }
- }
-
- return {
- names: Array.from(secretNames),
- types: Array.from(secretTypes),
- };
-};
-
-const formatToolLines = (
- tools: OverviewGenerationInput["toolkit"]["tools"]
-): string => {
- if (tools.length === 0) {
- return "None";
- }
-
- const sampled = tools.slice(0, 12);
- return sampled
- .map(
- (tool) =>
- `- ${tool.qualifiedName}: ${tool.description ?? "No description"}`
- )
- .join("\n");
-};
-
-const formatAuth = (toolkit: OverviewGenerationInput["toolkit"]): string => {
- if (!toolkit.auth) {
- return "none";
- }
- const scopes =
- toolkit.auth.allScopes.length > 0
- ? toolkit.auth.allScopes.join(", ")
- : "None";
- const provider = toolkit.auth.providerId ?? "unknown";
- return `${toolkit.auth.type}; provider: ${provider}; scopes: ${scopes}`;
-};
-
-const buildPrompt = (input: OverviewGenerationInput): string => {
- const { toolkit, instructions, previousOverview, mode } = input;
- const secrets = collectSecrets(toolkit.tools);
- const sources = instructions?.sources ?? [];
-
- const base = [
- "Write an overview section for Arcade toolkit docs.",
- 'Return JSON: {"shouldWrite": boolean, "overview": "", "reason": ""}',
- "",
- "Formatting requirements:",
- "- Start with a '## Overview' heading.",
- "- 1 short paragraph explaining what the toolkit enables.",
- "- Add a **Capabilities** list with 3 to 5 bullet points.",
- "- If auth type is oauth2 or mixed, add an **OAuth** section with provider and scopes.",
- "- If auth type is api_key or mixed, mention API key usage in **OAuth**.",
- "- If secrets exist, add a **Secrets** section describing secret types and examples.",
- "- Use concise developer-focused language.",
- "",
- ];
-
- const modeGuidance =
- mode === "auto"
- ? [
- "Decision rule:",
- '- If you are not confident you can write an accurate overview from the provided data, set shouldWrite=false and overview="".',
- "- Only use provided data and general public knowledge. Do not invent product-specific claims.",
- "",
- ]
- : [
- "Follow the toolkit-specific instructions below.",
- "Use the previous overview as context if provided.",
- "Set shouldWrite=true when you can comply with the instructions.",
- "",
- ];
-
- const instructionsBlock = [
- "Toolkit:",
- `Name: ${toolkit.label} (${toolkit.id})`,
- `Description: ${toolkit.description ?? "No description"}`,
- `Auth: ${formatAuth(toolkit)}`,
- `Secret types: ${secrets.types.length > 0 ? secrets.types.join(", ") : "None"}`,
- `Secret names: ${secrets.names.length > 0 ? secrets.names.join(", ") : "None"}`,
- `Tools (${toolkit.tools.length}, sample):`,
- formatToolLines(toolkit.tools),
- "",
- ];
-
- const referencesBlock =
- sources.length > 0
- ? ["References:", ...sources.map((source) => `- ${source}`), ""]
- : [];
-
- const customInstructions = instructions?.instructions
- ? ["Instructions:", instructions.instructions, ""]
- : [];
-
- const previousBlock = previousOverview
- ? ["Previous overview:", previousOverview, ""]
- : [];
-
- return [
- ...base,
- ...modeGuidance,
- ...instructionsBlock,
- ...referencesBlock,
- ...customInstructions,
- ...previousBlock,
- ].join("\n");
-};
-
-const extractJson = (text: string): string => {
- const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/);
- if (fenced?.[1]) {
- return fenced[1].trim();
- }
- return text.trim();
-};
-
-const parseJsonObject = (text: string): Record => {
- const jsonText = extractJson(text);
- const parsed = JSON.parse(jsonText) as unknown;
-
- if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
- throw new Error("LLM response is not a JSON object");
- }
-
- return parsed as Record;
-};
-
-const normalizeOverviewContent = (content: string): string => {
- const trimmed = content.trim();
- if (trimmed.toLowerCase().startsWith("## overview")) {
- return trimmed;
- }
- return `## Overview\n\n${trimmed}`;
-};
-
-const buildOverviewChunk = (content: string): DocumentationChunk => ({
- type: "markdown",
- location: "header",
- position: "before",
- content,
-});
-
-export class LlmToolkitOverviewGenerator implements ToolkitOverviewGenerator {
- private readonly client: LlmClient;
- private readonly model: string;
- private readonly temperature: number | undefined;
- private readonly maxTokens: number | undefined;
- private readonly systemPrompt: string;
-
- constructor(config: LlmToolkitOverviewGeneratorConfig) {
- this.client = config.client;
- this.model = config.model;
- this.temperature = config.temperature;
- this.maxTokens = config.maxTokens;
- this.systemPrompt = config.systemPrompt ?? defaultSystemPrompt;
- }
-
- async generate(
- input: OverviewGenerationInput
- ): Promise {
- const prompt = buildPrompt(input);
- const response = await this.client.generateText({
- model: this.model,
- prompt,
- system: this.systemPrompt,
- ...(this.temperature !== undefined
- ? { temperature: this.temperature }
- : {}),
- ...(this.maxTokens !== undefined ? { maxTokens: this.maxTokens } : {}),
- });
-
- const payload = parseJsonObject(response);
- const shouldWrite = (() => {
- const value = payload.shouldWrite;
- if (value === undefined) {
- return true;
- }
- if (typeof value === "boolean") {
- return value;
- }
- if (typeof value === "string") {
- return value.toLowerCase() === "true";
- }
- return Boolean(value);
- })();
- const overview = payload.overview;
- const reason =
- typeof payload.reason === "string" ? payload.reason : undefined;
-
- if (input.mode === "auto" && !shouldWrite) {
- return null;
- }
-
- if (typeof overview !== "string" || overview.trim().length === 0) {
- return null;
- }
-
- const content = normalizeOverviewContent(overview);
- return { chunk: buildOverviewChunk(content), reason };
- }
-}
diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts
index e1cad0517..8425b94ee 100644
--- a/toolkit-docs-generator/src/merger/data-merger.ts
+++ b/toolkit-docs-generator/src/merger/data-merger.ts
@@ -5,16 +5,7 @@
* into the final MergedToolkit format.
*/
-import type {
- OverviewGenerationInput,
- ToolkitOverviewGenerator,
- ToolkitOverviewInstructions,
-} from "../overview/types.js";
-import { createEmptyOverviewInstructionsSource } from "../sources/in-memory.js";
-import type {
- ICustomSectionsSource,
- IOverviewInstructionsSource,
-} from "../sources/interfaces.js";
+import type { ICustomSectionsSource } from "../sources/interfaces.js";
import type {
IToolkitDataSource,
ToolkitData,
@@ -44,9 +35,6 @@ import {
export interface DataMergerConfig {
toolkitDataSource: IToolkitDataSource;
customSectionsSource: ICustomSectionsSource;
- overviewInstructionsSource?: IOverviewInstructionsSource;
- overviewGenerator?: ToolkitOverviewGenerator;
- skipOverview?: boolean;
toolExampleGenerator?: ToolExampleGenerator;
toolkitSummaryGenerator?: ToolkitSummaryGenerator;
previousToolkits?: ReadonlyMap;
@@ -66,6 +54,8 @@ export interface DataMergerConfig {
onToolkitComplete?: ((result: MergeResult) => Promise) | undefined;
/** Set of toolkit IDs to skip (for resume support) */
skipToolkitIds?: ReadonlySet | undefined;
+ /** When true, only process toolkits with metadata and tools */
+ requireCompleteData?: boolean;
/** Fallback resolver: toolkit ID → OAuth provider ID (design system) */
resolveProviderId?: ((toolkitId: string) => string | null) | undefined;
}
@@ -100,9 +90,6 @@ interface MergeToolkitOptions {
previousToolkit?: MergedToolkit;
/** Maximum concurrent LLM calls for tool examples (default: 5) */
llmConcurrency?: number;
- overviewGenerator?: ToolkitOverviewGenerator;
- overviewInstructions?: ToolkitOverviewInstructions | null;
- skipOverview?: boolean;
/** Fallback resolver: toolkit ID → OAuth provider ID (design system) */
resolveProviderId?: (toolkitId: string) => string | null;
}
@@ -171,9 +158,34 @@ export const stableStringify = (value: unknown): string => {
return JSON.stringify(value);
};
+export type ToolSignatureInput = {
+ name: string;
+ qualifiedName: string;
+ description: string | null;
+ parameters: Array<{
+ name: string;
+ type: string;
+ innerType: string | null;
+ required: boolean;
+ description: string | null;
+ enum: string[] | null;
+ inferrable: boolean;
+ }>;
+ auth: {
+ providerId: string | null;
+ providerType: string;
+ scopes: string[];
+ } | null;
+ secrets: string[];
+ output: {
+ type: string;
+ description: string | null;
+ } | null;
+};
+
export const buildToolSignatureInput = (
tool: ToolDefinition | MergedTool
-): Record => ({
+): ToolSignatureInput => ({
name: tool.name,
qualifiedName: tool.qualifiedName,
description: tool.description ?? null,
@@ -207,6 +219,47 @@ export const buildToolSignatureInput = (
export const buildToolSignature = (tool: ToolDefinition | MergedTool): string =>
stableStringify(buildToolSignatureInput(tool));
+const normalizeOutputTypeForReuse = (value: string): string =>
+ value === "unknown" ? "string" : value;
+
+const buildToolReuseSignature = (tool: ToolDefinition | MergedTool): string => {
+ const signatureInput = buildToolSignatureInput(tool);
+ const parameters = signatureInput.parameters.map((parameter) => ({
+ ...parameter,
+ // Descriptions can vary by API source and should not force regeneration.
+ description: null,
+ // Treat [] and null as equivalent enum representations.
+ enum: parameter.enum && parameter.enum.length > 0 ? parameter.enum : null,
+ }));
+ const auth = signatureInput.auth
+ ? {
+ ...signatureInput.auth,
+ // OAuth provider IDs can vary by endpoint shape.
+ providerId:
+ signatureInput.auth.providerType === "oauth2"
+ ? null
+ : signatureInput.auth.providerId,
+ }
+ : null;
+ const output = signatureInput.output
+ ? {
+ ...signatureInput.output,
+ type: normalizeOutputTypeForReuse(signatureInput.output.type),
+ // Output descriptions vary by source and should not force regeneration.
+ description: null,
+ }
+ : null;
+
+ return stableStringify({
+ ...signatureInput,
+ // Tool descriptions are metadata and should not force regeneration.
+ description: null,
+ parameters,
+ auth,
+ output,
+ });
+};
+
export const buildToolkitSummarySignature = (toolkit: MergedToolkit): string =>
stableStringify({
id: toolkit.id,
@@ -237,7 +290,9 @@ const shouldReuseExample = (
return false;
}
- return buildToolSignature(tool) === buildToolSignature(previousTool);
+ return (
+ buildToolReuseSignature(tool) === buildToolReuseSignature(previousTool)
+ );
};
/**
@@ -287,10 +342,54 @@ export const extractVersion = (fullyQualifiedName: string): string => {
* Create default metadata for toolkits not found in Design System
*/
const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]/g;
+const TOOLKIT_ID_ACRONYM_BOUNDARY = /([A-Z]+)([A-Z][a-z])/g;
+const TOOLKIT_ID_WORD_BOUNDARY = /([a-z0-9])([A-Z])/g;
+const TOOLKIT_DESCRIPTION_LABEL_PREFIX = "Arcade.dev LLM tools for ";
const normalizeToolkitId = (toolkitId: string): string =>
toolkitId.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, "");
+const humanizeToolkitId = (toolkitId: string): string =>
+ toolkitId
+ .replace(TOOLKIT_ID_ACRONYM_BOUNDARY, "$1 $2")
+ .replace(TOOLKIT_ID_WORD_BOUNDARY, "$1 $2")
+ .replace(/\bApi\b/g, "API")
+ .trim();
+
+const extractLabelFromDescription = (
+ description: string | null
+): string | null => {
+ if (!description) {
+ return null;
+ }
+
+ const trimmed = description.trim();
+ if (!trimmed.startsWith(TOOLKIT_DESCRIPTION_LABEL_PREFIX)) {
+ return null;
+ }
+
+ const suffix = trimmed.slice(TOOLKIT_DESCRIPTION_LABEL_PREFIX.length).trim();
+ if (suffix.length === 0) {
+ return null;
+ }
+
+ const periodIndex = suffix.indexOf(".");
+ const candidate = (
+ periodIndex >= 0 ? suffix.slice(0, periodIndex) : suffix
+ ).trim();
+
+ return candidate.length > 0 ? candidate : null;
+};
+
+const resolveToolkitLabel = (options: {
+ toolkitId: string;
+ metadata: ToolkitMetadata | null;
+ description: string | null;
+}): string =>
+ options.metadata?.label ??
+ extractLabelFromDescription(options.description) ??
+ humanizeToolkitId(options.toolkitId);
+
const isStarterToolkitId = (toolkitId: string): boolean =>
normalizeToolkitId(toolkitId).endsWith("api");
@@ -371,38 +470,8 @@ const isOverviewChunk = (chunk: DocumentationChunk): boolean =>
chunk.type === "markdown" &&
chunk.content.trim().toLowerCase().startsWith("## overview");
-const getPreviousOverview = (toolkit?: MergedToolkit): string | null => {
- if (!toolkit) {
- return null;
- }
- const overviewChunk = toolkit.documentationChunks.find(isOverviewChunk);
- return overviewChunk?.content ?? null;
-};
-
-const applyOverviewChunk = (
- chunks: DocumentationChunk[],
- chunk: DocumentationChunk
-): DocumentationChunk[] => {
- const filtered = chunks.filter((item) => !isOverviewChunk(item));
- return [chunk, ...filtered];
-};
-
-const shouldAttemptOverview = (
- hasInstructions: boolean,
- isNewToolkit: boolean
-): boolean => hasInstructions || isNewToolkit;
-
-const buildOverviewInput = (
- toolkit: MergedToolkit,
- instructions: ToolkitOverviewInstructions | null,
- previousOverview: string | null,
- mode: OverviewGenerationInput["mode"]
-): OverviewGenerationInput => ({
- toolkit,
- instructions,
- previousOverview,
- mode,
-});
+const hasToolkitOverviewChunk = (toolkit: MergedToolkit): boolean =>
+ toolkit.documentationChunks.some(isOverviewChunk);
const mergeCustomSectionsArrays = (
fromSource: readonly T[] | undefined,
@@ -519,7 +588,11 @@ const buildMergedToolkit = (options: {
return {
id: options.toolkitId,
- label: options.metadata?.label ?? options.toolkitId,
+ label: resolveToolkitLabel({
+ toolkitId: options.toolkitId,
+ metadata: options.metadata,
+ description: options.description,
+ }),
version: options.version,
description: options.description,
metadata: mergedMetadata,
@@ -541,63 +614,6 @@ const buildMergedToolkit = (options: {
};
};
-const applyOverviewIfNeeded = async (options: {
- toolkit: MergedToolkit;
- toolkitId: string;
- mergeOptions: MergeToolkitOptions;
- warnings: string[];
-}): Promise => {
- const { toolkit, toolkitId, mergeOptions, warnings } = options;
- const isNewToolkit = !mergeOptions.previousToolkit;
- const hasInstructions = Boolean(mergeOptions.overviewInstructions);
-
- if (
- mergeOptions.skipOverview ||
- !shouldAttemptOverview(hasInstructions, isNewToolkit)
- ) {
- return;
- }
-
- if (!mergeOptions.overviewGenerator) {
- if (hasInstructions) {
- warnings.push(
- `Overview generation skipped for ${toolkitId}: missing LLM configuration`
- );
- }
- return;
- }
-
- const previousOverview = getPreviousOverview(mergeOptions.previousToolkit);
- const mode: OverviewGenerationInput["mode"] = hasInstructions
- ? "file"
- : "auto";
- try {
- const overviewResult = await mergeOptions.overviewGenerator.generate(
- buildOverviewInput(
- toolkit,
- mergeOptions.overviewInstructions ?? null,
- previousOverview,
- mode
- )
- );
- if (overviewResult?.chunk) {
- toolkit.documentationChunks = applyOverviewChunk(
- toolkit.documentationChunks,
- overviewResult.chunk
- );
- return;
- }
- if (hasInstructions) {
- warnings.push(
- `Overview generation skipped for ${toolkitId}: no overview produced`
- );
- }
- } catch (error) {
- const message = error instanceof Error ? error.message : String(error);
- warnings.push(`Overview generation failed for ${toolkitId}: ${message}`);
- }
-};
-
/**
* Transform a tool definition into a merged tool
*/
@@ -751,7 +767,7 @@ export const mergeToolkit = async (
const toolChunks = (customSections?.toolChunks ?? {}) as {
[key: string]: DocumentationChunk[];
};
- const llmConcurrency = options.llmConcurrency ?? 5;
+ const llmConcurrency = options.llmConcurrency ?? 10;
const previousToolByQualifiedName = buildPreviousToolMap(
options.previousToolkit
);
@@ -787,13 +803,6 @@ export const mergeToolkit = async (
warnings.push(...formatFreshnessWarnings(freshnessResult));
}
- await applyOverviewIfNeeded({
- toolkit,
- toolkitId,
- mergeOptions: options,
- warnings,
- });
-
return { toolkit, warnings, failedTools };
};
@@ -807,9 +816,6 @@ export const mergeToolkit = async (
export class DataMerger {
private readonly toolkitDataSource: IToolkitDataSource;
private readonly customSectionsSource: ICustomSectionsSource;
- private readonly overviewInstructionsSource: IOverviewInstructionsSource;
- private readonly overviewGenerator: ToolkitOverviewGenerator | undefined;
- private readonly skipOverview: boolean;
private readonly toolExampleGenerator: ToolExampleGenerator | undefined;
private readonly toolkitSummaryGenerator: ToolkitSummaryGenerator | undefined;
private readonly previousToolkits:
@@ -828,6 +834,7 @@ export class DataMerger {
| ((result: MergeResult) => Promise)
| undefined;
private readonly skipToolkitIds: ReadonlySet;
+ private readonly requireCompleteData: boolean;
private readonly resolveProviderId:
| ((toolkitId: string) => string | null)
| undefined;
@@ -835,19 +842,15 @@ export class DataMerger {
constructor(config: DataMergerConfig) {
this.toolkitDataSource = config.toolkitDataSource;
this.customSectionsSource = config.customSectionsSource;
- this.overviewInstructionsSource =
- config.overviewInstructionsSource ??
- createEmptyOverviewInstructionsSource();
- this.overviewGenerator = config.overviewGenerator;
- this.skipOverview = config.skipOverview ?? false;
this.toolExampleGenerator = config.toolExampleGenerator;
this.toolkitSummaryGenerator = config.toolkitSummaryGenerator;
this.previousToolkits = config.previousToolkits;
- this.llmConcurrency = config.llmConcurrency ?? 5;
- this.toolkitConcurrency = config.toolkitConcurrency ?? 3;
+ this.llmConcurrency = config.llmConcurrency ?? 10;
+ this.toolkitConcurrency = config.toolkitConcurrency ?? 5;
this.onToolkitProgress = config.onToolkitProgress;
this.onToolkitComplete = config.onToolkitComplete;
this.skipToolkitIds = config.skipToolkitIds ?? new Set();
+ this.requireCompleteData = config.requireCompleteData ?? false;
this.resolveProviderId = config.resolveProviderId;
}
@@ -901,10 +904,6 @@ export class DataMerger {
try {
const customSections =
await this.customSectionsSource.getCustomSections(toolkitId);
- const overviewInstructions =
- await this.overviewInstructionsSource.getOverviewInstructions(
- toolkitId
- );
const previousToolkit = this.getPreviousToolkit(toolkitId);
const result = await mergeToolkit(
@@ -916,11 +915,6 @@ export class DataMerger {
{
...(previousToolkit ? { previousToolkit } : {}),
llmConcurrency: this.llmConcurrency,
- ...(this.overviewGenerator
- ? { overviewGenerator: this.overviewGenerator }
- : {}),
- overviewInstructions,
- skipOverview: this.skipOverview,
...(this.resolveProviderId
? { resolveProviderId: this.resolveProviderId }
: {}),
@@ -944,6 +938,12 @@ export class DataMerger {
result: MergeResult,
previousToolkit?: MergedToolkit
): Promise {
+ if (hasToolkitOverviewChunk(result.toolkit)) {
+ // Keep overview as the canonical toolkit-level narrative.
+ result.toolkit.summary = undefined;
+ return;
+ }
+
if (previousToolkit?.summary) {
const currentSignature = buildToolkitSummarySignature(result.toolkit);
const previousSignature = buildToolkitSummarySignature(previousToolkit);
@@ -989,8 +989,6 @@ export class DataMerger {
// Fetch custom sections
const customSections =
await this.customSectionsSource.getCustomSections(toolkitId);
- const overviewInstructions =
- await this.overviewInstructionsSource.getOverviewInstructions(toolkitId);
const previousToolkit = this.getPreviousToolkit(toolkitId);
const result = await mergeToolkit(
@@ -1002,11 +1000,6 @@ export class DataMerger {
{
...(previousToolkit ? { previousToolkit } : {}),
llmConcurrency: this.llmConcurrency,
- ...(this.overviewGenerator
- ? { overviewGenerator: this.overviewGenerator }
- : {}),
- overviewInstructions,
- skipOverview: this.skipOverview,
...(this.resolveProviderId
? { resolveProviderId: this.resolveProviderId }
: {}),
@@ -1027,7 +1020,10 @@ export class DataMerger {
// Filter out toolkits that should be skipped (for resume support)
const filteredEntries = toolkitEntries.filter(
- ([toolkitId]) => !this.skipToolkitIds.has(toolkitId.toLowerCase())
+ ([toolkitId, toolkitData]) =>
+ !this.skipToolkitIds.has(toolkitId.toLowerCase()) &&
+ (!this.requireCompleteData ||
+ (toolkitData.metadata !== null && toolkitData.tools.length > 0))
);
const results = await mapWithConcurrency(
@@ -1061,8 +1057,11 @@ export class DataMerger {
}> {
const allToolkitsData = await this.toolkitDataSource.fetchAllToolkitsData();
const total = allToolkitsData.size;
- const skipped = Array.from(allToolkitsData.keys()).filter((id) =>
- this.skipToolkitIds.has(id.toLowerCase())
+ const skipped = Array.from(allToolkitsData.entries()).filter(
+ ([id, toolkitData]) =>
+ this.skipToolkitIds.has(id.toLowerCase()) ||
+ (this.requireCompleteData &&
+ (toolkitData.metadata === null || toolkitData.tools.length === 0))
).length;
return {
total,
diff --git a/toolkit-docs-generator/src/overview/types.ts b/toolkit-docs-generator/src/overview/types.ts
deleted file mode 100644
index 5bf1ec669..000000000
--- a/toolkit-docs-generator/src/overview/types.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import type { DocumentationChunk, MergedToolkit } from "../types/index.js";
-
-export type ToolkitOverviewInstructions = {
- readonly toolkitId: string;
- readonly label?: string;
- readonly sources?: readonly string[];
- readonly instructions?: string;
-};
-
-export type OverviewGenerationMode = "file" | "auto";
-
-export type OverviewGenerationInput = {
- readonly toolkit: MergedToolkit;
- readonly instructions?: ToolkitOverviewInstructions | null;
- readonly previousOverview?: string | null;
- readonly mode: OverviewGenerationMode;
-};
-
-export type OverviewGenerationResult = {
- readonly chunk: DocumentationChunk;
- readonly reason?: string;
-};
-
-export interface ToolkitOverviewGenerator {
- generate: (
- input: OverviewGenerationInput
- ) => Promise;
-}
diff --git a/toolkit-docs-generator/src/sources/design-system-loader.ts b/toolkit-docs-generator/src/sources/design-system-loader.ts
new file mode 100644
index 000000000..deb98fa23
--- /dev/null
+++ b/toolkit-docs-generator/src/sources/design-system-loader.ts
@@ -0,0 +1,17 @@
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+
+const require = createRequire(import.meta.url);
+
+export type DesignSystemModule = Record;
+
+export async function loadDesignSystemModule(): Promise {
+ try {
+ const designSystemEntry = require.resolve("@arcadeai/design-system");
+ return (await import(
+ pathToFileURL(designSystemEntry).href
+ )) as DesignSystemModule;
+ } catch {
+ return null;
+ }
+}
diff --git a/toolkit-docs-generator/src/sources/design-system-metadata.ts b/toolkit-docs-generator/src/sources/design-system-metadata.ts
index 03b50582b..160a73f0e 100644
--- a/toolkit-docs-generator/src/sources/design-system-metadata.ts
+++ b/toolkit-docs-generator/src/sources/design-system-metadata.ts
@@ -9,6 +9,7 @@
import { z } from "zod";
import type { ToolkitMetadata } from "../types/index.js";
import { ToolkitMetadataSchema } from "../types/index.js";
+import { loadDesignSystemModule } from "./design-system-loader.js";
import type { IMetadataSource } from "./internal.js";
// ============================================================================
@@ -131,10 +132,9 @@ export function createDesignSystemMetadataSourceFromToolkits(
}
export async function createDesignSystemMetadataSource(): Promise {
- // Use a dynamic import so the generator can still run in contexts where
- // @arcadeai/design-system isn't installed.
- const designSystem = await import("@arcadeai/design-system");
- const maybeToolkits = (designSystem as { TOOLKITS?: unknown }).TOOLKITS;
+ const designSystem = await loadDesignSystemModule();
+ const maybeToolkits = (designSystem as { TOOLKITS?: unknown } | null)
+ ?.TOOLKITS;
const toolkits = Array.isArray(maybeToolkits) ? maybeToolkits : [];
const parsed: ToolkitMetadata[] = [];
diff --git a/toolkit-docs-generator/src/sources/in-memory.ts b/toolkit-docs-generator/src/sources/in-memory.ts
index 49ffd22be..8c7c47973 100644
--- a/toolkit-docs-generator/src/sources/in-memory.ts
+++ b/toolkit-docs-generator/src/sources/in-memory.ts
@@ -5,17 +5,13 @@
* avoiding mocks while enabling isolated testing.
*/
-import type { ToolkitOverviewInstructions } from "../overview/types.js";
import type {
CustomSections,
ToolDefinition,
ToolkitMetadata,
} from "../types/index.js";
import { normalizeId } from "../utils/fp.js";
-import type {
- ICustomSectionsSource,
- IOverviewInstructionsSource,
-} from "./interfaces.js";
+import type { ICustomSectionsSource } from "./interfaces.js";
import type {
FetchOptions,
IMetadataSource,
@@ -212,56 +208,6 @@ export class InMemoryCustomSectionsSource implements ICustomSectionsSource {
}
}
-// ============================================================================
-// In-Memory Overview Instructions Source
-// ============================================================================
-
-export class InMemoryOverviewInstructionsSource
- implements IOverviewInstructionsSource
-{
- private readonly instructions: ReadonlyMap<
- string,
- ToolkitOverviewInstructions
- >;
-
- constructor(
- instructions: Readonly>
- ) {
- const map = new Map();
- for (const [key, value] of Object.entries(instructions)) {
- map.set(key, value);
- map.set(normalizeId(key), value);
- }
- this.instructions = map;
- }
-
- async getOverviewInstructions(
- toolkitId: string
- ): Promise {
- const exact = this.instructions.get(toolkitId);
- if (exact) return exact;
-
- const normalized = this.instructions.get(normalizeId(toolkitId));
- return normalized ?? null;
- }
-
- async getAllOverviewInstructions(): Promise<
- Readonly>
- > {
- const result: Record = {};
- const seen = new Set();
-
- for (const [key, value] of this.instructions) {
- if (!seen.has(normalizeId(key))) {
- seen.add(normalizeId(key));
- result[key] = value;
- }
- }
-
- return result;
- }
-}
-
// ============================================================================
// Empty Custom Sections Source
// ============================================================================
@@ -282,26 +228,6 @@ export class EmptyCustomSectionsSource implements ICustomSectionsSource {
}
}
-// ============================================================================
-// Empty Overview Instructions Source
-// ============================================================================
-
-export class EmptyOverviewInstructionsSource
- implements IOverviewInstructionsSource
-{
- async getOverviewInstructions(
- _toolkitId: string
- ): Promise {
- return null;
- }
-
- async getAllOverviewInstructions(): Promise<
- Readonly>
- > {
- return {};
- }
-}
-
// ============================================================================
// Factory Functions
// ============================================================================
@@ -332,9 +258,3 @@ export const createInMemoryCustomSectionsSource = (
*/
export const createEmptyCustomSectionsSource = (): ICustomSectionsSource =>
new EmptyCustomSectionsSource();
-
-/**
- * Create an empty overview instructions source
- */
-export const createEmptyOverviewInstructionsSource =
- (): IOverviewInstructionsSource => new EmptyOverviewInstructionsSource();
diff --git a/toolkit-docs-generator/src/sources/index.ts b/toolkit-docs-generator/src/sources/index.ts
index 9091c565d..5ce70d081 100644
--- a/toolkit-docs-generator/src/sources/index.ts
+++ b/toolkit-docs-generator/src/sources/index.ts
@@ -12,7 +12,6 @@ export * from "./interfaces.js";
export * from "./mock-engine-api.js";
export * from "./mock-metadata.js";
export * from "./oauth-provider-resolver.js";
-export * from "./overview-instructions-file.js";
export * from "./toolkit-data-source.js";
// Note: Design System source requires @arcadeai/design-system to be installed.
diff --git a/toolkit-docs-generator/src/sources/interfaces.ts b/toolkit-docs-generator/src/sources/interfaces.ts
index af3fa7272..8cead5925 100644
--- a/toolkit-docs-generator/src/sources/interfaces.ts
+++ b/toolkit-docs-generator/src/sources/interfaces.ts
@@ -3,7 +3,6 @@
*
*/
-import type { ToolkitOverviewInstructions } from "../overview/types.js";
import type { CustomSections } from "../types/index.js";
// ============================================================================
@@ -33,17 +32,3 @@ export interface ICustomSectionsSource {
Readonly>
>;
}
-
-// ============================================================================
-// Overview Instructions Source Interface
-// ============================================================================
-
-export interface IOverviewInstructionsSource {
- readonly getOverviewInstructions: (
- toolkitId: string
- ) => Promise;
-
- readonly getAllOverviewInstructions: () => Promise<
- Readonly>
- >;
-}
diff --git a/toolkit-docs-generator/src/sources/oauth-provider-resolver.ts b/toolkit-docs-generator/src/sources/oauth-provider-resolver.ts
index 10f82ea95..679862985 100644
--- a/toolkit-docs-generator/src/sources/oauth-provider-resolver.ts
+++ b/toolkit-docs-generator/src/sources/oauth-provider-resolver.ts
@@ -1,3 +1,5 @@
+import { loadDesignSystemModule } from "./design-system-loader.js";
+
/**
* OAuth Provider ID Resolver
*
@@ -97,21 +99,17 @@ export function buildProviderIdResolver(
* `@arcadeai/design-system` is not installed (returns null in that case).
*/
export async function createDesignSystemProviderIdResolver(): Promise {
- try {
- const designSystem = await import("@arcadeai/design-system");
- const catalogue = (
- designSystem as {
- OAUTH_PROVIDER_CATALOGUE?: Record;
- }
- ).OAUTH_PROVIDER_CATALOGUE;
-
- if (!catalogue || typeof catalogue !== "object") {
- return null;
- }
-
- const knownIds = new Set(Object.keys(catalogue));
- return buildProviderIdResolver(knownIds);
- } catch {
+ const designSystem = await loadDesignSystemModule();
+ const catalogue = (
+ designSystem as {
+ OAUTH_PROVIDER_CATALOGUE?: Record;
+ } | null
+ )?.OAUTH_PROVIDER_CATALOGUE;
+
+ if (!catalogue || typeof catalogue !== "object") {
return null;
}
+
+ const knownIds = new Set(Object.keys(catalogue));
+ return buildProviderIdResolver(knownIds);
}
diff --git a/toolkit-docs-generator/src/sources/overview-instructions-file.ts b/toolkit-docs-generator/src/sources/overview-instructions-file.ts
deleted file mode 100644
index ae1eee300..000000000
--- a/toolkit-docs-generator/src/sources/overview-instructions-file.ts
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * Overview Instructions File Source
- *
- * Loads overview instruction files from a directory or a single file.
- */
-import { access, readdir, readFile } from "fs/promises";
-import { basename, extname, join } from "path";
-import { z } from "zod";
-import type { ToolkitOverviewInstructions } from "../overview/types.js";
-import { normalizeId } from "../utils/fp.js";
-import type { IOverviewInstructionsSource } from "./interfaces.js";
-
-const OverviewInstructionsSchema = z.object({
- toolkitId: z.string().min(1).optional(),
- label: z.string().optional(),
- sources: z.array(z.string()).optional().default([]),
- instructions: z.string().optional().default(""),
-});
-
-const ToolkitsMapSchema = z.object({
- toolkits: z.record(z.string(), OverviewInstructionsSchema),
-});
-
-type OverviewFilePayload =
- | z.infer
- | z.infer;
-
-export interface OverviewInstructionsFileConfig {
- dirPath?: string;
- filePath?: string;
- allowMissing?: boolean;
-}
-
-const buildToolkitIdFromFile = (filePath: string): string =>
- basename(filePath, extname(filePath));
-
-const toInstructions = (
- raw: OverviewFilePayload,
- fallbackToolkitId?: string
-): ToolkitOverviewInstructions[] => {
- if ("toolkits" in raw) {
- return Object.entries(raw.toolkits).map(([key, value]) => ({
- ...value,
- toolkitId: value.toolkitId || key,
- }));
- }
-
- return [
- {
- ...raw,
- toolkitId: raw.toolkitId || fallbackToolkitId || raw.label || "unknown",
- },
- ];
-};
-
-export class OverviewInstructionsFileSource
- implements IOverviewInstructionsSource
-{
- private readonly dirPath?: string;
- private readonly filePath?: string;
- private readonly allowMissing: boolean;
- private cachedData: Record | null = null;
-
- constructor(config: OverviewInstructionsFileConfig) {
- this.dirPath = config.dirPath;
- this.filePath = config.filePath;
- this.allowMissing = config.allowMissing ?? true;
- }
-
- private isMissingAllowed(error: unknown): boolean {
- return (
- (error as NodeJS.ErrnoException).code === "ENOENT" && this.allowMissing
- );
- }
-
- private async resolveFiles(): Promise {
- if (this.filePath) {
- return [this.filePath];
- }
-
- if (!this.dirPath) {
- return [];
- }
- const dirPath = this.dirPath;
-
- try {
- await access(this.dirPath);
- } catch (error) {
- if (this.isMissingAllowed(error)) {
- return [];
- }
- throw error;
- }
-
- const entries = await readdir(dirPath);
- return entries
- .filter((entry) => extname(entry).toLowerCase() === ".json")
- .map((entry) => join(dirPath, entry));
- }
-
- private async readInstructionsFromFiles(
- files: readonly string[]
- ): Promise> {
- const result: Record = {};
-
- for (const filePath of files) {
- try {
- const payload = await this.readFileContent(filePath);
- const fallbackToolkitId = buildToolkitIdFromFile(filePath);
- const instructionsList = toInstructions(payload, fallbackToolkitId);
- for (const instruction of instructionsList) {
- const key = instruction.toolkitId;
- result[key] = instruction;
- }
- } catch (error) {
- if (this.isMissingAllowed(error)) {
- continue;
- }
- throw error;
- }
- }
-
- return result;
- }
-
- private async readFileContent(
- filePath: string
- ): Promise {
- const content = await readFile(filePath, "utf-8");
- const parsed = JSON.parse(content) as unknown;
-
- if (
- parsed &&
- typeof parsed === "object" &&
- !Array.isArray(parsed) &&
- "toolkits" in parsed
- ) {
- return ToolkitsMapSchema.parse(parsed);
- }
-
- return OverviewInstructionsSchema.parse(parsed);
- }
-
- private async loadFiles(): Promise<
- Record
- > {
- if (this.cachedData) {
- return this.cachedData;
- }
-
- const files = await this.resolveFiles();
- if (files.length === 0) {
- this.cachedData = {};
- return this.cachedData;
- }
- const result = await this.readInstructionsFromFiles(files);
- this.cachedData = result;
- return this.cachedData;
- }
-
- async getOverviewInstructions(
- toolkitId: string
- ): Promise {
- const data = await this.loadFiles();
- if (data[toolkitId]) {
- return data[toolkitId];
- }
-
- const normalizedId = normalizeId(toolkitId);
- const entry = Object.entries(data).find(
- ([key]) => normalizeId(key) === normalizedId
- );
-
- return entry ? entry[1] : null;
- }
-
- async getAllOverviewInstructions(): Promise<
- Readonly>
- > {
- const data = await this.loadFiles();
- return data;
- }
-}
-
-export const createOverviewInstructionsFileSource = (
- config: OverviewInstructionsFileConfig
-): IOverviewInstructionsSource => new OverviewInstructionsFileSource(config);
diff --git a/toolkit-docs-generator/src/sources/tool-metadata-schema.ts b/toolkit-docs-generator/src/sources/tool-metadata-schema.ts
index 996b476a8..9fa794581 100644
--- a/toolkit-docs-generator/src/sources/tool-metadata-schema.ts
+++ b/toolkit-docs-generator/src/sources/tool-metadata-schema.ts
@@ -27,8 +27,8 @@ const ToolMetadataInputSchema = z.object({
const ToolMetadataOutputSchema = z
.object({
- description: z.string().nullable(),
- value_schema: ToolMetadataValueSchema.nullable(),
+ description: z.string().nullable().optional(),
+ value_schema: ToolMetadataValueSchema.nullable().optional(),
})
.nullable()
.optional();
@@ -115,6 +115,9 @@ export type ToolMetadataSummary = {
}>;
};
+const normalizeEnum = (values: string[] | null | undefined): string[] | null =>
+ values && values.length > 0 ? values : null;
+
const transformParameter = (
apiParam: z.infer
): ToolParameter => ({
@@ -123,7 +126,7 @@ const transformParameter = (
innerType: apiParam.value_schema.inner_val_type ?? undefined,
required: apiParam.required,
description: apiParam.description,
- enum: apiParam.value_schema.enum ?? null,
+ enum: normalizeEnum(apiParam.value_schema.enum),
inferrable: apiParam.inferrable,
});
@@ -144,6 +147,7 @@ export const transformToolMetadataItem = (
): ToolDefinition => {
// authorization is now an array; pick the first entry (most tools have 0 or 1)
const authEntry = apiTool.requirements?.authorization?.[0] ?? null;
+ const providerType = authEntry?.provider_type ?? null;
return {
name: apiTool.name,
@@ -152,17 +156,19 @@ export const transformToolMetadataItem = (
description: apiTool.description,
toolkitDescription: apiTool.toolkit.description,
parameters: apiTool.input.parameters.map(transformParameter),
- auth: authEntry
- ? {
- providerId: authEntry.provider_id ?? null,
- providerType: authEntry.provider_type ?? "unknown",
- scopes: authEntry.scopes ?? [],
- }
- : null,
+ auth:
+ authEntry && providerType
+ ? {
+ providerId: authEntry.provider_id ?? null,
+ providerType,
+ scopes: authEntry.scopes ?? [],
+ }
+ : null,
secrets: normalizeSecrets(apiTool.requirements?.secrets),
output: apiTool.output
? {
- type: apiTool.output.value_schema?.val_type ?? "unknown",
+ // Keep parity with /v1/tools normalization for source-agnostic diffs.
+ type: apiTool.output.value_schema?.val_type ?? "string",
description: apiTool.output.description ?? null,
}
: null,
diff --git a/toolkit-docs-generator/src/sources/toolkit-data-source.ts b/toolkit-docs-generator/src/sources/toolkit-data-source.ts
index 4fb0d98d9..a2e887e83 100644
--- a/toolkit-docs-generator/src/sources/toolkit-data-source.ts
+++ b/toolkit-docs-generator/src/sources/toolkit-data-source.ts
@@ -167,13 +167,17 @@ export class CombinedToolkitDataSource implements IToolkitDataSource {
metadataMap.set(metadata.id, metadata);
}
- // Combine into ToolkitData map
+ // Combine into ToolkitData map.
+ // Use getToolkitMetadata for toolkits without a direct match so that
+ // fallback logic (e.g. "WeaviateApi" → "Weaviate") is applied consistently,
+ // matching the behaviour of fetchToolkitData.
const result = new Map();
for (const [toolkitId, tools] of toolkitGroups) {
- result.set(toolkitId, {
- tools,
- metadata: metadataMap.get(toolkitId) ?? null,
- });
+ const directMetadata = metadataMap.get(toolkitId) ?? null;
+ const metadata =
+ directMetadata ??
+ (await this.metadataSource.getToolkitMetadata(toolkitId));
+ result.set(toolkitId, { tools, metadata });
}
return result;
diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts
index d083ea2c9..8c5c59f2c 100644
--- a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts
+++ b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts
@@ -78,7 +78,7 @@ describe("toolkit static params", () => {
});
});
- it("prefers the design system category when available", async () => {
+ it("uses catalog category as fallback when JSON file is absent", async () => {
await withTempDir(async (dir) => {
await writeIndex(dir, [{ id: "Github", category: "development" }]);
@@ -86,6 +86,25 @@ describe("toolkit static params", () => {
{ id: "Github", category: "social" },
];
+ // No JSON file written — catalog is the only source for category
+ const routes = await listToolkitRoutes({ dataDir: dir, toolkitsCatalog });
+ expect(routes).toEqual([{ toolkitId: "github", category: "social" }]);
+ });
+ });
+
+ it("prefers JSON category over catalog when JSON file is present", async () => {
+ await withTempDir(async (dir) => {
+ await writeIndex(dir, [{ id: "Github", category: "development" }]);
+ // JSON says "social"; catalog says "development" — JSON wins
+ await writeToolkitData(dir, {
+ id: "Github",
+ metadata: { category: "social" },
+ });
+
+ const toolkitsCatalog: ToolkitCatalogEntry[] = [
+ { id: "Github", category: "development" },
+ ];
+
const routes = await listToolkitRoutes({ dataDir: dir, toolkitsCatalog });
expect(routes).toEqual([{ toolkitId: "github", category: "social" }]);
@@ -97,6 +116,35 @@ describe("toolkit static params", () => {
});
});
+ it("reads correct category from JSON for *Api toolkit with stale index", async () => {
+ await withTempDir(async (dir) => {
+ // Simulates WeaviateApi: index.json says "development" (stale), JSON says "databases"
+ await writeIndex(dir, [{ id: "WeaviateApi", category: "development" }]);
+ await writeToolkitData(dir, {
+ id: "WeaviateApi",
+ metadata: {
+ category: "databases",
+ docsLink:
+ "https://docs.arcade.dev/en/resources/integrations/databases/weaviate-api",
+ },
+ });
+
+ const routes = await listToolkitRoutes({
+ dataDir: dir,
+ toolkitsCatalog: [],
+ });
+ expect(routes).toEqual([
+ { toolkitId: "weaviate-api", category: "databases" },
+ ]);
+
+ const params = await getToolkitStaticParamsForCategory("databases", {
+ dataDir: dir,
+ toolkitsCatalog: [],
+ });
+ expect(params).toEqual([{ toolkitId: "weaviate-api" }]);
+ });
+ });
+
it("uses docsLink slugs when available", async () => {
await withTempDir(async (dir) => {
await writeIndex(dir, [
diff --git a/toolkit-docs-generator/tests/diff/previous-output.test.ts b/toolkit-docs-generator/tests/diff/previous-output.test.ts
new file mode 100644
index 000000000..402659fcf
--- /dev/null
+++ b/toolkit-docs-generator/tests/diff/previous-output.test.ts
@@ -0,0 +1,183 @@
+import { describe, expect, it } from "vitest";
+
+import { parsePreviousToolkitForDiff } from "../../src/diff/previous-output.js";
+import type { MergedToolkit } from "../../src/types/index.js";
+
+const createValidToolkit = (): MergedToolkit => ({
+ id: "Github",
+ label: "GitHub",
+ version: "1.0.0",
+ description: "GitHub toolkit",
+ metadata: {
+ category: "development",
+ iconUrl: "https://example.com/github.svg",
+ isBYOC: false,
+ isPro: false,
+ type: "arcade",
+ docsLink: "https://docs.example.com/github",
+ isComingSoon: false,
+ isHidden: false,
+ },
+ auth: null,
+ tools: [
+ {
+ name: "CreateIssue",
+ qualifiedName: "Github.CreateIssue",
+ fullyQualifiedName: "Github.CreateIssue@1.0.0",
+ description: null,
+ toolkitDescription: null,
+ parameters: [
+ {
+ name: "title",
+ type: "string",
+ required: true,
+ description: "Issue title",
+ enum: null,
+ inferrable: true,
+ },
+ ],
+ auth: null,
+ secrets: [],
+ output: {
+ type: "json",
+ description: null,
+ },
+ secretsInfo: [],
+ documentationChunks: [],
+ },
+ ],
+ documentationChunks: [],
+ customImports: [],
+ subPages: [],
+ generatedAt: "2026-01-01T00:00:00.000Z",
+});
+
+describe("parsePreviousToolkitForDiff", () => {
+ it("uses strict parsing when previous output already matches schema", () => {
+ const result = parsePreviousToolkitForDiff(createValidToolkit(), "github");
+
+ expect(result.usedFallback).toBe(false);
+ expect(result.toolkit?.id).toBe("Github");
+ expect(result.toolkit?.tools).toHaveLength(1);
+ });
+
+ it("falls back for legacy page fields and preserves tool signature fields", () => {
+ const result = parsePreviousToolkitForDiff(
+ {
+ id: "Jira",
+ label: "Jira",
+ version: "2.4.0",
+ description: "Jira toolkit",
+ metadata: {
+ category: "development",
+ iconUrl: "https://example.com/jira.svg",
+ isBYOC: false,
+ isPro: false,
+ type: "arcade",
+ docsLink: "https://docs.example.com/jira",
+ isComingSoon: false,
+ isHidden: false,
+ },
+ auth: null,
+ tools: [
+ {
+ name: "CreateIssue",
+ qualifiedName: "Jira.CreateIssue",
+ fullyQualifiedName: "Jira.CreateIssue@2.4.0",
+ description: null,
+ parameters: [
+ {
+ name: "title",
+ type: "string",
+ required: true,
+ description: "Issue title",
+ enum: null,
+ },
+ ],
+ auth: {
+ provider_type: "oauth2",
+ provider_id: null,
+ scopes: ["write:jira-work"],
+ },
+ secrets: [],
+ output: {
+ value_schema: {
+ val_type: "json",
+ },
+ },
+ },
+ ],
+ documentationChunks: [
+ {
+ heading: "How to use Jira",
+ },
+ ],
+ customImports: [{ component: "JiraOverview" }],
+ subPages: [{ slug: "advanced" }],
+ },
+ "jira"
+ );
+
+ expect(result.usedFallback).toBe(true);
+ expect(result.toolkit?.id).toBe("Jira");
+ expect(result.toolkit?.tools).toHaveLength(1);
+ expect(result.toolkit?.tools[0]?.output).toEqual({
+ type: "json",
+ description: null,
+ });
+ expect(result.toolkit?.tools[0]?.parameters[0]?.inferrable).toBe(true);
+ expect(result.toolkit?.tools[0]?.auth).toEqual({
+ providerId: null,
+ providerType: "oauth2",
+ scopes: ["write:jira-work"],
+ });
+ });
+
+ it("derives missing qualified names from toolkit and tool names", () => {
+ const result = parsePreviousToolkitForDiff(
+ {
+ id: "Notion",
+ label: "Notion",
+ version: "3.0.0",
+ description: null,
+ metadata: {
+ category: "productivity",
+ iconUrl: "https://example.com/notion.svg",
+ isBYOC: false,
+ isPro: false,
+ type: "arcade",
+ docsLink: "https://docs.example.com/notion",
+ isComingSoon: false,
+ isHidden: false,
+ },
+ auth: null,
+ tools: [
+ {
+ name: "CreatePage",
+ description: null,
+ parameters: [],
+ auth: null,
+ secrets: [],
+ output: null,
+ },
+ ],
+ },
+ "notion"
+ );
+
+ expect(result.usedFallback).toBe(true);
+ expect(result.toolkit?.tools).toHaveLength(1);
+ expect(result.toolkit?.tools[0]?.qualifiedName).toBe("Notion.CreatePage");
+ expect(result.toolkit?.tools[0]?.fullyQualifiedName).toBe(
+ "Notion.CreatePage@3.0.0"
+ );
+ });
+
+ it("returns null for non-object payloads", () => {
+ const result = parsePreviousToolkitForDiff("invalid-payload", "github");
+
+ expect(result.usedFallback).toBe(true);
+ expect(result.toolkit).toBeNull();
+ expect(result.reason).toContain("not a JSON object");
+ });
+});
diff --git a/toolkit-docs-generator/tests/diff/toolkit-diff.test.ts b/toolkit-docs-generator/tests/diff/toolkit-diff.test.ts
index 3845d4ec6..ce64d764f 100644
--- a/toolkit-docs-generator/tests/diff/toolkit-diff.test.ts
+++ b/toolkit-docs-generator/tests/diff/toolkit-diff.test.ts
@@ -15,6 +15,7 @@ import type {
MergedTool,
MergedToolkit,
ToolDefinition,
+ ToolkitMetadata,
} from "../../src/types/index.js";
// ============================================================================
@@ -112,6 +113,22 @@ const createMergedToolkit = (
...overrides,
});
+const createToolkitMetadata = (
+ overrides: Partial = {}
+): ToolkitMetadata => ({
+ id: "TestKit",
+ label: "Test Kit",
+ category: "development",
+ iconUrl: "https://example.com/icon.svg",
+ isBYOC: false,
+ isPro: false,
+ type: "arcade",
+ docsLink: "https://docs.example.com/test",
+ isComingSoon: false,
+ isHidden: false,
+ ...overrides,
+});
+
// ============================================================================
// compareTools Tests
// ============================================================================
@@ -173,7 +190,10 @@ describe("compareTools", () => {
createToolDefinition({
name: "Tool1",
qualifiedName: "TestKit.Tool1",
- description: "Updated description",
+ output: {
+ type: "array",
+ description: "Result",
+ },
}),
];
const previousToolkit = createMergedToolkit({
@@ -181,7 +201,10 @@ describe("compareTools", () => {
createMergedTool({
name: "Tool1",
qualifiedName: "TestKit.Tool1",
- description: "Original description",
+ output: {
+ type: "object",
+ description: "Result",
+ },
}),
],
});
@@ -196,6 +219,57 @@ describe("compareTools", () => {
});
});
+ it("ignores description-only differences", () => {
+ const currentTools = [
+ createToolDefinition({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ description: "Updated description text",
+ parameters: [
+ {
+ name: "param1",
+ type: "string",
+ required: true,
+ description: "Updated parameter description",
+ enum: null,
+ inferrable: true,
+ },
+ ],
+ output: {
+ type: "object",
+ description: "Updated output description",
+ },
+ }),
+ ];
+ const previousToolkit = createMergedToolkit({
+ tools: [
+ createMergedTool({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ description: "Original description text",
+ parameters: [
+ {
+ name: "param1",
+ type: "string",
+ required: true,
+ description: "Original parameter description",
+ enum: null,
+ inferrable: true,
+ },
+ ],
+ output: {
+ type: "object",
+ description: "Original output description",
+ },
+ }),
+ ],
+ });
+
+ const changes = compareTools(currentTools, previousToolkit);
+
+ expect(changes).toHaveLength(0);
+ });
+
it("should detect parameter changes as modifications", () => {
const currentTools = [
createToolDefinition({
@@ -278,6 +352,107 @@ describe("compareTools", () => {
expect(changes[0]?.changeType).toBe("modified");
});
+ it("ignores oauth provider ID differences across sources", () => {
+ const currentTools = [
+ createToolDefinition({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ auth: {
+ providerId: null,
+ providerType: "oauth2",
+ scopes: ["repo"],
+ },
+ }),
+ ];
+ const previousToolkit = createMergedToolkit({
+ tools: [
+ createMergedTool({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ auth: {
+ providerId: "github",
+ providerType: "oauth2",
+ scopes: ["repo"],
+ },
+ }),
+ ],
+ });
+
+ const changes = compareTools(currentTools, previousToolkit);
+
+ expect(changes).toHaveLength(0);
+ });
+
+ it("treats unknown output type as string for diff parity", () => {
+ const currentTools = [
+ createToolDefinition({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ output: {
+ type: "unknown",
+ description: "Result",
+ },
+ }),
+ ];
+ const previousToolkit = createMergedToolkit({
+ tools: [
+ createMergedTool({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ output: {
+ type: "string",
+ description: "Result",
+ },
+ }),
+ ],
+ });
+
+ const changes = compareTools(currentTools, previousToolkit);
+
+ expect(changes).toHaveLength(0);
+ });
+
+ it("treats empty enum arrays as null for diff parity", () => {
+ const currentTools = [
+ createToolDefinition({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ parameters: [
+ {
+ name: "mode",
+ type: "string",
+ required: true,
+ description: "Mode",
+ enum: [],
+ inferrable: true,
+ },
+ ],
+ }),
+ ];
+ const previousToolkit = createMergedToolkit({
+ tools: [
+ createMergedTool({
+ name: "Tool1",
+ qualifiedName: "TestKit.Tool1",
+ parameters: [
+ {
+ name: "mode",
+ type: "string",
+ required: true,
+ description: "Mode",
+ enum: null,
+ inferrable: true,
+ },
+ ],
+ }),
+ ],
+ });
+
+ const changes = compareTools(currentTools, previousToolkit);
+
+ expect(changes).toHaveLength(0);
+ });
+
it("should handle tools with no previous toolkit", () => {
const currentTools = [
createToolDefinition({ name: "Tool1", qualifiedName: "TestKit.Tool1" }),
@@ -323,7 +498,14 @@ describe("compareToolkit", () => {
});
it("should return modified when tools changed", () => {
- const currentTools = [createToolDefinition({ description: "Updated" })];
+ const currentTools = [
+ createToolDefinition({
+ output: {
+ type: "array",
+ description: "Result",
+ },
+ }),
+ ];
const previousToolkit = createMergedToolkit();
const change = compareToolkit("TestKit", currentTools, previousToolkit);
@@ -355,6 +537,26 @@ describe("compareToolkit", () => {
expect(change.currentVersion).toBe("2.0.0");
expect(change.previousVersion).toBe("1.0.0");
});
+
+ it("should return modified when metadata changes but tools do not", () => {
+ const currentTools = [createToolDefinition()];
+ const previousToolkit = createMergedToolkit();
+
+ const change = compareToolkit(
+ "TestKit",
+ currentTools,
+ previousToolkit,
+ createToolkitMetadata({
+ category: "databases",
+ docsLink: "https://docs.example.com/databases/test-kit",
+ })
+ );
+
+ expect(change.changeType).toBe("modified");
+ expect(change.toolChanges).toHaveLength(0);
+ expect(change.versionChanged).toBe(false);
+ expect(change.metadataChanged).toBe(true);
+ });
});
// ============================================================================
@@ -394,7 +596,14 @@ describe("detectChanges", () => {
const currentToolkitTools = new Map([
[
"TestKit",
- [createToolDefinition({ description: "Updated description" })],
+ [
+ createToolDefinition({
+ output: {
+ type: "array",
+ description: "Result",
+ },
+ }),
+ ],
],
]);
const previousToolkits = new Map([["TestKit", createMergedToolkit()]]);
@@ -455,7 +664,10 @@ describe("detectChanges", () => {
createMergedTool({
name: "CreateIssue",
qualifiedName: "Github.CreateIssue",
- description: "Old description",
+ output: {
+ type: "array",
+ description: "Result",
+ },
}),
createMergedTool({
name: "RemovedTool",
@@ -506,6 +718,30 @@ describe("detectChanges", () => {
expect(result.summary.modifiedToolkits).toBe(1);
expect(result.summary.versionOnlyToolkits).toBe(1);
});
+
+ it("should detect metadata-only changes", () => {
+ const currentToolkitData = new Map([
+ [
+ "TestKit",
+ {
+ tools: [createToolDefinition()],
+ metadata: createToolkitMetadata({
+ category: "databases",
+ docsLink: "https://docs.example.com/databases/test-kit",
+ }),
+ },
+ ],
+ ]);
+ const previousToolkits = new Map([["TestKit", createMergedToolkit()]]);
+
+ const result = detectChanges(currentToolkitData, previousToolkits);
+
+ expect(result.summary.modifiedToolkits).toBe(1);
+ expect(result.summary.versionOnlyToolkits).toBe(0);
+ expect(result.toolkitChanges[0]?.changeType).toBe("modified");
+ expect(result.toolkitChanges[0]?.metadataChanged).toBe(true);
+ expect(result.toolkitChanges[0]?.toolChanges).toHaveLength(0);
+ });
});
// ============================================================================
@@ -536,7 +772,17 @@ describe("hasChanges", () => {
it("should return true when there are modified toolkits", () => {
const result = detectChanges(
new Map([
- ["TestKit", [createToolDefinition({ description: "Changed" })]],
+ [
+ "TestKit",
+ [
+ createToolDefinition({
+ output: {
+ type: "array",
+ description: "Result",
+ },
+ }),
+ ],
+ ],
]),
new Map([["TestKit", createMergedToolkit()]])
);
@@ -567,7 +813,10 @@ describe("getChangedToolkitIds", () => {
[
createToolDefinition({
qualifiedName: "Changed.Tool",
- description: "New",
+ output: {
+ type: "array",
+ description: "Result",
+ },
}),
],
],
@@ -612,7 +861,10 @@ describe("formatChangeSummary", () => {
[
createToolDefinition({
qualifiedName: "Modified.Tool",
- description: "New desc",
+ output: {
+ type: "array",
+ description: "Result",
+ },
}),
],
],
@@ -707,6 +959,27 @@ describe("formatDetailedChanges", () => {
expect(lines.some((l) => l.includes("version update only"))).toBe(true);
});
+ it("should annotate metadata-only updates", () => {
+ const result = detectChanges(
+ new Map([
+ [
+ "TestKit",
+ {
+ tools: [createToolDefinition()],
+ metadata: createToolkitMetadata({
+ category: "databases",
+ docsLink: "https://docs.example.com/databases/test-kit",
+ }),
+ },
+ ],
+ ]),
+ new Map([["TestKit", createMergedToolkit()]])
+ );
+
+ const lines = formatDetailedChanges(result);
+ expect(lines.some((l) => l.includes("metadata update only"))).toBe(true);
+ });
+
it("should skip unchanged toolkits", () => {
const result = detectChanges(
new Map([["TestKit", [createToolDefinition()]]]),
diff --git a/toolkit-docs-generator/tests/generator/output-verifier.test.ts b/toolkit-docs-generator/tests/generator/output-verifier.test.ts
index c3c198fc6..3cdc95139 100644
--- a/toolkit-docs-generator/tests/generator/output-verifier.test.ts
+++ b/toolkit-docs-generator/tests/generator/output-verifier.test.ts
@@ -235,4 +235,110 @@ describe("verifyOutputDir", () => {
).toBe(true);
});
});
+
+ it("can verify legacy toolkit files when legacy fallback is enabled", async () => {
+ await withTempDir(async (dir) => {
+ const [githubToolkit] = await Promise.all([
+ loadFixture("github-toolkit.json"),
+ ]);
+ const generator = createJsonGenerator({
+ outputDir: dir,
+ prettyPrint: false,
+ generateIndex: true,
+ });
+
+ await generator.generateAll([githubToolkit]);
+
+ const toolkitPath = join(dir, "github.json");
+ const legacyToolkit = JSON.parse(
+ await readFile(toolkitPath, "utf-8")
+ ) as Record;
+ const chunks = legacyToolkit.documentationChunks as Record<
+ string,
+ unknown
+ >[];
+ if (Array.isArray(chunks) && chunks[0]) {
+ chunks[0].type = "legacy_note";
+ chunks[0].location = "overview";
+ }
+ legacyToolkit.subPages = [{ slug: "advanced" }];
+ await writeFile(
+ toolkitPath,
+ JSON.stringify(legacyToolkit, null, 2),
+ "utf-8"
+ );
+
+ const strictResult = await verifyOutputDir(dir);
+ expect(strictResult.valid).toBe(false);
+ expect(
+ strictResult.errors.some((error) =>
+ error.includes("Invalid toolkit schema in github.json")
+ )
+ ).toBe(true);
+
+ const fallbackResult = await verifyOutputDir(dir, undefined, {
+ allowLegacyFallback: true,
+ });
+ expect(fallbackResult.valid).toBe(true);
+ expect(fallbackResult.errors).toHaveLength(0);
+ expect(
+ fallbackResult.warnings.some((warning) =>
+ warning.includes("Loaded github.json with fallback parser")
+ )
+ ).toBe(true);
+ });
+ });
+
+ it("builds index from output when legacy files exist", async () => {
+ await withTempDir(async (dir) => {
+ const [githubToolkit, slackToolkit] = await Promise.all([
+ loadFixture("github-toolkit.json"),
+ loadFixture("slack-toolkit.json"),
+ ]);
+ const baselineGenerator = createJsonGenerator({
+ outputDir: dir,
+ prettyPrint: false,
+ generateIndex: true,
+ });
+ await baselineGenerator.generateAll([githubToolkit, slackToolkit]);
+
+ const toolkitPath = join(dir, "github.json");
+ const legacyToolkit = JSON.parse(
+ await readFile(toolkitPath, "utf-8")
+ ) as Record;
+ const chunks = legacyToolkit.documentationChunks as Record<
+ string,
+ unknown
+ >[];
+ if (Array.isArray(chunks) && chunks[0]) {
+ chunks[0].type = "legacy_note";
+ chunks[0].location = "overview";
+ }
+ legacyToolkit.subPages = [{ slug: "advanced" }];
+ await writeFile(
+ toolkitPath,
+ JSON.stringify(legacyToolkit, null, 2),
+ "utf-8"
+ );
+
+ const generator = createJsonGenerator({
+ outputDir: dir,
+ prettyPrint: false,
+ generateIndex: true,
+ indexSource: "output",
+ });
+ const result = await generator.generateAll([slackToolkit]);
+
+ expect(result.errors).toHaveLength(0);
+
+ const index = JSON.parse(
+ await readFile(join(dir, "index.json"), "utf-8")
+ ) as {
+ toolkits: Array<{ id: string }>;
+ };
+ const ids = new Set(index.toolkits.map((entry) => entry.id));
+ expect(ids.has("Github")).toBe(true);
+ expect(ids.has("Slack")).toBe(true);
+ });
+ });
});
diff --git a/toolkit-docs-generator/tests/llm/toolkit-overview-generator.test.ts b/toolkit-docs-generator/tests/llm/toolkit-overview-generator.test.ts
deleted file mode 100644
index c0d8891a6..000000000
--- a/toolkit-docs-generator/tests/llm/toolkit-overview-generator.test.ts
+++ /dev/null
@@ -1,114 +0,0 @@
-import { describe, expect, it } from "vitest";
-import type { LlmClient } from "../../src/llm/client.js";
-import { LlmToolkitOverviewGenerator } from "../../src/llm/toolkit-overview-generator.js";
-import type { MergedToolkit } from "../../src/types/index.js";
-
-const createToolkit = (): MergedToolkit => ({
- id: "TestKit",
- label: "Test Kit",
- version: "1.0.0",
- description: "Test toolkit description",
- metadata: {
- category: "development",
- iconUrl: "https://example.com/icon.svg",
- isBYOC: false,
- isPro: false,
- type: "arcade",
- docsLink: "https://docs.example.com",
- isComingSoon: false,
- isHidden: false,
- },
- auth: {
- type: "oauth2",
- providerId: "test",
- allScopes: ["read"],
- },
- tools: [
- {
- name: "TestTool",
- qualifiedName: "TestKit.TestTool",
- fullyQualifiedName: "TestKit.TestTool@1.0.0",
- description: "A tool.",
- parameters: [],
- auth: {
- providerId: "test",
- providerType: "oauth2",
- scopes: ["read"],
- },
- secrets: [],
- secretsInfo: [],
- output: null,
- documentationChunks: [],
- },
- ],
- documentationChunks: [],
- customImports: [],
- subPages: [],
- generatedAt: new Date().toISOString(),
-});
-
-const createClient = (payload: string): LlmClient => ({
- provider: "openai",
- generateText: async () => payload,
-});
-
-describe("LlmToolkitOverviewGenerator", () => {
- it("skips overview when auto mode says no", async () => {
- const client = createClient(
- JSON.stringify({ shouldWrite: false, overview: "", reason: "unknown" })
- );
- const generator = new LlmToolkitOverviewGenerator({
- client,
- model: "test-model",
- });
-
- const result = await generator.generate({
- toolkit: createToolkit(),
- instructions: null,
- previousOverview: null,
- mode: "auto",
- });
-
- expect(result).toBeNull();
- });
-
- it("builds a header overview chunk when present", async () => {
- const client = createClient(
- JSON.stringify({ shouldWrite: true, overview: "## Overview\n\nHello." })
- );
- const generator = new LlmToolkitOverviewGenerator({
- client,
- model: "test-model",
- });
-
- const result = await generator.generate({
- toolkit: createToolkit(),
- instructions: null,
- previousOverview: null,
- mode: "auto",
- });
-
- expect(result?.chunk.location).toBe("header");
- expect(result?.chunk.position).toBe("before");
- expect(result?.chunk.content.startsWith("## Overview")).toBe(true);
- });
-
- it("adds an Overview heading when missing", async () => {
- const client = createClient(
- JSON.stringify({ shouldWrite: true, overview: "Short overview." })
- );
- const generator = new LlmToolkitOverviewGenerator({
- client,
- model: "test-model",
- });
-
- const result = await generator.generate({
- toolkit: createToolkit(),
- instructions: null,
- previousOverview: null,
- mode: "file",
- });
-
- expect(result?.chunk.content.startsWith("## Overview")).toBe(true);
- });
-});
diff --git a/toolkit-docs-generator/tests/merger/data-merger.test.ts b/toolkit-docs-generator/tests/merger/data-merger.test.ts
index cf71a3358..c909d1e17 100644
--- a/toolkit-docs-generator/tests/merger/data-merger.test.ts
+++ b/toolkit-docs-generator/tests/merger/data-merger.test.ts
@@ -16,16 +16,17 @@ import {
type ToolExampleGenerator,
type ToolkitSummaryGenerator,
} from "../../src/merger/data-merger.js";
-import type {
- ToolkitOverviewGenerator,
- ToolkitOverviewInstructions,
-} from "../../src/overview/types.js";
import {
EmptyCustomSectionsSource,
+ InMemoryCustomSectionsSource,
InMemoryMetadataSource,
InMemoryToolDataSource,
} from "../../src/sources/in-memory.js";
-import { createCombinedToolkitDataSource } from "../../src/sources/toolkit-data-source.js";
+import {
+ createCombinedToolkitDataSource,
+ type IToolkitDataSource,
+ type ToolkitData,
+} from "../../src/sources/toolkit-data-source.js";
import type {
CustomSections,
ToolDefinition,
@@ -443,6 +444,46 @@ describe("mergeToolkit", () => {
);
});
+ it("infers a readable label from toolkit description without metadata", async () => {
+ const tools = [
+ createTool({
+ qualifiedName: "MicrosoftOnedrive.ListFolderItems",
+ fullyQualifiedName: "MicrosoftOnedrive.ListFolderItems@1.0.0",
+ toolkitDescription: "Arcade.dev LLM tools for Microsoft OneDrive",
+ }),
+ ];
+
+ const result = await mergeToolkit(
+ "MicrosoftOnedrive",
+ tools,
+ null,
+ null,
+ undefined
+ );
+
+ expect(result.toolkit.label).toBe("Microsoft OneDrive");
+ });
+
+ it("humanizes toolkit ID when metadata and description label are unavailable", async () => {
+ const tools = [
+ createTool({
+ qualifiedName: "MyCustomApi.Run",
+ fullyQualifiedName: "MyCustomApi.Run@1.0.0",
+ toolkitDescription: "",
+ }),
+ ];
+
+ const result = await mergeToolkit(
+ "MyCustomApi",
+ tools,
+ null,
+ null,
+ undefined
+ );
+
+ expect(result.toolkit.label).toBe("My Custom API");
+ });
+
it("marks *Api toolkits as arcade_starter", async () => {
const tools = [
createTool({
@@ -859,66 +900,87 @@ describe("mergeToolkit resolveProviderId fallback", () => {
});
});
-describe("mergeToolkit overview handling", () => {
- it("replaces existing overview when instructions are present", async () => {
- const tool = createTool();
- const metadata = createMetadata();
-
- const previous = await mergeToolkit(
+describe("mergeToolkit overview chunk handling", () => {
+ it("keeps toolkit-level overview chunks from source custom sections", async () => {
+ const result = await mergeToolkit(
"TestKit",
- [tool],
- metadata,
- null,
- undefined,
- {}
+ [createTool()],
+ createMetadata(),
+ createCustomSections({
+ documentationChunks: [
+ {
+ type: "markdown",
+ location: "header",
+ position: "before",
+ content: "## Overview\n\nOverview text.",
+ },
+ {
+ type: "warning",
+ location: "header",
+ position: "after",
+ content: "Keep this warning.",
+ },
+ ],
+ }),
+ undefined
);
- previous.toolkit.documentationChunks = [
- {
- type: "markdown",
- location: "header",
- position: "before",
- content: "## Overview\n\nOld overview.",
- },
- ];
- const overviewGenerator: ToolkitOverviewGenerator = {
- generate: async () => ({
- chunk: {
- type: "markdown",
- location: "header",
- position: "before",
- content: "## Overview\n\nNew overview.",
- },
- }),
- };
+ expect(result.toolkit.documentationChunks).toHaveLength(2);
+ expect(
+ result.toolkit.documentationChunks[0]?.content
+ .toLowerCase()
+ .startsWith("## overview")
+ ).toBe(true);
+ expect(result.toolkit.documentationChunks[1]?.content).toBe(
+ "Keep this warning."
+ );
+ });
- const overviewInstructions: ToolkitOverviewInstructions = {
- toolkitId: "TestKit",
- instructions: "Write a new overview.",
- sources: [],
- };
+ it("preserves previous toolkit overview chunks when source is empty", async () => {
+ const tools = [createTool({ qualifiedName: "TestKit.Tool1" })];
+ const previous = await mergeToolkit(
+ "TestKit",
+ tools,
+ createMetadata(),
+ createCustomSections({
+ documentationChunks: [
+ {
+ type: "markdown",
+ location: "header",
+ position: "before",
+ content: "## Overview\n\nOld overview.",
+ },
+ {
+ type: "info",
+ location: "header",
+ position: "after",
+ content: "Old tip.",
+ },
+ ],
+ }),
+ createStubGenerator()
+ );
const result = await mergeToolkit(
"TestKit",
- [tool],
- metadata,
+ tools,
+ createMetadata(),
null,
undefined,
- {
- previousToolkit: previous.toolkit,
- overviewGenerator,
- overviewInstructions,
- }
+ { previousToolkit: previous.toolkit }
);
- expect(result.toolkit.documentationChunks[0]?.content).toBe(
- "## Overview\n\nNew overview."
- );
+ expect(result.toolkit.documentationChunks).toHaveLength(2);
+ expect(
+ result.toolkit.documentationChunks.some(
+ (chunk) => chunk.content === "Old tip."
+ )
+ ).toBe(true);
expect(
result.toolkit.documentationChunks.some((chunk) =>
- chunk.content.includes("Old overview")
+ chunk.content.toLowerCase().startsWith("## overview")
)
- ).toBe(false);
+ ).toBe(true);
});
});
@@ -996,7 +1058,7 @@ describe("DataMerger", () => {
expect(result.toolkit.auth?.allScopes).toContain("public_repo");
});
- it("adds a summary when a summary generator is provided", async () => {
+ it("adds a summary when no overview chunk is present", async () => {
const toolkitDataSource = createCombinedToolkitDataSource({
toolSource: new InMemoryToolDataSource([githubTool1]),
metadataSource: new InMemoryMetadataSource([githubMetadata]),
@@ -1013,6 +1075,42 @@ describe("DataMerger", () => {
expect(result.toolkit.summary).toBe("Toolkit summary (Github)");
});
+ it("skips summary generation when an overview chunk is present", async () => {
+ const toolkitDataSource = createCombinedToolkitDataSource({
+ toolSource: new InMemoryToolDataSource([githubTool1]),
+ metadataSource: new InMemoryMetadataSource([githubMetadata]),
+ });
+ const countingSummary = createCountingSummaryGenerator();
+ const overviewCustomSectionsSource = new InMemoryCustomSectionsSource({
+ Github: createCustomSections({
+ documentationChunks: [
+ {
+ type: "markdown",
+ location: "header",
+ position: "before",
+ content: "## Overview\n\nGenerated overview.",
+ },
+ ],
+ }),
+ });
+ const merger = new DataMerger({
+ toolkitDataSource,
+ customSectionsSource: overviewCustomSectionsSource,
+ toolExampleGenerator: createStubGenerator(),
+ toolkitSummaryGenerator: countingSummary.generator,
+ });
+
+ const result = await merger.mergeToolkit("Github");
+
+ expect(
+ result.toolkit.documentationChunks.some((chunk) =>
+ chunk.content.toLowerCase().startsWith("## overview")
+ )
+ ).toBe(true);
+ expect(countingSummary.getCalls()).toBe(0);
+ expect(result.toolkit.summary).toBeUndefined();
+ });
+
it("reuses the previous summary when toolkit input is unchanged", async () => {
const toolkitDataSource = createCombinedToolkitDataSource({
toolSource: new InMemoryToolDataSource([githubTool1]),
@@ -1042,6 +1140,51 @@ describe("DataMerger", () => {
expect(result.toolkit.summary).toBe("Cached summary");
});
+ it("does not reuse previous summary when current toolkit has an overview chunk", async () => {
+ const toolkitDataSource = createCombinedToolkitDataSource({
+ toolSource: new InMemoryToolDataSource([githubTool1]),
+ metadataSource: new InMemoryMetadataSource([githubMetadata]),
+ });
+ const previousResult = await mergeToolkit(
+ "Github",
+ [githubTool1],
+ githubMetadata,
+ null,
+ createStubGenerator()
+ );
+ previousResult.toolkit.summary = "Cached summary";
+ const countingSummary = createCountingSummaryGenerator();
+ const overviewCustomSectionsSource = new InMemoryCustomSectionsSource({
+ Github: createCustomSections({
+ documentationChunks: [
+ {
+ type: "markdown",
+ location: "header",
+ position: "before",
+ content: "## Overview\n\nGenerated overview.",
+ },
+ ],
+ }),
+ });
+ const merger = new DataMerger({
+ toolkitDataSource,
+ customSectionsSource: overviewCustomSectionsSource,
+ toolExampleGenerator: createStubGenerator(),
+ toolkitSummaryGenerator: countingSummary.generator,
+ previousToolkits: new Map([["github", previousResult.toolkit]]),
+ });
+
+ const result = await merger.mergeToolkit("Github");
+
+ expect(
+ result.toolkit.documentationChunks.some((chunk) =>
+ chunk.content.toLowerCase().startsWith("## overview")
+ )
+ ).toBe(true);
+ expect(countingSummary.getCalls()).toBe(0);
+ expect(result.toolkit.summary).toBeUndefined();
+ });
+
it("reuses previous examples when the tool is unchanged", async () => {
const toolkitDataSource = createCombinedToolkitDataSource({
toolSource: new InMemoryToolDataSource([githubTool1]),
@@ -1070,7 +1213,7 @@ describe("DataMerger", () => {
);
});
- it("calls the generator when the tool definition changes", async () => {
+ it("does not call the generator when only descriptions change", async () => {
const updatedTool = createTool({
name: "CreateIssue",
qualifiedName: "Github.CreateIssue",
@@ -1101,9 +1244,71 @@ describe("DataMerger", () => {
previousToolkits: new Map([["github", previousResult.toolkit]]),
});
- await merger.mergeToolkit("Github");
+ const result = await merger.mergeToolkit("Github");
+
+ expect(counting.getCalls()).toBe(0);
+ expect(result.toolkit.tools[0]?.codeExample).toEqual(
+ previousResult.toolkit.tools[0]?.codeExample
+ );
+ });
+
+ it("calls the generator only for changed tools in mixed toolkits", async () => {
+ const unchangedTool = createTool({
+ name: "ListIssues",
+ qualifiedName: "Github.ListIssues",
+ fullyQualifiedName: "Github.ListIssues@1.0.0",
+ });
+ const changedToolPrevious = createTool({
+ name: "CreateIssue",
+ qualifiedName: "Github.CreateIssue",
+ fullyQualifiedName: "Github.CreateIssue@1.0.0",
+ });
+ const changedToolCurrent = createTool({
+ name: "CreateIssue",
+ qualifiedName: "Github.CreateIssue",
+ fullyQualifiedName: "Github.CreateIssue@1.0.0",
+ output: {
+ type: "array",
+ description: "Result",
+ },
+ });
+ const toolkitDataSource = createCombinedToolkitDataSource({
+ toolSource: new InMemoryToolDataSource([
+ unchangedTool,
+ changedToolCurrent,
+ ]),
+ metadataSource: new InMemoryMetadataSource([githubMetadata]),
+ });
+ const previousResult = await mergeToolkit(
+ "Github",
+ [unchangedTool, changedToolPrevious],
+ githubMetadata,
+ null,
+ createStubGenerator()
+ );
+ const counting = createCountingGenerator();
+ const merger = new DataMerger({
+ toolkitDataSource,
+ customSectionsSource: new EmptyCustomSectionsSource(),
+ toolExampleGenerator: counting.generator,
+ previousToolkits: new Map([["github", previousResult.toolkit]]),
+ });
+
+ const result = await merger.mergeToolkit("Github");
+ const toolByQualifiedName = new Map(
+ result.toolkit.tools.map((tool) => [tool.qualifiedName, tool])
+ );
+ const previousByQualifiedName = new Map(
+ previousResult.toolkit.tools.map((tool) => [tool.qualifiedName, tool])
+ );
expect(counting.getCalls()).toBe(1);
+ expect(toolByQualifiedName.get("Github.ListIssues")?.codeExample).toEqual(
+ previousByQualifiedName.get("Github.ListIssues")?.codeExample
+ );
+ expect(
+ toolByQualifiedName.get("Github.CreateIssue")?.codeExample
+ ).toBeDefined();
});
it("should merge data using unified toolkit data source", async () => {
@@ -1209,6 +1414,65 @@ describe("DataMerger", () => {
expect(slackResult?.toolkit.tools).toHaveLength(1);
});
+ it("skips toolkits missing metadata or tools when requireCompleteData is true", async () => {
+ const completeToolkitData: ToolkitData = {
+ tools: [githubTool1],
+ metadata: githubMetadata,
+ };
+ const missingMetadataToolkitData: ToolkitData = {
+ tools: [
+ createTool({
+ name: "Lookup",
+ qualifiedName: "Unknown.Lookup",
+ fullyQualifiedName: "Unknown.Lookup@1.0.0",
+ }),
+ ],
+ metadata: null,
+ };
+ const missingToolsToolkitData: ToolkitData = {
+ tools: [],
+ metadata: slackMetadata,
+ };
+
+ const toolkitDataSource: IToolkitDataSource = {
+ fetchToolkitData: async (toolkitId: string) => {
+ if (toolkitId === "Github") {
+ return completeToolkitData;
+ }
+ if (toolkitId === "Unknown") {
+ return missingMetadataToolkitData;
+ }
+ if (toolkitId === "Slack") {
+ return missingToolsToolkitData;
+ }
+ return { tools: [], metadata: null };
+ },
+ fetchAllToolkitsData: async () =>
+ new Map([
+ ["Github", completeToolkitData],
+ ["Unknown", missingMetadataToolkitData],
+ ["Slack", missingToolsToolkitData],
+ ]),
+ isAvailable: async () => true,
+ };
+
+ const merger = new DataMerger({
+ toolkitDataSource,
+ customSectionsSource: new EmptyCustomSectionsSource(),
+ toolExampleGenerator: createStubGenerator(),
+ requireCompleteData: true,
+ });
+
+ const count = await merger.getToolkitCount();
+ const results = await merger.mergeAllToolkits();
+
+ expect(count.total).toBe(3);
+ expect(count.toProcess).toBe(1);
+ expect(count.skipped).toBe(2);
+ expect(results).toHaveLength(1);
+ expect(results[0]?.toolkit.id).toBe("Github");
+ });
+
it("should return empty array when no tools", async () => {
const toolkitDataSource = createCombinedToolkitDataSource({
toolSource: new InMemoryToolDataSource([]),
diff --git a/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts b/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts
index 28f967663..c09caf0c8 100644
--- a/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts
+++ b/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts
@@ -93,7 +93,10 @@ describe("Scenario: Skip unchanged toolkits", () => {
createTool({
name: "Tool1",
qualifiedName: "Github.Tool1",
- description: "Updated",
+ output: {
+ type: "array",
+ description: null,
+ },
}),
],
],
@@ -192,4 +195,39 @@ describe("Scenario: Skip unchanged toolkits", () => {
expect(result.toolkitChanges[0]?.versionChanged).toBe(true);
expect(result.toolkitChanges[0]?.toolChanges).toHaveLength(0); // No tool-level changes
});
+
+ it("includes metadata-only changes in changed IDs", () => {
+ const currentToolkitData = new Map([
+ [
+ "Github",
+ {
+ tools: [createTool({ name: "Tool1", qualifiedName: "Github.Tool1" })],
+ metadata: {
+ id: "Github",
+ label: "Github",
+ category: "databases",
+ iconUrl: "https://example.com/icon.svg",
+ isBYOC: false,
+ isPro: false,
+ type: "arcade",
+ docsLink: "https://docs.example.com/databases/github",
+ isComingSoon: false,
+ isHidden: false,
+ },
+ },
+ ],
+ ]);
+ const previousToolkits = new Map([
+ ["Github", createMergedToolkit("Github")],
+ ]);
+
+ const result = detectChanges(currentToolkitData, previousToolkits);
+
+ const changedIds = getChangedToolkitIds(result);
+ expect(changedIds).toContain("Github");
+ expect(result.summary.modifiedToolkits).toBe(1);
+ expect(result.summary.versionOnlyToolkits).toBe(0);
+ expect(result.toolkitChanges[0]?.toolChanges).toHaveLength(0);
+ expect(result.toolkitChanges[0]?.metadataChanged).toBe(true);
+ });
});
diff --git a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts
index 8c6b466f6..f142fc376 100644
--- a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts
+++ b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts
@@ -7,7 +7,8 @@
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { getToolkitStaticParamsForCategory } from "../../../app/_lib/toolkit-static-params";
import {
buildToolkitInfoList,
generateCategoryMeta,
@@ -17,30 +18,31 @@ import {
getToolkitLabel,
getToolkitLabelFromJson,
groupByCategory,
+ parseBooleanCliFlag,
+ resolveRemoveEmptySections,
+ setToolkitsForTesting,
syncToolkitSidebar,
type ToolkitInfo,
} from "../../scripts/sync-toolkit-sidebar";
-// Mock the design system
-vi.mock("@arcadeai/design-system", () => ({
- TOOLKITS: [
- { id: "Gmail", label: "Gmail", category: "productivity" },
- { id: "Slack", label: "Slack", category: "social" },
- { id: "Github", label: "GitHub", category: "development" },
- { id: "Stripe", label: "Stripe", category: "payments" },
- { id: "Zendesk", label: "Zendesk", category: "customer-support" },
- { id: "GoogleSearch", label: "Google Search", category: "search" },
- { id: "Hubspot", label: "HubSpot", category: "sales" },
- { id: "Spotify", label: "Spotify", category: "entertainment" },
- { id: "Postgres", label: "Postgres", category: "databases" },
- {
- id: "HiddenToolkit",
- label: "Hidden",
- category: "productivity",
- isHidden: true,
- },
- ],
-}));
+setToolkitsForTesting([
+ { id: "Gmail", label: "Gmail", category: "productivity" },
+ { id: "Slack", label: "Slack", category: "social" },
+ { id: "Github", label: "GitHub", category: "development" },
+ { id: "Stripe", label: "Stripe", category: "payments" },
+ { id: "Zendesk", label: "Zendesk", category: "customer-support" },
+ { id: "GoogleSearch", label: "Google Search", category: "search" },
+ { id: "Hubspot", label: "HubSpot", category: "sales" },
+ { id: "Spotify", label: "Spotify", category: "entertainment" },
+ { id: "Postgres", label: "Postgres", category: "databases" },
+ { id: "WeaviateApi", label: "Weaviate API", category: "development" },
+ {
+ id: "HiddenToolkit",
+ label: "Hidden",
+ category: "productivity",
+ isHidden: true,
+ },
+]);
// Test directory setup
const TEST_DIR = join(process.cwd(), ".test-sync-sidebar");
@@ -292,6 +294,59 @@ describe("buildToolkitInfoList", () => {
const matches = result.filter((item) => item.slug === "clickup-api");
expect(matches).toHaveLength(1);
});
+
+ it("keeps sidebar href categories consistent with static params", async () => {
+ createToolkitJson("weaviateapi", {
+ id: "WeaviateApi",
+ label: "Weaviate API",
+ metadata: {
+ category: "databases",
+ docsLink:
+ "https://docs.arcade.dev/en/mcp-servers/databases/weaviate-api",
+ },
+ });
+
+ const result = buildToolkitInfoList(TEST_DATA_DIR);
+ const weaviate = result.find((item) => item.id === "WeaviateApi");
+ expect(weaviate).toBeDefined();
+ if (!weaviate) {
+ throw new Error("Expected WeaviateApi toolkit in sidebar data");
+ }
+ expect(weaviate.category).toBe("databases");
+ expect(weaviate.slug).toBe("weaviate-api");
+
+ const sidebarMeta = generateCategoryMeta(
+ [weaviate],
+ weaviate.category,
+ "/en/resources/integrations"
+ );
+ expect(sidebarMeta).toContain(
+ 'href: "/en/resources/integrations/databases/weaviate-api"'
+ );
+
+ const toolkitsCatalog = [
+ { id: "WeaviateApi", category: "development", docsLink: undefined },
+ ];
+ const databasesParams = await getToolkitStaticParamsForCategory(
+ "databases",
+ {
+ dataDir: TEST_DATA_DIR,
+ toolkitsCatalog,
+ }
+ );
+ const developmentParams = await getToolkitStaticParamsForCategory(
+ "development",
+ {
+ dataDir: TEST_DATA_DIR,
+ toolkitsCatalog,
+ }
+ );
+
+ expect(databasesParams).toContainEqual({ toolkitId: "weaviate-api" });
+ expect(developmentParams).not.toContainEqual({
+ toolkitId: "weaviate-api",
+ });
+ });
});
// ============================================================================
@@ -349,6 +404,60 @@ describe("groupByCategory", () => {
});
});
+// ============================================================================
+// Unit Tests: remove empty section flags
+// ============================================================================
+
+describe("remove empty section flags", () => {
+ it("defaults to false when no flag is provided", () => {
+ expect(resolveRemoveEmptySections({})).toBe(false);
+ });
+
+ it("supports the explicit removeEmptySections option", () => {
+ expect(resolveRemoveEmptySections({ removeEmptySections: true })).toBe(
+ true
+ );
+ expect(resolveRemoveEmptySections({ removeEmptySections: false })).toBe(
+ false
+ );
+ });
+
+ it("supports prune as a backward-compatible alias", () => {
+ expect(resolveRemoveEmptySections({ prune: true })).toBe(true);
+ expect(resolveRemoveEmptySections({ prune: false })).toBe(false);
+ });
+
+ it("prefers removeEmptySections over prune when both are set", () => {
+ expect(
+ resolveRemoveEmptySections({ removeEmptySections: false, prune: true })
+ ).toBe(false);
+ expect(
+ resolveRemoveEmptySections({ removeEmptySections: true, prune: false })
+ ).toBe(true);
+ });
+
+ it("parses boolean CLI flags in value and shorthand formats", () => {
+ expect(
+ parseBooleanCliFlag(
+ ["--remove-empty-sections=true"],
+ "--remove-empty-sections"
+ )
+ ).toBe(true);
+ expect(
+ parseBooleanCliFlag(
+ ["--remove-empty-sections=false"],
+ "--remove-empty-sections"
+ )
+ ).toBe(false);
+ expect(
+ parseBooleanCliFlag(
+ ["--remove-empty-sections"],
+ "--remove-empty-sections"
+ )
+ ).toBe(true);
+ });
+});
+
// ============================================================================
// Unit Tests: generateCategoryMeta
// ============================================================================
diff --git a/toolkit-docs-generator/tests/sources/engine-api.test.ts b/toolkit-docs-generator/tests/sources/engine-api.test.ts
index 3936eef5f..8575c2c89 100644
--- a/toolkit-docs-generator/tests/sources/engine-api.test.ts
+++ b/toolkit-docs-generator/tests/sources/engine-api.test.ts
@@ -198,10 +198,90 @@ describe("EngineApiSource", () => {
expect(tools).toHaveLength(2);
expect(tools[0]?.toolkitDescription).toBe("GitHub toolkit");
expect(tools[0]?.secrets).toEqual(["GITHUB_API_KEY"]);
- expect(tools[1]?.output?.type).toBe("unknown");
+ expect(tools[1]?.output?.type).toBe("string");
expect(tools[1]?.auth?.providerId).toBeNull();
});
+ it("handles tool metadata output objects with missing fields", async () => {
+ const items: ToolMetadataItem[] = [
+ {
+ fully_qualified_name: "Github.CreateIssue@1.0.0",
+ qualified_name: "Github.CreateIssue",
+ name: "CreateIssue",
+ description: "Create issue",
+ toolkit: {
+ name: "Github",
+ version: "1.0.0",
+ description: "GitHub toolkit",
+ },
+ input: { parameters: [] },
+ output: {} as ToolMetadataItem["output"],
+ requirements: {
+ authorization: null,
+ secrets: [],
+ },
+ },
+ ];
+ const source = new EngineApiSource({
+ baseUrl: "https://api.arcade.dev",
+ apiKey: "test",
+ fetchFn: createFetchStub(items),
+ });
+
+ const tools = await source.fetchAllTools();
+
+ expect(tools).toHaveLength(1);
+ expect(tools[0]?.output).toEqual({
+ type: "string",
+ description: null,
+ });
+ });
+
+ it("normalizes empty enum arrays to null", async () => {
+ const items: ToolMetadataItem[] = [
+ {
+ fully_qualified_name: "Github.CreateIssue@1.0.0",
+ qualified_name: "Github.CreateIssue",
+ name: "CreateIssue",
+ description: "Create issue",
+ toolkit: {
+ name: "Github",
+ version: "1.0.0",
+ description: "GitHub toolkit",
+ },
+ input: {
+ parameters: [
+ {
+ name: "mode",
+ required: true,
+ description: "Execution mode",
+ value_schema: {
+ val_type: "string",
+ inner_val_type: null,
+ enum: [],
+ },
+ inferrable: true,
+ },
+ ],
+ },
+ output: null,
+ requirements: {
+ authorization: null,
+ secrets: [],
+ },
+ },
+ ];
+ const source = new EngineApiSource({
+ baseUrl: "https://api.arcade.dev",
+ apiKey: "test",
+ fetchFn: createFetchStub(items),
+ });
+
+ const tools = await source.fetchAllTools();
+
+ expect(tools[0]?.parameters[0]?.enum).toBeNull();
+ });
+
it("filters tools by toolkit and provider", async () => {
const items = createItems();
const source = new EngineApiSource({
diff --git a/toolkit-docs-generator/tests/sources/overview-instructions-file.test.ts b/toolkit-docs-generator/tests/sources/overview-instructions-file.test.ts
deleted file mode 100644
index 72ca894f3..000000000
--- a/toolkit-docs-generator/tests/sources/overview-instructions-file.test.ts
+++ /dev/null
@@ -1,113 +0,0 @@
-import { mkdtemp, rm, writeFile } from "fs/promises";
-import { tmpdir } from "os";
-import { join } from "path";
-import { afterEach, describe, expect, it } from "vitest";
-import { createOverviewInstructionsFileSource } from "../../src/sources/overview-instructions-file.js";
-
-const createTempDir = async (): Promise =>
- mkdtemp(join(tmpdir(), "overview-input-"));
-
-describe("OverviewInstructionsFileSource", () => {
- let tempDir: string | null = null;
-
- afterEach(async () => {
- if (tempDir) {
- await rm(tempDir, { recursive: true, force: true });
- tempDir = null;
- }
- });
-
- it("returns null when directory is missing", async () => {
- const source = createOverviewInstructionsFileSource({
- dirPath: join(tmpdir(), "does-not-exist"),
- allowMissing: true,
- });
-
- const result = await source.getOverviewInstructions("Github");
- expect(result).toBeNull();
- });
-
- it("loads instructions from directory files", async () => {
- tempDir = await createTempDir();
- const filePath = join(tempDir, "github.json");
- await writeFile(
- filePath,
- JSON.stringify(
- {
- toolkitId: "Github",
- label: "GitHub",
- sources: ["https://docs.github.com"],
- instructions: "Write an overview.",
- },
- null,
- 2
- )
- );
-
- const source = createOverviewInstructionsFileSource({
- dirPath: tempDir,
- allowMissing: true,
- });
-
- const result = await source.getOverviewInstructions("github");
- expect(result?.toolkitId).toBe("Github");
- expect(result?.label).toBe("GitHub");
- expect(result?.sources).toEqual(["https://docs.github.com"]);
- });
-
- it("uses filename as fallback toolkitId", async () => {
- tempDir = await createTempDir();
- const filePath = join(tempDir, "slack.json");
- await writeFile(
- filePath,
- JSON.stringify(
- {
- label: "Slack",
- instructions: "Write an overview for Slack.",
- },
- null,
- 2
- )
- );
-
- const source = createOverviewInstructionsFileSource({
- dirPath: tempDir,
- allowMissing: true,
- });
-
- const result = await source.getOverviewInstructions("Slack");
- expect(result?.toolkitId).toBe("slack");
- expect(result?.label).toBe("Slack");
- });
-
- it("reads instructions from toolkits map files", async () => {
- tempDir = await createTempDir();
- const filePath = join(tempDir, "toolkits.json");
- await writeFile(
- filePath,
- JSON.stringify(
- {
- toolkits: {
- Github: {
- toolkitId: "Github",
- label: "GitHub",
- sources: ["https://docs.github.com"],
- instructions: "Write an overview.",
- },
- },
- },
- null,
- 2
- )
- );
-
- const source = createOverviewInstructionsFileSource({
- dirPath: tempDir,
- allowMissing: true,
- });
-
- const result = await source.getOverviewInstructions("Github");
- expect(result?.toolkitId).toBe("Github");
- expect(result?.label).toBe("GitHub");
- });
-});
diff --git a/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts b/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts
index 6c9b3630f..989464219 100644
--- a/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts
+++ b/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts
@@ -5,6 +5,7 @@
* abstraction returns tools + metadata together.
*/
import { describe, expect, it } from "vitest";
+import { createDesignSystemMetadataSourceFromToolkits } from "../../src/sources/design-system-metadata.js";
import {
InMemoryMetadataSource,
InMemoryToolDataSource,
@@ -125,6 +126,40 @@ describe("CombinedToolkitDataSource", () => {
expect(result.get("Slack")?.metadata?.label).toBe("Slack");
});
+ it("fetchAllToolkitsData resolves metadata for *Api toolkits whose design system id omits the Api suffix", async () => {
+ // Simulates WeaviateApi (Engine API name) vs Weaviate (design system id).
+ // getAllToolkitsMetadata returns { id: "Weaviate" }, but the toolkit group key
+ // is "WeaviateApi" — the direct map lookup misses and must fall through to
+ // getToolkitMetadata which has the "api" suffix fallback.
+ const toolSource = new InMemoryToolDataSource([
+ createTool({
+ qualifiedName: "WeaviateApi.ActivateUser",
+ fullyQualifiedName: "WeaviateApi.ActivateUser@2.0.0",
+ }),
+ ]);
+
+ const metadataSource = createDesignSystemMetadataSourceFromToolkits([
+ createMetadata({
+ id: "Weaviate",
+ label: "Weaviate",
+ category: "databases",
+ iconUrl: "https://design-system.arcade.dev/icons/weaviate.svg",
+ }),
+ ]);
+
+ const dataSource = createCombinedToolkitDataSource({
+ toolSource,
+ metadataSource,
+ });
+
+ const result = await dataSource.fetchAllToolkitsData();
+ const weaviate = result.get("WeaviateApi");
+
+ expect(weaviate).toBeDefined();
+ expect(weaviate?.metadata?.category).toBe("databases");
+ expect(weaviate?.metadata?.label).toBe("Weaviate API");
+ });
+
it("should fall back to providerId metadata for *Api toolkits", async () => {
const toolSource = new InMemoryToolDataSource([
createTool({
diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts
index 9f95afad7..5ba776a36 100644
--- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts
+++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts
@@ -15,11 +15,24 @@ test("porter workflow includes required triggers", () => {
expect(workflowContents).toContain("repository_dispatch");
expect(workflowContents).toContain("porter_deploy_succeeded");
expect(workflowContents).toContain("workflow_dispatch");
+ expect(workflowContents).toContain("schedule:");
+ expect(workflowContents).toContain('cron: "0 11 * * *"');
});
test("porter workflow generates docs and opens a PR", () => {
- expect(workflowContents).toContain("pnpm start generate");
+ expect(workflowContents).toContain("pnpm dlx tsx src/cli/index.ts generate");
expect(workflowContents).toContain("--skip-unchanged");
+ expect(workflowContents).toContain("--require-complete");
+ expect(workflowContents).toContain("--verbose");
+ expect(workflowContents).toContain("--api-source tool-metadata");
+ expect(workflowContents).toContain("--tool-metadata-url");
+ expect(workflowContents).toContain("--tool-metadata-key");
+ expect(workflowContents).toContain("--llm-provider openai");
+ expect(workflowContents).toContain("--llm-model");
+ expect(workflowContents).toContain("--llm-api-key");
+ expect(workflowContents).toContain("--remove-empty-sections=false");
expect(workflowContents).toContain("peter-evans/create-pull-request");
+ expect(workflowContents).toContain("HUSKY: 0");
+ expect(workflowContents).toContain("[AUTO] Adding MCP Servers docs update");
expect(workflowContents).toContain("pull-requests: write");
});
diff --git a/toolkit-docs-generator/vitest.config.ts b/toolkit-docs-generator/vitest.config.ts
index 06efcd802..f7e5aeeda 100644
--- a/toolkit-docs-generator/vitest.config.ts
+++ b/toolkit-docs-generator/vitest.config.ts
@@ -1,17 +1,6 @@
-import { createRequire } from "node:module";
import { defineConfig } from "vitest/config";
-const require = createRequire(import.meta.url);
-const designSystemEntry = require.resolve("@arcadeai/design-system");
-
export default defineConfig({
- resolve: {
- alias: {
- // Use the Node-resolved entry to avoid broken "development" export paths
- // in some published design-system versions during Vitest transform.
- "@arcadeai/design-system": designSystemEntry,
- },
- },
test: {
// Enable globals like describe, it, expect without imports
globals: true,
diff --git a/vitest.config.ts b/vitest.config.ts
index ce69e8ffe..8fb6f2dcf 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -1,16 +1,3 @@
-import { createRequire } from "node:module";
import { defineConfig } from "vitest/config";
-const require = createRequire(import.meta.url);
-const designSystemEntry = require.resolve("@arcadeai/design-system");
-
-export default defineConfig({
- resolve: {
- alias: {
- // Force Vitest to use the Node-resolved entrypoint instead of the
- // package "development" condition, which some published versions point
- // to non-shipped source files.
- "@arcadeai/design-system": designSystemEntry,
- },
- },
-});
+export default defineConfig({});