From b5a79bffbfc6a3192f9fc433fdd04722d5b50fd9 Mon Sep 17 00:00:00 2001 From: mvm Date: Tue, 24 Feb 2026 16:00:05 -0600 Subject: [PATCH 1/6] Add per-product llms.txt pages and update directory structure - Create dynamic llms.txt pages for each product at /[product]/llms.txt - Update root llms.txt to group products by category and link to product-specific llms.txt files - Add wildcard route pattern "**/llms.txt" to astro.config.ts - Update sidebar links to point to product-specific llms.txt instead of root - Filter directory entries to only include products with actual documentation pages --- astro.config.ts | 1 + src/pages/[product]/llms.txt.ts | 62 +++++++++++++++++++++++++++++++++ src/pages/llms.txt.ts | 61 +++++++++++++++++--------------- src/util/sidebar.ts | 2 +- 4 files changed, 96 insertions(+), 30 deletions(-) create mode 100644 src/pages/[product]/llms.txt.ts diff --git a/astro.config.ts b/astro.config.ts index 1a67a778247057..c21e9591cc01b0 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -144,6 +144,7 @@ export default defineConfig({ "/http/resources/**", "/llms.txt", "/llms-full.txt", + "**/llms.txt", "{props.*}", "/", "/glossary/", diff --git a/src/pages/[product]/llms.txt.ts b/src/pages/[product]/llms.txt.ts new file mode 100644 index 00000000000000..b02a1ceb6ed831 --- /dev/null +++ b/src/pages/[product]/llms.txt.ts @@ -0,0 +1,62 @@ +import type { APIRoute, GetStaticPaths, InferGetStaticPropsType } from "astro"; +import { getCollection } from "astro:content"; +import dedent from "dedent"; + +export const getStaticPaths = (async () => { + const directory = await getCollection("directory", (p) => { + return !!p.data.entry.group; + }); + + const docs = await getCollection("docs"); + + return directory + .map((entry) => { + const prefix = entry.data.entry.url.slice(1, -1); + const pages = docs.filter( + (e) => e.id.startsWith(prefix + "/") || e.id === prefix, + ); + + if (pages.length === 0) return null; + + return { + params: { product: entry.id }, + props: { entry, pages }, + }; + }) + .filter((p) => p !== null); +}) satisfies GetStaticPaths; + +type Props = InferGetStaticPropsType; + +export const GET: APIRoute = async ({ props }) => { + const { entry, pages } = props; + const { title, url } = entry.data.entry; + const description = entry.data.meta?.description; + + const pageLinks = pages + .map((e) => { + const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.id}/index.md)`; + return e.data.description ? line.concat(`: ${e.data.description}`) : line; + }) + .join("\n"); + + const markdown = dedent(` + # ${title} + + ${description ?? ""} + + > Use [${title} llms-full.txt](https://developers.cloudflare.com${url}llms-full.txt) for the complete ${title} documentation in a single file. That file is intended for offline indexing, bulk vectorization, or large-context models. + > + > For other Cloudflare products, see the [Cloudflare documentation directory](https://developers.cloudflare.com/llms.txt). + + ## ${title} documentation pages + + ${pageLinks} + `); + + return new Response(markdown, { + headers: { + "content-type": "text/plain", + }, + }); +}; diff --git a/src/pages/llms.txt.ts b/src/pages/llms.txt.ts index e162367f470ccc..f36e277404ac12 100644 --- a/src/pages/llms.txt.ts +++ b/src/pages/llms.txt.ts @@ -3,48 +3,51 @@ import { getCollection } from "astro:content"; import dedent from "dedent"; export const GET: APIRoute = async () => { - const products = await getCollection("directory", (p) => { - return p.data.entry.group?.toLowerCase() === "developer platform"; + const directory = await getCollection("directory", (p) => { + return !!p.data.entry.group; }); - const docs = await getCollection("docs", (e) => { - return products.some((p) => e.id.startsWith(p.data.entry.url.slice(1, -1))); - }); - - const grouped = Object.entries( - Object.groupBy(docs, (e) => { - const product = products.find((p) => - e.id.startsWith(p.data.entry.url.slice(1, -1)), - ); + const docs = await getCollection("docs"); - if (!product) throw new Error(`Unable to find product for ${e.id}`); - - return product.data.entry.title; - }), + // Build a set of product IDs that actually have docs pages + const productsWithDocs = new Set( + directory + .filter((entry) => { + const prefix = entry.data.entry.url.slice(1, -1); + return docs.some( + (e) => e.id.startsWith(prefix + "/") || e.id === prefix, + ); + }) + .map((entry) => entry.id), ); + // Group products by their group, skipping any without docs pages + const grouped = Object.entries( + Object.groupBy( + directory.filter((entry) => productsWithDocs.has(entry.id)), + (entry) => entry.data.entry.group as string, + ), + ).sort(([a], [b]) => a.localeCompare(b)); + const markdown = dedent(` # Cloudflare Developer Documentation - Easily build and deploy full-stack applications everywhere, - thanks to integrated compute, storage, and networking. + Cloudflare's platform spans developer tools, network security, application performance, Zero Trust, and more. + + > For the complete documentation archive in a single file, use the [Full Documentation Archive](https://developers.cloudflare.com/llms-full.txt). That file is intended for offline indexing, bulk vectorization, or large-context models. + > + > For a specific product's full page index, follow that product's llms.txt link below. ${grouped - .map(([product, entries]) => { + .map(([group, entries]) => { return dedent(` - ## ${product} + ## ${group} ${entries - ?.map((e) => { - const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.id}/index.md)`; - - const description = e.data.description; - - if (description) { - return line.concat(`: ${description}`); - } - - return line; + ?.map((entry) => { + const line = `- [${entry.data.entry.title}](https://developers.cloudflare.com/${entry.id}/llms.txt)`; + const description = entry.data.meta?.description; + return description ? line.concat(`: ${description}`) : line; }) .join("\n")} `); diff --git a/src/util/sidebar.ts b/src/util/sidebar.ts index 65337a7bc63ffe..3187f97cf97c29 100644 --- a/src/util/sidebar.ts +++ b/src/util/sidebar.ts @@ -96,7 +96,7 @@ export async function generateSidebar(group: Group) { const product = directory.find((p) => p.id === group.label); if (product && product.data.entry.group === "Developer platform") { const links = [ - ["llms.txt", "/llms.txt"], + ["llms.txt", `/${product.id}/llms.txt`], ["prompt.txt", "/workers/prompt.txt"], [`${product.data.name} llms-full.txt`, `/${product.id}/llms-full.txt`], ["Developer Platform llms-full.txt", "/developer-platform/llms-full.txt"], From 7c6772f230cc5307abb365e0fb28f42a2ccecbcc Mon Sep 17 00:00:00 2001 From: mvm Date: Tue, 24 Feb 2026 16:25:45 -0600 Subject: [PATCH 2/6] use the origin url instead of hardcoded prod url --- src/pages/[product]/llms.txt.ts | 11 ++++++----- src/pages/llms.txt.ts | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/pages/[product]/llms.txt.ts b/src/pages/[product]/llms.txt.ts index b02a1ceb6ed831..ef1ae2d94f394e 100644 --- a/src/pages/[product]/llms.txt.ts +++ b/src/pages/[product]/llms.txt.ts @@ -28,14 +28,15 @@ export const getStaticPaths = (async () => { type Props = InferGetStaticPropsType; -export const GET: APIRoute = async ({ props }) => { +export const GET: APIRoute = async ({ props, url }) => { + const base = url.origin; const { entry, pages } = props; - const { title, url } = entry.data.entry; + const { title, url: productUrl } = entry.data.entry; const description = entry.data.meta?.description; const pageLinks = pages .map((e) => { - const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.id}/index.md)`; + const line = `- [${e.data.title}](${base}/${e.id}/index.md)`; return e.data.description ? line.concat(`: ${e.data.description}`) : line; }) .join("\n"); @@ -45,9 +46,9 @@ export const GET: APIRoute = async ({ props }) => { ${description ?? ""} - > Use [${title} llms-full.txt](https://developers.cloudflare.com${url}llms-full.txt) for the complete ${title} documentation in a single file. That file is intended for offline indexing, bulk vectorization, or large-context models. + > Use [${title} llms-full.txt](${base}${productUrl}llms-full.txt) for the complete ${title} documentation in a single file. That file is intended for offline indexing, bulk vectorization, or large-context models. > - > For other Cloudflare products, see the [Cloudflare documentation directory](https://developers.cloudflare.com/llms.txt). + > For other Cloudflare products, see the [Cloudflare documentation directory](${base}/llms.txt). ## ${title} documentation pages diff --git a/src/pages/llms.txt.ts b/src/pages/llms.txt.ts index f36e277404ac12..f7c582dbb8b18b 100644 --- a/src/pages/llms.txt.ts +++ b/src/pages/llms.txt.ts @@ -2,7 +2,8 @@ import type { APIRoute } from "astro"; import { getCollection } from "astro:content"; import dedent from "dedent"; -export const GET: APIRoute = async () => { +export const GET: APIRoute = async ({ url }) => { + const base = url.origin; const directory = await getCollection("directory", (p) => { return !!p.data.entry.group; }); @@ -32,9 +33,9 @@ export const GET: APIRoute = async () => { const markdown = dedent(` # Cloudflare Developer Documentation - Cloudflare's platform spans developer tools, network security, application performance, Zero Trust, and more. + Explore guides and tutorials to start building on Cloudflare's platform. - > For the complete documentation archive in a single file, use the [Full Documentation Archive](https://developers.cloudflare.com/llms-full.txt). That file is intended for offline indexing, bulk vectorization, or large-context models. + > For the complete documentation archive in a single file, use the [Full Documentation Archive](${base}/llms-full.txt). That file is intended for offline indexing, bulk vectorization, or large-context models. > > For a specific product's full page index, follow that product's llms.txt link below. @@ -45,7 +46,7 @@ export const GET: APIRoute = async () => { ${entries ?.map((entry) => { - const line = `- [${entry.data.entry.title}](https://developers.cloudflare.com/${entry.id}/llms.txt)`; + const line = `- [${entry.data.entry.title}](${base}/${entry.id}/llms.txt)`; const description = entry.data.meta?.description; return description ? line.concat(`: ${description}`) : line; }) From 63a48d22ba1946faeb4eb90c41dfe78b50aa2640 Mon Sep 17 00:00:00 2001 From: mvm Date: Wed, 25 Feb 2026 09:05:08 -0600 Subject: [PATCH 3/6] update style guide --- src/content/partials/style-guide/llms-txt.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/content/partials/style-guide/llms-txt.mdx b/src/content/partials/style-guide/llms-txt.mdx index 56d63449453fc0..06eca120d983f0 100644 --- a/src/content/partials/style-guide/llms-txt.mdx +++ b/src/content/partials/style-guide/llms-txt.mdx @@ -6,8 +6,9 @@ import { Width } from "~/components"; We have implemented `llms.txt` and `llms-full.txt` as follows: -- [`llms.txt`](/llms.txt) — An LLM-friendly directory of all available documentation pages for Dev Platform products in Markdown format. -- [`llms-full.txt`](/llms-full.txt) — The full contents of the Developer Docs for LLM consumption in Markdown format. We also provide a `llms-full.txt` file on a per-product basis - for example, [`/workers/llms-full.txt`](/workers/llms-full.txt). +- [`llms.txt`](/llms.txt) — A directory of all Cloudflare documentation products, grouped by category. Each entry links to that product's own `llms.txt` file. Use this as the starting point for discovering what documentation is available. +- `/{product}/llms.txt` — A per-product page index listing every documentation page for that product in Markdown format. For example, [`/workers/llms.txt`](/workers/llms.txt). Use this when you need to find a specific page within a product. +- [`llms-full.txt`](/llms-full.txt) — The full contents of all Cloudflare documentation in a single file, intended for offline indexing, bulk vectorization, or large-context models. We also provide a `llms-full.txt` file on a per-product basis — for example, [`/workers/llms-full.txt`](/workers/llms-full.txt). To obtain a Markdown version of a single documentation page, you can: From 9c060ce68bfa3e2b44db178a0362379c120cd28f Mon Sep 17 00:00:00 2001 From: mvm Date: Wed, 25 Feb 2026 09:06:32 -0600 Subject: [PATCH 4/6] match syntax --- src/content/partials/style-guide/llms-txt.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/partials/style-guide/llms-txt.mdx b/src/content/partials/style-guide/llms-txt.mdx index 06eca120d983f0..e4f6181d4c2a68 100644 --- a/src/content/partials/style-guide/llms-txt.mdx +++ b/src/content/partials/style-guide/llms-txt.mdx @@ -7,7 +7,7 @@ import { Width } from "~/components"; We have implemented `llms.txt` and `llms-full.txt` as follows: - [`llms.txt`](/llms.txt) — A directory of all Cloudflare documentation products, grouped by category. Each entry links to that product's own `llms.txt` file. Use this as the starting point for discovering what documentation is available. -- `/{product}/llms.txt` — A per-product page index listing every documentation page for that product in Markdown format. For example, [`/workers/llms.txt`](/workers/llms.txt). Use this when you need to find a specific page within a product. +- `/$product/llms.txt` — A per-product page index listing every documentation page for that product in Markdown format. For example, [`/workers/llms.txt`](/workers/llms.txt). Use this when you need to find a specific page within a product. - [`llms-full.txt`](/llms-full.txt) — The full contents of all Cloudflare documentation in a single file, intended for offline indexing, bulk vectorization, or large-context models. We also provide a `llms-full.txt` file on a per-product basis — for example, [`/workers/llms-full.txt`](/workers/llms-full.txt). To obtain a Markdown version of a single documentation page, you can: From 41b720b1ab3e57a1bf4afce2d021e3aea54e627d Mon Sep 17 00:00:00 2001 From: mvm Date: Wed, 25 Feb 2026 09:17:32 -0600 Subject: [PATCH 5/6] refactor ai style guide changes --- src/content/partials/style-guide/llms-txt.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/content/partials/style-guide/llms-txt.mdx b/src/content/partials/style-guide/llms-txt.mdx index e4f6181d4c2a68..f084a87a144ea6 100644 --- a/src/content/partials/style-guide/llms-txt.mdx +++ b/src/content/partials/style-guide/llms-txt.mdx @@ -6,8 +6,7 @@ import { Width } from "~/components"; We have implemented `llms.txt` and `llms-full.txt` as follows: -- [`llms.txt`](/llms.txt) — A directory of all Cloudflare documentation products, grouped by category. Each entry links to that product's own `llms.txt` file. Use this as the starting point for discovering what documentation is available. -- `/$product/llms.txt` — A per-product page index listing every documentation page for that product in Markdown format. For example, [`/workers/llms.txt`](/workers/llms.txt). Use this when you need to find a specific page within a product. +- [`llms.txt`](/llms.txt) — A directory of all Cloudflare documentation products, grouped by category. Each entry links to that product's own `llms.txt` — for example, [`/workers/llms.txt`](/workers/llms.txt) — which lists every page for that product in Markdown format. - [`llms-full.txt`](/llms-full.txt) — The full contents of all Cloudflare documentation in a single file, intended for offline indexing, bulk vectorization, or large-context models. We also provide a `llms-full.txt` file on a per-product basis — for example, [`/workers/llms-full.txt`](/workers/llms-full.txt). To obtain a Markdown version of a single documentation page, you can: From 26aa75ee140744e9c5a55ae50f364cb0db6221c7 Mon Sep 17 00:00:00 2001 From: mvm Date: Wed, 25 Feb 2026 09:53:52 -0600 Subject: [PATCH 6/6] tweak root llms.txt block quote verbage to not be confusing --- src/pages/llms.txt.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/llms.txt.ts b/src/pages/llms.txt.ts index f7c582dbb8b18b..c5df24bbc302bb 100644 --- a/src/pages/llms.txt.ts +++ b/src/pages/llms.txt.ts @@ -35,9 +35,9 @@ export const GET: APIRoute = async ({ url }) => { Explore guides and tutorials to start building on Cloudflare's platform. - > For the complete documentation archive in a single file, use the [Full Documentation Archive](${base}/llms-full.txt). That file is intended for offline indexing, bulk vectorization, or large-context models. + > Each product below links to its own llms.txt, which contains a full index of that product's documentation pages and is the recommended way to explore a specific product's content. > - > For a specific product's full page index, follow that product's llms.txt link below. + > For the complete documentation archive in a single file, use the [Full Documentation Archive](${base}/llms-full.txt). That file is intended for offline indexing, bulk vectorization, or large-context models. Each product's llms.txt also links to a product-scoped llms-full.txt. ${grouped .map(([group, entries]) => {