diff --git a/astro.config.ts b/astro.config.ts index 1a67a778247057..c21e9591cc01b0 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -144,6 +144,7 @@ export default defineConfig({ "/http/resources/**", "/llms.txt", "/llms-full.txt", + "**/llms.txt", "{props.*}", "/", "/glossary/", diff --git a/src/content/partials/style-guide/llms-txt.mdx b/src/content/partials/style-guide/llms-txt.mdx index 56d63449453fc0..f084a87a144ea6 100644 --- a/src/content/partials/style-guide/llms-txt.mdx +++ b/src/content/partials/style-guide/llms-txt.mdx @@ -6,8 +6,8 @@ import { Width } from "~/components"; We have implemented `llms.txt` and `llms-full.txt` as follows: -- [`llms.txt`](/llms.txt) — An LLM-friendly directory of all available documentation pages for Dev Platform products in Markdown format. -- [`llms-full.txt`](/llms-full.txt) — The full contents of the Developer Docs for LLM consumption in Markdown format. We also provide a `llms-full.txt` file on a per-product basis - for example, [`/workers/llms-full.txt`](/workers/llms-full.txt). +- [`llms.txt`](/llms.txt) — A directory of all Cloudflare documentation products, grouped by category. Each entry links to that product's own `llms.txt` — for example, [`/workers/llms.txt`](/workers/llms.txt) — which lists every page for that product in Markdown format. +- [`llms-full.txt`](/llms-full.txt) — The full contents of all Cloudflare documentation in a single file, intended for offline indexing, bulk vectorization, or large-context models. We also provide a `llms-full.txt` file on a per-product basis — for example, [`/workers/llms-full.txt`](/workers/llms-full.txt). To obtain a Markdown version of a single documentation page, you can: diff --git a/src/pages/[product]/llms.txt.ts b/src/pages/[product]/llms.txt.ts new file mode 100644 index 00000000000000..ef1ae2d94f394e --- /dev/null +++ b/src/pages/[product]/llms.txt.ts @@ -0,0 +1,63 @@ +import type { APIRoute, GetStaticPaths, InferGetStaticPropsType } from "astro"; +import { getCollection } from "astro:content"; +import dedent from "dedent"; + +export const getStaticPaths = (async () => { + const directory = await getCollection("directory", (p) => { + return !!p.data.entry.group; + }); + + const docs = await getCollection("docs"); + + return directory + .map((entry) => { + const prefix = entry.data.entry.url.slice(1, -1); + const pages = docs.filter( + (e) => e.id.startsWith(prefix + "/") || e.id === prefix, + ); + + if (pages.length === 0) return null; + + return { + params: { product: entry.id }, + props: { entry, pages }, + }; + }) + .filter((p) => p !== null); +}) satisfies GetStaticPaths; + +type Props = InferGetStaticPropsType; + +export const GET: APIRoute = async ({ props, url }) => { + const base = url.origin; + const { entry, pages } = props; + const { title, url: productUrl } = entry.data.entry; + const description = entry.data.meta?.description; + + const pageLinks = pages + .map((e) => { + const line = `- [${e.data.title}](${base}/${e.id}/index.md)`; + return e.data.description ? line.concat(`: ${e.data.description}`) : line; + }) + .join("\n"); + + const markdown = dedent(` + # ${title} + + ${description ?? ""} + + > Use [${title} llms-full.txt](${base}${productUrl}llms-full.txt) for the complete ${title} documentation in a single file. That file is intended for offline indexing, bulk vectorization, or large-context models. + > + > For other Cloudflare products, see the [Cloudflare documentation directory](${base}/llms.txt). + + ## ${title} documentation pages + + ${pageLinks} + `); + + return new Response(markdown, { + headers: { + "content-type": "text/plain", + }, + }); +}; diff --git a/src/pages/llms.txt.ts b/src/pages/llms.txt.ts index e162367f470ccc..c5df24bbc302bb 100644 --- a/src/pages/llms.txt.ts +++ b/src/pages/llms.txt.ts @@ -2,49 +2,53 @@ import type { APIRoute } from "astro"; import { getCollection } from "astro:content"; import dedent from "dedent"; -export const GET: APIRoute = async () => { - const products = await getCollection("directory", (p) => { - return p.data.entry.group?.toLowerCase() === "developer platform"; +export const GET: APIRoute = async ({ url }) => { + const base = url.origin; + const directory = await getCollection("directory", (p) => { + return !!p.data.entry.group; }); - const docs = await getCollection("docs", (e) => { - return products.some((p) => e.id.startsWith(p.data.entry.url.slice(1, -1))); - }); - - const grouped = Object.entries( - Object.groupBy(docs, (e) => { - const product = products.find((p) => - e.id.startsWith(p.data.entry.url.slice(1, -1)), - ); + const docs = await getCollection("docs"); - if (!product) throw new Error(`Unable to find product for ${e.id}`); - - return product.data.entry.title; - }), + // Build a set of product IDs that actually have docs pages + const productsWithDocs = new Set( + directory + .filter((entry) => { + const prefix = entry.data.entry.url.slice(1, -1); + return docs.some( + (e) => e.id.startsWith(prefix + "/") || e.id === prefix, + ); + }) + .map((entry) => entry.id), ); + // Group products by their group, skipping any without docs pages + const grouped = Object.entries( + Object.groupBy( + directory.filter((entry) => productsWithDocs.has(entry.id)), + (entry) => entry.data.entry.group as string, + ), + ).sort(([a], [b]) => a.localeCompare(b)); + const markdown = dedent(` # Cloudflare Developer Documentation - Easily build and deploy full-stack applications everywhere, - thanks to integrated compute, storage, and networking. + Explore guides and tutorials to start building on Cloudflare's platform. + + > Each product below links to its own llms.txt, which contains a full index of that product's documentation pages and is the recommended way to explore a specific product's content. + > + > For the complete documentation archive in a single file, use the [Full Documentation Archive](${base}/llms-full.txt). That file is intended for offline indexing, bulk vectorization, or large-context models. Each product's llms.txt also links to a product-scoped llms-full.txt. ${grouped - .map(([product, entries]) => { + .map(([group, entries]) => { return dedent(` - ## ${product} + ## ${group} ${entries - ?.map((e) => { - const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.id}/index.md)`; - - const description = e.data.description; - - if (description) { - return line.concat(`: ${description}`); - } - - return line; + ?.map((entry) => { + const line = `- [${entry.data.entry.title}](${base}/${entry.id}/llms.txt)`; + const description = entry.data.meta?.description; + return description ? line.concat(`: ${description}`) : line; }) .join("\n")} `); diff --git a/src/util/sidebar.ts b/src/util/sidebar.ts index 65337a7bc63ffe..3187f97cf97c29 100644 --- a/src/util/sidebar.ts +++ b/src/util/sidebar.ts @@ -96,7 +96,7 @@ export async function generateSidebar(group: Group) { const product = directory.find((p) => p.id === group.label); if (product && product.data.entry.group === "Developer platform") { const links = [ - ["llms.txt", "/llms.txt"], + ["llms.txt", `/${product.id}/llms.txt`], ["prompt.txt", "/workers/prompt.txt"], [`${product.data.name} llms-full.txt`, `/${product.id}/llms-full.txt`], ["Developer Platform llms-full.txt", "/developer-platform/llms-full.txt"],