Refactor llms.txt and add per-product llms.txt

astro.config.ts

@@ -144,6 +144,7 @@ export default defineConfig({
 									"/http/resources/**",
 									"/llms.txt",
 									"/llms-full.txt",
+									"**/llms.txt",
 									"{props.*}",
 									"/",
 									"/glossary/",

src/content/partials/style-guide/llms-txt.mdx

@@ -6,8 +6,8 @@ import { Width } from "~/components";
 
 We have implemented `llms.txt` and `llms-full.txt` as follows:
 
-- [`llms.txt`](/llms.txt) — An LLM-friendly directory of all available documentation pages for Dev Platform products in Markdown format.
-- [`llms-full.txt`](/llms-full.txt) — The full contents of the Developer Docs for LLM consumption in Markdown format. We also provide a `llms-full.txt` file on a per-product basis - for example, [`/workers/llms-full.txt`](/workers/llms-full.txt).
+- [`llms.txt`](/llms.txt) — A directory of all Cloudflare documentation products, grouped by category. Each entry links to that product's own `llms.txt` — for example, [`/workers/llms.txt`](/workers/llms.txt) — which lists every page for that product in Markdown format.
+- [`llms-full.txt`](/llms-full.txt) — The full contents of all Cloudflare documentation in a single file, intended for offline indexing, bulk vectorization, or large-context models. We also provide a `llms-full.txt` file on a per-product basis — for example, [`/workers/llms-full.txt`](/workers/llms-full.txt).
 
 To obtain a Markdown version of a single documentation page, you can:
 

src/pages/[product]/llms.txt.ts

@@ -0,0 +1,63 @@
+import type { APIRoute, GetStaticPaths, InferGetStaticPropsType } from "astro";
+import { getCollection } from "astro:content";
+import dedent from "dedent";
+
+export const getStaticPaths = (async () => {
+	const directory = await getCollection("directory", (p) => {
+		return !!p.data.entry.group;
+	});
+
+	const docs = await getCollection("docs");
+
+	return directory
+		.map((entry) => {
+			const prefix = entry.data.entry.url.slice(1, -1);
+			const pages = docs.filter(
+				(e) => e.id.startsWith(prefix + "/") || e.id === prefix,
+			);
+
+			if (pages.length === 0) return null;
+
+			return {
+				params: { product: entry.id },
+				props: { entry, pages },
+			};
+		})
+		.filter((p) => p !== null);
+}) satisfies GetStaticPaths;
+
+type Props = InferGetStaticPropsType<typeof getStaticPaths>;
+
+export const GET: APIRoute<Props> = async ({ props, url }) => {
+	const base = url.origin;
+	const { entry, pages } = props;
+	const { title, url: productUrl } = entry.data.entry;
+	const description = entry.data.meta?.description;
+
+	const pageLinks = pages
+		.map((e) => {
+			const line = `- [${e.data.title}](${base}/${e.id}/index.md)`;
+			return e.data.description ? line.concat(`: ${e.data.description}`) : line;
+		})
+		.join("\n");
+
+	const markdown = dedent(`
+		# ${title}
+
+		${description ?? ""}
+
+		> Use [${title} llms-full.txt](${base}${productUrl}llms-full.txt) for the complete ${title} documentation in a single file. That file is intended for offline indexing, bulk vectorization, or large-context models.
+		>
+		> For other Cloudflare products, see the [Cloudflare documentation directory](${base}/llms.txt).
+
+		## ${title} documentation pages
+
+		${pageLinks}
+	`);
+
+	return new Response(markdown, {
+		headers: {
+			"content-type": "text/plain",
+		},
+	});
+};

src/pages/llms.txt.ts

@@ -2,49 +2,53 @@ import type { APIRoute } from "astro";
 import { getCollection } from "astro:content";
 import dedent from "dedent";
 
-export const GET: APIRoute = async () => {
-	const products = await getCollection("directory", (p) => {
-		return p.data.entry.group?.toLowerCase() === "developer platform";
+export const GET: APIRoute = async ({ url }) => {
+	const base = url.origin;
+	const directory = await getCollection("directory", (p) => {
+		return !!p.data.entry.group;
 	});
 
-	const docs = await getCollection("docs", (e) => {
-		return products.some((p) => e.id.startsWith(p.data.entry.url.slice(1, -1)));
-	});
-
-	const grouped = Object.entries(
-		Object.groupBy(docs, (e) => {
-			const product = products.find((p) =>
-				e.id.startsWith(p.data.entry.url.slice(1, -1)),
-			);
+	const docs = await getCollection("docs");
 
-			if (!product) throw new Error(`Unable to find product for ${e.id}`);
-
-			return product.data.entry.title;
-		}),
+	// Build a set of product IDs that actually have docs pages
+	const productsWithDocs = new Set(
+		directory
+			.filter((entry) => {
+				const prefix = entry.data.entry.url.slice(1, -1);
+				return docs.some(
+					(e) => e.id.startsWith(prefix + "/") || e.id === prefix,
+				);
+			})
+			.map((entry) => entry.id),
 	);
 
+	// Group products by their group, skipping any without docs pages
+	const grouped = Object.entries(
+		Object.groupBy(
+			directory.filter((entry) => productsWithDocs.has(entry.id)),
+			(entry) => entry.data.entry.group as string,
+		),
+	).sort(([a], [b]) => a.localeCompare(b));
+
 	const markdown = dedent(`
 		# Cloudflare Developer Documentation
 
-		Easily build and deploy full-stack applications everywhere,
-		thanks to integrated compute, storage, and networking.
+		Explore guides and tutorials to start building on Cloudflare's platform.
+
+		> Each product below links to its own llms.txt, which contains a full index of that product's documentation pages and is the recommended way to explore a specific product's content.
+		>
+		> For the complete documentation archive in a single file, use the [Full Documentation Archive](${base}/llms-full.txt). That file is intended for offline indexing, bulk vectorization, or large-context models. Each product's llms.txt also links to a product-scoped llms-full.txt.
 
 		${grouped
-			.map(([product, entries]) => {
+			.map(([group, entries]) => {
 				return dedent(`
-				## ${product}
+				## ${group}
 
 				${entries
-					?.map((e) => {
-						const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.id}/index.md)`;
-
-						const description = e.data.description;
-
-						if (description) {
-							return line.concat(`: ${description}`);
-						}
-
-						return line;
+					?.map((entry) => {
+						const line = `- [${entry.data.entry.title}](${base}/${entry.id}/llms.txt)`;
+						const description = entry.data.meta?.description;
+						return description ? line.concat(`: ${description}`) : line;
 					})
 					.join("\n")}
 			`);

src/util/sidebar.ts

@@ -96,7 +96,7 @@ export async function generateSidebar(group: Group) {
 	const product = directory.find((p) => p.id === group.label);
 	if (product && product.data.entry.group === "Developer platform") {
 		const links = [
-			["llms.txt", "/llms.txt"],
+			["llms.txt", `/${product.id}/llms.txt`],
 			["prompt.txt", "/workers/prompt.txt"],
 			[`${product.data.name} llms-full.txt`, `/${product.id}/llms-full.txt`],
 			["Developer Platform llms-full.txt", "/developer-platform/llms-full.txt"],