feat(llms): add LLM text processing and routing for MDX and TXT files (#470)

Author: adiologydev

apps/docs/app/llms.mdx/[[...slug]]/route.ts

@@ -0,0 +1,18 @@
+import { notFound } from 'next/navigation'
+import { type NextRequest, NextResponse } from 'next/server'
+import { getLLMText } from '@/lib/llms'
+import { source } from '@/lib/source'
+
+export const revalidate = false
+
+export async function GET(_req: NextRequest, { params }: { params: Promise<{ slug?: string[] }> }) {
+  const { slug } = await params
+  const page = source.getPage(slug)
+  if (!page) notFound()
+
+  return new NextResponse(await getLLMText(page))
+}
+
+export function generateStaticParams() {
+  return source.generateParams()
+}

apps/docs/app/llms.txt/route.ts

@@ -0,0 +1,12 @@
+import { getLLMText } from '@/lib/llms'
+import { source } from '@/lib/source'
+
+// cached forever
+export const revalidate = false
+
+export async function GET() {
+  const scan = source.getPages().map(getLLMText)
+  const scanned = await Promise.all(scan)
+
+  return new Response(scanned.join('\n\n'))
+}

apps/docs/lib/llms.ts

@@ -0,0 +1,22 @@
+import type { InferPageType } from 'fumadocs-core/source'
+import { remarkInclude } from 'fumadocs-mdx/config'
+import { remark } from 'remark'
+import remarkGfm from 'remark-gfm'
+import remarkMdx from 'remark-mdx'
+import type { source } from '@/lib/source'
+
+const processor = remark().use(remarkMdx).use(remarkInclude).use(remarkGfm)
+
+export async function getLLMText(page: InferPageType<typeof source>) {
+  const processed = await processor.process({
+    path: page.data._file.absolutePath,
+    value: page.data.content,
+  })
+
+  return `# ${page.data.title}
+URL: ${page.url}
+
+${page.data.description}
+
+${processed.value}`
+}

apps/docs/next.config.ts

@@ -12,6 +12,11 @@ const config = {
         destination: '/introduction',
         permanent: true,
       },
+      {
+        source: '/docs/:path*.mdx',
+        destination: '/llms.mdx/:path*',
+        permanent: true,
+      },
     ]
   },
 }