From 916f4700aef79744e1cec050c79ff4abc70431a8 Mon Sep 17 00:00:00 2001 From: Darius Cepulis Date: Tue, 3 Mar 2026 17:18:26 -0600 Subject: [PATCH] feat(site): split llms.txt into per-framework and blog sub-indexes (#697) --- site/astro.config.mjs | 9 +- site/integrations/llms-markdown.ts | 198 +++++++++++++++-------------- 2 files changed, 112 insertions(+), 95 deletions(-) diff --git a/site/astro.config.mjs b/site/astro.config.mjs index f538e114..7e0ccf40 100644 --- a/site/astro.config.mjs +++ b/site/astro.config.mjs @@ -57,7 +57,14 @@ export default defineConfig({ }), mdx({ extendMarkdownConfig: true }), sitemap({ - customPages: [`${SITE_URL}/llms.txt`], + // llms-markdown.ts auto-generates per-framework sub-indexes, but sitemap + // entries are hardcoded here. Add a new line when adding a framework. + customPages: [ + `${SITE_URL}/llms.txt`, + `${SITE_URL}/blog/llms.txt`, + `${SITE_URL}/docs/framework/html/llms.txt`, + `${SITE_URL}/docs/framework/react/llms.txt`, + ], }), pagefind(), llmsMarkdown(), diff --git a/site/integrations/llms-markdown.ts b/site/integrations/llms-markdown.ts index 6a41b826..1a28772b 100644 --- a/site/integrations/llms-markdown.ts +++ b/site/integrations/llms-markdown.ts @@ -5,6 +5,14 @@ import type { AstroIntegration } from 'astro'; import { JSDOM } from 'jsdom'; import TurndownService from 'turndown'; +interface PageEntry { + pathname: string; + title: string; + description?: string; + sort?: string; + framework?: string; +} + export default function llmsMarkdown(): AstroIntegration { return { name: 'llms-markdown', @@ -18,9 +26,9 @@ export default function llmsMarkdown(): AstroIntegration { }); // Track all docs and blog pages for llms.txt index - const docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = []; - const blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = []; - const otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = []; + const docsPages: PageEntry[] = []; + const blogPages: PageEntry[] = []; + const otherPages: PageEntry[] = []; logger.info('Generating LLM-optimized markdown files...'); @@ -71,15 +79,18 @@ export default function llmsMarkdown(): AstroIntegration { const sortAttr = contentElements[0]?.getAttribute('data-llms-sort'); const sort = sortAttr || undefined; + const frameworkAttr = contentElements[0]?.getAttribute('data-framework'); + const framework = frameworkAttr || undefined; + // Write markdown file as sibling to the directory - // docs/framework/html/style/css/slug -> docs/framework/html/style/css/slug.md + // docs/framework/html/how-to/slug -> docs/framework/html/how-to/slug.md const mdPath = join(siteDir, `${pathname}.md`); await mkdir(dirname(mdPath), { recursive: true }); await writeFile(mdPath, markdown, 'utf-8'); // Track for llms.txt index (with leading slash for URLs) if (pathname.startsWith('docs/')) { - docsPages.push({ pathname: `/${pathname}`, title, description, sort }); + docsPages.push({ pathname: `/${pathname}`, title, description, sort, framework }); } else if (pathname.startsWith('blog/')) { blogPages.push({ pathname: `/${pathname}`, title, description, sort }); } else { @@ -90,113 +101,72 @@ export default function llmsMarkdown(): AstroIntegration { } } - // Generate llms.txt index file - const llmsTxt = generateLlmsTxt(docsPages, blogPages, otherPages); - const llmsTxtPath = join(siteDir, 'llms.txt'); - await writeFile(llmsTxtPath, llmsTxt, 'utf-8'); + // Group docs by framework + const docsByFramework = new Map(); + for (const doc of docsPages) { + const fw = doc.framework ?? 'unknown'; + if (!docsByFramework.has(fw)) { + docsByFramework.set(fw, []); + } + docsByFramework.get(fw)!.push(doc); + } + // Write per-framework docs sub-indexes + const frameworks: string[] = []; + for (const [fw, fwPages] of docsByFramework) { + frameworks.push(fw); + const subIndex = generateDocsIndex(fw, fwPages); + const subIndexPath = join(siteDir, 'docs', 'framework', fw, 'llms.txt'); + await mkdir(dirname(subIndexPath), { recursive: true }); + await writeFile(subIndexPath, subIndex, 'utf-8'); + } + + // Write blog sub-index + if (blogPages.length > 0) { + const blogIndex = generateBlogIndex(blogPages); + const blogIndexPath = join(siteDir, 'blog', 'llms.txt'); + await mkdir(dirname(blogIndexPath), { recursive: true }); + await writeFile(blogIndexPath, blogIndex, 'utf-8'); + } + + // Write root llms.txt index + const rootIndex = generateRootIndex(frameworks, blogPages.length > 0, otherPages); + const rootIndexPath = join(siteDir, 'llms.txt'); + await writeFile(rootIndexPath, rootIndex, 'utf-8'); + + const subIndexCount = frameworks.length + (blogPages.length > 0 ? 1 : 0); logger.info( - `Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files and llms.txt index` + `Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files, llms.txt root index, and ${subIndexCount} sub-indexes` ); }, }, }; } -function generateLlmsTxt( - docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>, - blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>, - otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> -): string { - // Group docs by framework and style - const docsByFrameworkStyle = new Map< - string, - Array<{ pathname: string; title: string; description?: string; sort?: string }> - >(); - - for (const doc of docsPages) { - // Extract framework and style from pathname - // Pattern: /docs/framework/{framework}/style/{style}/{...slug} - const match = doc.pathname.match(/^\/docs\/framework\/([^/]+)\/style\/([^/]+)\//); - if (match) { - const [, framework, style] = match; - const key = `${framework}/${style}`; - if (!docsByFrameworkStyle.has(key)) { - docsByFrameworkStyle.set(key, []); - } - docsByFrameworkStyle.get(key)!.push(doc); - } - } - - // Build llms.txt content +function generateRootIndex(frameworks: string[], hasBlog: boolean, otherPages: PageEntry[]): string { let content = `# Video.js v10\n\n`; content += `> Modern video player framework with multi-platform support\n\n`; - // Add documentation sections grouped by framework/style - if (docsByFrameworkStyle.size > 0) { - content += `## Documentation\n\n`; + content += `## Documentation\n\n`; + for (const fw of [...frameworks].sort()) { + const label = fw.charAt(0).toUpperCase() + fw.slice(1); + content += `- [${label} Docs](/docs/framework/${fw}/llms.txt)\n`; + } + content += `\n`; - // Sort by framework/style for consistent output - const sortedKeys = Array.from(docsByFrameworkStyle.keys()).sort(); - - for (const key of sortedKeys) { - const [framework, style] = key.split('/'); - const frameworkLabel = framework.charAt(0).toUpperCase() + framework.slice(1); - const styleLabel = style.toUpperCase(); - - content += `### ${frameworkLabel} + ${styleLabel}\n\n`; - - const docs = docsByFrameworkStyle.get(key)!; - // Sort docs by pathname for consistent output - docs.sort((a, b) => a.pathname.localeCompare(b.pathname)); - - for (const doc of docs) { - if (doc.description) { - content += `- [${doc.title}](${doc.pathname}): ${doc.description}\n`; - } else { - content += `- [${doc.title}](${doc.pathname})\n`; - } - } - content += `\n`; - } + if (hasBlog) { + content += `## Blog\n\n`; + content += `- [Blog Posts](/blog/llms.txt)\n\n`; } - // Add blog posts section - if (blogPages.length > 0) { - content += `## Blog Posts\n\n`; - - // Sort by date using data-llms-sort attribute in reverse order (newest first) - const sortedBlogPages = [...blogPages].sort((a, b) => { - // If both have sort attributes, compare them (reverse for newest first) - if (a.sort && b.sort) { - return b.sort.localeCompare(a.sort); - } - // Fallback to pathname comparison if sort is missing - return b.pathname.localeCompare(a.pathname); - }); - - for (const post of sortedBlogPages) { - if (post.description) { - content += `- [${post.title}](${post.pathname}): ${post.description}\n`; - } else { - content += `- [${post.title}](${post.pathname})\n`; - } - } - content += `\n`; - } - - // Add other pages section if (otherPages.length > 0) { content += `## Other\n\n`; - - // Sort by pathname - const sortedOtherPages = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname)); - - for (const page of sortedOtherPages) { + const sorted = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname)); + for (const page of sorted) { if (page.description) { - content += `- [${page.title}](${page.pathname}): ${page.description}\n`; + content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`; } else { - content += `- [${page.title}](${page.pathname})\n`; + content += `- [${page.title}](${page.pathname}.md)\n`; } } content += `\n`; @@ -204,3 +174,43 @@ function generateLlmsTxt( return content; } + +function generateDocsIndex(framework: string, pages: PageEntry[]): string { + const label = framework.charAt(0).toUpperCase() + framework.slice(1); + let content = `# Video.js v10 — ${label} Documentation\n\n`; + + const sorted = [...pages].sort((a, b) => a.pathname.localeCompare(b.pathname)); + for (const page of sorted) { + if (page.description) { + content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`; + } else { + content += `- [${page.title}](${page.pathname}.md)\n`; + } + } + content += `\n`; + + return content; +} + +function generateBlogIndex(pages: PageEntry[]): string { + let content = `# Video.js v10 — Blog\n\n`; + + // Newest first + const sorted = [...pages].sort((a, b) => { + if (a.sort && b.sort) { + return b.sort.localeCompare(a.sort); + } + return b.pathname.localeCompare(a.pathname); + }); + + for (const post of sorted) { + if (post.description) { + content += `- [${post.title}](${post.pathname}.md): ${post.description}\n`; + } else { + content += `- [${post.title}](${post.pathname}.md)\n`; + } + } + content += `\n`; + + return content; +}