feat(site): split llms.txt into per-framework and blog sub-indexes (#697)

site/astro.config.mjs

@@ -57,7 +57,14 @@ export default defineConfig({
     }),
     mdx({ extendMarkdownConfig: true }),
     sitemap({
-      customPages: [`${SITE_URL}/llms.txt`],
+      // llms-markdown.ts auto-generates per-framework sub-indexes, but sitemap
+      // entries are hardcoded here. Add a new line when adding a framework.
+      customPages: [
+        `${SITE_URL}/llms.txt`,
+        `${SITE_URL}/blog/llms.txt`,
+        `${SITE_URL}/docs/framework/html/llms.txt`,
+        `${SITE_URL}/docs/framework/react/llms.txt`,
+      ],
     }),
     pagefind(),
     llmsMarkdown(),

site/integrations/llms-markdown.ts

@@ -5,6 +5,14 @@ import type { AstroIntegration } from 'astro';
 import { JSDOM } from 'jsdom';
 import TurndownService from 'turndown';
 
+interface PageEntry {
+  pathname: string;
+  title: string;
+  description?: string;
+  sort?: string;
+  framework?: string;
+}
+
 export default function llmsMarkdown(): AstroIntegration {
   return {
     name: 'llms-markdown',
@@ -18,9 +26,9 @@ export default function llmsMarkdown(): AstroIntegration {
         });
 
         // Track all docs and blog pages for llms.txt index
-        const docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
+        const docsPages: PageEntry[] = [];
-        const blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
+        const blogPages: PageEntry[] = [];
-        const otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
+        const otherPages: PageEntry[] = [];
 
         logger.info('Generating LLM-optimized markdown files...');
 
@@ -71,15 +79,18 @@ export default function llmsMarkdown(): AstroIntegration {
             const sortAttr = contentElements[0]?.getAttribute('data-llms-sort');
             const sort = sortAttr || undefined;
 
+            const frameworkAttr = contentElements[0]?.getAttribute('data-framework');
+            const framework = frameworkAttr || undefined;
+
             // Write markdown file as sibling to the directory
-            // docs/framework/html/style/css/slug -> docs/framework/html/style/css/slug.md
+            // docs/framework/html/how-to/slug -> docs/framework/html/how-to/slug.md
             const mdPath = join(siteDir, `${pathname}.md`);
             await mkdir(dirname(mdPath), { recursive: true });
             await writeFile(mdPath, markdown, 'utf-8');
 
             // Track for llms.txt index (with leading slash for URLs)
             if (pathname.startsWith('docs/')) {
-              docsPages.push({ pathname: `/${pathname}`, title, description, sort });
+              docsPages.push({ pathname: `/${pathname}`, title, description, sort, framework });
             } else if (pathname.startsWith('blog/')) {
               blogPages.push({ pathname: `/${pathname}`, title, description, sort });
             } else {
@@ -90,113 +101,72 @@ export default function llmsMarkdown(): AstroIntegration {
           }
         }
 
-        // Generate llms.txt index file
+        // Group docs by framework
-        const llmsTxt = generateLlmsTxt(docsPages, blogPages, otherPages);
+        const docsByFramework = new Map<string, PageEntry[]>();
-        const llmsTxtPath = join(siteDir, 'llms.txt');
+        for (const doc of docsPages) {
-        await writeFile(llmsTxtPath, llmsTxt, 'utf-8');
+          const fw = doc.framework ?? 'unknown';
+          if (!docsByFramework.has(fw)) {
+            docsByFramework.set(fw, []);
+          }
+          docsByFramework.get(fw)!.push(doc);
+        }
 
+        // Write per-framework docs sub-indexes
+        const frameworks: string[] = [];
+        for (const [fw, fwPages] of docsByFramework) {
+          frameworks.push(fw);
+          const subIndex = generateDocsIndex(fw, fwPages);
+          const subIndexPath = join(siteDir, 'docs', 'framework', fw, 'llms.txt');
+          await mkdir(dirname(subIndexPath), { recursive: true });
+          await writeFile(subIndexPath, subIndex, 'utf-8');
+        }
+
+        // Write blog sub-index
+        if (blogPages.length > 0) {
+          const blogIndex = generateBlogIndex(blogPages);
+          const blogIndexPath = join(siteDir, 'blog', 'llms.txt');
+          await mkdir(dirname(blogIndexPath), { recursive: true });
+          await writeFile(blogIndexPath, blogIndex, 'utf-8');
+        }
+
+        // Write root llms.txt index
+        const rootIndex = generateRootIndex(frameworks, blogPages.length > 0, otherPages);
+        const rootIndexPath = join(siteDir, 'llms.txt');
+        await writeFile(rootIndexPath, rootIndex, 'utf-8');
+
+        const subIndexCount = frameworks.length + (blogPages.length > 0 ? 1 : 0);
         logger.info(
-          `Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files and llms.txt index`
+          `Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files, llms.txt root index, and ${subIndexCount} sub-indexes`
         );
       },
     },
   };
 }
 
-function generateLlmsTxt(
+function generateRootIndex(frameworks: string[], hasBlog: boolean, otherPages: PageEntry[]): string {
-  docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
-  blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
-  otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>
-): string {
-  // Group docs by framework and style
-  const docsByFrameworkStyle = new Map<
-    string,
-    Array<{ pathname: string; title: string; description?: string; sort?: string }>
-  >();
-
-  for (const doc of docsPages) {
-    // Extract framework and style from pathname
-    // Pattern: /docs/framework/{framework}/style/{style}/{...slug}
-    const match = doc.pathname.match(/^\/docs\/framework\/([^/]+)\/style\/([^/]+)\//);
-    if (match) {
-      const [, framework, style] = match;
-      const key = `${framework}/${style}`;
-      if (!docsByFrameworkStyle.has(key)) {
-        docsByFrameworkStyle.set(key, []);
-      }
-      docsByFrameworkStyle.get(key)!.push(doc);
-    }
-  }
-
-  // Build llms.txt content
   let content = `# Video.js v10\n\n`;
   content += `> Modern video player framework with multi-platform support\n\n`;
 
-  // Add documentation sections grouped by framework/style
+  content += `## Documentation\n\n`;
-  if (docsByFrameworkStyle.size > 0) {
+  for (const fw of [...frameworks].sort()) {
-    content += `## Documentation\n\n`;
+    const label = fw.charAt(0).toUpperCase() + fw.slice(1);
+    content += `- [${label} Docs](/docs/framework/${fw}/llms.txt)\n`;
+  }
+  content += `\n`;
 
-    // Sort by framework/style for consistent output
+  if (hasBlog) {
-    const sortedKeys = Array.from(docsByFrameworkStyle.keys()).sort();
+    content += `## Blog\n\n`;
-
+    content += `- [Blog Posts](/blog/llms.txt)\n\n`;
-    for (const key of sortedKeys) {
-      const [framework, style] = key.split('/');
-      const frameworkLabel = framework.charAt(0).toUpperCase() + framework.slice(1);
-      const styleLabel = style.toUpperCase();
-
-      content += `### ${frameworkLabel} + ${styleLabel}\n\n`;
-
-      const docs = docsByFrameworkStyle.get(key)!;
-      // Sort docs by pathname for consistent output
-      docs.sort((a, b) => a.pathname.localeCompare(b.pathname));
-
-      for (const doc of docs) {
-        if (doc.description) {
-          content += `- [${doc.title}](${doc.pathname}): ${doc.description}\n`;
-        } else {
-          content += `- [${doc.title}](${doc.pathname})\n`;
-        }
-      }
-      content += `\n`;
-    }
   }
 
-  // Add blog posts section
-  if (blogPages.length > 0) {
-    content += `## Blog Posts\n\n`;
-
-    // Sort by date using data-llms-sort attribute in reverse order (newest first)
-    const sortedBlogPages = [...blogPages].sort((a, b) => {
-      // If both have sort attributes, compare them (reverse for newest first)
-      if (a.sort && b.sort) {
-        return b.sort.localeCompare(a.sort);
-      }
-      // Fallback to pathname comparison if sort is missing
-      return b.pathname.localeCompare(a.pathname);
-    });
-
-    for (const post of sortedBlogPages) {
-      if (post.description) {
-        content += `- [${post.title}](${post.pathname}): ${post.description}\n`;
-      } else {
-        content += `- [${post.title}](${post.pathname})\n`;
-      }
-    }
-    content += `\n`;
-  }
-
-  // Add other pages section
   if (otherPages.length > 0) {
     content += `## Other\n\n`;
-
+    const sorted = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
-    // Sort by pathname
+    for (const page of sorted) {
-    const sortedOtherPages = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
-
-    for (const page of sortedOtherPages) {
       if (page.description) {
-        content += `- [${page.title}](${page.pathname}): ${page.description}\n`;
+        content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
       } else {
-        content += `- [${page.title}](${page.pathname})\n`;
+        content += `- [${page.title}](${page.pathname}.md)\n`;
       }
     }
     content += `\n`;
@@ -204,3 +174,43 @@ function generateLlmsTxt(
 
   return content;
 }
+
+function generateDocsIndex(framework: string, pages: PageEntry[]): string {
+  const label = framework.charAt(0).toUpperCase() + framework.slice(1);
+  let content = `# Video.js v10 — ${label} Documentation\n\n`;
+
+  const sorted = [...pages].sort((a, b) => a.pathname.localeCompare(b.pathname));
+  for (const page of sorted) {
+    if (page.description) {
+      content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
+    } else {
+      content += `- [${page.title}](${page.pathname}.md)\n`;
+    }
+  }
+  content += `\n`;
+
+  return content;
+}
+
+function generateBlogIndex(pages: PageEntry[]): string {
+  let content = `# Video.js v10 — Blog\n\n`;
+
+  // Newest first
+  const sorted = [...pages].sort((a, b) => {
+    if (a.sort && b.sort) {
+      return b.sort.localeCompare(a.sort);
+    }
+    return b.pathname.localeCompare(a.pathname);
+  });
+
+  for (const post of sorted) {
+    if (post.description) {
+      content += `- [${post.title}](${post.pathname}.md): ${post.description}\n`;
+    } else {
+      content += `- [${post.title}](${post.pathname}.md)\n`;
+    }
+  }
+  content += `\n`;
+
+  return content;
+}