feat(site): organize llms.txt indexes by sidebar structure (#1203)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

site/integrations/llms-markdown.ts

@@ -5,6 +5,10 @@ import type { AstroIntegration } from 'astro';
 import { JSDOM } from 'jsdom';
 import TurndownService from 'turndown';
 
+import { sidebar } from '../src/docs.config';
+import type { Sidebar, SupportedFramework } from '../src/types/docs';
+import { isSection, isValidFramework } from '../src/types/docs';
+
 interface PageEntry {
   pathname: string;
   title: string;
@@ -211,16 +215,80 @@ function generateRootIndex(frameworks: string[], hasBlog: boolean, otherPages: P
 
 function generateDocsIndex(framework: string, pages: PageEntry[], siteUrl: string): string {
   let content = `# Video.js v10 — ${capitalize(framework)} Documentation\n\n`;
-  const sorted = [...pages].sort((a, b) => a.pathname.localeCompare(b.pathname));
-  for (const page of sorted) {
-    content += page.description
-      ? `- [${page.title}](${siteUrl}${page.pathname}.md): ${page.description}\n`
-      : `- [${page.title}](${siteUrl}${page.pathname}.md)\n`;
+
+  // Build slug → page lookup
+  const prefix = `/docs/framework/${framework}/`;
+  const pageBySlug = new Map<string, PageEntry>();
+  for (const page of pages) {
+    if (page.pathname.startsWith(prefix)) {
+      const slug = page.pathname.slice(prefix.length).replace(/\/$/, '');
+      pageBySlug.set(slug, page);
+    }
   }
+
+  // Get sidebar filtered for this framework (production only)
+  if (!isValidFramework(framework)) return content;
+  const filtered = filterSidebarForLlms(sidebar, framework);
+
+  content += renderSidebarToMarkdown(filtered, pageBySlug, siteUrl);
   content += generateIndexFooter(siteUrl);
+
   return content;
 }
 
+function renderSidebarToMarkdown(
+  items: Sidebar,
+  pageBySlug: Map<string, PageEntry>,
+  siteUrl: string,
+  depth: number = 0
+): string {
+  let content = '';
+
+  for (const item of items) {
+    if (isSection(item)) {
+      const heading = '#'.repeat(depth + 2);
+      content += `${heading} ${item.sidebarLabel}\n\n`;
+      if (item.llmsDescription) {
+        content += `${item.llmsDescription}\n\n`;
+      }
+      content += renderSidebarToMarkdown(item.contents, pageBySlug, siteUrl, depth + 1);
+    } else {
+      const page = pageBySlug.get(item.slug);
+      if (!page) continue;
+      content += page.description
+        ? `- [${page.title}](${siteUrl}${page.pathname}.md): ${page.description}\n`
+        : `- [${page.title}](${siteUrl}${page.pathname}.md)\n`;
+    }
+  }
+
+  if (content.length > 0 && !content.endsWith('\n\n')) {
+    content += '\n';
+  }
+
+  return content;
+}
+
+/**
+ * Inline sidebar filter for the integration context where `@/` path aliases
+ * aren't available (can't import `filterSidebar` from `src/utils/docs/sidebar`).
+ * Filters out `devOnly` items and sections restricted to other frameworks,
+ * then removes empty sections.
+ */
+function filterSidebarForLlms(items: Sidebar, framework: SupportedFramework): Sidebar {
+  return items
+    .filter((item) => {
+      if (item.devOnly) return false;
+      return !item.frameworks || item.frameworks.includes(framework);
+    })
+    .map((item) => {
+      if (isSection(item)) {
+        return { ...item, contents: filterSidebarForLlms(item.contents, framework) };
+      }
+      return item;
+    })
+    .filter((item) => !isSection(item) || item.contents.length > 0);
+}
+
 function generateBlogIndex(pages: PageEntry[], siteUrl: string): string {
   let content = `# Video.js v10 — Blog\n\n`;
   // Newest first

site/src/docs.config.ts

@@ -19,6 +19,8 @@ export const sidebar: Sidebar = [
   },
   {
     sidebarLabel: 'Getting started',
+    // May change when we revisit this section's boundary with Concepts (#1105)
+    llmsDescription: 'Installation, project setup, and introductory guides.',
     contents: [
       { slug: 'how-to/installation' },
       { slug: 'concepts/overview' },
@@ -29,6 +31,8 @@ export const sidebar: Sidebar = [
   },
   {
     sidebarLabel: 'Concepts',
+    llmsDescription:
+      'Understanding-oriented pages that explain how and why things work. Read these to build a mental model of the library.',
     contents: [
       { slug: 'concepts/features' },
       { slug: 'concepts/skins' },
@@ -39,10 +43,13 @@ export const sidebar: Sidebar = [
   },
   {
     sidebarLabel: 'How to',
+    llmsDescription:
+      'Task-oriented guides with step-by-step instructions to achieve a specific outcome by applying one or more concepts. Each guide may assume you already understand the relevant concepts.',
     contents: [{ slug: 'how-to/customize-skins' }],
   },
   {
     sidebarLabel: 'Components',
+    llmsDescription: 'API Reference for UI components for building media player interfaces.',
     contents: [
       { slug: 'reference/player-provider' },
       { slug: 'reference/player-container' },
@@ -68,6 +75,7 @@ export const sidebar: Sidebar = [
   },
   {
     sidebarLabel: 'Hooks & Utilities',
+    llmsDescription: 'API Reference for React hooks and utilities for player integration.',
     frameworks: ['react'],
     contents: [
       { slug: 'reference/create-player' },
@@ -91,6 +99,7 @@ export const sidebar: Sidebar = [
   },
   {
     sidebarLabel: 'Controllers & Mixins',
+    llmsDescription: 'API Reference for controllers and mixins for HTML custom element integration.',
     frameworks: ['html'],
     contents: [
       { slug: 'reference/html-create-player', sidebarLabel: 'createPlayer' },
@@ -111,6 +120,7 @@ export const sidebar: Sidebar = [
   },
   {
     sidebarLabel: 'Features',
+    llmsDescription: 'API reference for feature modules that provide player capabilities and state.',
     contents: [
       { slug: 'reference/create-selector' },
       { slug: 'reference/feature-buffer' },

site/src/types/docs.ts

@@ -54,6 +54,7 @@ export interface Guide {
 
 export interface Section {
   sidebarLabel: string;
+  llmsDescription?: string;
   frameworks?: SupportedFramework[];
   devOnly?: boolean; // only visible in development mode
   defaultOpen?: boolean;