feat(site): llms.txt (#184)

pnpm-lock.yaml

@@ -390,6 +390,9 @@ importers:
       '@types/react-dom':
         specifier: ^19.2.1
         version: 19.2.1(@types/react@19.2.2)
+      '@types/turndown':
+        specifier: ^5.0.6
+        version: 5.0.6
       '@vitejs/plugin-react':
         specifier: ^5.0.3
         version: 5.0.3(vite@7.1.6(@types/node@22.18.6)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.1))
@@ -408,6 +411,9 @@ importers:
       sirv:
         specifier: ^3.0.2
         version: 3.0.2
+      turndown:
+        specifier: ^7.2.2
+        version: 7.2.2
       vitest:
         specifier: ^3.2.4
         version: 3.2.4(@types/debug@4.1.12)(@types/node@22.18.6)(@vitest/ui@3.2.4)(jiti@2.6.1)(jsdom@27.0.0(postcss@8.5.6))(lightningcss@1.30.2)(yaml@2.8.1)
@@ -1272,6 +1278,9 @@ packages:
   '@mdx-js/mdx@3.1.1':
     resolution: {integrity: sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==}
 
+  '@mixmark-io/domino@2.2.0':
+    resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
+
   '@nanostores/react@1.0.0':
     resolution: {integrity: sha512-eDduyNy+lbQJMg6XxZ/YssQqF6b4OXMFEZMYKPJCCmBevp1lg0g+4ZRi94qGHirMtsNfAWKNwsjOhC+q1gvC+A==}
     engines: {node: ^20.0.0 || >=22.0.0}
@@ -2155,6 +2164,9 @@ packages:
   '@types/sax@1.2.7':
     resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==}
 
+  '@types/turndown@5.0.6':
+    resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==}
+
   '@types/unist@2.0.11':
     resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
 
@@ -6046,6 +6058,9 @@ packages:
     resolution: {integrity: sha512-5c9Fdsr9qfpT3hA0EyYSFRZj1dVVsb6KIWubA9JBYZ/9ZEAijgUEae0BBR/Xl/wekt4w65/lYLTFaP3JmwSO8w==}
     hasBin: true
 
+  turndown@7.2.2:
+    resolution: {integrity: sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==}
+
   type-check@0.4.0:
     resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==}
     engines: {node: '>= 0.8.0'}
@@ -7588,6 +7603,8 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  '@mixmark-io/domino@2.2.0': {}
+
   '@nanostores/react@1.0.0(nanostores@1.0.1)(react@18.3.1)':
     dependencies:
       nanostores: 1.0.1
@@ -8330,6 +8347,8 @@ snapshots:
     dependencies:
       '@types/node': 22.18.6
 
+  '@types/turndown@5.0.6': {}
+
   '@types/unist@2.0.11': {}
 
   '@types/unist@3.0.3': {}
@@ -13253,6 +13272,10 @@ snapshots:
       turbo-windows-64: 2.5.8
       turbo-windows-arm64: 2.5.8
 
+  turndown@7.2.2:
+    dependencies:
+      '@mixmark-io/domino': 2.2.0
+
   type-check@0.4.0:
     dependencies:
       prelude-ls: 1.2.1

site/astro.config.mjs

@@ -9,15 +9,18 @@ import vercel from '@astrojs/vercel';
 import tailwindcss from '@tailwindcss/vite';
 import { defineConfig, fontProviders } from 'astro/config';
 import checkV8Urls from './integrations/check-v8-urls';
+import llmsMarkdown from './integrations/llms-markdown';
 import pagefind from './integrations/pagefind';
 import rehypePrepareCodeBlocks from './src/utils/rehypePrepareCodeBlocks';
 import remarkConditionalHeadings from './src/utils/remarkConditionalHeadings';
 import { remarkReadingTime } from './src/utils/remarkReadingTime.mjs';
 import shikiTransformMetadata from './src/utils/shikiTransformMetadata';
 
+const SITE_URL = 'https://v10.videojs.org';
+
 // https://astro.build/config
 export default defineConfig({
-  site: 'https://v10.videojs.org',
+  site: SITE_URL,
   trailingSlash: 'never',
   adapter: vercel(),
   redirects: {
@@ -25,8 +28,11 @@ export default defineConfig({
   },
   integrations: [
     mdx({ extendMarkdownConfig: true }),
-    sitemap(),
+    sitemap({
+      customPages: [`${SITE_URL}/llms.txt`],
+    }),
     pagefind(),
+    llmsMarkdown(),
     checkV8Urls(),
     react({
       babel: {

site/integrations/llms-markdown.ts

@@ -0,0 +1,197 @@
+import type { AstroIntegration } from 'astro';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { JSDOM } from 'jsdom';
+import TurndownService from 'turndown';
+
+export default function llmsMarkdown(): AstroIntegration {
+  return {
+    name: 'llms-markdown',
+    hooks: {
+      'astro:build:done': async ({ dir, pages, logger }) => {
+        const siteDir = fileURLToPath(dir);
+        const turndown = new TurndownService({
+          headingStyle: 'atx',
+          codeBlockStyle: 'fenced',
+          emDelimiter: '*',
+        });
+
+        // Track all docs and blog pages for llms.txt index
+        const docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
+        const blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
+        const otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
+
+        logger.info('Generating LLM-optimized markdown files...');
+
+        for (const page of pages) {
+          const { pathname } = page;
+
+          try {
+            // Construct path to HTML file
+            const htmlPath = join(siteDir, pathname, 'index.html');
+            const html = await readFile(htmlPath, 'utf-8');
+
+            // Parse HTML with jsdom
+            const dom = new JSDOM(html);
+            const document = dom.window.document;
+
+            // Check if page has llms content
+            const contentElements = document.querySelectorAll('[data-llms-content]');
+
+            if (contentElements.length === 0) {
+              // No llms content, skip silently
+              continue;
+            }
+
+            // For each content element, remove [data-llms-ignore] descendants
+            const contentParts: string[] = [];
+            contentElements.forEach((contentEl) => {
+              const clone = contentEl.cloneNode(true) as Element;
+              const ignoreElements = clone.querySelectorAll('[data-llms-ignore]');
+              ignoreElements.forEach(el => el.remove());
+              contentParts.push(clone.innerHTML);
+            });
+
+            // Combine all content parts
+            const combinedHtml = contentParts.join('\n\n');
+            const markdown = turndown.turndown(combinedHtml);
+
+            // Extract title and description for llms.txt index
+            const titleElement = document.querySelector('h1');
+            const title = titleElement?.textContent?.trim() || 'Untitled';
+
+            const descriptionAttr = contentElements[0]?.getAttribute('data-llms-description');
+            const description = descriptionAttr || undefined;
+
+            const sortAttr = contentElements[0]?.getAttribute('data-llms-sort');
+            const sort = sortAttr || undefined;
+
+            // Write markdown file as sibling to the directory
+            // docs/framework/html/style/css/slug -> docs/framework/html/style/css/slug.md
+            const mdPath = join(siteDir, `${pathname}.md`);
+            await mkdir(dirname(mdPath), { recursive: true });
+            await writeFile(mdPath, markdown, 'utf-8');
+
+            // Track for llms.txt index (with leading slash for URLs)
+            if (pathname.startsWith('docs/')) {
+              docsPages.push({ pathname: `/${pathname}`, title, description, sort });
+            } else if (pathname.startsWith('blog/')) {
+              blogPages.push({ pathname: `/${pathname}`, title, description, sort });
+            } else {
+              otherPages.push({ pathname: `/${pathname}`, title, description, sort });
+            }
+          } catch (error) {
+            logger.error(`Failed to process ${pathname}: ${error instanceof Error ? error.message : String(error)}`);
+          }
+        }
+
+        // Generate llms.txt index file
+        const llmsTxt = generateLlmsTxt(docsPages, blogPages, otherPages);
+        const llmsTxtPath = join(siteDir, 'llms.txt');
+        await writeFile(llmsTxtPath, llmsTxt, 'utf-8');
+
+        logger.info(`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files and llms.txt index`);
+      },
+    },
+  };
+}
+
+function generateLlmsTxt(
+  docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
+  blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
+  otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
+): string {
+  // Group docs by framework and style
+  const docsByFrameworkStyle = new Map<string, Array<{ pathname: string; title: string; description?: string; sort?: string }>>();
+
+  for (const doc of docsPages) {
+    // Extract framework and style from pathname
+    // Pattern: /docs/framework/{framework}/style/{style}/{...slug}
+    const match = doc.pathname.match(/^\/docs\/framework\/([^/]+)\/style\/([^/]+)\//);
+    if (match) {
+      const [, framework, style] = match;
+      const key = `${framework}/${style}`;
+      if (!docsByFrameworkStyle.has(key)) {
+        docsByFrameworkStyle.set(key, []);
+      }
+      docsByFrameworkStyle.get(key)!.push(doc);
+    }
+  }
+
+  // Build llms.txt content
+  let content = `# Video.js v10\n\n`;
+  content += `> Modern video player framework with multi-platform support\n\n`;
+
+  // Add documentation sections grouped by framework/style
+  if (docsByFrameworkStyle.size > 0) {
+    content += `## Documentation\n\n`;
+
+    // Sort by framework/style for consistent output
+    const sortedKeys = Array.from(docsByFrameworkStyle.keys()).sort();
+
+    for (const key of sortedKeys) {
+      const [framework, style] = key.split('/');
+      const frameworkLabel = framework.charAt(0).toUpperCase() + framework.slice(1);
+      const styleLabel = style.toUpperCase();
+
+      content += `### ${frameworkLabel} + ${styleLabel}\n\n`;
+
+      const docs = docsByFrameworkStyle.get(key)!;
+      // Sort docs by pathname for consistent output
+      docs.sort((a, b) => a.pathname.localeCompare(b.pathname));
+
+      for (const doc of docs) {
+        if (doc.description) {
+          content += `- [${doc.title}](${doc.pathname}): ${doc.description}\n`;
+        } else {
+          content += `- [${doc.title}](${doc.pathname})\n`;
+        }
+      }
+      content += `\n`;
+    }
+  }
+
+  // Add blog posts section
+  if (blogPages.length > 0) {
+    content += `## Blog Posts\n\n`;
+
+    // Sort by date using data-llms-sort attribute in reverse order (newest first)
+    const sortedBlogPages = [...blogPages].sort((a, b) => {
+      // If both have sort attributes, compare them (reverse for newest first)
+      if (a.sort && b.sort) {
+        return b.sort.localeCompare(a.sort);
+      }
+      // Fallback to pathname comparison if sort is missing
+      return b.pathname.localeCompare(a.pathname);
+    });
+
+    for (const post of sortedBlogPages) {
+      if (post.description) {
+        content += `- [${post.title}](${post.pathname}): ${post.description}\n`;
+      } else {
+        content += `- [${post.title}](${post.pathname})\n`;
+      }
+    }
+    content += `\n`;
+  }
+
+  // Add other pages section
+  if (otherPages.length > 0) {
+    content += `## Other\n\n`;
+
+    // Sort by pathname
+    const sortedOtherPages = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
+
+    for (const page of sortedOtherPages) {
+      if (page.description) {
+        content += `- [${page.title}](${page.pathname}): ${page.description}\n`;
+      } else {
+        content += `- [${page.title}](${page.pathname})\n`;
+      }
+    }
+    content += `\n`;
+  }
+
+  return content;
+}

site/package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "dev": " astro dev",
     "build": "astro build",
-    "preview": "astro preview",
+    "preview": "echo 'Preview server is not supported on with the Vercel adapter.'",
     "astro": "astro",
     "test": "vitest run",
     "test:watch": "vitest",
@@ -49,12 +49,14 @@
     "@testing-library/user-event": "^14.6.1",
     "@types/react": "^19.2.2",
     "@types/react-dom": "^19.2.1",
+    "@types/turndown": "^5.0.6",
     "@vitejs/plugin-react": "^5.0.3",
     "@vitest/coverage-v8": "^3.2.4",
     "@vitest/ui": "^3.2.4",
     "babel-plugin-react-compiler": "1.0.0",
     "jsdom": "^27.0.0",
     "sirv": "^3.0.2",
+    "turndown": "^7.2.2",
     "vitest": "^3.2.4"
   }
 }

site/src/components/CopyMarkdownButton.tsx

@@ -0,0 +1,124 @@
+import clsx from 'clsx';
+import { CheckIcon } from 'lucide-react';
+import { useState } from 'react';
+import useIsHydrated from '@/utils/useIsHydrated';
+
+export interface CopyMarkdownButtonProps {
+  className?: string;
+  style?: React.CSSProperties;
+  timeout?: number;
+}
+
+type CopyState
+  = | { status: 'idle' }
+    | { status: 'loading' }
+    | { status: 'success' }
+    | { status: 'error'; message: string };
+
+export default function CopyMarkdownButton({
+  className,
+  style,
+}: CopyMarkdownButtonProps) {
+  const [state, setState] = useState<CopyState>({ status: 'idle' });
+  const isHydrated = useIsHydrated();
+  const disabled = !isHydrated || state.status === 'loading';
+
+  const handleCopy = async () => {
+    try {
+      setState({ status: 'loading' });
+
+      // Get current pathname and construct markdown URL
+      const pathname = window.location.pathname;
+      const mdUrl = `${pathname}.md`;
+
+      // Create fetch promise - in dev mode, return helpful message
+      const markdownBlobPromise = import.meta.env.DEV
+        ? Promise.resolve(new Blob([
+            'Markdown source files are only available in production builds.\n\n'
+            + 'Run `pnpm build` and `pnpm preview` to test this feature.',
+          ], { type: 'text/plain' }))
+        : fetch(mdUrl)
+            .then((response) => {
+              if (!response.ok) {
+                throw new Error(`Failed to fetch markdown: ${response.status} ${response.statusText}`);
+              }
+              return response.text();
+            })
+            .then(text => new Blob([text], { type: 'text/plain' }));
+
+      // Feature detection: ClipboardItem required for Safari compatibility
+      if (typeof ClipboardItem === 'undefined') {
+        // Fallback for very old browsers (pre-2024)
+        const blob = await markdownBlobPromise;
+        const text = await blob.text();
+        await navigator.clipboard.writeText(text);
+      } else {
+        // Safari requires passing async operation TO clipboard API
+        // (not awaiting first) to preserve user gesture context
+        const clipboardItem = new ClipboardItem({
+          'text/plain': markdownBlobPromise,
+        });
+        await navigator.clipboard.write([clipboardItem]);
+      }
+
+      setState({ status: 'success' });
+      setTimeout(() => {
+        setState({ status: 'idle' });
+      }, 2000);
+    } catch (err) {
+      console.error('Failed to copy markdown:', err);
+      const message = err instanceof Error ? err.message : 'Failed to copy markdown';
+      setState({ status: 'error', message });
+      setTimeout(() => {
+        setState({ status: 'idle' });
+      }, 2000);
+    }
+  };
+
+  const ariaLabel
+    = state.status === 'success'
+      ? 'Copied'
+      : 'Copy markdown to clipboard';
+
+  return (
+    <button
+      type="button"
+      disabled={disabled}
+      onClick={handleCopy}
+      className={clsx(
+        'relative border border-light-40 dark:border-dark-80 px-3 py-1 rounded-lg whitespace-nowrap text-sm',
+        state.status === 'idle' && 'intent:border-dark-40 dark:intent:border-dark-40',
+        state.status === 'loading' ? 'opacity-70' : 'cursor-100',
+        disabled ? 'cursor-wait' : 'cursor-pointer',
+        className,
+      )}
+      style={style}
+      aria-label={ariaLabel}
+      data-llms-ignore
+    >
+      <span className={clsx(
+        state.status !== 'idle' && state.status !== 'loading' ? 'opacity-0 pointer-events-none' : 'opacity-100',
+        'inline-flex items-center justify-center',
+      )}
+      >
+        Copy page
+      </span>
+      <span className={clsx(
+        state.status !== 'success' ? 'opacity-0 pointer-events-none' : 'opacity-100',
+        'absolute inset-0 inline-flex items-center justify-center',
+      )}
+      >
+        Copied
+        {' '}
+        <CheckIcon className="ml-1 w-4 h-4" />
+      </span>
+      <span className={clsx(
+        state.status !== 'error' ? 'opacity-0 pointer-events-none' : 'opacity-100',
+        'absolute inset-0 inline-flex items-center justify-center',
+      )}
+      >
+        Error
+      </span>
+    </button>
+  );
+}

site/src/components/docs/FrameworkCase.astro

@@ -26,6 +26,11 @@ const shouldRender = !frameworks || frameworks.includes(framework as SupportedFr
     So, while I debug those... let's do this
   */
 }
-<div class="contents" hidden={!shouldRender} data-pagefind-ignore={shouldRender ? undefined : 'all'}>
+<div
+  class="contents"
+  hidden={!shouldRender}
+  data-pagefind-ignore={shouldRender ? undefined : 'all'}
+  data-llms-ignore={shouldRender ? undefined : 'all'}
+>
   <slot />
 </div>

site/src/components/docs/StyleCase.astro

@@ -26,6 +26,11 @@ const shouldRender = !styles || styles.includes(style as AnySupportedStyle);
     So, while I debug those... let's do this
   */
 }
-<div class="contents" hidden={!shouldRender} data-pagefind-ignore={shouldRender ? undefined : 'all'}>
+<div
+  class="contents"
+  hidden={!shouldRender}
+  data-pagefind-ignore={shouldRender ? undefined : 'all'}
+  data-llms-ignore={shouldRender ? undefined : 'all'}
+>
   <slot />
 </div>

site/src/pages/blog/[...slug].astro

@@ -38,12 +38,20 @@ const jsonLdSchema = createBlogPostingSchema({
 
 <Blog title={[post.data.title, 'Blog']} description={post.data.description}>
   <JsonLd slot="head" schema={jsonLdSchema} />
-  <article class="@container" data-pagefind-body data-site="blog" data-pagefind-filter="site[data-site]">
-    <header class="border-b border-light-40 dark:border-dark-80 max-w-3xl mx-auto pb-10 mb-10">
+  <article
+    class="@container"
+    data-pagefind-body
+    data-llms-content
+    data-llms-description={post.data.description}
+    data-llms-sort={post.data.pubDate.toISOString()}
+    data-site="blog"
+    data-pagefind-filter="site[data-site]"
+  >
+    <header class="border-b border-light-40 dark:border-dark-80 max-w-3xl mx-auto pb-10 mb-10 relative">
       <div class="font-medium md:text-lg text-dark-40 dark:text-light-40 flex gap-4 mb-1">
         <FormattedDate data-pagefind-sort={`date:${post.data.pubDate.toISOString()}`} date={post.data.pubDate} />
         <span>&bull;</span>
-        <span data-pagefind-ignore>{remarkPluginFrontmatter.minutesRead}</span>
+        <span data-pagefind-ignore data-llms-ignore>{remarkPluginFrontmatter.minutesRead}</span>
       </div>
       <h1 class="text-h4 md:text-h2 mb-2">{post.data.title}</h1>
       <div class="font-medium md:text-lg text-dark-40 dark:text-light-40">

site/src/pages/docs/framework/[framework]/style/[style]/[...slug].astro

@@ -16,6 +16,7 @@ import EditPageButton from '@/components/docs/EditPageButton.astro';
 import JsonLd from '@/components/JsonLd.astro';
 import { createTechArticleSchema } from '@/utils/jsonLd/schemas';
 import { getDocTitle } from '@/utils/docs/title';
+import CopyMarkdownButton from '@/components/CopyMarkdownButton';
 
 export async function getStaticPaths() {
   const docsCollection = await getCollection('docs');
@@ -126,15 +127,25 @@ const jsonLdSchema = createTechArticleSchema({
       <article
         class="mb-18 flex-1"
         data-pagefind-body
+        data-llms-content
+        data-llms-description={doc.data.description}
         data-framework={framework}
         data-style={style}
         data-site="docs"
         data-category={sections[0]}
         data-pagefind-filter="framework[data-framework], style[data-style], section[data-site]"
       >
-        <header class="border-b border-light-40 dark:border-dark-80 max-w-3xl mx-auto py-8 mb-8">
-          <H3 as="h1" class="mt-0 mb-2">{getDocTitle(doc, framework)}</H3>
-          <p class="@lg:font-medium">{doc.data.description}</p>
+        <header
+          class:list={[
+            'border-b border-light-40 dark:border-dark-80 max-w-3xl mx-auto py-8 mb-8',
+            'flex flex-col md:flex-row gap-4 items-start md:items-end justify-between',
+          ]}
+        >
+          <div>
+            <H3 as="h1" class="mt-0 mb-2">{getDocTitle(doc, framework)}</H3>
+            <p class="@lg:font-medium">{doc.data.description}</p>
+          </div>
+          <CopyMarkdownButton client:load />
         </header>
         <Content components={{ ...defaultMarkdownComponents }} />
       </article>