From 1a249ce00df6d0d2fe91a927709279aa63d0453f Mon Sep 17 00:00:00 2001 From: Darius Cepulis Date: Wed, 12 Nov 2025 14:22:34 -0800 Subject: [PATCH] feat(site): llms.txt (#184) --- pnpm-lock.yaml | 23 ++ site/astro.config.mjs | 10 +- site/integrations/llms-markdown.ts | 197 ++++++++++++++++++ site/package.json | 4 +- site/src/components/CopyMarkdownButton.tsx | 124 +++++++++++ site/src/components/docs/FrameworkCase.astro | 7 +- site/src/components/docs/StyleCase.astro | 7 +- site/src/pages/blog/[...slug].astro | 14 +- .../[framework]/style/[style]/[...slug].astro | 17 +- 9 files changed, 392 insertions(+), 11 deletions(-) create mode 100644 site/integrations/llms-markdown.ts create mode 100644 site/src/components/CopyMarkdownButton.tsx diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d961ab9..4b786574 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -390,6 +390,9 @@ importers: '@types/react-dom': specifier: ^19.2.1 version: 19.2.1(@types/react@19.2.2) + '@types/turndown': + specifier: ^5.0.6 + version: 5.0.6 '@vitejs/plugin-react': specifier: ^5.0.3 version: 5.0.3(vite@7.1.6(@types/node@22.18.6)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.1)) @@ -408,6 +411,9 @@ importers: sirv: specifier: ^3.0.2 version: 3.0.2 + turndown: + specifier: ^7.2.2 + version: 7.2.2 vitest: specifier: ^3.2.4 version: 3.2.4(@types/debug@4.1.12)(@types/node@22.18.6)(@vitest/ui@3.2.4)(jiti@2.6.1)(jsdom@27.0.0(postcss@8.5.6))(lightningcss@1.30.2)(yaml@2.8.1) @@ -1272,6 +1278,9 @@ packages: '@mdx-js/mdx@3.1.1': resolution: {integrity: sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==} + '@mixmark-io/domino@2.2.0': + resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} + '@nanostores/react@1.0.0': resolution: {integrity: sha512-eDduyNy+lbQJMg6XxZ/YssQqF6b4OXMFEZMYKPJCCmBevp1lg0g+4ZRi94qGHirMtsNfAWKNwsjOhC+q1gvC+A==} engines: {node: ^20.0.0 || >=22.0.0} @@ -2155,6 +2164,9 @@ packages: '@types/sax@1.2.7': resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} + '@types/turndown@5.0.6': + resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} + '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -6046,6 +6058,9 @@ packages: resolution: {integrity: sha512-5c9Fdsr9qfpT3hA0EyYSFRZj1dVVsb6KIWubA9JBYZ/9ZEAijgUEae0BBR/Xl/wekt4w65/lYLTFaP3JmwSO8w==} hasBin: true + turndown@7.2.2: + resolution: {integrity: sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -7588,6 +7603,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@mixmark-io/domino@2.2.0': {} + '@nanostores/react@1.0.0(nanostores@1.0.1)(react@18.3.1)': dependencies: nanostores: 1.0.1 @@ -8330,6 +8347,8 @@ snapshots: dependencies: '@types/node': 22.18.6 + '@types/turndown@5.0.6': {} + '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -13253,6 +13272,10 @@ snapshots: turbo-windows-64: 2.5.8 turbo-windows-arm64: 2.5.8 + turndown@7.2.2: + dependencies: + '@mixmark-io/domino': 2.2.0 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 diff --git a/site/astro.config.mjs b/site/astro.config.mjs index bc03756a..94e98947 100644 --- a/site/astro.config.mjs +++ b/site/astro.config.mjs @@ -9,15 +9,18 @@ import vercel from '@astrojs/vercel'; import tailwindcss from '@tailwindcss/vite'; import { defineConfig, fontProviders } from 'astro/config'; import checkV8Urls from './integrations/check-v8-urls'; +import llmsMarkdown from './integrations/llms-markdown'; import pagefind from './integrations/pagefind'; import rehypePrepareCodeBlocks from './src/utils/rehypePrepareCodeBlocks'; import remarkConditionalHeadings from './src/utils/remarkConditionalHeadings'; import { remarkReadingTime } from './src/utils/remarkReadingTime.mjs'; import shikiTransformMetadata from './src/utils/shikiTransformMetadata'; +const SITE_URL = 'https://v10.videojs.org'; + // https://astro.build/config export default defineConfig({ - site: 'https://v10.videojs.org', + site: SITE_URL, trailingSlash: 'never', adapter: vercel(), redirects: { @@ -25,8 +28,11 @@ export default defineConfig({ }, integrations: [ mdx({ extendMarkdownConfig: true }), - sitemap(), + sitemap({ + customPages: [`${SITE_URL}/llms.txt`], + }), pagefind(), + llmsMarkdown(), checkV8Urls(), react({ babel: { diff --git a/site/integrations/llms-markdown.ts b/site/integrations/llms-markdown.ts new file mode 100644 index 00000000..801347b1 --- /dev/null +++ b/site/integrations/llms-markdown.ts @@ -0,0 +1,197 @@ +import type { AstroIntegration } from 'astro'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { JSDOM } from 'jsdom'; +import TurndownService from 'turndown'; + +export default function llmsMarkdown(): AstroIntegration { + return { + name: 'llms-markdown', + hooks: { + 'astro:build:done': async ({ dir, pages, logger }) => { + const siteDir = fileURLToPath(dir); + const turndown = new TurndownService({ + headingStyle: 'atx', + codeBlockStyle: 'fenced', + emDelimiter: '*', + }); + + // Track all docs and blog pages for llms.txt index + const docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = []; + const blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = []; + const otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = []; + + logger.info('Generating LLM-optimized markdown files...'); + + for (const page of pages) { + const { pathname } = page; + + try { + // Construct path to HTML file + const htmlPath = join(siteDir, pathname, 'index.html'); + const html = await readFile(htmlPath, 'utf-8'); + + // Parse HTML with jsdom + const dom = new JSDOM(html); + const document = dom.window.document; + + // Check if page has llms content + const contentElements = document.querySelectorAll('[data-llms-content]'); + + if (contentElements.length === 0) { + // No llms content, skip silently + continue; + } + + // For each content element, remove [data-llms-ignore] descendants + const contentParts: string[] = []; + contentElements.forEach((contentEl) => { + const clone = contentEl.cloneNode(true) as Element; + const ignoreElements = clone.querySelectorAll('[data-llms-ignore]'); + ignoreElements.forEach(el => el.remove()); + contentParts.push(clone.innerHTML); + }); + + // Combine all content parts + const combinedHtml = contentParts.join('\n\n'); + const markdown = turndown.turndown(combinedHtml); + + // Extract title and description for llms.txt index + const titleElement = document.querySelector('h1'); + const title = titleElement?.textContent?.trim() || 'Untitled'; + + const descriptionAttr = contentElements[0]?.getAttribute('data-llms-description'); + const description = descriptionAttr || undefined; + + const sortAttr = contentElements[0]?.getAttribute('data-llms-sort'); + const sort = sortAttr || undefined; + + // Write markdown file as sibling to the directory + // docs/framework/html/style/css/slug -> docs/framework/html/style/css/slug.md + const mdPath = join(siteDir, `${pathname}.md`); + await mkdir(dirname(mdPath), { recursive: true }); + await writeFile(mdPath, markdown, 'utf-8'); + + // Track for llms.txt index (with leading slash for URLs) + if (pathname.startsWith('docs/')) { + docsPages.push({ pathname: `/${pathname}`, title, description, sort }); + } else if (pathname.startsWith('blog/')) { + blogPages.push({ pathname: `/${pathname}`, title, description, sort }); + } else { + otherPages.push({ pathname: `/${pathname}`, title, description, sort }); + } + } catch (error) { + logger.error(`Failed to process ${pathname}: ${error instanceof Error ? error.message : String(error)}`); + } + } + + // Generate llms.txt index file + const llmsTxt = generateLlmsTxt(docsPages, blogPages, otherPages); + const llmsTxtPath = join(siteDir, 'llms.txt'); + await writeFile(llmsTxtPath, llmsTxt, 'utf-8'); + + logger.info(`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files and llms.txt index`); + }, + }, + }; +} + +function generateLlmsTxt( + docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>, + blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>, + otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>, +): string { + // Group docs by framework and style + const docsByFrameworkStyle = new Map>(); + + for (const doc of docsPages) { + // Extract framework and style from pathname + // Pattern: /docs/framework/{framework}/style/{style}/{...slug} + const match = doc.pathname.match(/^\/docs\/framework\/([^/]+)\/style\/([^/]+)\//); + if (match) { + const [, framework, style] = match; + const key = `${framework}/${style}`; + if (!docsByFrameworkStyle.has(key)) { + docsByFrameworkStyle.set(key, []); + } + docsByFrameworkStyle.get(key)!.push(doc); + } + } + + // Build llms.txt content + let content = `# Video.js v10\n\n`; + content += `> Modern video player framework with multi-platform support\n\n`; + + // Add documentation sections grouped by framework/style + if (docsByFrameworkStyle.size > 0) { + content += `## Documentation\n\n`; + + // Sort by framework/style for consistent output + const sortedKeys = Array.from(docsByFrameworkStyle.keys()).sort(); + + for (const key of sortedKeys) { + const [framework, style] = key.split('/'); + const frameworkLabel = framework.charAt(0).toUpperCase() + framework.slice(1); + const styleLabel = style.toUpperCase(); + + content += `### ${frameworkLabel} + ${styleLabel}\n\n`; + + const docs = docsByFrameworkStyle.get(key)!; + // Sort docs by pathname for consistent output + docs.sort((a, b) => a.pathname.localeCompare(b.pathname)); + + for (const doc of docs) { + if (doc.description) { + content += `- [${doc.title}](${doc.pathname}): ${doc.description}\n`; + } else { + content += `- [${doc.title}](${doc.pathname})\n`; + } + } + content += `\n`; + } + } + + // Add blog posts section + if (blogPages.length > 0) { + content += `## Blog Posts\n\n`; + + // Sort by date using data-llms-sort attribute in reverse order (newest first) + const sortedBlogPages = [...blogPages].sort((a, b) => { + // If both have sort attributes, compare them (reverse for newest first) + if (a.sort && b.sort) { + return b.sort.localeCompare(a.sort); + } + // Fallback to pathname comparison if sort is missing + return b.pathname.localeCompare(a.pathname); + }); + + for (const post of sortedBlogPages) { + if (post.description) { + content += `- [${post.title}](${post.pathname}): ${post.description}\n`; + } else { + content += `- [${post.title}](${post.pathname})\n`; + } + } + content += `\n`; + } + + // Add other pages section + if (otherPages.length > 0) { + content += `## Other\n\n`; + + // Sort by pathname + const sortedOtherPages = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname)); + + for (const page of sortedOtherPages) { + if (page.description) { + content += `- [${page.title}](${page.pathname}): ${page.description}\n`; + } else { + content += `- [${page.title}](${page.pathname})\n`; + } + } + content += `\n`; + } + + return content; +} diff --git a/site/package.json b/site/package.json index c3525af0..fbee660c 100644 --- a/site/package.json +++ b/site/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": " astro dev", "build": "astro build", - "preview": "astro preview", + "preview": "echo 'Preview server is not supported on with the Vercel adapter.'", "astro": "astro", "test": "vitest run", "test:watch": "vitest", @@ -49,12 +49,14 @@ "@testing-library/user-event": "^14.6.1", "@types/react": "^19.2.2", "@types/react-dom": "^19.2.1", + "@types/turndown": "^5.0.6", "@vitejs/plugin-react": "^5.0.3", "@vitest/coverage-v8": "^3.2.4", "@vitest/ui": "^3.2.4", "babel-plugin-react-compiler": "1.0.0", "jsdom": "^27.0.0", "sirv": "^3.0.2", + "turndown": "^7.2.2", "vitest": "^3.2.4" } } diff --git a/site/src/components/CopyMarkdownButton.tsx b/site/src/components/CopyMarkdownButton.tsx new file mode 100644 index 00000000..45cd9654 --- /dev/null +++ b/site/src/components/CopyMarkdownButton.tsx @@ -0,0 +1,124 @@ +import clsx from 'clsx'; +import { CheckIcon } from 'lucide-react'; +import { useState } from 'react'; +import useIsHydrated from '@/utils/useIsHydrated'; + +export interface CopyMarkdownButtonProps { + className?: string; + style?: React.CSSProperties; + timeout?: number; +} + +type CopyState + = | { status: 'idle' } + | { status: 'loading' } + | { status: 'success' } + | { status: 'error'; message: string }; + +export default function CopyMarkdownButton({ + className, + style, +}: CopyMarkdownButtonProps) { + const [state, setState] = useState({ status: 'idle' }); + const isHydrated = useIsHydrated(); + const disabled = !isHydrated || state.status === 'loading'; + + const handleCopy = async () => { + try { + setState({ status: 'loading' }); + + // Get current pathname and construct markdown URL + const pathname = window.location.pathname; + const mdUrl = `${pathname}.md`; + + // Create fetch promise - in dev mode, return helpful message + const markdownBlobPromise = import.meta.env.DEV + ? Promise.resolve(new Blob([ + 'Markdown source files are only available in production builds.\n\n' + + 'Run `pnpm build` and `pnpm preview` to test this feature.', + ], { type: 'text/plain' })) + : fetch(mdUrl) + .then((response) => { + if (!response.ok) { + throw new Error(`Failed to fetch markdown: ${response.status} ${response.statusText}`); + } + return response.text(); + }) + .then(text => new Blob([text], { type: 'text/plain' })); + + // Feature detection: ClipboardItem required for Safari compatibility + if (typeof ClipboardItem === 'undefined') { + // Fallback for very old browsers (pre-2024) + const blob = await markdownBlobPromise; + const text = await blob.text(); + await navigator.clipboard.writeText(text); + } else { + // Safari requires passing async operation TO clipboard API + // (not awaiting first) to preserve user gesture context + const clipboardItem = new ClipboardItem({ + 'text/plain': markdownBlobPromise, + }); + await navigator.clipboard.write([clipboardItem]); + } + + setState({ status: 'success' }); + setTimeout(() => { + setState({ status: 'idle' }); + }, 2000); + } catch (err) { + console.error('Failed to copy markdown:', err); + const message = err instanceof Error ? err.message : 'Failed to copy markdown'; + setState({ status: 'error', message }); + setTimeout(() => { + setState({ status: 'idle' }); + }, 2000); + } + }; + + const ariaLabel + = state.status === 'success' + ? 'Copied' + : 'Copy markdown to clipboard'; + + return ( + + ); +} diff --git a/site/src/components/docs/FrameworkCase.astro b/site/src/components/docs/FrameworkCase.astro index 34783954..5345e595 100644 --- a/site/src/components/docs/FrameworkCase.astro +++ b/site/src/components/docs/FrameworkCase.astro @@ -26,6 +26,11 @@ const shouldRender = !frameworks || frameworks.includes(framework as SupportedFr So, while I debug those... let's do this */ } -