zoriya/v10
1
0
Fork 0
mirror of https://github.com/zoriya/v10.git synced 2026-08-05 21:57:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(site): split llms.txt into per-framework and blog sub-indexes (#697)

Browse Source
This commit is contained in:
Darius Cepulis
2026-03-03 17:18:26 -06:00
committed by GitHub
parent cc5b37e351
commit 916f4700ae
2 changed files with 112 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
site/astro.config.mjs
+8 -1
View File
@@ -57,7 +57,14 @@ export default defineConfig({
}),
mdx({ extendMarkdownConfig: true }),
sitemap({
customPages: [`${SITE_URL}/llms.txt`],
// llms-markdown.ts auto-generates per-framework sub-indexes, but sitemap
// entries are hardcoded here. Add a new line when adding a framework.
customPages: [
`${SITE_URL}/llms.txt`,
`${SITE_URL}/blog/llms.txt`,
`${SITE_URL}/docs/framework/html/llms.txt`,
`${SITE_URL}/docs/framework/react/llms.txt`,
],
}),
pagefind(),
llmsMarkdown(),
site/integrations/llms-markdown.ts
+104 -94
View File
@@ -5,6 +5,14 @@ import type { AstroIntegration } from 'astro';
import { JSDOM } from 'jsdom';
import TurndownService from 'turndown';
interface PageEntry {
pathname: string;
title: string;
description?: string;
sort?: string;
framework?: string;
}
export default function llmsMarkdown(): AstroIntegration {
return {
name: 'llms-markdown',
@@ -18,9 +26,9 @@ export default function llmsMarkdown(): AstroIntegration {
});
// Track all docs and blog pages for llms.txt index
const docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
const blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
const otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
const docsPages: PageEntry[] = [];
const blogPages: PageEntry[] = [];
const otherPages: PageEntry[] = [];
logger.info('Generating LLM-optimized markdown files...');
@@ -71,15 +79,18 @@ export default function llmsMarkdown(): AstroIntegration {
const sortAttr = contentElements[0]?.getAttribute('data-llms-sort');
const sort = sortAttr || undefined;
const frameworkAttr = contentElements[0]?.getAttribute('data-framework');
const framework = frameworkAttr || undefined;
// Write markdown file as sibling to the directory
// docs/framework/html/style/css/slug -> docs/framework/html/style/css/slug.md
// docs/framework/html/how-to/slug -> docs/framework/html/how-to/slug.md
const mdPath = join(siteDir, `${pathname}.md`);
await mkdir(dirname(mdPath), { recursive: true });
await writeFile(mdPath, markdown, 'utf-8');
// Track for llms.txt index (with leading slash for URLs)
if (pathname.startsWith('docs/')) {
docsPages.push({ pathname: `/${pathname}`, title, description, sort });
docsPages.push({ pathname: `/${pathname}`, title, description, sort, framework });
} else if (pathname.startsWith('blog/')) {
blogPages.push({ pathname: `/${pathname}`, title, description, sort });
} else {
@@ -90,113 +101,72 @@ export default function llmsMarkdown(): AstroIntegration {
}
}
// Generate llms.txt index file
const llmsTxt = generateLlmsTxt(docsPages, blogPages, otherPages);
const llmsTxtPath = join(siteDir, 'llms.txt');
await writeFile(llmsTxtPath, llmsTxt, 'utf-8');
// Group docs by framework
const docsByFramework = new Map<string, PageEntry[]>();
for (const doc of docsPages) {
const fw = doc.framework ?? 'unknown';
if (!docsByFramework.has(fw)) {
docsByFramework.set(fw, []);
}
docsByFramework.get(fw)!.push(doc);
}
// Write per-framework docs sub-indexes
const frameworks: string[] = [];
for (const [fw, fwPages] of docsByFramework) {
frameworks.push(fw);
const subIndex = generateDocsIndex(fw, fwPages);
const subIndexPath = join(siteDir, 'docs', 'framework', fw, 'llms.txt');
await mkdir(dirname(subIndexPath), { recursive: true });
await writeFile(subIndexPath, subIndex, 'utf-8');
}
// Write blog sub-index
if (blogPages.length > 0) {
const blogIndex = generateBlogIndex(blogPages);
const blogIndexPath = join(siteDir, 'blog', 'llms.txt');
await mkdir(dirname(blogIndexPath), { recursive: true });
await writeFile(blogIndexPath, blogIndex, 'utf-8');
}
// Write root llms.txt index
const rootIndex = generateRootIndex(frameworks, blogPages.length > 0, otherPages);
const rootIndexPath = join(siteDir, 'llms.txt');
await writeFile(rootIndexPath, rootIndex, 'utf-8');
const subIndexCount = frameworks.length + (blogPages.length > 0 ? 1 : 0);
logger.info(
`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files and llms.txt index`
`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files, llms.txt root index, and ${subIndexCount} sub-indexes`
);
},
},
};
}
function generateLlmsTxt(
docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>
): string {
// Group docs by framework and style
const docsByFrameworkStyle = new Map<
string,
Array<{ pathname: string; title: string; description?: string; sort?: string }>
>();
for (const doc of docsPages) {
// Extract framework and style from pathname
// Pattern: /docs/framework/{framework}/style/{style}/{...slug}
const match = doc.pathname.match(/^\/docs\/framework\/([^/]+)\/style\/([^/]+)\//);
if (match) {
const [, framework, style] = match;
const key = `${framework}/${style}`;
if (!docsByFrameworkStyle.has(key)) {
docsByFrameworkStyle.set(key, []);
}
docsByFrameworkStyle.get(key)!.push(doc);
}
}
// Build llms.txt content
function generateRootIndex(frameworks: string[], hasBlog: boolean, otherPages: PageEntry[]): string {
let content = `# Video.js v10\n\n`;
content += `> Modern video player framework with multi-platform support\n\n`;
// Add documentation sections grouped by framework/style
if (docsByFrameworkStyle.size > 0) {
content += `## Documentation\n\n`;
content += `## Documentation\n\n`;
for (const fw of [...frameworks].sort()) {
const label = fw.charAt(0).toUpperCase() + fw.slice(1);
content += `- [${label} Docs](/docs/framework/${fw}/llms.txt)\n`;
}
content += `\n`;
// Sort by framework/style for consistent output
const sortedKeys = Array.from(docsByFrameworkStyle.keys()).sort();
for (const key of sortedKeys) {
const [framework, style] = key.split('/');
const frameworkLabel = framework.charAt(0).toUpperCase() + framework.slice(1);
const styleLabel = style.toUpperCase();
content += `### ${frameworkLabel} + ${styleLabel}\n\n`;
const docs = docsByFrameworkStyle.get(key)!;
// Sort docs by pathname for consistent output
docs.sort((a, b) => a.pathname.localeCompare(b.pathname));
for (const doc of docs) {
if (doc.description) {
content += `- [${doc.title}](${doc.pathname}): ${doc.description}\n`;
} else {
content += `- [${doc.title}](${doc.pathname})\n`;
}
}
content += `\n`;
}
if (hasBlog) {
content += `## Blog\n\n`;
content += `- [Blog Posts](/blog/llms.txt)\n\n`;
}
// Add blog posts section
if (blogPages.length > 0) {
content += `## Blog Posts\n\n`;
// Sort by date using data-llms-sort attribute in reverse order (newest first)
const sortedBlogPages = [...blogPages].sort((a, b) => {
// If both have sort attributes, compare them (reverse for newest first)
if (a.sort && b.sort) {
return b.sort.localeCompare(a.sort);
}
// Fallback to pathname comparison if sort is missing
return b.pathname.localeCompare(a.pathname);
});
for (const post of sortedBlogPages) {
if (post.description) {
content += `- [${post.title}](${post.pathname}): ${post.description}\n`;
} else {
content += `- [${post.title}](${post.pathname})\n`;
}
}
content += `\n`;
}
// Add other pages section
if (otherPages.length > 0) {
content += `## Other\n\n`;
// Sort by pathname
const sortedOtherPages = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
for (const page of sortedOtherPages) {
const sorted = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
for (const page of sorted) {
if (page.description) {
content += `- [${page.title}](${page.pathname}): ${page.description}\n`;
content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
} else {
content += `- [${page.title}](${page.pathname})\n`;
content += `- [${page.title}](${page.pathname}.md)\n`;
}
}
content += `\n`;
@@ -204,3 +174,43 @@ function generateLlmsTxt(
return content;
}
function generateDocsIndex(framework: string, pages: PageEntry[]): string {
const label = framework.charAt(0).toUpperCase() + framework.slice(1);
let content = `# Video.js v10 — ${label} Documentation\n\n`;
const sorted = [...pages].sort((a, b) => a.pathname.localeCompare(b.pathname));
for (const page of sorted) {
if (page.description) {
content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
} else {
content += `- [${page.title}](${page.pathname}.md)\n`;
}
}
content += `\n`;
return content;
}
function generateBlogIndex(pages: PageEntry[]): string {
let content = `# Video.js v10 — Blog\n\n`;
// Newest first
const sorted = [...pages].sort((a, b) => {
if (a.sort && b.sort) {
return b.sort.localeCompare(a.sort);
}
return b.pathname.localeCompare(a.pathname);
});
for (const post of sorted) {
if (post.description) {
content += `- [${post.title}](${post.pathname}.md): ${post.description}\n`;
} else {
content += `- [${post.title}](${post.pathname}.md)\n`;
}
}
content += `\n`;
return content;
}