diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3bc023ad..d55ab856 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -664,6 +664,9 @@ importers: jsdom: specifier: ^27.0.0 version: 27.4.0 + linkedom: + specifier: ^0.18.12 + version: 0.18.12 prettier: specifier: ^3.8.1 version: 3.8.1 @@ -4240,6 +4243,9 @@ packages: resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + cssom@0.5.0: + resolution: {integrity: sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==} + cssstyle@4.6.0: resolution: {integrity: sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==} engines: {node: '>=18'} @@ -5070,6 +5076,9 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + htmlparser2@10.1.0: + resolution: {integrity: sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==} + http-cache-semantics@4.2.0: resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} @@ -5677,6 +5686,15 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + linkedom@0.18.12: + resolution: {integrity: sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==} + engines: {node: '>=16'} + peerDependencies: + canvas: '>= 2' + peerDependenciesMeta: + canvas: + optional: true + lint-staged@16.3.2: resolution: {integrity: sha512-xKqhC2AeXLwiAHXguxBjuChoTTWFC6Pees0SHPwOpwlvI3BH7ZADFPddAdN3pgo3aiKgPUx/bxE78JfUnxQnlg==} engines: {node: '>=20.17'} @@ -7412,6 +7430,9 @@ packages: ufo@1.6.3: resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==} + uhyphen@0.2.0: + resolution: {integrity: sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==} + ulid@3.0.2: resolution: {integrity: sha512-yu26mwteFYzBAot7KVMqFGCVpsF6g8wXfJzQUHvu1no3+rRRSFcSV2nKeYvNPLD2J4b08jYBDhHUjeH0ygIl9w==} hasBin: true @@ -11118,7 +11139,7 @@ snapshots: '@vitest/mocker': 4.1.0(vite@8.0.0(@types/node@24.12.2)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2)) playwright: 1.59.1 tinyrainbow: 3.1.0 - vitest: 4.1.0(@opentelemetry/api@1.9.1)(@types/node@24.12.2)(@vitest/browser-playwright@4.1.0)(@vitest/ui@4.1.0)(happy-dom@18.0.1)(jsdom@26.1.0)(vite@8.0.0(@types/node@24.12.2)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2)) + vitest: 4.1.0(@opentelemetry/api@1.9.1)(@types/node@24.12.2)(@vitest/browser-playwright@4.1.0)(@vitest/ui@4.1.0)(happy-dom@18.0.1)(jsdom@27.4.0)(vite@8.0.0(@types/node@24.12.2)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2)) transitivePeerDependencies: - bufferutil - msw @@ -11289,7 +11310,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.1.0 - vitest: 4.1.0(@opentelemetry/api@1.9.1)(@types/node@24.12.2)(@vitest/browser-playwright@4.1.0)(@vitest/ui@4.1.0)(happy-dom@18.0.1)(jsdom@26.1.0)(vite@8.0.0(@types/node@24.12.2)(esbuild@0.27.3)(jiti@2.6.1)(tsx@4.21.0)(yaml@2.8.2)) + vitest: 4.1.0(@opentelemetry/api@1.9.1)(@types/node@24.12.2)(@vitest/browser-playwright@4.1.0)(@vitest/ui@4.1.0)(happy-dom@18.0.1)(jsdom@27.4.0)(vite@7.3.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/utils@4.1.0': dependencies: @@ -12088,6 +12109,8 @@ snapshots: dependencies: css-tree: 2.2.1 + cssom@0.5.0: {} + cssstyle@4.6.0: dependencies: '@asamuzakjp/css-color': 3.2.0 @@ -13134,6 +13157,13 @@ snapshots: html-void-elements@3.0.0: {} + htmlparser2@10.1.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + entities: 7.0.1 + http-cache-semantics@4.2.0: {} http-proxy-agent@7.0.2: @@ -13728,6 +13758,14 @@ snapshots: lines-and-columns@1.2.4: {} + linkedom@0.18.12: + dependencies: + css-select: 5.2.2 + cssom: 0.5.0 + html-escaper: 3.0.3 + htmlparser2: 10.1.0 + uhyphen: 0.2.0 + lint-staged@16.3.2: dependencies: commander: 14.0.3 @@ -15852,6 +15890,8 @@ snapshots: ufo@1.6.3: {} + uhyphen@0.2.0: {} + ulid@3.0.2: {} ultrahtml@1.6.0: {} diff --git a/site/astro.config.mjs b/site/astro.config.mjs index 109c3261..7c825cc8 100644 --- a/site/astro.config.mjs +++ b/site/astro.config.mjs @@ -65,11 +65,19 @@ export default defineConfig({ // Redirects are configured in netlify.toml }, integrations: [ - sentry({ - project: 'videojsorg', - org: 'mux', - authToken: process.env.SENTRY_AUTH_TOKEN, - }), + // Only register Sentry when the upload token is present (i.e. production + // deploys). Without a token the integration still initializes the vite + // plugin and emits telemetry/init noise during every local and PR build, + // all for a "can't upload source maps" warning. Gate it at the source. + ...(process.env.SENTRY_AUTH_TOKEN + ? [ + sentry({ + project: 'videojsorg', + org: 'mux', + authToken: process.env.SENTRY_AUTH_TOKEN, + }), + ] + : []), mdx({ extendMarkdownConfig: true }), sitemap({ // llms-markdown.ts auto-generates per-framework sub-indexes, but sitemap @@ -102,6 +110,25 @@ export default defineConfig({ light: 'gruvbox-dark-hard', dark: 'gruvbox-dark-soft', }, + // Pre-declare only the languages used in MDX code fences. Without this, + // Astro ships a highlighter that lazily loads grammars on first use, + // which serializes per-block initialization during build. + langs: [ + 'tsx', + 'ts', + 'css', + 'html', + 'js', + 'jsx', + 'javascript', + 'bash', + 'markdown', + 'mdx', + 'json', + 'yaml', + 'http', + 'astro', + ], transformers: [shikiTransformMetadata, ...shikiNotationTransformers], }, remarkPlugins: [remarkConditionalHeadings, remarkReadingTime], diff --git a/site/integrations/llms-markdown.ts b/site/integrations/llms-markdown.ts index 2a3e0174..5d8f41c5 100644 --- a/site/integrations/llms-markdown.ts +++ b/site/integrations/llms-markdown.ts @@ -2,7 +2,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { AstroIntegration } from 'astro'; -import { JSDOM } from 'jsdom'; +import { parseHTML } from 'linkedom'; import TurndownService from 'turndown'; import { sidebar } from '../src/docs.config'; @@ -59,31 +59,24 @@ export default function llmsMarkdown(): AstroIntegration { // Standalone error pages emit e.g. 404.html, not 404/index.html const SKIP_PAGES = new Set(['404', '500']); - for (const page of pages) { - const { pathname } = page; - - if (SKIP_PAGES.has(pathname.replace(/\/$/, ''))) continue; + async function processPage(pathname: string): Promise { + if (SKIP_PAGES.has(pathname.replace(/\/$/, ''))) return; try { // Construct path to HTML file const htmlPath = join(siteDir, pathname, 'index.html'); const html = await readFile(htmlPath, 'utf-8'); - // Strip styles before JSDOM to avoid "Could not parse CSS stylesheet" warnings - const cleanHtml = html - .replace(//gi, '') - .replace(/]*rel=["']stylesheet["'][^>]*\/?>/gi, ''); - - // Parse HTML with jsdom - const dom = new JSDOM(cleanHtml); - const document = dom.window.document; + // linkedom is a lightweight DOM-compatible parser — no CSS engine, + // no script execution, just enough DOM to run querySelector/cloneNode. + const { document } = parseHTML(html); // Check if page has llms content const contentElements = document.querySelectorAll('[data-llms-content]'); if (contentElements.length === 0) { // No llms content, skip silently - continue; + return; } // For each content element, strip non-content elements before conversion @@ -136,6 +129,21 @@ export default function llmsMarkdown(): AstroIntegration { } } + // Process pages with a concurrency cap. Work is mostly CPU-bound + // (linkedom parse + Turndown) so parallelism only buys overlap with + // the readFile/writeFile I/O; 8 is enough to keep that overlap busy + // without flooding the event loop. + const CONCURRENCY = 8; + const queue = pages.map((page) => page.pathname); + const workers = Array.from({ length: CONCURRENCY }, async () => { + while (queue.length > 0) { + const pathname = queue.shift(); + if (pathname === undefined) return; + await processPage(pathname); + } + }); + await Promise.all(workers); + // Group docs by framework const docsByFramework = new Map(); for (const doc of docsPages) { diff --git a/site/package.json b/site/package.json index 9e198c59..e83e5c77 100644 --- a/site/package.json +++ b/site/package.json @@ -71,6 +71,7 @@ "@vitest/ui": "^4.1.0", "babel-plugin-react-compiler": "1.0.0", "jsdom": "^27.0.0", + "linkedom": "^0.18.12", "prettier": "^3.8.1", "prettier-plugin-astro": "^0.14.1", "tsx": "^4.21.0", diff --git a/site/src/components/Code/Shared.tsx b/site/src/components/Code/Shared.tsx index c3825984..2745b456 100644 --- a/site/src/components/Code/Shared.tsx +++ b/site/src/components/Code/Shared.tsx @@ -10,7 +10,24 @@ export interface SharedProps { highlighter: Highlighter; } -export default function Shared({ code, lang, highlighter }: SharedProps) { +interface Highlighted { + html: string; + preClassName: unknown; + codeClassName: unknown; +} + +// Build-time memo: identical (code, lang) pairs repeat across pages +// (e.g. the same ejected skin block on both /concepts/skins and +// /how-to/customize-skins). Shiki's codeToHast + hastToHtml is the +// dominant cost per ServerCode; caching the rendered output reuses +// it across every page in a single build. +const highlightCache = new Map(); + +function highlight(code: string, lang: BundledLanguage, highlighter: Highlighter): Highlighted { + const cacheKey = `${lang}\0${code}`; + const cached = highlightCache.get(cacheKey); + if (cached) return cached; + const hast = highlighter.codeToHast(code, { lang, themes: { @@ -22,24 +39,32 @@ export default function Shared({ code, lang, highlighter }: SharedProps) { // shiki gives us a root > pre > code > text structure // since we want to define pre and code ourselves, let's extract the text - let preProps: Record = {}; - let codeProps: Record = {}; + let preProps: Record = {}; + let codeProps: Record = {}; if (hast.type === 'root') { const pre = hast.children[0]; if (pre && pre.type === 'element' && pre.tagName === 'pre') { preProps = pre.properties; - const code = pre.children[0]; - if (code && code.type === 'element' && code.tagName === 'code') { - codeProps = code.properties; + const codeNode = pre.children[0]; + if (codeNode && codeNode.type === 'element' && codeNode.tagName === 'code') { + codeProps = codeNode.properties; // everything looked as expected! Let's use the code's children as the new root - hast.children = code.children; + hast.children = codeNode.children; } } } - const html = hastToHtml(hast); - const { class: preClassName } = preProps; - const { class: codeClassName } = codeProps; + const result: Highlighted = { + html: hastToHtml(hast), + preClassName: preProps.class, + codeClassName: codeProps.class, + }; + highlightCache.set(cacheKey, result); + return result; +} + +export default function Shared({ code, lang, highlighter }: SharedProps) { + const { html, preClassName, codeClassName } = highlight(code, lang, highlighter); return ( 
diff --git a/site/src/components/Code/serverHighlighter.ts b/site/src/components/Code/serverHighlighter.ts
index 058ec53d..5172cb1a 100644
--- a/site/src/components/Code/serverHighlighter.ts
+++ b/site/src/components/Code/serverHighlighter.ts
@@ -1,8 +1,18 @@
-import { bundledLanguages } from 'shiki';
+import bash from 'shiki/langs/bash.mjs';
+import css from 'shiki/langs/css.mjs';
+import html from 'shiki/langs/html.mjs';
+import javascript from 'shiki/langs/javascript.mjs';
+import tsx from 'shiki/langs/tsx.mjs';
+import ts from 'shiki/langs/typescript.mjs';
 import createHighlighter from './createHighlighter';
 
+// The highlighter is a long-lived singleton at module scope; it is never
+// disposed, so every grammar and theme we hand it stays in memory for the
+// process lifetime. Only pre-load the languages `` actually
+// renders — adding `bundledLanguages` here would drag ~300 grammars into the
+// build and dominate first-render cost on code-heavy pages.
 const serverHighlighter = await createHighlighter({
-  langs: Object.values(bundledLanguages),
+  langs: [bash, css, html, javascript, ts, tsx],
 });
-// TODO memory leak?
+
 export default serverHighlighter;