diff --git a/jest.config.mjs b/jest.config.mjs index e726fae197..021d56c471 100644 --- a/jest.config.mjs +++ b/jest.config.mjs @@ -37,5 +37,10 @@ export default { moduleNameMapper: { '^@docusaurus/Link$': '/jest/mockComponent.js', }, + // The llms-txt plugin's dependencies (cheerio, unified, rehype, remark) ship as + // ESM only, and Jest does not transform node_modules by default. Transforming + // everything costs about two seconds on the full suite and avoids maintaining a + // brittle allowlist of the whole unified ecosystem. + transformIgnorePatterns: [], roots: ['/src'], }; diff --git a/src/plugins/docusaurus-plugin-llms-txt/__test__/conversion.test.js b/src/plugins/docusaurus-plugin-llms-txt/__test__/conversion.test.js new file mode 100644 index 0000000000..13af4fea66 --- /dev/null +++ b/src/plugins/docusaurus-plugin-llms-txt/__test__/conversion.test.js @@ -0,0 +1,133 @@ +import { extractFromHtmlString } from '../extract.js'; +import { convertToMarkdown } from '../convert.js'; +import { shiftHeadings } from '../generate.js'; + +const SITE_URL = 'https://docs.tigera.io'; + +/** + * Build a page in the shape Docusaurus and Prism actually emit: the title in a + *
, headings trailed by a zero-width hash-link anchor, and every code line + * wrapped in a block-level `token-line` div that also ends in a
. + */ +function page(body) { + return ` +
+

Page title

${body} +
footer
`; +} + +function codeBlock(lines, lang = 'yaml') { + const rendered = lines + .map((l) => `
${l}
`) + .join(''); + return `
${rendered}
`; +} + +function codeBlockPage(lines) { + return page(codeBlock(lines)); +} + +function heading(level, id, text) { + return `${text}​`; +} + +describe('extractFromHtmlString', () => { + it('keeps the page title as metadata and out of the body', () => { + const extracted = extractFromHtmlString(page('

Body text.

')); + + expect(extracted.title).toBe('Page title'); + expect(extracted.description).toBe('A description.'); + expect(extracted.html).not.toContain('Page title'); + }); + + it('drops site chrome', () => { + const extracted = extractFromHtmlString(page('

Body text.

')); + + expect(extracted.html).not.toContain('nav'); + expect(extracted.html).not.toContain('footer'); + }); + + it('returns null for HTML with no doc content', () => { + expect(extractFromHtmlString('')).toBeNull(); + }); +}); + +describe('convertToMarkdown', () => { + it('strips the zero-width hash-link anchor from headings', async () => { + const extracted = extractFromHtmlString(page(heading(2, 'before-you-begin', 'Before you begin'))); + const markdown = await convertToMarkdown(extracted.html, SITE_URL); + + expect(markdown).toBe('## Before you begin'); + expect(markdown).not.toContain('​'); + }); + + it('emits one line per source line, not two', async () => { + const extracted = extractFromHtmlString(page(codeBlock(['kind: Cluster', 'nodes:']))); + const markdown = await convertToMarkdown(extracted.html, SITE_URL); + + expect(markdown).toBe('```yaml\nkind: Cluster\nnodes:\n```'); + }); + + it('preserves indentation and column alignment inside code', async () => { + const extracted = extractFromHtmlString( + codeBlockPage(['nodes:', ' - role: control-plane', 'NAME STATUS']) + ); + const markdown = await convertToMarkdown(extracted.html, SITE_URL); + + expect(markdown).toContain('\n - role: control-plane\n'); + expect(markdown).toContain('NAME STATUS'); + }); + + it('preserves genuinely blank lines inside code', async () => { + const extracted = extractFromHtmlString(codeBlockPage(['first', '', 'third'])); + const markdown = await convertToMarkdown(extracted.html, SITE_URL); + + expect(markdown).toBe('```yaml\nfirst\n\nthird\n```'); + }); + + it('resolves root-relative links against the site URL', async () => { + const extracted = extractFromHtmlString(page('

About

')); + const markdown = await convertToMarkdown(extracted.html, SITE_URL); + + expect(markdown).toBe('[About](https://docs.tigera.io/calico/latest/about)'); + }); +}); + +describe('shiftHeadings', () => { + it('shifts headings down by the given delta', () => { + expect(shiftHeadings('## A\ntext\n### B', 2)).toBe('#### A\ntext\n##### B'); + }); + + it('caps at h6', () => { + expect(shiftHeadings('##### A\n###### B', 2)).toBe('###### A\n###### B'); + }); + + it('is a no-op for a delta of zero', () => { + expect(shiftHeadings('## A', 0)).toBe('## A'); + }); + + it('leaves comments inside fenced code alone', () => { + const input = '## Real\n\n```bash\n# not a heading\n## also not\n```\n\n## Real again'; + const want = '#### Real\n\n```bash\n# not a heading\n## also not\n```\n\n#### Real again'; + + expect(shiftHeadings(input, 2)).toBe(want); + }); + + it('leaves comments inside a fence indented under a list item alone', () => { + const input = '## H\n\n1. step\n\n ```yaml\n # comment\n ```\n\n## H2'; + const want = '#### H\n\n1. step\n\n ```yaml\n # comment\n ```\n\n#### H2'; + + expect(shiftHeadings(input, 2)).toBe(want); + }); + + it('handles tilde fences, including backticks nested inside one', () => { + const input = '## H\n~~~\n```\n# no\n```\n~~~\n## H2'; + const want = '#### H\n~~~\n```\n# no\n```\n~~~\n#### H2'; + + expect(shiftHeadings(input, 2)).toBe(want); + }); + + it('ignores a hash run with no following space', () => { + expect(shiftHeadings('##NotAHeading\n## Yes', 2)).toBe('##NotAHeading\n#### Yes'); + }); +}); diff --git a/src/plugins/docusaurus-plugin-llms-txt/convert.js b/src/plugins/docusaurus-plugin-llms-txt/convert.js index 238d0a0668..e9840689aa 100644 --- a/src/plugins/docusaurus-plugin-llms-txt/convert.js +++ b/src/plugins/docusaurus-plugin-llms-txt/convert.js @@ -125,7 +125,13 @@ function createHandlers(siteUrl) { if (codeLangMatch) lang = codeLangMatch[1]; } - const value = codeNode ? toText(codeNode) : toText(node); + // Prism renders each source line as a block-level `token-line` div that also + // ends in a
. Without `whitespace: 'pre'`, to-text adds a newline for the + // block boundary on top of the one from the
, double-spacing every fence + // and collapsing leading indentation. + const value = codeNode + ? toText(codeNode, { whitespace: 'pre' }) + : toText(node, { whitespace: 'pre' }); return { type: 'code', diff --git a/src/plugins/docusaurus-plugin-llms-txt/extract.js b/src/plugins/docusaurus-plugin-llms-txt/extract.js index c07532ebba..4f60282ea4 100644 --- a/src/plugins/docusaurus-plugin-llms-txt/extract.js +++ b/src/plugins/docusaurus-plugin-llms-txt/extract.js @@ -23,6 +23,7 @@ const REMOVE_SELECTORS = [ '.theme-doc-breadcrumbs', '.pagination-nav', 'button[class*="copyButton"]', + 'a.hash-link', 'svg.iconExternalLink', '.table-of-contents', 'nav.navbar', @@ -31,6 +32,25 @@ const REMOVE_SELECTORS = [ 'header', ]; +/** + * Preprocess Prism code blocks: demote each `token-line` from a block-level div + * to an inline span. + * + * Prism emits every source line as `
…
`, so the + * line break is represented twice — once by the block boundary and once by the + *
. hast-util-to-text honours both, double-spacing every fence. Demoting the + * div to a span leaves the
as the only line break. Doing it this way (rather + * than dropping the
) keeps genuinely blank source lines, which are empty + * divs whose only content is the
. + * + * @param {cheerio.CheerioAPI} $ + */ +function preprocessCodeBlocks($) { + $('pre .token-line').each(function () { + this.tagName = 'span'; + }); +} + /** * Preprocess tab containers: expand all panels and annotate with group info. * Docusaurus tabs use role="tablist" for the button bar and role="tabpanel" @@ -93,6 +113,16 @@ export async function extractFromHtml(htmlPath) { return null; } + return extractFromHtmlString(rawHtml); +} + +/** + * Extract content and metadata from a rendered Docusaurus page. + * + * @param {string} rawHtml - Full page HTML + * @returns {{ html: string, title: string, description: string } | null} + */ +export function extractFromHtmlString(rawHtml) { const $ = cheerio.load(rawHtml); // Extract metadata before stripping elements @@ -109,7 +139,8 @@ export async function extractFromHtml(htmlPath) { $(selector).remove(); } - // Preprocess tabs before extraction + // Preprocess code blocks and tabs before extraction + preprocessCodeBlocks($); preprocessTabs($); // Extract content using priority selectors diff --git a/src/plugins/docusaurus-plugin-llms-txt/generate.js b/src/plugins/docusaurus-plugin-llms-txt/generate.js index ecfd0e40fe..b3e29fa592 100644 --- a/src/plugins/docusaurus-plugin-llms-txt/generate.js +++ b/src/plugins/docusaurus-plugin-llms-txt/generate.js @@ -7,6 +7,56 @@ * @typedef {{ title: string, description: string, permalink: string, markdown: string, sectionLabel: string }} ProcessedDoc */ +/** + * Shift every ATX heading in a Markdown body down by `delta` levels, capped at h6. + * + * Page bodies come out of the converter with `##` as their top level, because the + * page

lives in the `
` we strip and is carried as metadata instead. + * Nesting such a body under a doc-title heading inverts the hierarchy unless the + * body is shifted to match. + * + * Fenced code is skipped — `#` starts a comment in most of the shell and YAML + * samples in these docs. + * + * @param {string} markdown + * @param {number} delta + * @returns {string} + */ +export function shiftHeadings(markdown, delta) { + if (delta <= 0) { + return markdown; + } + + const lines = markdown.split('\n'); + let openFence = null; + + for (let i = 0; i < lines.length; i++) { + const fenceMatch = lines[i].match(/^\s*(`{3,}|~{3,})/); + + if (fenceMatch) { + const marker = fenceMatch[1]; + if (openFence === null) { + openFence = marker[0]; + } else if (marker[0] === openFence) { + openFence = null; + } + continue; + } + + if (openFence !== null) { + continue; + } + + const headingMatch = lines[i].match(/^(#{1,6})(\s)/); + if (headingMatch) { + const level = Math.min(6, headingMatch[1].length + delta); + lines[i] = '#'.repeat(level) + lines[i].slice(headingMatch[1].length); + } + } + + return lines.join('\n'); +} + /** * Group docs by their section label, preserving insertion order. * @@ -91,28 +141,29 @@ export function generateProductIndex(productName, description, docs, siteUrl, op * @param {string} description - Blockquote description * @param {string} versionLabel - Version string (e.g., "3.31") * @param {ProcessedDoc[]} docs - All processed docs with markdown content + * @param {string} siteUrl - Site base URL, for the per-doc Source line * @returns {string} */ -export function generateProductFull(productName, description, versionLabel, docs) { - const sections = groupBySection(docs); +export function generateProductFull(productName, description, versionLabel, docs, siteUrl) { const lines = []; lines.push(`# ${productName} - Full Documentation`); lines.push(''); lines.push(`> Complete documentation for ${productName} (version ${versionLabel}).`); - for (const [sectionLabel, sectionDocs] of sections) { + for (const doc of docs) { lines.push(''); lines.push('---'); lines.push(''); - lines.push(`## ${sectionLabel}`); - - for (const doc of sectionDocs) { - lines.push(''); - lines.push(`### ${doc.title}`); - lines.push(''); - lines.push(doc.markdown); + lines.push(`## ${doc.title}`); + lines.push(''); + lines.push(`Source: ${siteUrl}${doc.permalink}`); + if (doc.sectionLabel) { + lines.push(`Section: ${doc.sectionLabel}`); } + lines.push(''); + // Body headings start at h2; nest them one level under the h2 doc title. + lines.push(shiftHeadings(doc.markdown, 1)); } lines.push(''); diff --git a/src/plugins/docusaurus-plugin-llms-txt/index.js b/src/plugins/docusaurus-plugin-llms-txt/index.js index d35cb2c75a..b3247780f9 100644 --- a/src/plugins/docusaurus-plugin-llms-txt/index.js +++ b/src/plugins/docusaurus-plugin-llms-txt/index.js @@ -197,7 +197,8 @@ export default function llmsTxtPlugin(context, options) { productName, description, result.versionLabel, - result.docs + result.docs, + siteUrl ); const fullPath = path.join(outDir, instanceId, 'llms-full.txt'); await fs.writeFile(fullPath, fullContent);