tigera · ctauchen · Aug 24, 2026 · Aug 24, 2026
@@ -37,5 +37,10 @@ export default {
     moduleNameMapper: {
       '^@docusaurus/Link$': '<rootDir>/jest/mockComponent.js',
     },
+    // The llms-txt plugin's dependencies (cheerio, unified, rehype, remark) ship as
+    // ESM only, and Jest does not transform node_modules by default. Transforming
+    // everything costs about two seconds on the full suite and avoids maintaining a
+    // brittle allowlist of the whole unified ecosystem.
+    transformIgnorePatterns: [],
     roots: ['<rootDir>/src'],
 };
@@ -0,0 +1,133 @@
+import { extractFromHtmlString } from '../extract.js';
+import { convertToMarkdown } from '../convert.js';
+import { shiftHeadings } from '../generate.js';
+
+const SITE_URL = 'https://docs.tigera.io';
+
+/**
+ * Build a page in the shape Docusaurus and Prism actually emit: the title in a
+ * <header>, headings trailed by a zero-width hash-link anchor, and every code line
+ * wrapped in a block-level `token-line` div that also ends in a <br>.
+ */
+function page(body) {
+  return `<!DOCTYPE html><html><head><meta name="description" content="A description."></head>
+    <body><nav class="navbar">nav</nav><main><article><div class="theme-doc-markdown markdown">
+    <header><h1>Page title</h1></header>${body}
+    </div></article></main><footer class="footer">footer</footer></body></html>`;
+}
+
+function codeBlock(lines, lang = 'yaml') {
+  const rendered = lines
+    .map((l) => `<div class="token-line"><span class="token plain">${l}</span><br/></div>`)
+    .join('');
+  return `<pre class="prism-code language-${lang}"><code class="codeBlockLines_vJ6I">${rendered}</code></pre>`;
+}
+
+function codeBlockPage(lines) {
+  return page(codeBlock(lines));
+}
+
+function heading(level, id, text) {
+  return `<h${level} class="anchor" id="${id}">${text}<a href="#${id}" class="hash-link" aria-label="Direct link to ${text}" title="Direct link to ${text}"></a></h${level}>`;
+}
+
+describe('extractFromHtmlString', () => {
+  it('keeps the page title as metadata and out of the body', () => {
+    const extracted = extractFromHtmlString(page('<p>Body text.</p>'));
+
+    expect(extracted.title).toBe('Page title');
+    expect(extracted.description).toBe('A description.');
+    expect(extracted.html).not.toContain('Page title');
+  });
+
+  it('drops site chrome', () => {
+    const extracted = extractFromHtmlString(page('<p>Body text.</p>'));
+
+    expect(extracted.html).not.toContain('nav');
+    expect(extracted.html).not.toContain('footer');
+  });
+
+  it('returns null for HTML with no doc content', () => {
+    expect(extractFromHtmlString('<html><body></body></html>')).toBeNull();
+  });
+});
+
+describe('convertToMarkdown', () => {
+  it('strips the zero-width hash-link anchor from headings', async () => {
+    const extracted = extractFromHtmlString(page(heading(2, 'before-you-begin', 'Before you begin')));
+    const markdown = await convertToMarkdown(extracted.html, SITE_URL);
+
+    expect(markdown).toBe('## Before you begin');
+    expect(markdown).not.toContain('');
+  });
+
+  it('emits one line per source line, not two', async () => {
+    const extracted = extractFromHtmlString(page(codeBlock(['kind: Cluster', 'nodes:'])));
+    const markdown = await convertToMarkdown(extracted.html, SITE_URL);
+
+    expect(markdown).toBe('```yaml\nkind: Cluster\nnodes:\n```');
+  });
+
+  it('preserves indentation and column alignment inside code', async () => {
+    const extracted = extractFromHtmlString(
+      codeBlockPage(['nodes:', '  - role: control-plane', 'NAME     STATUS'])
+    );
+    const markdown = await convertToMarkdown(extracted.html, SITE_URL);
+
+    expect(markdown).toContain('\n  - role: control-plane\n');
+    expect(markdown).toContain('NAME     STATUS');
+  });
+
+  it('preserves genuinely blank lines inside code', async () => {
+    const extracted = extractFromHtmlString(codeBlockPage(['first', '', 'third']));
+    const markdown = await convertToMarkdown(extracted.html, SITE_URL);
+
+    expect(markdown).toBe('```yaml\nfirst\n\nthird\n```');
+  });
+
+  it('resolves root-relative links against the site URL', async () => {
+    const extracted = extractFromHtmlString(page('<p><a href="/calico/latest/about">About</a></p>'));
+    const markdown = await convertToMarkdown(extracted.html, SITE_URL);
+
+    expect(markdown).toBe('[About](https://docs.tigera.io/calico/latest/about)');
+  });
+});
+
+describe('shiftHeadings', () => {
+  it('shifts headings down by the given delta', () => {
+    expect(shiftHeadings('## A\ntext\n### B', 2)).toBe('#### A\ntext\n##### B');
+  });
+
+  it('caps at h6', () => {
+    expect(shiftHeadings('##### A\n###### B', 2)).toBe('###### A\n###### B');
+  });
+
+  it('is a no-op for a delta of zero', () => {
+    expect(shiftHeadings('## A', 0)).toBe('## A');
+  });
+
+  it('leaves comments inside fenced code alone', () => {
+    const input = '## Real\n\n```bash\n# not a heading\n## also not\n```\n\n## Real again';
+    const want = '#### Real\n\n```bash\n# not a heading\n## also not\n```\n\n#### Real again';
+
+    expect(shiftHeadings(input, 2)).toBe(want);
+  });
+
+  it('leaves comments inside a fence indented under a list item alone', () => {
+    const input = '## H\n\n1. step\n\n   ```yaml\n   # comment\n   ```\n\n## H2';
+    const want = '#### H\n\n1. step\n\n   ```yaml\n   # comment\n   ```\n\n#### H2';
+
+    expect(shiftHeadings(input, 2)).toBe(want);
+  });
+
+  it('handles tilde fences, including backticks nested inside one', () => {
+    const input = '## H\n~~~\n```\n# no\n```\n~~~\n## H2';
+    const want = '#### H\n~~~\n```\n# no\n```\n~~~\n#### H2';
+
+    expect(shiftHeadings(input, 2)).toBe(want);
+  });
+
+  it('ignores a hash run with no following space', () => {
+    expect(shiftHeadings('##NotAHeading\n## Yes', 2)).toBe('##NotAHeading\n#### Yes');
+  });
+});
@@ -125,7 +125,13 @@ function createHandlers(siteUrl) {
         if (codeLangMatch) lang = codeLangMatch[1];
       }
 
-      const value = codeNode ? toText(codeNode) : toText(node);
+      // Prism renders each source line as a block-level `token-line` div that also
+      // ends in a <br>. Without `whitespace: 'pre'`, to-text adds a newline for the
+      // block boundary on top of the one from the <br>, double-spacing every fence
+      // and collapsing leading indentation.
+      const value = codeNode
+        ? toText(codeNode, { whitespace: 'pre' })
+        : toText(node, { whitespace: 'pre' });
 
       return {
         type: 'code',

@@ -23,6 +23,7 @@ const REMOVE_SELECTORS = [
   '.theme-doc-breadcrumbs',
   '.pagination-nav',
   'button[class*="copyButton"]',
+  'a.hash-link',
   'svg.iconExternalLink',
   '.table-of-contents',
   'nav.navbar',
@@ -31,6 +32,25 @@ const REMOVE_SELECTORS = [
   'header',
 ];
 
+/**
+ * Preprocess Prism code blocks: demote each `token-line` from a block-level div
+ * to an inline span.
+ *
+ * Prism emits every source line as `<div class="token-line">…<br/></div>`, so the
+ * line break is represented twice — once by the block boundary and once by the
+ * <br>. hast-util-to-text honours both, double-spacing every fence. Demoting the
+ * div to a span leaves the <br> as the only line break. Doing it this way (rather
+ * than dropping the <br>) keeps genuinely blank source lines, which are empty
+ * divs whose only content is the <br>.
+ *
+ * @param {cheerio.CheerioAPI} $
+ */
+function preprocessCodeBlocks($) {
+  $('pre .token-line').each(function () {
+    this.tagName = 'span';
+  });
+}
+
 /**
  * Preprocess tab containers: expand all panels and annotate with group info.
  * Docusaurus tabs use role="tablist" for the button bar and role="tabpanel"
@@ -93,6 +113,16 @@ export async function extractFromHtml(htmlPath) {
     return null;
   }
 
+  return extractFromHtmlString(rawHtml);
+}
+
+/**
+ * Extract content and metadata from a rendered Docusaurus page.
+ *
+ * @param {string} rawHtml - Full page HTML
+ * @returns {{ html: string, title: string, description: string } | null}
+ */
+export function extractFromHtmlString(rawHtml) {
   const $ = cheerio.load(rawHtml);
 
   // Extract metadata before stripping elements
@@ -109,7 +139,8 @@ export async function extractFromHtml(htmlPath) {
     $(selector).remove();
   }
 
-  // Preprocess tabs before extraction
+  // Preprocess code blocks and tabs before extraction
+  preprocessCodeBlocks($);
   preprocessTabs($);
 
   // Extract content using priority selectors

@@ -7,6 +7,56 @@
  * @typedef {{ title: string, description: string, permalink: string, markdown: string, sectionLabel: string }} ProcessedDoc
  */
 
+/**
+ * Shift every ATX heading in a Markdown body down by `delta` levels, capped at h6.
+ *
+ * Page bodies come out of the converter with `##` as their top level, because the
+ * page <h1> lives in the `<header>` we strip and is carried as metadata instead.
+ * Nesting such a body under a doc-title heading inverts the hierarchy unless the
+ * body is shifted to match.
+ *
+ * Fenced code is skipped — `#` starts a comment in most of the shell and YAML
+ * samples in these docs.
+ *
+ * @param {string} markdown
+ * @param {number} delta
+ * @returns {string}
+ */
+export function shiftHeadings(markdown, delta) {
+  if (delta <= 0) {
+    return markdown;
+  }
+
+  const lines = markdown.split('\n');
+  let openFence = null;
+
+  for (let i = 0; i < lines.length; i++) {
+    const fenceMatch = lines[i].match(/^\s*(`{3,}|~{3,})/);
+
+    if (fenceMatch) {
+      const marker = fenceMatch[1];
+      if (openFence === null) {
+        openFence = marker[0];
+      } else if (marker[0] === openFence) {
+        openFence = null;
+      }
+      continue;
+    }
+
+    if (openFence !== null) {
+      continue;
+    }
+
+    const headingMatch = lines[i].match(/^(#{1,6})(\s)/);
+    if (headingMatch) {
+      const level = Math.min(6, headingMatch[1].length + delta);
+      lines[i] = '#'.repeat(level) + lines[i].slice(headingMatch[1].length);
+    }
+  }
+
+  return lines.join('\n');
+}
+
 /**
  * Group docs by their section label, preserving insertion order.
  *
@@ -91,28 +141,29 @@ export function generateProductIndex(productName, description, docs, siteUrl, op
  * @param {string} description - Blockquote description
  * @param {string} versionLabel - Version string (e.g., "3.31")
  * @param {ProcessedDoc[]} docs - All processed docs with markdown content
+ * @param {string} siteUrl - Site base URL, for the per-doc Source line
  * @returns {string}
  */
-export function generateProductFull(productName, description, versionLabel, docs) {
-  const sections = groupBySection(docs);
+export function generateProductFull(productName, description, versionLabel, docs, siteUrl) {
   const lines = [];
 
   lines.push(`# ${productName} - Full Documentation`);
   lines.push('');
   lines.push(`> Complete documentation for ${productName} (version ${versionLabel}).`);
 
-  for (const [sectionLabel, sectionDocs] of sections) {
+  for (const doc of docs) {
     lines.push('');
     lines.push('---');
     lines.push('');
-    lines.push(`## ${sectionLabel}`);
-
-    for (const doc of sectionDocs) {
-      lines.push('');
-      lines.push(`### ${doc.title}`);
-      lines.push('');
-      lines.push(doc.markdown);
+    lines.push(`## ${doc.title}`);
+    lines.push('');
+    lines.push(`Source: ${siteUrl}${doc.permalink}`);
+    if (doc.sectionLabel) {
+      lines.push(`Section: ${doc.sectionLabel}`);
     }
+    lines.push('');
+    // Body headings start at h2; nest them one level under the h2 doc title.
+    lines.push(shiftHeadings(doc.markdown, 1));
   }
 
   lines.push('');

@@ -197,7 +197,8 @@ export default function llmsTxtPlugin(context, options) {
             productName,
             description,
             result.versionLabel,
-            result.docs
+            result.docs,
+            siteUrl
           );
           const fullPath = path.join(outDir, instanceId, 'llms-full.txt');
           await fs.writeFile(fullPath, fullContent);