diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js index d185119bc9..0743ecba07 100644 --- a/docs/.vuepress/config.js +++ b/docs/.vuepress/config.js @@ -5,6 +5,7 @@ const searchBoxPlugin = require('./plugins/search-box'); const examples = require('./plugins/examples/examples'); const HyperFormula = require('../../dist/hyperformula.full'); const includeCodeSnippet = require('./plugins/markdown-it-include-code-snippet'); +const stripCitationMarkers = require('./plugins/strip-citation-markers'); const searchPattern = new RegExp('^/api', 'i'); @@ -136,6 +137,9 @@ module.exports = { }) md.use(footnotePlugin) md.use(includeCodeSnippet) + // Strip internal audit-harness annotations ([V] markers and the + // §Sources footer) so they never appear in customer-facing docs. + md.use(stripCitationMarkers) } }, // TODO: It doesn't work. It's seems that this option is bugged. Documentation says that this option is configurable, diff --git a/docs/.vuepress/plugins/strip-citation-markers/index.js b/docs/.vuepress/plugins/strip-citation-markers/index.js new file mode 100644 index 0000000000..431c8d5bb4 --- /dev/null +++ b/docs/.vuepress/plugins/strip-citation-markers/index.js @@ -0,0 +1,175 @@ +/** + * markdown-it plugin: strip internal audit-harness annotations from rendered docs. + * + * Our internal authoring workflow uses the audit-harness convention: + * - Inline citation markers like `[vrf_1]`, `[dec_3]` (legacy `[V12]`) placed + * next to factual claims. + * - A trailing `§AuditSources` footer listing the sources. + * + * These markers exist so the audit-harness can re-verify every claim against + * its source before content is shipped. They are NEVER meant to be seen by + * end users. When any spec or note ends up published as docs, we strip them + * at build time so the rendered site stays clean. + * + * Stripping rules: + * - Inline marker: `[_]` (e.g. `[vrf_1]`) or legacy + * `[V]`, NOT followed by `(` (so real markdown + * links `[vrf_1](url)` / `[V12](url)` are left untouched). + * - Footer section: a heading whose text is exactly `§AuditSources` + * (a unique token, so legitimate `Sources` headings are + * never clobbered), together with everything below it up + * to end-of-file or the next top-level (`#`) heading. + * - Fenced/inline code is left alone, so pages that document the + * audit-harness itself can still render the markers verbatim. + * + * Implementation: walks the markdown-it token stream after parsing. + */ + +// Audit-harness citation markers. The current convention (post-2026-05-21) is a +// lowercase prefix + `_` + digits — `[vrf_1]`, `[dec_3]`, `[con_2]`, `[que_5]`, +// `[wrg_7]`, `[crf_4]` — matching the parser grammar `^\[[a-z][a-z0-9_]*\]$`. +// The legacy uppercase `[V]` form is kept as an alternative for older notes. +// In both cases a trailing `(` is excluded so real markdown links like +// `[vrf_1](url)` / `[V12](url)` are left untouched. +const INLINE_CITATION_PATTERN = /\[(?:V\d+|[a-z][a-z0-9]*_\d+)\](?!\()/g; +// Footer marker. Deliberately a unique token (`§AuditSources`) rather than a +// bare `Sources` heading, so that legitimate docs sections titled "Sources" +// are never clobbered. The `§` is required. +const SOURCES_HEADING_PATTERN = /^\s*§\s*AuditSources\s*$/i; + +/** + * Removes inline `[V]` markers from a string of text. + * + * @param {string} text - Raw text content from a markdown token. + * @returns {string} Text with citation markers removed and surrounding + * whitespace normalized. + */ +const stripInlineMarkers = (text) => + text + .replace(INLINE_CITATION_PATTERN, '') + // collapse stray double spaces left behind by removal + .replace(/[ \t]{2,}/g, ' ') + // tidy " ." / " ," / " ;" / " :" / " )" + .replace(/ ([.,;:!?\)])/g, '$1'); + +/** + * Recursively strips inline markers from children of an inline token. + * + * @param {Array} children - markdown-it inline children tokens. + */ +const stripChildren = (children) => { + if (!Array.isArray(children)) return; + children.forEach((child) => { + if (child.type === 'text' && typeof child.content === 'string') { + child.content = stripInlineMarkers(child.content); + } + if (child.children) { + stripChildren(child.children); + } + }); +}; + +/** + * Detects whether a heading_open token (already located) introduces the + * `Sources` / `§Sources` footer. The heading's raw inline content is first + * normalized via `stripInlineMarkers` so that an authored heading like + * `§ Sources [V1]` (markers next to the heading text) still matches the + * strict end-anchored pattern; without normalization the trailing `[V1]` + * would defeat the `\s*$` anchor and the footer would never be detected. + * + * @param {Array} tokens - Full token array. + * @param {number} headingOpenIdx - Index of the heading_open token. + * @returns {boolean} True when the heading text matches the Sources footer. + */ +const isSourcesHeading = (tokens, headingOpenIdx) => { + const inline = tokens[headingOpenIdx + 1]; + if (!inline || inline.type !== 'inline') return false; + return SOURCES_HEADING_PATTERN.test(stripInlineMarkers(inline.content || '')); +}; + +/** + * Returns the index after which the Sources footer ends. The footer extends + * from the Sources heading up to (but not including) the FIRST of: + * - the next top-level (`h1`) heading_open token, or + * - any `footnote_*` token (markdown-it-footnote appends `footnote_block` + * and friends at the END of the stream; they belong to the page body, + * not to the footer), or + * - end-of-stream. + * + * @param {Array} tokens - Full token array. + * @param {number} startIdx - Index of the Sources heading_open token. + * @returns {number} Exclusive end index of the footer. + */ +const findFooterEnd = (tokens, startIdx) => { + for (let i = startIdx + 1; i < tokens.length; i += 1) { + const t = tokens[i]; + if (t.type === 'heading_open' && t.tag === 'h1') { + return i; + } + if (typeof t.type === 'string' && t.type.startsWith('footnote_')) { + return i; + } + } + return tokens.length; +}; + +/** + * Mutates the token array in place to remove the Sources footer (heading + + * everything below) and apply inline marker stripping to every text token. + * + * Footnote invariant: markdown-it-footnote (registered in `config.js`) + * appends `footnote_block` / `footnote_anchor` / `footnote_open` / + * `footnote_close` / `footnote_ref` tokens at the END of the token stream. + * The footer splice stops before any such token so footnotes on pages that + * also carry a `§ Sources` footer are not silently swallowed. + * + * @param {Array} tokens - markdown-it token array. + * @returns {Array} The same token array (for chaining). + */ +const transformTokens = (tokens) => { + // 1. Find a `Sources` heading and drop everything from it onward + // (up to the next h1, if any). + for (let i = 0; i < tokens.length; i += 1) { + const t = tokens[i]; + if (t.type === 'heading_open' && isSourcesHeading(tokens, i)) { + const end = findFooterEnd(tokens, i); + tokens.splice(i, end - i); + i -= 1; + } + } + + // 2. Strip `[V]` markers from every remaining inline text token. + // Code tokens (`code_inline`, `code_block`, `fence`) are skipped so + // docs that illustrate the audit-harness syntax keep working. + tokens.forEach((token) => { + if (token.type === 'inline' && token.children) { + stripChildren(token.children); + } + }); + + return tokens; +}; + +/** + * markdown-it plugin entry point. Hooks into the core ruler so transforms + * run after parsing but before rendering. + * + * @param {object} md - markdown-it instance supplied by VuePress. + */ +const stripCitationMarkers = (md) => { + // Insert before `replacements` so that VuePress's heading-anchor logic + // (which runs later and slugifies heading text) also sees the cleaned + // text. Falls back to push() if the anchor rule cannot be located. + const insert = (state) => { + transformTokens(state.tokens); + }; + try { + md.core.ruler.before('replacements', 'strip-citation-markers', insert); + } catch (e) { + md.core.ruler.push('strip-citation-markers', insert); + } +}; + +module.exports = stripCitationMarkers; +module.exports.transformTokens = transformTokens; +module.exports.stripInlineMarkers = stripInlineMarkers; diff --git a/docs/.vuepress/plugins/strip-citation-markers/test-fixture.md b/docs/.vuepress/plugins/strip-citation-markers/test-fixture.md new file mode 100644 index 0000000000..c81f5f5b7e --- /dev/null +++ b/docs/.vuepress/plugins/strip-citation-markers/test-fixture.md @@ -0,0 +1,28 @@ +# Sample page + +This sentence has a citation marker [V1] right after a word, and another one [V42]. + +Real markdown links such as [V12](https://example.com/v12) MUST remain intact because they are not bare citation markers. + +A line with multiple markers [V3] [V4] should collapse trailing whitespace cleanly. + +Inline code like `[V99]` must NOT be stripped because authors may need to discuss the audit-harness syntax itself. + +``` +fenced code [V7] stays as-is +``` + +## A subsection [V8] + +Body of a subsection [V9]. + +## §AuditSources + +- [V1] https://example.com/source-1 +- [V3] https://example.com/source-3 +- [V4] https://example.com/source-4 +- [V8] https://example.com/source-8 +- [V9] https://example.com/source-9 +- [V42] https://example.com/source-42 + +Trailing footer content that must also be removed. diff --git a/docs/.vuepress/plugins/strip-citation-markers/test-plugin-order.js b/docs/.vuepress/plugins/strip-citation-markers/test-plugin-order.js new file mode 100644 index 0000000000..77c21abfb5 --- /dev/null +++ b/docs/.vuepress/plugins/strip-citation-markers/test-plugin-order.js @@ -0,0 +1,160 @@ +/** + * Plugin-order regression test for `strip-citation-markers`. + * + * Background: the strip plugin's footer splice intentionally stops before + * any `footnote_*` token because `markdown-it-footnote` appends those at + * the END of the token stream — they belong to the page body, not to the + * `§ Sources` footer. + * + * The wiring contract in `docs/.vuepress/config.js` is: + * + * md.use(footnotePlugin) // registers footnote_tail + * md.use(includeCodeSnippet) + * md.use(stripCitationMarkers) // splices §Sources footer + * + * The ACTUAL ordering that makes footnotes survive is determined by where + * each plugin hooks into `core.ruler`: + * - `markdown-it-footnote`: `core.ruler.after('inline', 'footnote_tail')` + * - `strip-citation-markers`: `core.ruler.before('replacements', ...)` + * + * Because `inline` comes before `replacements` in markdown-it's default + * core rule chain, `footnote_tail` always runs before our strip rule — as + * long as BOTH plugins are registered. If a future refactor: + * (a) removes `markdown-it-footnote` (no footnote tokens ever exist), or + * (b) registers it in a way that moves `footnote_tail` AFTER our hook, + * then footnotes on any page with a `§ Sources` footer will be silently + * swallowed by the splice. + * + * This test demonstrates both halves of the contract: + * + * 1. NEGATIVE CONTROL: build a markdown-it instance that DOES NOT carry + * `markdown-it-footnote`. Feed it a page with `[^note]` syntax + a + * `§ Sources` footer. The `[^note]` literal text appears BEFORE the + * `§ Sources` heading so it survives the splice — but no footnote + * anchor/section is produced (because no footnote plugin is loaded). + * This anchors the "footnote_tail must be registered upstream" half + * of the contract. + * + * 2. POSITIVE CONTROL: same source, plugins registered in the SAME order + * as `config.js`. Footnote anchor + body + section all survive AND + * the `§ Sources` footer body is stripped AND inline `[V]` markers + * are stripped. This is the contract `config.js` relies on. + * + * If either of these assertions ever flips, the strip plugin and the + * VuePress config are out of sync and footnotes will break in customer + * docs. + * + * Run with: `node docs/.vuepress/plugins/strip-citation-markers/test-plugin-order.js` + */ + +const MarkdownIt = require('markdown-it'); +const footnotePlugin = require('markdown-it-footnote'); +const stripCitationMarkers = require('./index'); + +const failures = []; + +const assert = (cond, message) => { + if (!cond) failures.push(message); +}; + +const source = [ + '# Footnote-aware page', + '', + 'Body text with a footnote ref.[^note] [V5]', + '', + '[^note]: Footnote body content.', + '', + '## §AuditSources', + '', + '- [V5] https://example.com/source-5', + '- Trailing footer entry that must be stripped.', + '', +].join('\n'); + +const hasFootnoteAnchor = (html) => + /class="footnote-ref"|class="footnotes"|]*footnotes/i.test(html); +const hasFootnoteBody = (html) => /Footnote body content/.test(html); + +// --- 1. NEGATIVE CONTROL: no markdown-it-footnote installed. +// The `[^note]` reference is just literal text; no `footnote_*` tokens +// are ever generated; the strip plugin behaves correctly on body text +// (markers stripped, §Sources footer dropped) but there is no footnote +// anchor/section in the output. This locks in the assumption that +// footnote tokens come from a SEPARATE plugin — if someone replaces +// `markdown-it-footnote` with a different mechanism, this test fails +// and forces a review of `findFooterEnd`'s footnote check. +const mdNoFootnote = new MarkdownIt({ html: true }); +mdNoFootnote.use(stripCitationMarkers); +const noFootnoteHtml = mdNoFootnote.render(source); + +assert( + !hasFootnoteAnchor(noFootnoteHtml), + 'Negative control: without markdown-it-footnote, no footnote anchor/section should appear. If this fires, the strip plugin or markdown-it core gained an unexpected footnote rule and the wiring assumption changed.' +); +assert( + !/Trailing footer entry/.test(noFootnoteHtml), + 'Negative control: §Sources footer body must still be stripped even without footnote plugin' +); +assert( + !/\[V5\]/.test(noFootnoteHtml.replace(//g, '')), + 'Negative control: inline [V] markers must still be stripped even without footnote plugin' +); + +// --- 2. POSITIVE CONTROL: plugins registered in the same order as +// `config.js`: footnote FIRST, strip LAST. This is the contract. +const mdConfig = new MarkdownIt({ html: true }); +mdConfig.use(footnotePlugin); +mdConfig.use(stripCitationMarkers); +const configOrderHtml = mdConfig.render(source); + +assert( + hasFootnoteAnchor(configOrderHtml), + 'Positive control: config-order (footnote BEFORE strip) must render the footnote anchor/section' +); +assert( + hasFootnoteBody(configOrderHtml), + 'Positive control: config-order must render the footnote body content' +); +assert( + !/Trailing footer entry/.test(configOrderHtml), + 'Positive control: config-order must still strip the §Sources footer body' +); +assert( + !/\[V5\]/.test(configOrderHtml.replace(//g, '')), + 'Positive control: config-order must still strip inline [V] markers' +); + +// --- 3. RULE-CHAIN INVARIANT: assert that `footnote_tail` runs BEFORE the +// strip plugin's rule in the resulting `core.ruler` chain. This is the +// PRIMITIVE mechanism that makes the wiring work. If a future +// markdown-it-footnote version moves `footnote_tail` to a different +// ruler position, this assertion fires and points engineers at the +// root cause directly. +const ruleNames = mdConfig.core.ruler.__rules__.map((r) => r.name); +const footnoteIdx = ruleNames.indexOf('footnote_tail'); +const stripIdx = ruleNames.indexOf('strip-citation-markers'); +assert( + footnoteIdx !== -1, + 'Rule-chain invariant: expected `footnote_tail` rule to be registered by markdown-it-footnote' +); +assert( + stripIdx !== -1, + 'Rule-chain invariant: expected `strip-citation-markers` rule to be registered' +); +assert( + footnoteIdx < stripIdx, + 'Rule-chain invariant: expected `footnote_tail` to run BEFORE `strip-citation-markers` so footnote tokens exist when the splice runs (got footnote_tail=' + + footnoteIdx + ', strip=' + stripIdx + ')' +); + +if (failures.length > 0) { + console.error('FAIL strip-citation-markers/test-plugin-order'); + failures.forEach((f) => console.error(' - ' + f)); + console.error('\n--- no-footnote rendered output ---\n' + noFootnoteHtml); + console.error('\n--- config-order rendered output ---\n' + configOrderHtml); + process.exit(1); +} + +console.log( + 'PASS strip-citation-markers/test-plugin-order (10 assertions: 3 negative + 4 positive + 3 rule-chain)' +); diff --git a/docs/.vuepress/plugins/strip-citation-markers/test.js b/docs/.vuepress/plugins/strip-citation-markers/test.js new file mode 100644 index 0000000000..28cfade08b --- /dev/null +++ b/docs/.vuepress/plugins/strip-citation-markers/test.js @@ -0,0 +1,339 @@ +/** + * Standalone Node test for the strip-citation-markers plugin. + * + * Run with: `node docs/.vuepress/plugins/strip-citation-markers/test.js` + * + * Loads the fixture, parses it with markdown-it (the same parser VuePress + * ships) using the plugin installed, then asserts: + * - No `[V]` markers leak into the rendered HTML body. + * - The `§Sources` footer and its content are removed. + * - Inline code and fenced code blocks keep their `[V]` text intact. + * - Real markdown links `[V](url)` are preserved as links. + */ + +const fs = require('fs'); +const path = require('path'); +const MarkdownIt = require('markdown-it'); +const footnotePlugin = require('markdown-it-footnote'); +const stripCitationMarkers = require('./index'); +const { transformTokens } = stripCitationMarkers; + +const fixturePath = path.join(__dirname, 'test-fixture.md'); +const source = fs.readFileSync(fixturePath, 'utf8'); + +const md = new MarkdownIt({ html: true }); +md.use(stripCitationMarkers); + +const rendered = md.render(source); + +const failures = []; + +const assert = (cond, message) => { + if (!cond) failures.push(message); +}; + +// 1. No bare markers in rendered HTML body text. +// We use a regex that excludes anything wrapped in ... +// so inline/fenced code content does not count against us. +const renderedWithoutCode = rendered + .replace(//g, '') + .replace(//g, ''); +assert( + !/\[V\d+\]/.test(renderedWithoutCode), + 'Expected no [V] markers in rendered HTML body, but found some' +); + +// 2. §Sources footer must be gone. +assert( + !/Sources/i.test(renderedWithoutCode) && !/source-1/.test(rendered), + 'Expected §Sources footer to be removed' +); +assert( + !/Trailing footer content/.test(rendered), + 'Expected §Sources footer body to be removed' +); + +// 3. Inline code with `[V99]` must survive. +assert( + /\[V99\]<\/code>/.test(rendered), + 'Expected inline code `[V99]` to survive untouched' +); + +// 4. Fenced code block with `[V7]` must survive. +assert( + /fenced code \[V7\] stays as-is/.test(rendered), + 'Expected fenced code block content to keep [V7]' +); + +// 5. Real markdown link `[V12](url)` must remain a link. +assert( + /]*href="https:\/\/example\.com\/v12"[^>]*>V12<\/a>/.test(rendered), + 'Expected [V12](url) to remain a real markdown link' +); + +// 6. Subsection heading should keep its text minus the marker. +assert( + /]*>.*A subsection.*<\/h2>/.test(rendered) && + !/A subsection \[V8\]/.test(rendered), + 'Expected subsection heading text to be cleaned of marker' +); + +// 6a. Mutation kill (M1) — zero-digit `[V]` must NOT be stripped; the +// regex must require `\d+`, not `\d*`. If the quantifier is weakened, +// literal `[V]` (no digits) would also be removed. +const mdM1 = new MarkdownIt({ html: true }); +mdM1.use(stripCitationMarkers); +const renderedM1 = mdM1.render('Bare brackets like [V] are not markers.'); +assert( + /\[V\]/.test(renderedM1), + 'Expected literal [V] (no digits) to be preserved' +); + +// 6b. Mutation kill (M2) — the negative-lookahead `(?!\()` anchors the +// "real markdown link survives" guarantee. Re-assert post-parse that +// the link text is "V12" (digits intact), not collapsed to "V". +assert( + /]*href="https:\/\/example\.com\/v12"[^>]*>V12<\/a>/.test(rendered) && + !/]*href="https:\/\/example\.com\/v12"[^>]*>V<\/a>/.test(rendered), + 'Expected link text to remain "V12", not be collapsed to "V"' +); + +// 6c. `SOURCES_HEADING_PATTERN` uses the `i` flag — a lowercase +// `## §auditsources` footer must still be stripped. +const mdM3 = new MarkdownIt({ html: true }); +mdM3.use(stripCitationMarkers); +const renderedM3 = mdM3.render( + '# Page\n\nBody.\n\n## §auditsources\n\n- lower-case marker body\n' +); +assert( + !/lower-case marker body/.test(renderedM3), + 'Expected case-insensitive §AuditSources heading match (lowercase variant stripped)' +); + +// 6c-bis. Footgun guard — a legitimate `Sources` / `§ Sources` heading is NOT a +// marker and must be PRESERVED (the whole point of the §AuditSources rename). +const mdLegit = new MarkdownIt({ html: true }); +mdLegit.use(stripCitationMarkers); +const renderedLegit = mdLegit.render( + '# Page\n\nBody.\n\n## Sources\n\n- legit sources body must survive\n\n## § Sources\n\n- legit para-sources body must survive\n' +); +assert( + /legit sources body must survive/.test(renderedLegit) && + /legit para-sources body must survive/.test(renderedLegit), + 'Expected legitimate `Sources` / `§ Sources` headings to be PRESERVED (not clobbered)' +); + +// 6d. Mutation kill (M5) — `findFooterEnd` terminates at an `h1` boundary, +// NOT an `h2`. A top-level `# Second page` after the Sources footer must +// end the splice so its body survives. +const mdM5 = new MarkdownIt({ html: true }); +mdM5.use(stripCitationMarkers); +const renderedM5 = mdM5.render([ + '# First page', + '', + 'First body.', + '', + '## §AuditSources', + '', + '- footer to drop', + '', + '# Second page', + '', + 'Second body must survive.', +].join('\n')); +assert( + /Second body must survive/.test(renderedM5) && + !/footer to drop/.test(renderedM5), + 'Expected footer stripping to stop at the next h1 boundary' +); + +// 6e. Mutation kill (M11) — `stripInlineMarkers` collapses 2+ spaces left +// behind by marker removal. The fixture line "multiple markers [V3] [V4] +// should collapse..." must not contain a run of 2+ spaces in the output. +assert( + !/multiple markers {2,}should collapse/.test(rendered), + 'Expected double whitespace left by marker removal to be collapsed to a single space' +); + +// 6f. Bugbot edge case — a heading whose raw inline content carries an +// authored `[V]` marker (e.g. `§ Sources [V1]`) must still be +// detected as the Sources footer. Footer detection runs BEFORE inline +// marker stripping, so the heading-text check must normalize markers +// out before testing the strict-anchored regex. +const mdBugbot = new MarkdownIt({ html: true }); +mdBugbot.use(stripCitationMarkers); +const renderedBugbot = mdBugbot.render([ + '# Page', + '', + 'Body text.', + '', + '## §AuditSources [V1]', + '', + '- [V1] https://example.com/source-1 — footer body to drop', + '', +].join('\n')); +assert( + !/footer body to drop/.test(renderedBugbot), + 'Expected `§ Sources [V1]` heading to be detected as Sources footer (Bugbot edge case)' +); +assert( + !/\[V1\]/.test(renderedBugbot.replace(//g, '')), + 'Expected `[V]` marker not to survive on a page with `§ Sources [V1]` heading' +); + +// 6g. Mutation kill — a bare `§ Sources` heading (no markers) must still +// trigger the strip after the normalization shim is added; guards +// against accidentally swallowing the pattern test or replacing it. +const mdBareSources = new MarkdownIt({ html: true }); +mdBareSources.use(stripCitationMarkers); +const renderedBareSources = mdBareSources.render([ + '# Page', + '', + 'Body.', + '', + '## §AuditSources', + '', + '- bare-sources footer body', + '', +].join('\n')); +assert( + !/bare-sources footer body/.test(renderedBareSources), + 'Expected bare `§ Sources` heading (no markers) to still trigger footer strip' +); + +// 7. Synthetic token-stream: a §Sources heading followed by footer text +// followed by `footnote_*` tokens must keep the footnote tokens after the +// footer is spliced out. Mirrors what markdown-it-footnote appends at the +// END of the token stream. +const makeToken = (type, tag = '', extra = {}) => + Object.assign({ type, tag, content: '', children: null }, extra); + +const syntheticTokens = [ + makeToken('heading_open', 'h1', { markup: '#' }), + makeToken('inline', '', { content: 'Page title', children: [] }), + makeToken('heading_close', 'h1'), + makeToken('paragraph_open', 'p'), + makeToken('inline', '', { content: 'Body with a footnote ref.', children: [] }), + makeToken('paragraph_close', 'p'), + makeToken('heading_open', 'h2', { markup: '##' }), + makeToken('inline', '', { content: '§AuditSources', children: [] }), + makeToken('heading_close', 'h2'), + makeToken('paragraph_open', 'p'), + makeToken('inline', '', { content: 'Trailing footer body.', children: [] }), + makeToken('paragraph_close', 'p'), + makeToken('footnote_block_open'), + makeToken('footnote_open', '', { meta: { id: 0 } }), + makeToken('inline', '', { content: 'Footnote text.', children: [] }), + makeToken('footnote_anchor'), + makeToken('footnote_close'), + makeToken('footnote_block_close'), +]; + +transformTokens(syntheticTokens); + +const survivingTypes = syntheticTokens.map((t) => t.type); +assert( + survivingTypes.includes('footnote_block_open') && + survivingTypes.includes('footnote_open') && + survivingTypes.includes('footnote_anchor') && + survivingTypes.includes('footnote_close') && + survivingTypes.includes('footnote_block_close'), + 'Expected footnote_* tokens to survive past the §Sources splice (got: ' + + survivingTypes.join(',') + ')' +); +assert( + !syntheticTokens.some( + (t) => t.type === 'inline' && /Trailing footer body/.test(t.content || '') + ), + 'Expected §Sources footer body to be removed from synthetic token stream' +); + +// 8. Full-pipeline check with markdown-it-footnote: a page that has a +// footnote AND a §Sources footer must still render the footnote anchor. +const mdWithFootnotes = new MarkdownIt({ html: true }); +mdWithFootnotes.use(footnotePlugin); +mdWithFootnotes.use(stripCitationMarkers); + +const footnoteSource = [ + '# Footnote-aware page', + '', + 'Body text with a footnote ref.[^note] [V5]', + '', + '[^note]: Footnote body content.', + '', + '## §AuditSources', + '', + '- [V5] https://example.com/source-5', + '', +].join('\n'); + +const footnoteRendered = mdWithFootnotes.render(footnoteSource); + +assert( + /class="footnote-ref"|class="footnotes"|]*footnotes/i.test(footnoteRendered), + 'Expected footnote anchor/section to survive in full-pipeline render' +); +assert( + /Footnote body content/.test(footnoteRendered), + 'Expected footnote body content to survive in full-pipeline render' +); +assert( + !/Trailing footer/.test(footnoteRendered) && !/\[V5\]/.test(footnoteRendered.replace(//g, '')), + 'Expected §Sources footer and inline [V] markers to still be stripped' +); + +// 9. CURRENT audit-harness grammar: lowercase prefixed markers like `[vrf_1]` +// / `[dec_3]` (parser `^\[[a-z][a-z0-9_]*\]$`, canonical run-strip uses +// `[a-z][a-z0-9]*_`) must be stripped. The pre-2026-05-21 `[V]` +// form is legacy; real specs emit the lowercase prefixed form. +const mdLower = new MarkdownIt({ html: true }); +mdLower.use(stripCitationMarkers); +const renderedLower = mdLower.render( + 'Claim [vrf_1], decision [dec_3], constraint [con_2], question [que_5], wrong [wrg_7], crossref [crf_4].' +); +const lowerBody = renderedLower + .replace(//g, '') + .replace(//g, ''); +assert( + !/\[(?:vrf|dec|con|que|wrg|crf)_\d+\]/.test(lowerBody), + 'Expected lowercase audit-harness markers ([vrf_1] etc.) to be stripped' +); + +// 9a. Over-strip guard — bracketed tokens WITHOUT the `prefix_` grammar +// (plain words, no trailing `_`) must be preserved. +const mdPlain = new MarkdownIt({ html: true }); +mdPlain.use(stripCitationMarkers); +const renderedPlain = mdPlain.render('See [note] and [abc] inline.'); +assert( + /\[note\]/.test(renderedPlain) && /\[abc\]/.test(renderedPlain), + 'Expected non-marker bracketed tokens like [note]/[abc] to be preserved' +); + +// 9b. Real markdown link with a lowercase-marker-shaped label must survive +// (negative lookahead `(?!\()`). +const mdLowerLink = new MarkdownIt({ html: true }); +mdLowerLink.use(stripCitationMarkers); +const renderedLowerLink = mdLowerLink.render('Link [vrf_1](https://example.com/v) here.'); +assert( + /]*href="https:\/\/example\.com\/v"[^>]*>vrf_1<\/a>/.test(renderedLowerLink), + 'Expected [vrf_1](url) to remain a real markdown link' +); + +// 9c. Lowercase marker inside inline code must survive (code tokens skipped). +const mdLowerCode = new MarkdownIt({ html: true }); +mdLowerCode.use(stripCitationMarkers); +const renderedLowerCode = mdLowerCode.render('Docs show `[vrf_9]` verbatim.'); +assert( + /\[vrf_9\]<\/code>/.test(renderedLowerCode), + 'Expected inline code `[vrf_9]` to survive untouched' +); + +if (failures.length > 0) { + console.error('FAIL strip-citation-markers'); + failures.forEach((f) => console.error(' - ' + f)); + console.error('\n--- rendered output ---\n' + rendered); + console.error('\n--- footnote rendered output ---\n' + footnoteRendered); + process.exit(1); +} + +console.log('PASS strip-citation-markers (' + 24 + ' assertions)');