660 lines
22 KiB
JavaScript
660 lines
22 KiB
JavaScript
|
// One-off content migration: legacy index.html (tri-lingual data-l spans)
|
||
|
|
// -> per-locale MDX section files + language-neutral tools.json + QA inventories.
|
||
|
|
//
|
||
|
|
// Usage:
|
||
|
|
// node migration/extract.mjs # extract MDX + tools.json + anchors + legacy inventories
|
||
|
|
// node migration/extract.mjs --built # dump built-page inventories from dist/ (run after pnpm build)
|
||
|
|
//
|
||
|
|
// Fidelity rules implemented here:
|
||
|
|
// - every visible prose character of the chosen language is preserved
|
||
|
|
// (whitespace-collapsed); inline markup becomes markdown (**/*) in
|
||
|
|
// markdown contexts and JSX-safe HTML in JSX contexts;
|
||
|
|
// - MDX-hostile characters ({ } < > * _ [ ] ` ~ \ |) are emitted as
|
||
|
|
// HTML character references so they render byte-identical;
|
||
|
|
// - external links keep target="_blank" and get rel="noopener noreferrer";
|
||
|
|
// - "www." in plain text is emitted as "www." so remark-gfm cannot
|
||
|
|
// invent autolinks that the legacy page did not have.
|
||
|
|
//
|
||
|
|
// Deliberate exclusions (per migration plan, all listed in diff-report.md):
|
||
|
|
// - topbar / hero / status banner / TOC / footer (UI chrome rebuilt separately);
|
||
|
|
// - the legislative status banner and adopted/extended claims
|
||
|
|
// (#menace table rows "Durée/Duration" and "Statut/Status");
|
||
|
|
// - the runtime simpleicons CDN script and theme/lang toggle JS.
|
||
|
|
|
||
|
|
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
||
|
|
import { dirname, join } from 'node:path'
|
||
|
|
import { fileURLToPath } from 'node:url'
|
||
|
|
import * as cheerio from 'cheerio'
|
||
|
|
import * as simpleIcons from 'simple-icons'
|
||
|
|
|
||
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..')
|
||
|
|
const LEGACY = join(ROOT, 'index.html')
|
||
|
|
const OUT_SECTIONS = join(ROOT, 'src/content/sections')
|
||
|
|
const OUT_TOOLS = join(ROOT, 'src/data/tools.json')
|
||
|
|
const OUT_INVENTORY = join(ROOT, 'migration/inventory')
|
||
|
|
const LANGS = ['en', 'fr', 'nl']
|
||
|
|
|
||
|
|
const warnings = []
|
||
|
|
const exclusions = []
|
||
|
|
const nlGaps = []
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// text escaping
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/** Escape a text node so MDX renders it byte-identical, in md or jsx context. */
|
||
|
|
function escText(s) {
|
||
|
|
return (
|
||
|
|
s
|
||
|
|
.replace(/&/g, '&')
|
||
|
|
.replace(/</g, '<')
|
||
|
|
.replace(/>/g, '>')
|
||
|
|
.replace(/\{/g, '{')
|
||
|
|
.replace(/\}/g, '}')
|
||
|
|
.replace(/\*/g, '*')
|
||
|
|
.replace(/_/g, '_')
|
||
|
|
.replace(/`/g, '`')
|
||
|
|
.replace(/\[/g, '[')
|
||
|
|
.replace(/\]/g, ']')
|
||
|
|
.replace(/~/g, '~')
|
||
|
|
.replace(/\\/g, '\')
|
||
|
|
.replace(/\|/g, '|')
|
||
|
|
// keep non-breaking spaces (French punctuation) visible in the source
|
||
|
|
.replace(/\u00A0/g, ' ')
|
||
|
|
// remark-gfm autolink literals: "www.x" in plain text would become a link
|
||
|
|
.replace(/\bwww\./gi, 'www.')
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
function escAttr(s) {
|
||
|
|
return s.replace(/&/g, '&').replace(/"/g, '"')
|
||
|
|
}
|
||
|
|
|
||
|
|
// collapse runs of whitespace EXCEPT non-breaking spaces (French punctuation)
|
||
|
|
function collapse(s) {
|
||
|
|
return s.replace(/[^\S\u00A0]+/g, ' ')
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Guard the first character of a markdown block against list/quote/heading parsing. */
|
||
|
|
function guardBlockStart(s) {
|
||
|
|
const m = /^(\d+[.)] |[-+>#=] )/.exec(s)
|
||
|
|
if (!m) return s
|
||
|
|
const c = s.charAt(0)
|
||
|
|
return `&#${c.charCodeAt(0)};` + s.slice(1)
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// language filtering
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Reduce a cloned DOM subtree to one language: in every run of consecutive
|
||
|
|
* [data-l] siblings, unwrap the spans of `lang` and drop the others.
|
||
|
|
* If a run carries no span for `lang`, fall back to English and log it.
|
||
|
|
*/
|
||
|
|
function filterLang($, root, lang, context) {
|
||
|
|
const parents = new Set()
|
||
|
|
$(root)
|
||
|
|
.find('[data-l]')
|
||
|
|
.each((_, el) => parents.add(el.parent))
|
||
|
|
if ($(root).is('[data-l]')) throw new Error('unexpected data-l on root')
|
||
|
|
|
||
|
|
for (const parent of parents) {
|
||
|
|
const runs = []
|
||
|
|
let run = []
|
||
|
|
for (const node of parent.children) {
|
||
|
|
if (node.type === 'tag' && $(node).attr('data-l')) {
|
||
|
|
run.push(node)
|
||
|
|
} else if (node.type === 'text' && node.data.trim() === '') {
|
||
|
|
continue // whitespace between alternates does not break a run
|
||
|
|
} else if (run.length) {
|
||
|
|
runs.push(run)
|
||
|
|
run = []
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (run.length) runs.push(run)
|
||
|
|
|
||
|
|
for (const r of runs) {
|
||
|
|
let chosen = r.filter((n) => $(n).attr('data-l') === lang)
|
||
|
|
if (chosen.length === 0) {
|
||
|
|
chosen = r.filter((n) => $(n).attr('data-l') === 'en')
|
||
|
|
nlGaps.push({ lang, context, fallback: $(chosen).text().trim().slice(0, 80) })
|
||
|
|
}
|
||
|
|
for (const n of r) {
|
||
|
|
if (chosen.includes(n)) $(n).replaceWith($(n).contents())
|
||
|
|
else $(n).remove()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if ($(root).find('[data-l]').length) {
|
||
|
|
throw new Error(`unfiltered data-l spans left in ${context}`)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// inline serialization
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
function serializeLink($, a) {
|
||
|
|
const href = $(a).attr('href') ?? ''
|
||
|
|
const inner = inlineChildren($, a, 'jsx')
|
||
|
|
if (href.startsWith('#')) return `<a href="${escAttr(href)}">${inner}</a>`
|
||
|
|
return `<a href="${escAttr(href)}" target="_blank" rel="noopener noreferrer">${inner}</a>`
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Serialize the children of `el` as inline MDX content. mode: 'md' | 'jsx'. */
|
||
|
|
function inlineChildren($, el, mode) {
|
||
|
|
let out = ''
|
||
|
|
for (const node of el.children) {
|
||
|
|
if (node.type === 'text') {
|
||
|
|
out += escText(collapse(node.data))
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if (node.type === 'comment') continue
|
||
|
|
if (node.type !== 'tag') throw new Error(`unexpected node type ${node.type}`)
|
||
|
|
const tag = node.tagName
|
||
|
|
const inner = () => inlineChildren($, node, mode)
|
||
|
|
if (tag === 'b' || tag === 'strong') {
|
||
|
|
const hasChildEl = node.children.some((n) => n.type === 'tag')
|
||
|
|
if (mode === 'md' && !hasChildEl) {
|
||
|
|
out += wrapMd(inner(), '**')
|
||
|
|
} else {
|
||
|
|
out += `<${tag}>${inner()}</${tag}>`
|
||
|
|
}
|
||
|
|
} else if (tag === 'em' || tag === 'i') {
|
||
|
|
const hasChildEl = node.children.some((n) => n.type === 'tag')
|
||
|
|
if (mode === 'md' && !hasChildEl) {
|
||
|
|
out += wrapMd(inner(), '*')
|
||
|
|
} else {
|
||
|
|
out += `<${tag}>${inner()}</${tag}>`
|
||
|
|
}
|
||
|
|
} else if (tag === 'sup') {
|
||
|
|
out += `<sup>${inner()}</sup>`
|
||
|
|
} else if (tag === 'br') {
|
||
|
|
out += '<br />'
|
||
|
|
} else if (tag === 'a') {
|
||
|
|
out += serializeLink($, node)
|
||
|
|
} else if (tag === 'span') {
|
||
|
|
const cls = $(node).attr('class')
|
||
|
|
out += cls ? `<span class="${escAttr(cls)}">${inner()}</span>` : `<span>${inner()}</span>`
|
||
|
|
} else {
|
||
|
|
throw new Error(`unhandled inline tag <${tag}> in ${$(el).attr('class') ?? el.tagName}`)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return out
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Wrap trimmed content in markdown emphasis markers, keeping outer spaces outside. */
|
||
|
|
function wrapMd(inner, marker) {
|
||
|
|
const lead = /^\s/.test(inner) ? ' ' : ''
|
||
|
|
const trail = /\s$/.test(inner) ? ' ' : ''
|
||
|
|
const core = inner.trim()
|
||
|
|
if (!core) return inner
|
||
|
|
return `${lead}${marker}${core}${marker}${trail}`
|
||
|
|
}
|
||
|
|
|
||
|
|
function inlineMd($, el) {
|
||
|
|
return guardBlockStart(inlineChildren($, el, 'md').trim())
|
||
|
|
}
|
||
|
|
|
||
|
|
function inlineJsx($, el) {
|
||
|
|
return inlineChildren($, el, 'jsx').trim()
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// JSX block serialization (tables, callouts, grids… anything with structure)
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
// Elements whose content is serialized on a single line (phrasing content).
|
||
|
|
const JSX_LEAF = new Set(['p', 'li', 'h3', 'h4', 'td', 'th', 'span', 'blockquote', 'small', 'b'])
|
||
|
|
// Elements serialized as multi-line containers of element children.
|
||
|
|
const JSX_CONTAINER = new Set(['div', 'ul', 'ol', 'table', 'thead', 'tbody', 'tr', 'details'])
|
||
|
|
|
||
|
|
function jsxAttrs($, el, extra = {}) {
|
||
|
|
const keep = {}
|
||
|
|
const cls = $(el).attr('class')
|
||
|
|
if (cls) keep.class = cls
|
||
|
|
for (const name of ['aria-label', 'aria-hidden', 'colspan', 'rowspan', 'start']) {
|
||
|
|
const v = $(el).attr(name)
|
||
|
|
if (v !== undefined) keep[name] = v
|
||
|
|
}
|
||
|
|
const style = $(el).attr('style')
|
||
|
|
if (style) {
|
||
|
|
// presentation only; styling is rebuilt later
|
||
|
|
warnings.push(`dropped inline style "${style}" on <${el.tagName} class="${cls ?? ''}">`)
|
||
|
|
}
|
||
|
|
Object.assign(keep, extra)
|
||
|
|
return Object.entries(keep)
|
||
|
|
.map(([k, v]) => ` ${k}="${escAttr(String(v))}"`)
|
||
|
|
.join('')
|
||
|
|
}
|
||
|
|
|
||
|
|
function jsxBlock($, el, indent = '', extraAttrs = {}) {
|
||
|
|
const tag = el.tagName
|
||
|
|
if (tag === 'a') return indent + serializeLink($, el)
|
||
|
|
const attrs = jsxAttrs($, el, extraAttrs)
|
||
|
|
const elementKids = el.children.filter((n) => n.type === 'tag')
|
||
|
|
const significantText = el.children.some((n) => n.type === 'text' && n.data.trim() !== '')
|
||
|
|
if (JSX_LEAF.has(tag) || significantText || elementKids.length === 0) {
|
||
|
|
return `${indent}<${tag}${attrs}>${inlineJsx($, el)}</${tag}>`
|
||
|
|
}
|
||
|
|
if (!JSX_CONTAINER.has(tag)) throw new Error(`unhandled jsx block <${tag}>`)
|
||
|
|
const inner = elementKids.map((k) => jsxBlock($, k, indent + ' ')).join('\n')
|
||
|
|
return `${indent}<${tag}${attrs}>\n${inner}\n${indent}</${tag}>`
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// tool cards
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
const LEVELS = { Beginner: 1, Intermediate: 2, Advanced: 3 }
|
||
|
|
|
||
|
|
/** Extract the language-neutral facts of a tool card (called on the raw, unfiltered card). */
|
||
|
|
function extractToolFacts($, tg, iconMap, iconSlugs) {
|
||
|
|
const id = $(tg).attr('id')
|
||
|
|
const logo = $(tg).find('.tg-logo').first()
|
||
|
|
const nameEl = $(tg).find('h3.tg-name').first()
|
||
|
|
const nameLocalized = nameEl.find('[data-l]').length > 0
|
||
|
|
// EN plain text of the name, used as the stable language-neutral name
|
||
|
|
const clone = nameEl.clone()
|
||
|
|
clone.find('[data-l]').each((_, s) => {
|
||
|
|
if ($(s).attr('data-l') !== 'en') $(s).remove()
|
||
|
|
})
|
||
|
|
const nameEn = collapse(clone.text()).trim()
|
||
|
|
|
||
|
|
// level: from the tag whose EN text is a known level word
|
||
|
|
let level = null
|
||
|
|
$(tg)
|
||
|
|
.find('.tg-tags .tg-tag')
|
||
|
|
.each((_, t) => {
|
||
|
|
const en = collapse($(t).find('[data-l="en"]').text()).trim()
|
||
|
|
if (LEVELS[en]) level = LEVELS[en]
|
||
|
|
})
|
||
|
|
|
||
|
|
// neutral footer links (localized ones stay in the MDX, tracked by href)
|
||
|
|
const links = []
|
||
|
|
const localizedLinkHrefs = []
|
||
|
|
$(tg)
|
||
|
|
.find('.tg-foot a.tg-link')
|
||
|
|
.each((_, a) => {
|
||
|
|
if ($(a).find('[data-l]').length) {
|
||
|
|
localizedLinkHrefs.push($(a).attr('href') ?? '')
|
||
|
|
return
|
||
|
|
}
|
||
|
|
const label = collapse($(a).text()).trim().replace(/\s*→$/, '')
|
||
|
|
links.push({ label, href: $(a).attr('href') })
|
||
|
|
})
|
||
|
|
|
||
|
|
const note = collapse($(tg).find('.tg-foot .tg-note').text()).trim()
|
||
|
|
const profiles = [...note]
|
||
|
|
|
||
|
|
const legacySlug = iconMap[id] ?? null
|
||
|
|
const candidate = legacySlug && (ICON_RENAMES[legacySlug] ?? legacySlug)
|
||
|
|
const iconSlug = candidate && iconSlugs.has(candidate) ? candidate : null
|
||
|
|
if (legacySlug && !iconSlug) {
|
||
|
|
warnings.push(`icon slug "${legacySlug}" (${id}) not found in simple-icons — set to null`)
|
||
|
|
}
|
||
|
|
|
||
|
|
return {
|
||
|
|
nameLocalized,
|
||
|
|
localizedLinkHrefs,
|
||
|
|
facts: {
|
||
|
|
id,
|
||
|
|
name: nameEn,
|
||
|
|
url: links[0]?.href ?? null,
|
||
|
|
links,
|
||
|
|
iconSlug,
|
||
|
|
monogram: collapse(logo.text()).trim(),
|
||
|
|
color: (/--b:\s*([^;"]+)/.exec(logo.attr('style') ?? '') ?? [])[1]?.trim() ?? null,
|
||
|
|
level,
|
||
|
|
profiles,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Serialize one (already language-filtered) tool card as a <ToolCard> MDX block. */
|
||
|
|
function toolCardMdx($, tg) {
|
||
|
|
const id = $(tg).attr('id')
|
||
|
|
const lines = [`<ToolCard tool="${id}">`]
|
||
|
|
|
||
|
|
// name slot only when the legacy name carried localized content
|
||
|
|
if (tg.__nameLocalized) {
|
||
|
|
const nameEl = $(tg).find('h3.tg-name').first()
|
||
|
|
lines.push(` <Fragment slot="name">${inlineJsx($, nameEl[0])}</Fragment>`)
|
||
|
|
}
|
||
|
|
|
||
|
|
const tags = $(tg)
|
||
|
|
.find('.tg-tags .tg-tag')
|
||
|
|
.map((_, t) => `<span class="tool-tag">${inlineJsx($, t)}</span>`)
|
||
|
|
.get()
|
||
|
|
if (tags.length) {
|
||
|
|
lines.push(` <Fragment slot="tags">${tags.join('')}</Fragment>`)
|
||
|
|
}
|
||
|
|
|
||
|
|
for (const [slot, sel] of [
|
||
|
|
['what', '.lbl-what'],
|
||
|
|
['why', '.lbl-why'],
|
||
|
|
['who', '.lbl-who'],
|
||
|
|
]) {
|
||
|
|
const p = $(tg).find(`p.tg-field > b${sel}`).parent()
|
||
|
|
if (p.length !== 1) throw new Error(`tool ${id}: missing ${slot} field`)
|
||
|
|
const cl = p.clone()
|
||
|
|
cl.find('b.lbl').remove()
|
||
|
|
lines.push(` <Fragment slot="${slot}">${inlineJsx($, cl[0])}</Fragment>`)
|
||
|
|
}
|
||
|
|
|
||
|
|
// extra blocks (e.g. the unaudited-warning box inside t-bitchat)
|
||
|
|
$(tg)
|
||
|
|
.children('div.box')
|
||
|
|
.each((_, box) => {
|
||
|
|
lines.push(jsxBlock($, box, ' ', { slot: 'extra' }))
|
||
|
|
})
|
||
|
|
|
||
|
|
const steps = $(tg).find('details.tg-install > ol.tg-steps')
|
||
|
|
if (steps.length) {
|
||
|
|
const items = steps
|
||
|
|
.children('li')
|
||
|
|
.map((_, li) => ` <li>${inlineJsx($, li)}</li>`)
|
||
|
|
.get()
|
||
|
|
lines.push(` <ol slot="install">\n${items.join('\n')}\n </ol>`)
|
||
|
|
}
|
||
|
|
|
||
|
|
// localized footer links (neutral ones come from tools.json)
|
||
|
|
const localizedHrefs = tg.__localizedLinkHrefs ?? []
|
||
|
|
$(tg)
|
||
|
|
.find('.tg-foot a.tg-link')
|
||
|
|
.each((_, a) => {
|
||
|
|
const href = $(a).attr('href') ?? ''
|
||
|
|
if (!localizedHrefs.includes(href)) return
|
||
|
|
const inner = inlineJsx($, a)
|
||
|
|
lines.push(` <a slot="links" class="tool-link" href="${escAttr(href)}">${inner}</a>`)
|
||
|
|
})
|
||
|
|
|
||
|
|
lines.push('</ToolCard>')
|
||
|
|
return lines.join('\n')
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// section blocks -> MDX
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
function blockToMdx($, el, sectionId) {
|
||
|
|
const tag = el.tagName
|
||
|
|
const cls = $(el).attr('class') ?? ''
|
||
|
|
if (tag === 'div' && cls === 'part-head') {
|
||
|
|
const parts = []
|
||
|
|
for (const child of el.children.filter((n) => n.type === 'tag')) {
|
||
|
|
if (child.tagName === 'div' && $(child).attr('class') === 'num') {
|
||
|
|
parts.push(`<p class="part-num">${inlineJsx($, child)}</p>`)
|
||
|
|
} else if (child.tagName === 'h2') {
|
||
|
|
parts.push(`## ${inlineMd($, child)}`)
|
||
|
|
} else if (child.tagName === 'p') {
|
||
|
|
parts.push(inlineMd($, child))
|
||
|
|
} else {
|
||
|
|
throw new Error(`unhandled part-head child <${child.tagName}> in #${sectionId}`)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return parts
|
||
|
|
}
|
||
|
|
if (tag === 'h3') return [`### ${inlineMd($, el)}`]
|
||
|
|
if (tag === 'h4') return [`#### ${inlineMd($, el)}`]
|
||
|
|
if (tag === 'p') return [inlineMd($, el)]
|
||
|
|
if (tag === 'ul' || tag === 'ol') {
|
||
|
|
const items = $(el)
|
||
|
|
.children('li')
|
||
|
|
.map((_, li) => inlineChildren($, li, 'md').trim())
|
||
|
|
.get()
|
||
|
|
const lines = items.map((t, i) => (tag === 'ol' ? `${i + 1}. ${t}` : `- ${t}`))
|
||
|
|
return [lines.join('\n')]
|
||
|
|
}
|
||
|
|
if (tag === 'article' && cls.includes('tg')) return [toolCardMdx($, el)]
|
||
|
|
if (
|
||
|
|
(tag === 'div' &&
|
||
|
|
/^(tablewrap|box( (warn|honest|ok|info))?|swap|namewall|profiles|eco|manifesto)$/.test(
|
||
|
|
cls,
|
||
|
|
)) ||
|
||
|
|
(tag === 'blockquote' && cls === 'pull')
|
||
|
|
) {
|
||
|
|
return [jsxBlock($, el)]
|
||
|
|
}
|
||
|
|
throw new Error(`unhandled block <${tag} class="${cls}"> in #${sectionId}`)
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// plain-text inventories (shared by legacy and built dumps)
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
const TEXT_BLOCKS = new Set([
|
||
|
|
'address',
|
||
|
|
'article',
|
||
|
|
'aside',
|
||
|
|
'blockquote',
|
||
|
|
'body',
|
||
|
|
'button',
|
||
|
|
'details',
|
||
|
|
'div',
|
||
|
|
'dl',
|
||
|
|
'dt',
|
||
|
|
'dd',
|
||
|
|
'fieldset',
|
||
|
|
'figure',
|
||
|
|
'footer',
|
||
|
|
'form',
|
||
|
|
'h1',
|
||
|
|
'h2',
|
||
|
|
'h3',
|
||
|
|
'h4',
|
||
|
|
'h5',
|
||
|
|
'h6',
|
||
|
|
'header',
|
||
|
|
'hr',
|
||
|
|
'li',
|
||
|
|
'main',
|
||
|
|
'nav',
|
||
|
|
'ol',
|
||
|
|
'p',
|
||
|
|
'pre',
|
||
|
|
'section',
|
||
|
|
'summary',
|
||
|
|
'table',
|
||
|
|
'tbody',
|
||
|
|
'td',
|
||
|
|
'tfoot',
|
||
|
|
'th',
|
||
|
|
'thead',
|
||
|
|
'tr',
|
||
|
|
'ul',
|
||
|
|
'small',
|
||
|
|
])
|
||
|
|
|
||
|
|
/** One line of normalized text per block-level element. */
|
||
|
|
function textLines($, root) {
|
||
|
|
const lines = []
|
||
|
|
let buf = ''
|
||
|
|
const flush = () => {
|
||
|
|
const t = buf.replace(/[\s\u00A0]+/g, ' ').trim()
|
||
|
|
if (t) lines.push(t)
|
||
|
|
buf = ''
|
||
|
|
}
|
||
|
|
const walk = (node) => {
|
||
|
|
if (node.type === 'text') {
|
||
|
|
buf += node.data
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if (node.type !== 'tag') return
|
||
|
|
const tag = node.tagName
|
||
|
|
// noscript is excluded: parse5 keeps its children as raw text, which would
|
||
|
|
// leak markup into the dump (it is new chrome, absent from the legacy page)
|
||
|
|
if (['script', 'style', 'svg', 'template', 'noscript'].includes(tag)) return
|
||
|
|
if (tag === 'br') {
|
||
|
|
buf += ' '
|
||
|
|
return
|
||
|
|
}
|
||
|
|
// label <b> elements get their own line so prose lines stay comparable
|
||
|
|
const cls = ($(node).attr('class') ?? '').split(/\s+/)
|
||
|
|
const isBlock = TEXT_BLOCKS.has(tag) || cls.includes('lbl') || cls.includes('tool-label')
|
||
|
|
if (isBlock) flush()
|
||
|
|
for (const c of node.children) walk(c)
|
||
|
|
if (isBlock) flush()
|
||
|
|
}
|
||
|
|
walk(root)
|
||
|
|
flush()
|
||
|
|
return lines
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
// main passes
|
||
|
|
// ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
function loadLegacy() {
|
||
|
|
return cheerio.load(readFileSync(LEGACY, 'utf8'))
|
||
|
|
}
|
||
|
|
|
||
|
|
function parseIconMap(html) {
|
||
|
|
const script = /var ICONS=\{([\s\S]*?)\};/.exec(html)
|
||
|
|
if (!script) throw new Error('legacy ICONS map not found')
|
||
|
|
const map = {}
|
||
|
|
for (const m of script[1].matchAll(/'([^']+)':'([^']+)'/g)) map[m[1]] = m[2]
|
||
|
|
return map
|
||
|
|
}
|
||
|
|
|
||
|
|
function simpleIconSlugs() {
|
||
|
|
return new Set(
|
||
|
|
Object.values(simpleIcons)
|
||
|
|
.map((i) => i && i.slug)
|
||
|
|
.filter(Boolean),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Legacy CDN slugs whose brand was renamed in current simple-icons releases.
|
||
|
|
const ICON_RENAMES = { tutanota: 'tuta', aegis: 'aegisauthenticator' }
|
||
|
|
|
||
|
|
function applyExclusions($, log = false) {
|
||
|
|
// Rule: adopted/extended legislative-status claims are rewritten from
|
||
|
|
// verified sources separately -> drop the two claim rows of the #menace table.
|
||
|
|
$('#menace .tablewrap tbody tr').each((_, tr) => {
|
||
|
|
const firstCell = collapse($(tr).find('td').first().find('[data-l="en"]').text()).trim()
|
||
|
|
if (firstCell === 'Duration' || /^Status/.test(firstCell)) {
|
||
|
|
if (log) {
|
||
|
|
exclusions.push({
|
||
|
|
where: '#menace comparison table',
|
||
|
|
why: 'adopted/extended legislative-status claim, deferred for verified rewrite',
|
||
|
|
text: collapse($(tr).text()).trim(),
|
||
|
|
})
|
||
|
|
}
|
||
|
|
$(tr).remove()
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
function extract() {
|
||
|
|
const html = readFileSync(LEGACY, 'utf8')
|
||
|
|
const $ = loadLegacy()
|
||
|
|
if ($('[data-l] [data-l]').length) throw new Error('nested data-l spans — unsupported')
|
||
|
|
|
||
|
|
mkdirSync(OUT_INVENTORY, { recursive: true })
|
||
|
|
|
||
|
|
// ---- anchors.json: every id present in the legacy file, in document order
|
||
|
|
const anchors = $('[id]')
|
||
|
|
.map((_, el) => $(el).attr('id'))
|
||
|
|
.get()
|
||
|
|
writeFileSync(join(OUT_INVENTORY, 'anchors.json'), JSON.stringify(anchors, null, 2) + '\n')
|
||
|
|
|
||
|
|
// ---- legacy per-language inventories (full document, before exclusions)
|
||
|
|
for (const lang of LANGS) {
|
||
|
|
const $full = loadLegacy()
|
||
|
|
filterLang($full, $full('body')[0], lang, 'full-document')
|
||
|
|
const lines = textLines($full, $full('body')[0])
|
||
|
|
writeFileSync(join(OUT_INVENTORY, `legacy-${lang}.txt`), lines.join('\n') + '\n')
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- tools.json (language-neutral facts, from the raw DOM)
|
||
|
|
const iconMap = parseIconMap(html)
|
||
|
|
const slugs = simpleIconSlugs()
|
||
|
|
const extracted = $('article.tg')
|
||
|
|
.map((_, tg) => extractToolFacts($, tg, iconMap, slugs))
|
||
|
|
.get()
|
||
|
|
const toolsOut = extracted.map((e) => e.facts)
|
||
|
|
mkdirSync(dirname(OUT_TOOLS), { recursive: true })
|
||
|
|
writeFileSync(OUT_TOOLS, JSON.stringify(toolsOut, null, 2) + '\n')
|
||
|
|
const localizedNames = new Set(extracted.filter((e) => e.nameLocalized).map((e) => e.facts.id))
|
||
|
|
const localizedLinks = new Map(extracted.map((e) => [e.facts.id, e.localizedLinkHrefs]))
|
||
|
|
|
||
|
|
// ---- sections -> MDX
|
||
|
|
applyExclusions($, true)
|
||
|
|
const sections = $('main > section').get()
|
||
|
|
let lastPart = 0
|
||
|
|
const written = { en: 0, fr: 0, nl: 0 }
|
||
|
|
sections.forEach((section, idx) => {
|
||
|
|
const id = $(section).attr('id')
|
||
|
|
const order = idx + 1
|
||
|
|
const numEn = collapse($(section).find('.part-head .num [data-l="en"]').text())
|
||
|
|
const m = /(?:PART)\s+(\d+)/.exec(numEn)
|
||
|
|
const part = m ? Number(m[1]) : lastPart
|
||
|
|
lastPart = part
|
||
|
|
|
||
|
|
for (const lang of LANGS) {
|
||
|
|
const $lang = loadLegacy()
|
||
|
|
applyExclusions($lang)
|
||
|
|
const sec = $lang(`main > section#${id.replace(/([^\w-])/g, '\\$1')}`)[0]
|
||
|
|
filterLang($lang, sec, lang, `#${id}`)
|
||
|
|
// flag localized tool names / footer links for the MDX slots
|
||
|
|
$lang(sec)
|
||
|
|
.find('article.tg')
|
||
|
|
.each((_, tg) => {
|
||
|
|
const tgId = $lang(tg).attr('id')
|
||
|
|
tg.__nameLocalized = localizedNames.has(tgId)
|
||
|
|
tg.__localizedLinkHrefs = localizedLinks.get(tgId) ?? []
|
||
|
|
})
|
||
|
|
|
||
|
|
const title = collapse($lang(sec).find('.part-head h2').first().text()).trim()
|
||
|
|
const blocks = []
|
||
|
|
for (const child of sec.children.filter((n) => n.type === 'tag')) {
|
||
|
|
blocks.push(...blockToMdx($lang, child, id))
|
||
|
|
}
|
||
|
|
const fm = [
|
||
|
|
'---',
|
||
|
|
`id: ${JSON.stringify(id)}`,
|
||
|
|
`part: ${part}`,
|
||
|
|
`order: ${order}`,
|
||
|
|
`title: ${JSON.stringify(title)}`,
|
||
|
|
'---',
|
||
|
|
].join('\n')
|
||
|
|
const nn = String(order).padStart(2, '0')
|
||
|
|
const dir = join(OUT_SECTIONS, lang)
|
||
|
|
mkdirSync(dir, { recursive: true })
|
||
|
|
writeFileSync(join(dir, `${nn}-${id}.mdx`), fm + '\n\n' + blocks.join('\n\n') + '\n')
|
||
|
|
written[lang]++
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
// ---- run log for the diff report
|
||
|
|
const log = {
|
||
|
|
sections: written,
|
||
|
|
tools: toolsOut.length,
|
||
|
|
anchors: anchors.length,
|
||
|
|
nlGaps,
|
||
|
|
exclusions,
|
||
|
|
warnings,
|
||
|
|
}
|
||
|
|
writeFileSync(join(OUT_INVENTORY, 'extract-log.json'), JSON.stringify(log, null, 2) + '\n')
|
||
|
|
console.log(`sections written: en=${written.en} fr=${written.fr} nl=${written.nl}`)
|
||
|
|
console.log(`tools: ${toolsOut.length}, anchors: ${anchors.length}`)
|
||
|
|
console.log(`NL gaps: ${nlGaps.length}`)
|
||
|
|
console.log(`exclusions: ${exclusions.length}`)
|
||
|
|
for (const w of warnings) console.log('WARN:', w)
|
||
|
|
}
|
||
|
|
|
||
|
|
function dumpBuilt() {
|
||
|
|
const pages = { en: 'dist/index.html', fr: 'dist/fr/index.html', nl: 'dist/nl/index.html' }
|
||
|
|
for (const [lang, rel] of Object.entries(pages)) {
|
||
|
|
const $ = cheerio.load(readFileSync(join(ROOT, rel), 'utf8'))
|
||
|
|
const lines = textLines($, $('body')[0])
|
||
|
|
writeFileSync(join(OUT_INVENTORY, `built-${lang}.txt`), lines.join('\n') + '\n')
|
||
|
|
console.log(`built-${lang}.txt: ${lines.length} lines (from ${rel})`)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (process.argv.includes('--built')) dumpBuilt()
|
||
|
|
else extract()
|