// Generates lib/entities.weft (the HTML5 named character references) and lib/unicode.weft (Unicode // punctuation and symbols, what CommonMark's delimiter rules call punctuation). // // curl -o entities.json https://html.spec.whatwg.org/entities.json // node packages/markdown/scripts/gen-tables.mjs entities.json import { readFileSync, writeFileSync } from "node:fs" import { dirname, join } from "node:path" import { fileURLToPath } from "node:url" const lib = join(dirname(fileURLToPath(import.meta.url)), "..", "lib") const entities = JSON.parse(readFileSync(process.argv[2], "utf8")) const pairs = Object.entries(entities) .filter(([name]) => name.endsWith(";")) .map(([name, v]) => `${name.slice(1, -1)}=${v.codepoints.map((c) => c.toString(16)).join(",")}`) .sort() writeFileSync( join(lib, "entities.weft"), `// Generated by scripts/gen-tables.mjs from https://html.spec.whatwg.org/entities.json. Don't edit it. import { parseInt } from "std/math" let DATA = "${pairs.join(" ")}" fn build(): Map { let mut table = Map() for entry of DATA.split(" ") { let at = entry.indexOf("=") let mut text = "" for hex of entry.slice(at + 1).split(",") { text += String.fromCodePoint(try parseInt(hex, 16) else 0xfffd) } table.set(entry.slice(0, at), text) } return table } let ENTITIES = build() /// The text of a named character reference (\`amp\` for \`&\`), or \`null\` when HTML has no such name. export fn entity(name: string): string? { return ENTITIES.get(name) } `, ) const punct = /[\p{P}\p{S}]/u const ranges = [] let start = -1 for (let c = 0; c <= 0x110000; c++) { const is = c < 0x110000 && !(c >= 0xd800 && c <= 0xdfff) && punct.test(String.fromCodePoint(c)) if (is && start < 0) start = c if (!is && start >= 0) { ranges.push(start, c - 1) start = -1 } } const rows = [] for (let i = 0; i < ranges.length; i += 16) rows.push(" " + ranges.slice(i, i + 16).join(", ") + ",") writeFileSync( join(lib, "unicode.weft"), `// Generated by scripts/gen-tables.mjs from the engine's Unicode tables (general categories P and S). Don't edit it. /// Sorted, inclusive ranges of code points: start, end, start, end, … export let PUNCTUATION: number[] = [ ${rows.join("\n")} ] `, ) console.log(`${pairs.length} entities, ${ranges.length / 2} punctuation ranges`)