packages/ at markup-kit dd1e6eb5270e
A snapshot of packages/ in the Weft repository; the official packages are published from these commits. Markup-Kit-Commit: dd1e6eb5270e62e0c8b6c202b3d01f0da6e00ddc
This commit is contained in:
commit
306b143104
72 changed files with 10915 additions and 0 deletions
49
markdown/scripts/bench.mjs
Normal file
49
markdown/scripts/bench.mjs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
// Times @weft/markdown on the VM against markdown-it on Node (JIT and --jitless), on the same text: the CommonMark
|
||||
// spec's examples, joined and repeated. Needs markdown-it: MARKDOWN_IT=<path to its folder>.
|
||||
//
|
||||
// node packages/markdown/scripts/bench.mjs commonmark.json 1,10,50
|
||||
import { spawnSync } from "node:child_process"
|
||||
import { readFileSync, writeFileSync } from "node:fs"
|
||||
import { dirname, join } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..")
|
||||
const weft = process.env.WEFT_BIN ?? join(root, "../../core/target/release/weft")
|
||||
const spec = process.argv[2]
|
||||
const sizes = (process.argv[3] ?? "1,10,50").split(",").map(Number)
|
||||
const lib = process.env.MARKDOWN_IT
|
||||
|
||||
const script = `
|
||||
const MarkdownIt = require(${JSON.stringify(lib)})
|
||||
const fs = require("fs")
|
||||
const cases = JSON.parse(fs.readFileSync(${JSON.stringify(spec)}, "utf8"))
|
||||
const text = cases.map((c) => c.markdown).join("\\n\\n").repeat(Number(process.argv[2]))
|
||||
const md = new MarkdownIt({ html: false, linkify: true })
|
||||
const start = process.hrtime.bigint()
|
||||
const html = md.render(text)
|
||||
console.log(Number(process.hrtime.bigint() - start) / 1e6, text.length, html.length)
|
||||
`
|
||||
const file = join(process.env.TMPDIR ?? "/workspace/core/target/tools", "bench-markdown-it.cjs")
|
||||
writeFileSync(file, script)
|
||||
|
||||
function ms(cmd, args, env = {}) {
|
||||
const start = process.hrtime.bigint()
|
||||
const r = spawnSync(cmd, args, { cwd: root, encoding: "utf8", env: { ...process.env, ...env } })
|
||||
if (r.status !== 0) throw new Error(r.stderr)
|
||||
return { ms: Number(process.hrtime.bigint() - start) / 1e6, out: r.stdout.trim() }
|
||||
}
|
||||
|
||||
const limits = { WEFT_FUEL: "100000000000", WEFT_MEMORY_MB: "4000" }
|
||||
const base = Math.min(...[0, 1, 2].map(() => ms(weft, ["run", "bench/bench.weft", "0"], limits).ms))
|
||||
console.log(`VM start-up (loading the package and the spec's text): ${base.toFixed(0)} ms`)
|
||||
console.log("size KB | weft VM ms | markdown-it JIT ms | markdown-it --jitless ms | VM / JIT | VM / jitless")
|
||||
for (const n of sizes) {
|
||||
const vm = Math.min(...[0, 1, 2].map(() => ms(weft, ["run", "bench/bench.weft", String(n)], limits).ms)) - base
|
||||
const best = (flags) => {
|
||||
const runs = [0, 1, 2].map(() => ms("node", [...flags, file, String(n)]).out.split(" ").map(Number))
|
||||
return { ms: Math.min(...runs.map((r) => r[0])), size: runs[0][1] }
|
||||
}
|
||||
const jit = best([])
|
||||
const nojit = best(["--jitless"])
|
||||
console.log(`${(jit.size / 1024).toFixed(0).padStart(7)} | ${vm.toFixed(0).padStart(10)} | ${jit.ms.toFixed(1).padStart(18)} | ${nojit.ms.toFixed(1).padStart(24)} | ${(vm / jit.ms).toFixed(1).padStart(8)} | ${(vm / nojit.ms).toFixed(1).padStart(12)}`)
|
||||
}
|
||||
69
markdown/scripts/gen-spec.mjs
Normal file
69
markdown/scripts/gen-spec.mjs
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
// Generates tests/cases.weft from the CommonMark spec's examples and cmark-gfm's spec text.
|
||||
//
|
||||
// curl -o commonmark.json https://spec.commonmark.org/0.31.2/spec.json
|
||||
// curl -o gfm.txt https://raw.githubusercontent.com/github/cmark-gfm/master/test/spec.txt
|
||||
// node packages/markdown/scripts/gen-spec.mjs commonmark.json gfm.txt
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs"
|
||||
import { dirname, join } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const tests = join(dirname(fileURLToPath(import.meta.url)), "..", "tests")
|
||||
mkdirSync(tests, { recursive: true })
|
||||
|
||||
const commonmark = JSON.parse(readFileSync(process.argv[2], "utf8")).map((c) => ({
|
||||
id: c.example,
|
||||
section: c.section,
|
||||
markdown: c.markdown,
|
||||
html: c.html,
|
||||
}))
|
||||
|
||||
// cmark-gfm's spec.txt: examples are fenced by 32 backticks, `example` then the extension's name.
|
||||
const gfm = []
|
||||
let section = ""
|
||||
let number = 0
|
||||
const lines = readFileSync(process.argv[3], "utf8").split("\n")
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const heading = lines[i].match(/^#{1,6} (.*)$/)
|
||||
if (heading) section = heading[1]
|
||||
const open = lines[i].match(/^`{32} example(.*)$/)
|
||||
if (!open) continue
|
||||
number++
|
||||
const extensions = open[1].trim().split(/\s+/).filter(Boolean)
|
||||
let j = i + 1
|
||||
const source = []
|
||||
while (lines[j] !== ".") source.push(lines[j++])
|
||||
j++
|
||||
const html = []
|
||||
while (!lines[j].startsWith("`".repeat(32))) html.push(lines[j++])
|
||||
i = j
|
||||
// `disabled` examples describe behaviour cmark-gfm itself doesn't have. `tagfilter` is for pages that embed raw HTML:
|
||||
// here `html: "escape"` and `"sanitize"` already remove those tags, and `"trust"` means what it says.
|
||||
if (extensions.includes("disabled") || extensions.includes("tagfilter")) continue
|
||||
const text = (l) => l.map((x) => x + "\n").join("").replaceAll("→", "\t")
|
||||
gfm.push({ id: number, section, markdown: text(source), html: text(html), extensions })
|
||||
}
|
||||
|
||||
const str = (s) => JSON.stringify(s).replaceAll("\u2028", "\\u2028").replaceAll("\u2029", "\\u2029").replaceAll("$", "\\$")
|
||||
|
||||
const row = (c) =>
|
||||
` { id: ${c.id}, section: ${str(c.section)}, markdown: ${str(c.markdown)}, html: ${str(c.html)}, gfm: ${(c.extensions ?? []).length > 0} },`
|
||||
|
||||
writeFileSync(
|
||||
join(tests, "cases.weft"),
|
||||
`// Generated by scripts/gen-spec.mjs. Don't edit it.
|
||||
|
||||
/// One example. With gfm set it needs GFM's extensions; cmark-gfm runs the others without them.
|
||||
export type Case = { id: number, section: string, markdown: string, html: string, gfm: boolean }
|
||||
|
||||
/// The examples of the CommonMark spec, version 0.31.2.
|
||||
export let COMMONMARK: Case[] = [
|
||||
${commonmark.map(row).join("\n")}
|
||||
]
|
||||
|
||||
/// The examples of the GitHub Flavored Markdown spec.
|
||||
export let GFM: Case[] = [
|
||||
${gfm.map(row).join("\n")}
|
||||
]
|
||||
`,
|
||||
)
|
||||
console.log(`${commonmark.length} CommonMark examples, ${gfm.length} GFM examples`)
|
||||
73
markdown/scripts/gen-tables.mjs
Normal file
73
markdown/scripts/gen-tables.mjs
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
// Generates lib/entities.weft (the HTML5 named character references) and lib/unicode.weft (Unicode
|
||||
// punctuation and symbols, what CommonMark's delimiter rules call punctuation).
|
||||
//
|
||||
// curl -o entities.json https://html.spec.whatwg.org/entities.json
|
||||
// node packages/markdown/scripts/gen-tables.mjs entities.json
|
||||
import { readFileSync, writeFileSync } from "node:fs"
|
||||
import { dirname, join } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const lib = join(dirname(fileURLToPath(import.meta.url)), "..", "lib")
|
||||
|
||||
const entities = JSON.parse(readFileSync(process.argv[2], "utf8"))
|
||||
const pairs = Object.entries(entities)
|
||||
.filter(([name]) => name.endsWith(";"))
|
||||
.map(([name, v]) => `${name.slice(1, -1)}=${v.codepoints.map((c) => c.toString(16)).join(",")}`)
|
||||
.sort()
|
||||
|
||||
writeFileSync(
|
||||
join(lib, "entities.weft"),
|
||||
`// Generated by scripts/gen-tables.mjs from https://html.spec.whatwg.org/entities.json. Don't edit it.
|
||||
import { parseInt } from "std/math"
|
||||
|
||||
let DATA = "${pairs.join(" ")}"
|
||||
|
||||
fn build(): Map<string, string> {
|
||||
let mut table = Map<string, string>()
|
||||
for entry of DATA.split(" ") {
|
||||
let at = entry.indexOf("=")
|
||||
let mut text = ""
|
||||
for hex of entry.slice(at + 1).split(",") {
|
||||
text += String.fromCodePoint(try parseInt(hex, 16) else 0xfffd)
|
||||
}
|
||||
table.set(entry.slice(0, at), text)
|
||||
}
|
||||
return table
|
||||
}
|
||||
|
||||
let ENTITIES = build()
|
||||
|
||||
/// The text of a named character reference (\`amp\` for \`&\`), or \`null\` when HTML has no such name.
|
||||
export fn entity(name: string): string? {
|
||||
return ENTITIES.get(name)
|
||||
}
|
||||
`,
|
||||
)
|
||||
|
||||
const punct = /[\p{P}\p{S}]/u
|
||||
const ranges = []
|
||||
let start = -1
|
||||
for (let c = 0; c <= 0x110000; c++) {
|
||||
const is = c < 0x110000 && !(c >= 0xd800 && c <= 0xdfff) && punct.test(String.fromCodePoint(c))
|
||||
if (is && start < 0) start = c
|
||||
if (!is && start >= 0) {
|
||||
ranges.push(start, c - 1)
|
||||
start = -1
|
||||
}
|
||||
}
|
||||
|
||||
const rows = []
|
||||
for (let i = 0; i < ranges.length; i += 16) rows.push(" " + ranges.slice(i, i + 16).join(", ") + ",")
|
||||
|
||||
writeFileSync(
|
||||
join(lib, "unicode.weft"),
|
||||
`// Generated by scripts/gen-tables.mjs from the engine's Unicode tables (general categories P and S). Don't edit it.
|
||||
|
||||
/// Sorted, inclusive ranges of code points: start, end, start, end, …
|
||||
export let PUNCTUATION: number[] = [
|
||||
${rows.join("\n")}
|
||||
]
|
||||
`,
|
||||
)
|
||||
|
||||
console.log(`${pairs.length} entities, ${ranges.length / 2} punctuation ranges`)
|
||||
39
markdown/scripts/hostile.mjs
Normal file
39
markdown/scripts/hostile.mjs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
// Runs tests/hostile.weft: each input at n and 2n, timed from outside the VM (the clock doesn't move within a turn).
|
||||
// A size that doubles must not take much more than twice as long.
|
||||
//
|
||||
// node packages/markdown/scripts/hostile.mjs [n]
|
||||
import { spawnSync } from "node:child_process"
|
||||
import { dirname, join } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..")
|
||||
const weft = process.env.WEFT_BIN ?? join(root, "../../core/target/release/weft")
|
||||
const n = Number(process.argv[2] ?? 8000)
|
||||
// The limits are lifted so that a slow path shows up as time rather than as a stop.
|
||||
const env = { ...process.env, WEFT_FUEL: "20000000000", WEFT_MEMORY_MB: "1500" }
|
||||
|
||||
function run(args) {
|
||||
const start = process.hrtime.bigint()
|
||||
const r = spawnSync(weft, ["run", "tests/hostile.weft", ...args], { cwd: root, env, encoding: "utf8", timeout: 120000 })
|
||||
return { ms: Number(process.hrtime.bigint() - start) / 1e6, out: r.stdout.trim(), err: r.stderr.trim(), status: r.status }
|
||||
}
|
||||
|
||||
const names = run(["--list"]).out.split("\n")
|
||||
const base = run(["lines", "0"]).ms
|
||||
let bad = 0
|
||||
for (const name of names) {
|
||||
const a = run([name, String(n)])
|
||||
const b = run([name, String(n * 2)])
|
||||
const [ta, tb] = [Math.max(a.ms - base, 1), Math.max(b.ms - base, 1)]
|
||||
const stopped = a.status !== 0 || b.status !== 0
|
||||
const quadratic = tb > 1500 && tb / ta > 3.2
|
||||
if (stopped || quadratic) bad++
|
||||
const note = stopped ? ` STOPPED: ${(b.err || a.err).split("\n")[0]}` : quadratic ? " QUADRATIC" : ""
|
||||
console.log(`${name.padEnd(24)} ${ta.toFixed(0).padStart(7)} ms ${tb.toFixed(0).padStart(7)} ms${note}`)
|
||||
}
|
||||
const expansion = run(["expansion"])
|
||||
const size = Number(expansion.out)
|
||||
if (!(size > 0 && size < 1000000)) bad++
|
||||
console.log(`reference expansion: ${size} bytes of HTML from 50 KB of Markdown`)
|
||||
console.log(bad === 0 ? "ok" : `${bad} failed`)
|
||||
process.exit(bad === 0 ? 0 : 1)
|
||||
Loading…
Reference in a new issue