Watch
1
0
Fork
You've already forked packages
0

packages/ at markup-kit dd1e6eb5270e

A snapshot of packages/ in the Weft repository; the official packages are published from these commits.

Markup-Kit-Commit: dd1e6eb5270e62e0c8b6c202b3d01f0da6e00ddc
This commit is contained in:
Weft packages 2026-11-19 20:43:26 +00:00
commit 306b143104
72 changed files with 10915 additions and 0 deletions

View file

@ -0,0 +1,49 @@
// Times @weft/markdown on the VM against markdown-it on Node (JIT and --jitless), on the same text: the CommonMark
// spec's examples, joined and repeated. Needs markdown-it: MARKDOWN_IT=<path to its folder>.
//
// node packages/markdown/scripts/bench.mjs commonmark.json 1,10,50
import { spawnSync } from "node:child_process"
import { readFileSync, writeFileSync } from "node:fs"
import { dirname, join } from "node:path"
import { fileURLToPath } from "node:url"
const root = join(dirname(fileURLToPath(import.meta.url)), "..")
const weft = process.env.WEFT_BIN ?? join(root, "../../core/target/release/weft")
const spec = process.argv[2]
const sizes = (process.argv[3] ?? "1,10,50").split(",").map(Number)
const lib = process.env.MARKDOWN_IT
const script = `
const MarkdownIt = require(${JSON.stringify(lib)})
const fs = require("fs")
const cases = JSON.parse(fs.readFileSync(${JSON.stringify(spec)}, "utf8"))
const text = cases.map((c) => c.markdown).join("\\n\\n").repeat(Number(process.argv[2]))
const md = new MarkdownIt({ html: false, linkify: true })
const start = process.hrtime.bigint()
const html = md.render(text)
console.log(Number(process.hrtime.bigint() - start) / 1e6, text.length, html.length)
`
const file = join(process.env.TMPDIR ?? "/workspace/core/target/tools", "bench-markdown-it.cjs")
writeFileSync(file, script)
function ms(cmd, args, env = {}) {
const start = process.hrtime.bigint()
const r = spawnSync(cmd, args, { cwd: root, encoding: "utf8", env: { ...process.env, ...env } })
if (r.status !== 0) throw new Error(r.stderr)
return { ms: Number(process.hrtime.bigint() - start) / 1e6, out: r.stdout.trim() }
}
const limits = { WEFT_FUEL: "100000000000", WEFT_MEMORY_MB: "4000" }
const base = Math.min(...[0, 1, 2].map(() => ms(weft, ["run", "bench/bench.weft", "0"], limits).ms))
console.log(`VM start-up (loading the package and the spec's text): ${base.toFixed(0)} ms`)
console.log("size KB | weft VM ms | markdown-it JIT ms | markdown-it --jitless ms | VM / JIT | VM / jitless")
for (const n of sizes) {
const vm = Math.min(...[0, 1, 2].map(() => ms(weft, ["run", "bench/bench.weft", String(n)], limits).ms)) - base
const best = (flags) => {
const runs = [0, 1, 2].map(() => ms("node", [...flags, file, String(n)]).out.split(" ").map(Number))
return { ms: Math.min(...runs.map((r) => r[0])), size: runs[0][1] }
}
const jit = best([])
const nojit = best(["--jitless"])
console.log(`${(jit.size / 1024).toFixed(0).padStart(7)} | ${vm.toFixed(0).padStart(10)} | ${jit.ms.toFixed(1).padStart(18)} | ${nojit.ms.toFixed(1).padStart(24)} | ${(vm / jit.ms).toFixed(1).padStart(8)} | ${(vm / nojit.ms).toFixed(1).padStart(12)}`)
}

View file

@ -0,0 +1,69 @@
// Generates tests/cases.weft from the CommonMark spec's examples and cmark-gfm's spec text.
//
// curl -o commonmark.json https://spec.commonmark.org/0.31.2/spec.json
// curl -o gfm.txt https://raw.githubusercontent.com/github/cmark-gfm/master/test/spec.txt
// node packages/markdown/scripts/gen-spec.mjs commonmark.json gfm.txt
import { mkdirSync, readFileSync, writeFileSync } from "node:fs"
import { dirname, join } from "node:path"
import { fileURLToPath } from "node:url"
const tests = join(dirname(fileURLToPath(import.meta.url)), "..", "tests")
mkdirSync(tests, { recursive: true })
const commonmark = JSON.parse(readFileSync(process.argv[2], "utf8")).map((c) => ({
id: c.example,
section: c.section,
markdown: c.markdown,
html: c.html,
}))
// cmark-gfm's spec.txt: examples are fenced by 32 backticks, `example` then the extension's name.
const gfm = []
let section = ""
let number = 0
const lines = readFileSync(process.argv[3], "utf8").split("\n")
for (let i = 0; i < lines.length; i++) {
const heading = lines[i].match(/^#{1,6} (.*)$/)
if (heading) section = heading[1]
const open = lines[i].match(/^`{32} example(.*)$/)
if (!open) continue
number++
const extensions = open[1].trim().split(/\s+/).filter(Boolean)
let j = i + 1
const source = []
while (lines[j] !== ".") source.push(lines[j++])
j++
const html = []
while (!lines[j].startsWith("`".repeat(32))) html.push(lines[j++])
i = j
// `disabled` examples describe behaviour cmark-gfm itself doesn't have. `tagfilter` is for pages that embed raw HTML:
// here `html: "escape"` and `"sanitize"` already remove those tags, and `"trust"` means what it says.
if (extensions.includes("disabled") || extensions.includes("tagfilter")) continue
const text = (l) => l.map((x) => x + "\n").join("").replaceAll("→", "\t")
gfm.push({ id: number, section, markdown: text(source), html: text(html), extensions })
}
const str = (s) => JSON.stringify(s).replaceAll("\u2028", "\\u2028").replaceAll("\u2029", "\\u2029").replaceAll("$", "\\$")
const row = (c) =>
` { id: ${c.id}, section: ${str(c.section)}, markdown: ${str(c.markdown)}, html: ${str(c.html)}, gfm: ${(c.extensions ?? []).length > 0} },`
writeFileSync(
join(tests, "cases.weft"),
`// Generated by scripts/gen-spec.mjs. Don't edit it.
/// One example. With gfm set it needs GFM's extensions; cmark-gfm runs the others without them.
export type Case = { id: number, section: string, markdown: string, html: string, gfm: boolean }
/// The examples of the CommonMark spec, version 0.31.2.
export let COMMONMARK: Case[] = [
${commonmark.map(row).join("\n")}
]
/// The examples of the GitHub Flavored Markdown spec.
export let GFM: Case[] = [
${gfm.map(row).join("\n")}
]
`,
)
console.log(`${commonmark.length} CommonMark examples, ${gfm.length} GFM examples`)

View file

@ -0,0 +1,73 @@
// Generates lib/entities.weft (the HTML5 named character references) and lib/unicode.weft (Unicode
// punctuation and symbols, what CommonMark's delimiter rules call punctuation).
//
// curl -o entities.json https://html.spec.whatwg.org/entities.json
// node packages/markdown/scripts/gen-tables.mjs entities.json
import { readFileSync, writeFileSync } from "node:fs"
import { dirname, join } from "node:path"
import { fileURLToPath } from "node:url"
const lib = join(dirname(fileURLToPath(import.meta.url)), "..", "lib")
const entities = JSON.parse(readFileSync(process.argv[2], "utf8"))
const pairs = Object.entries(entities)
.filter(([name]) => name.endsWith(";"))
.map(([name, v]) => `${name.slice(1, -1)}=${v.codepoints.map((c) => c.toString(16)).join(",")}`)
.sort()
writeFileSync(
join(lib, "entities.weft"),
`// Generated by scripts/gen-tables.mjs from https://html.spec.whatwg.org/entities.json. Don't edit it.
import { parseInt } from "std/math"
let DATA = "${pairs.join(" ")}"
fn build(): Map<string, string> {
let mut table = Map<string, string>()
for entry of DATA.split(" ") {
let at = entry.indexOf("=")
let mut text = ""
for hex of entry.slice(at + 1).split(",") {
text += String.fromCodePoint(try parseInt(hex, 16) else 0xfffd)
}
table.set(entry.slice(0, at), text)
}
return table
}
let ENTITIES = build()
/// The text of a named character reference (\`amp\` for \`&amp;\`), or \`null\` when HTML has no such name.
export fn entity(name: string): string? {
return ENTITIES.get(name)
}
`,
)
const punct = /[\p{P}\p{S}]/u
const ranges = []
let start = -1
for (let c = 0; c <= 0x110000; c++) {
const is = c < 0x110000 && !(c >= 0xd800 && c <= 0xdfff) && punct.test(String.fromCodePoint(c))
if (is && start < 0) start = c
if (!is && start >= 0) {
ranges.push(start, c - 1)
start = -1
}
}
const rows = []
for (let i = 0; i < ranges.length; i += 16) rows.push(" " + ranges.slice(i, i + 16).join(", ") + ",")
writeFileSync(
join(lib, "unicode.weft"),
`// Generated by scripts/gen-tables.mjs from the engine's Unicode tables (general categories P and S). Don't edit it.
/// Sorted, inclusive ranges of code points: start, end, start, end, …
export let PUNCTUATION: number[] = [
${rows.join("\n")}
]
`,
)
console.log(`${pairs.length} entities, ${ranges.length / 2} punctuation ranges`)

View file

@ -0,0 +1,39 @@
// Runs tests/hostile.weft: each input at n and 2n, timed from outside the VM (the clock doesn't move within a turn).
// A size that doubles must not take much more than twice as long.
//
// node packages/markdown/scripts/hostile.mjs [n]
import { spawnSync } from "node:child_process"
import { dirname, join } from "node:path"
import { fileURLToPath } from "node:url"
const root = join(dirname(fileURLToPath(import.meta.url)), "..")
const weft = process.env.WEFT_BIN ?? join(root, "../../core/target/release/weft")
const n = Number(process.argv[2] ?? 8000)
// The limits are lifted so that a slow path shows up as time rather than as a stop.
const env = { ...process.env, WEFT_FUEL: "20000000000", WEFT_MEMORY_MB: "1500" }
function run(args) {
const start = process.hrtime.bigint()
const r = spawnSync(weft, ["run", "tests/hostile.weft", ...args], { cwd: root, env, encoding: "utf8", timeout: 120000 })
return { ms: Number(process.hrtime.bigint() - start) / 1e6, out: r.stdout.trim(), err: r.stderr.trim(), status: r.status }
}
const names = run(["--list"]).out.split("\n")
const base = run(["lines", "0"]).ms
let bad = 0
for (const name of names) {
const a = run([name, String(n)])
const b = run([name, String(n * 2)])
const [ta, tb] = [Math.max(a.ms - base, 1), Math.max(b.ms - base, 1)]
const stopped = a.status !== 0 || b.status !== 0
const quadratic = tb > 1500 && tb / ta > 3.2
if (stopped || quadratic) bad++
const note = stopped ? ` STOPPED: ${(b.err || a.err).split("\n")[0]}` : quadratic ? " QUADRATIC" : ""
console.log(`${name.padEnd(24)} ${ta.toFixed(0).padStart(7)} ms ${tb.toFixed(0).padStart(7)} ms${note}`)
}
const expansion = run(["expansion"])
const size = Number(expansion.out)
if (!(size > 0 && size < 1000000)) bad++
console.log(`reference expansion: ${size} bytes of HTML from 50 KB of Markdown`)
console.log(bad === 0 ? "ok" : `${bad} failed`)
process.exit(bad === 0 ? 0 : 1)