// Character classes and the text transformations CommonMark defines: escapes, entities, URLs. import { entity } from "./entities" import { PUNCTUATION } from "./unicode" /// The code unit at `i`, or -1 past the end. export fn unit(s: string, i: int): int { return s.codeUnitAt(i) ?? -1 } export fn isSpaceOrTab(c: float): boolean { return c == 32 || c == 9 } export fn isDigit(c: float): boolean { return c >= 48 && c <= 57 } export fn isAlpha(c: float): boolean { return (c >= 65 && c <= 90) || (c >= 97 && c <= 122) } export fn isAlnum(c: float): boolean { return isAlpha(c) || isDigit(c) } export fn isHex(c: float): boolean { return isDigit(c) || (c >= 65 && c <= 70) || (c >= 97 && c <= 102) } /// ASCII punctuation: what a backslash can escape. export fn isAsciiPunct(c: float): boolean { return (c >= 33 && c <= 47) || (c >= 58 && c <= 64) || (c >= 91 && c <= 96) || (c >= 123 && c <= 126) } /// Unicode whitespace: the `Zs` category, tab, line feed, form feed and carriage return. export fn isUnicodeSpace(c: float): boolean { if c < 128 { return c == 32 || c == 9 || c == 10 || c == 12 || c == 13 } return c == 0xa0 || c == 0x1680 || (c >= 0x2000 && c <= 0x200a) || c == 0x202f || c == 0x205f || c == 0x3000 } /// Unicode punctuation: the `P` and `S` categories. export fn isUnicodePunct(c: float): boolean { if c < 128 { return isAsciiPunct(c) } let mut lo = 0 let mut hi = PUNCTUATION.length.div(2) - 1 while lo <= hi { let mid = (lo + hi) >> 1 if c < PUNCTUATION[mid * 2] { hi = mid - 1 } else if c > PUNCTUATION[mid * 2 + 1] { lo = mid + 1 } else { return true } } return false } /// Whether `s` holds only spaces and tabs (or nothing). export fn isBlank(s: string): boolean { for i of 0..s.length { let c = s.codeUnitAt(i) ?? 32 if c != 32 && c != 9 && c != 10 && c != 13 { return false } } return true } /// The code point ending just before `pos`, or a line feed at the start of `s`. export fn codePointBefore(s: string, pos: int): int { if pos <= 0 { return 10 } let low = s.codeUnitAt(pos - 1) ?? 10 if low >= 0xdc00 && low <= 0xdfff && pos >= 2 { let high = s.codeUnitAt(pos - 2) ?? 0 if high >= 0xd800 && high <= 0xdbff { return (high - 0xd800) * 0x400 + (low - 0xdc00) + 0x10000 } } return low } /// The character reference starting at the `&` at `pos`: its text and where it ends, or `null` when there is none. export fn scanEntity(s: string, pos: int): [string, int]? { let mut i = pos + 1 if s.codeUnitAt(i) == 35 { i += 1 let c = s.codeUnitAt(i) ?? -1 let hex = c == 120 || c == 88 if hex { i += 1 } let start = i while i - start < (hex ? 6 : 7) && (hex ? isHex(s.codeUnitAt(i) ?? -1) : isDigit(s.codeUnitAt(i) ?? -1)) { i += 1 } if i == start || s.codeUnitAt(i) != 59 { return null } let n = try int(s.slice(start, i), hex ? 16 : 10) else 0 let valid = n != 0 && n <= 0x10ffff && !(n >= 0xd800 && n <= 0xdfff) return [String.fromCodePoint(valid ? n : 0xfffd), i + 1] } let start = i while i - start < 32 && isAlnum(s.codeUnitAt(i) ?? -1) { i += 1 } if i == start || s.codeUnitAt(i) != 59 { return null } let text = entity(s.slice(start, i)) if text == null { return null } return [text, i + 1] } /// `s` with its backslash escapes and character references replaced by what they stand for. export fn unescape(s: string): string { if !s.includes("\\") && !s.includes("&") { return s } let mut parts: string[] = [] let mut from = 0 let mut i = 0 while i < s.length { let c = s.codeUnitAt(i) ?? 0 if c == 92 && isAsciiPunct(s.codeUnitAt(i + 1) ?? 0) { parts.push(s.slice(from, i)) from = i + 1 i += 2 } else if c == 38 { let found = scanEntity(s, i) if found != null { parts.push(s.slice(from, i)) parts.push(found[0]) i = found[1] from = i } else { i += 1 } } else { i += 1 } } parts.push(s.slice(from)) return parts.join("") } /// `s` escaped for HTML text and attribute values. export fn escapeHtml(s: string): string { let mut i = 0 while i < s.length { let c = s.codeUnitAt(i) ?? 0 if c == 38 || c == 60 || c == 62 || c == 34 { break } i += 1 } if i == s.length { return s } let mut parts: string[] = [s.slice(0, i)] let mut from = i while i < s.length { let c = s.codeUnitAt(i) ?? 0 let rep = c == 38 ? "&" : c == 60 ? "<" : c == 62 ? ">" : c == 34 ? """ : "" if rep != "" { parts.push(s.slice(from, i)) parts.push(rep) from = i + 1 } i += 1 } parts.push(s.slice(from)) return parts.join("") } fn keptInUrl(c: int): boolean { // Letters, digits and what URLs use as syntax; everything else is percent-encoded. return isAlnum(c) || (c < 128 && ";/?:@&=+$,-_.!~*'()#".includes(String.fromCharCode(c))) } /// `url` percent-encoded for an `href`: what is already encoded stays, and a lone surrogate becomes U+FFFD. export fn normalizeUrl(url: string): string { let mut parts: string[] = [] let mut from = 0 let mut i = 0 while i < url.length { let c = url.codeUnitAt(i) ?? 0 if c == 37 && isHex(url.codeUnitAt(i + 1) ?? -1) && isHex(url.codeUnitAt(i + 2) ?? -1) { i += 3 } else if keptInUrl(c) { i += 1 } else { parts.push(url.slice(from, i)) let point = url.codePointAt(i) ?? 0xfffd let width = point > 0xffff ? 2 : 1 let safe = point >= 0xd800 && point <= 0xdfff ? 0xfffd : point for b of Bytes.fromUtf8(String.fromCodePoint(safe)).toList() { parts.push("%" + str(b, { radix: 16 }).toUpperCase().padStart(2, "0")) } i += width from = i } } parts.push(url.slice(from)) return parts.join("") } /// A link label as references are matched: case-folded, with whitespace collapsed. export fn normalizeLabel(label: string): string { let mut out: string[] = [] let mut space = false for ch of label.trim() { if ch == " " || ch == "\t" || ch == "\n" || ch == "\r" { space = true } else { if space { out.push(" ") space = false } out.push(ch) } } return out.join("").toLowerCase().toUpperCase() }