userland: add archive tools and pinned GoAWK, esbuild, and QuickJS

userland/esbuild/source/scripts/gen-unicode-table.js

@@ -0,0 +1,133 @@
+const fs = require('fs')
+const path = require('path')
+
+// ES5 reference: https://es5.github.io/
+//
+// A conforming implementation of this International standard shall interpret
+// characters in conformance with the Unicode Standard, Version 3.0 or later
+// and ISO/IEC 10646-1 with either UCS-2 or UTF-16 as the adopted encoding
+// form, implementation level 3. If the adopted ISO/IEC 10646-1 subset is not
+// otherwise specified, it is presumed to be the BMP subset, collection 300.
+//
+// UnicodeLetter: any character in the Unicode categories “Uppercase letter (Lu)”,
+// “Lowercase letter (Ll)”, “Titlecase letter (Lt)”, “Modifier letter (Lm)”,
+// “Other letter (Lo)”, or “Letter number (Nl)”.
+const idStartES5 = [].concat(
+  require('@unicode/unicode-3.0.0/General_Category/Uppercase_Letter/code-points'),
+  require('@unicode/unicode-3.0.0/General_Category/Lowercase_Letter/code-points'),
+  require('@unicode/unicode-3.0.0/General_Category/Titlecase_Letter/code-points'),
+  require('@unicode/unicode-3.0.0/General_Category/Modifier_Letter/code-points'),
+  require('@unicode/unicode-3.0.0/General_Category/Other_Letter/code-points'),
+
+  // The "letter number" category is not included because old versions of Safari
+  // had a bug where they didn't include it. This means it does not match ES5.
+  // We need to make sure we escape these characters so Safari can read them.
+  // See https://github.com/evanw/esbuild/issues/1349 for more information.
+  // require('@unicode/unicode-3.0.0/General_Category/Letter_Number/code-points'),
+).sort((a, b) => a - b)
+
+// UnicodeCombiningMark: any character in the Unicode categories “Non-spacing mark (Mn)”
+// or “Combining spacing mark (Mc)”
+// UnicodeDigit: any character in the Unicode category “Decimal number (Nd)”
+// UnicodeConnectorPunctuation: any character in the Unicode category “Connector punctuation (Pc)”
+const idContinueES5 = idStartES5.concat(
+  require('@unicode/unicode-3.0.0/General_Category/Nonspacing_Mark/code-points'),
+  require('@unicode/unicode-3.0.0/General_Category/Spacing_Mark/code-points'),
+  require('@unicode/unicode-3.0.0/General_Category/Decimal_Number/code-points'),
+  require('@unicode/unicode-3.0.0/General_Category/Connector_Punctuation/code-points'),
+).sort((a, b) => a - b)
+
+// ESNext reference: https://tc39.es/ecma262/
+//
+// A conforming implementation of ECMAScript must interpret source text input
+// in conformance with the Unicode Standard, Version 5.1.0 or later and ISO/IEC
+// 10646. If the adopted ISO/IEC 10646-1 subset is not otherwise specified, it
+// is presumed to be the Unicode set, collection 10646.
+//
+// UnicodeIDStart: any Unicode code point with the Unicode property “ID_Start”
+const idStartESNext = require('@unicode/unicode-15.1.0/Binary_Property/ID_Start/code-points')
+const idStartESNextSet = new Set(idStartESNext)
+
+// Unicode 4.1 through Unicode 15 omitted these two characters from ID_Continue
+// by accident. However, this accident was corrected in Unicode 15.1. Any JS VM
+// that supports ES6+ but that uses a version of Unicode earlier than 15.1 will
+// consider these to be a syntax error, so we deliberately omit these characters
+// from the set of identifiers that are valid in both ES5 and ES6+. For more info
+// see 2.2 in https://www.unicode.org/L2/L2023/23160-utc176-properties-recs.pdf
+const ID_Continue_mistake = new Set([0x30FB, 0xFF65])
+
+// UnicodeIDContinue: any Unicode code point with the Unicode property “ID_Continue”
+const idContinueESNext = require('@unicode/unicode-15.1.0/Binary_Property/ID_Continue/code-points')
+const idContinueESNextSet = new Set(idContinueESNext)
+
+// These identifiers are valid in both ES5 and ES6+ (i.e. an intersection of both)
+const idStartES5AndESNext = idStartES5.filter(n => idStartESNextSet.has(n))
+const idContinueES5AndESNext = idContinueES5.filter(n => idContinueESNextSet.has(n) && !ID_Continue_mistake.has(n))
+
+// These identifiers are valid in either ES5 or ES6+ (i.e. a union of both)
+const idStartES5OrESNext = [...new Set(idStartES5.concat(idStartESNext))].sort((a, b) => a - b)
+const idContinueES5OrESNext = [...new Set(idContinueES5.concat(idContinueESNext))].sort((a, b) => a - b)
+
+function generateRangeTable(codePoints) {
+  let lines = []
+  let index = 0
+  let latinOffset = 0
+
+  while (latinOffset < codePoints.length && codePoints[latinOffset] <= 0xFF) {
+    latinOffset++
+  }
+
+  lines.push(
+    `&unicode.RangeTable{`,
+    `\tLatinOffset: ${latinOffset},`,
+    `\tR16: []unicode.Range16{`,
+  )
+
+  // 16-bit code points
+  while (index < codePoints.length && codePoints[index] < 0x1000) {
+    let start = codePoints[index]
+    index++
+    while (index < codePoints.length && codePoints[index] < 0x1000 && codePoints[index] === codePoints[index - 1] + 1) {
+      index++
+    }
+    let end = codePoints[index - 1]
+    lines.push(`\t\t{Lo: 0x${start.toString(16)}, Hi: 0x${end.toString(16)}, Stride: 1},`)
+  }
+
+  lines.push(
+    `\t},`,
+    `\tR32: []unicode.Range32{`,
+  )
+
+  // 32-bit code points
+  while (index < codePoints.length) {
+    let start = codePoints[index]
+    index++
+    while (index < codePoints.length && codePoints[index] === codePoints[index - 1] + 1) {
+      index++
+    }
+    let end = codePoints[index - 1]
+    lines.push(`\t\t{Lo: 0x${start.toString(16)}, Hi: 0x${end.toString(16)}, Stride: 1},`)
+  }
+
+  lines.push(
+    `\t},`,
+    `}`,
+  )
+  return lines.join('\n')
+}
+
+fs.writeFileSync(path.join(__dirname, '..', 'internal', 'js_ast', 'unicode.go'),
+  `// This file was automatically generated by ${path.basename(__filename)}. Do not edit.
+package js_ast
+
+import "unicode"
+
+var idStartES5AndESNext = ${generateRangeTable(idStartES5AndESNext)}
+
+var idContinueES5AndESNext = ${generateRangeTable(idContinueES5AndESNext)}
+
+var idStartES5OrESNext = ${generateRangeTable(idStartES5OrESNext)}
+
+var idContinueES5OrESNext = ${generateRangeTable(idContinueES5OrESNext)}
+`)