esprima-compare.js
2.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
// Copyright 2015 Simon Lydell
// X11 (“MIT”) Licensed. (See LICENSE.)
var fs = require("fs")
var esprima = require("esprima")
var jsTokens = require("./")
var typeMap = {
Boolean: "name",
Identifier: "name",
Keyword: "name",
Null: "name",
Numeric: "number",
Punctuator: "punctuator",
RegularExpression: "regex",
String: "string"
}
function getEsprimaTokens(code) {
var tokens = esprima.tokenize(code, {loc: true})
tokens.forEach(function(token) { token.type = typeMap[token.type] })
return tokens
}
function jsTokensTokenize(string) {
jsTokens.lastIndex = 0
if (string === "") return []
var tokens = []
var match
while (match = jsTokens.exec(string)) {
tokens.push(jsTokens.matchToToken(match))
}
return tokens
}
var exclusionMap = {
comment: true,
whitespace: true
}
function getJsTokensTokens(code) {
return jsTokensTokenize(code)
.filter(function(token) { return !exclusionMap.hasOwnProperty(token.type) })
}
function compare(file) {
var code = fs.readFileSync(require.resolve(file)).toString()
var esprimaTokens = getEsprimaTokens(code)
var jsTokensTokens = getJsTokensTokens(code)
var length = Math.min(esprimaTokens.length, jsTokensTokens.length)
for (var index = 0; index < length; index++) {
var esprimaToken = esprimaTokens[index]
var jsTokensToken = jsTokensTokens[index]
if (
esprimaToken.type !== jsTokensToken.type ||
esprimaToken.value !== jsTokensToken.value
) {
var loc = esprimaToken.loc.start
console.error(
file + ":" + loc.line + ":" + (loc.column + 1) + ": " +
"(token #" + (index + 1) + ")\n" +
" esprima: '" + esprimaToken.type + "': " + esprimaToken.value + "\n" +
" jsTokens: '" + jsTokensToken.type + "': " + jsTokensToken.value
)
return false
}
}
if (esprimaTokens.length !== jsTokensTokens.length) {
console.error(
file + ': Number of tokens mismatch.\n' +
" esprima: " + (esprimaTokens.length + 1) + "\n" +
" jsTokens: " + (jsTokensTokens.length + 1)
)
return false
}
return true
}
var results = process.argv.slice(2).map(compare)
if (results.every(Boolean)) {
console.log(
"Comparison succeeded: esprima and jsTokens produced the same tokens!"
)
} else {
console.error("Comparison failed.")
}