dbcs-codec.js
21.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
"use strict";
var Buffer = require("safer-buffer").Buffer;
// Multibyte codec. In this scheme, a character is represented by 1 or more bytes.
// Our codec supports UTF-16 surrogates, extensions for GB18030 and unicode sequences.
// To save memory and loading time, we read table files only when requested.
exports._dbcs = DBCSCodec;
var UNASSIGNED = -1,
GB18030_CODE = -2,
SEQ_START = -10,
NODE_START = -1000,
UNASSIGNED_NODE = new Array(0x100),
DEF_CHAR = -1;
for (var i = 0; i < 0x100; i++)
UNASSIGNED_NODE[i] = UNASSIGNED;
// Class DBCSCodec reads and initializes mapping tables.
function DBCSCodec(codecOptions, iconv) {
this.encodingName = codecOptions.encodingName;
if (!codecOptions)
throw new Error("DBCS codec is called without the data.")
if (!codecOptions.table)
throw new Error("Encoding '" + this.encodingName + "' has no data.");
// Load tables.
var mappingTable = codecOptions.table();
// Decode tables: MBCS -> Unicode.
// decodeTables is a trie, encoded as an array of arrays of integers. Internal arrays are trie nodes and all have len = 256.
// Trie root is decodeTables[0].
// Values: >= 0 -> unicode character code. can be > 0xFFFF
// == UNASSIGNED -> unknown/unassigned sequence.
// == GB18030_CODE -> this is the end of a GB18030 4-byte sequence.
// <= NODE_START -> index of the next node in our trie to process next byte.
// <= SEQ_START -> index of the start of a character code sequence, in decodeTableSeq.
this.decodeTables = [];
this.decodeTables[0] = UNASSIGNED_NODE.slice(0); // Create root node.
// Sometimes a MBCS char corresponds to a sequence of unicode chars. We store them as arrays of integers here.
this.decodeTableSeq = [];
// Actual mapping tables consist of chunks. Use them to fill up decode tables.
for (var i = 0; i < mappingTable.length; i++)
this._addDecodeChunk(mappingTable[i]);
this.defaultCharUnicode = iconv.defaultCharUnicode;
// Encode tables: Unicode -> DBCS.
// `encodeTable` is array mapping from unicode char to encoded char. All its values are integers for performance.
// Because it can be sparse, it is represented as array of buckets by 256 chars each. Bucket can be null.
// Values: >= 0 -> it is a normal char. Write the value (if <=256 then 1 byte, if <=65536 then 2 bytes, etc.).
// == UNASSIGNED -> no conversion found. Output a default char.
// <= SEQ_START -> it's an index in encodeTableSeq, see below. The character starts a sequence.
this.encodeTable = [];
// `encodeTableSeq` is used when a sequence of unicode characters is encoded as a single code. We use a tree of
// objects where keys correspond to characters in sequence and leafs are the encoded dbcs values. A special DEF_CHAR key
// means end of sequence (needed when one sequence is a strict subsequence of another).
// Objects are kept separately from encodeTable to increase performance.
this.encodeTableSeq = [];
// Some chars can be decoded, but need not be encoded.
var skipEncodeChars = {};
if (codecOptions.encodeSkipVals)
for (var i = 0; i < codecOptions.encodeSkipVals.length; i++) {
var val = codecOptions.encodeSkipVals[i];
if (typeof val === 'number')
skipEncodeChars[val] = true;
else
for (var j = val.from; j <= val.to; j++)
skipEncodeChars[j] = true;
}
// Use decode trie to recursively fill out encode tables.
this._fillEncodeTable(0, 0, skipEncodeChars);
// Add more encoding pairs when needed.
if (codecOptions.encodeAdd) {
for (var uChar in codecOptions.encodeAdd)
if (Object.prototype.hasOwnProperty.call(codecOptions.encodeAdd, uChar))
this._setEncodeChar(uChar.charCodeAt(0), codecOptions.encodeAdd[uChar]);
}
this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)];
if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?'];
if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0);
// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 decode tables.
var thirdByteNodeIdx = this.decodeTables.length;
var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0);
var fourthByteNodeIdx = this.decodeTables.length;
var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0);
for (var i = 0x81; i <= 0xFE; i++) {
var secondByteNodeIdx = NODE_START - this.decodeTables[0][i];
var secondByteNode = this.decodeTables[secondByteNodeIdx];
for (var j = 0x30; j <= 0x39; j++)
secondByteNode[j] = NODE_START - thirdByteNodeIdx;
}
for (var i = 0x81; i <= 0xFE; i++)
thirdByteNode[i] = NODE_START - fourthByteNodeIdx;
for (var i = 0x30; i <= 0x39; i++)
fourthByteNode[i] = GB18030_CODE
}
}
DBCSCodec.prototype.encoder = DBCSEncoder;
DBCSCodec.prototype.decoder = DBCSDecoder;
// Decoder helpers
DBCSCodec.prototype._getDecodeTrieNode = function(addr) {
var bytes = [];
for (; addr > 0; addr >>= 8)
bytes.push(addr & 0xFF);
if (bytes.length == 0)
bytes.push(0);
var node = this.decodeTables[0];
for (var i = bytes.length-1; i > 0; i--) { // Traverse nodes deeper into the trie.
var val = node[bytes[i]];
if (val == UNASSIGNED) { // Create new node.
node[bytes[i]] = NODE_START - this.decodeTables.length;
this.decodeTables.push(node = UNASSIGNED_NODE.slice(0));
}
else if (val <= NODE_START) { // Existing node.
node = this.decodeTables[NODE_START - val];
}
else
throw new Error("Overwrite byte in " + this.encodingName + ", addr: " + addr.toString(16));
}
return node;
}
DBCSCodec.prototype._addDecodeChunk = function(chunk) {
// First element of chunk is the hex mbcs code where we start.
var curAddr = parseInt(chunk[0], 16);
// Choose the decoding node where we'll write our chars.
var writeTable = this._getDecodeTrieNode(curAddr);
curAddr = curAddr & 0xFF;
// Write all other elements of the chunk to the table.
for (var k = 1; k < chunk.length; k++) {
var part = chunk[k];
if (typeof part === "string") { // String, write as-is.
for (var l = 0; l < part.length;) {
var code = part.charCodeAt(l++);
if (0xD800 <= code && code < 0xDC00) { // Decode surrogate
var codeTrail = part.charCodeAt(l++);
if (0xDC00 <= codeTrail && codeTrail < 0xE000)
writeTable[curAddr++] = 0x10000 + (code - 0xD800) * 0x400 + (codeTrail - 0xDC00);
else
throw new Error("Incorrect surrogate pair in " + this.encodingName + " at chunk " + chunk[0]);
}
else if (0x0FF0 < code && code <= 0x0FFF) { // Character sequence (our own encoding used)
var len = 0xFFF - code + 2;
var seq = [];
for (var m = 0; m < len; m++)
seq.push(part.charCodeAt(l++)); // Simple variation: don't support surrogates or subsequences in seq.
writeTable[curAddr++] = SEQ_START - this.decodeTableSeq.length;
this.decodeTableSeq.push(seq);
}
else
writeTable[curAddr++] = code; // Basic char
}
}
else if (typeof part === "number") { // Integer, meaning increasing sequence starting with prev character.
var charCode = writeTable[curAddr - 1] + 1;
for (var l = 0; l < part; l++)
writeTable[curAddr++] = charCode++;
}
else
throw new Error("Incorrect type '" + typeof part + "' given in " + this.encodingName + " at chunk " + chunk[0]);
}
if (curAddr > 0xFF)
throw new Error("Incorrect chunk in " + this.encodingName + " at addr " + chunk[0] + ": too long" + curAddr);
}
// Encoder helpers
DBCSCodec.prototype._getEncodeBucket = function(uCode) {
var high = uCode >> 8; // This could be > 0xFF because of astral characters.
if (this.encodeTable[high] === undefined)
this.encodeTable[high] = UNASSIGNED_NODE.slice(0); // Create bucket on demand.
return this.encodeTable[high];
}
DBCSCodec.prototype._setEncodeChar = function(uCode, dbcsCode) {
var bucket = this._getEncodeBucket(uCode);
var low = uCode & 0xFF;
if (bucket[low] <= SEQ_START)
this.encodeTableSeq[SEQ_START-bucket[low]][DEF_CHAR] = dbcsCode; // There's already a sequence, set a single-char subsequence of it.
else if (bucket[low] == UNASSIGNED)
bucket[low] = dbcsCode;
}
DBCSCodec.prototype._setEncodeSequence = function(seq, dbcsCode) {
// Get the root of character tree according to first character of the sequence.
var uCode = seq[0];
var bucket = this._getEncodeBucket(uCode);
var low = uCode & 0xFF;
var node;
if (bucket[low] <= SEQ_START) {
// There's already a sequence with - use it.
node = this.encodeTableSeq[SEQ_START-bucket[low]];
}
else {
// There was no sequence object - allocate a new one.
node = {};
if (bucket[low] !== UNASSIGNED) node[DEF_CHAR] = bucket[low]; // If a char was set before - make it a single-char subsequence.
bucket[low] = SEQ_START - this.encodeTableSeq.length;
this.encodeTableSeq.push(node);
}
// Traverse the character tree, allocating new nodes as needed.
for (var j = 1; j < seq.length-1; j++) {
var oldVal = node[uCode];
if (typeof oldVal === 'object')
node = oldVal;
else {
node = node[uCode] = {}
if (oldVal !== undefined)
node[DEF_CHAR] = oldVal
}
}
// Set the leaf to given dbcsCode.
uCode = seq[seq.length-1];
node[uCode] = dbcsCode;
}
DBCSCodec.prototype._fillEncodeTable = function(nodeIdx, prefix, skipEncodeChars) {
var node = this.decodeTables[nodeIdx];
for (var i = 0; i < 0x100; i++) {
var uCode = node[i];
var mbCode = prefix + i;
if (skipEncodeChars[mbCode])
continue;
if (uCode >= 0)
this._setEncodeChar(uCode, mbCode);
else if (uCode <= NODE_START)
this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars);
else if (uCode <= SEQ_START)
this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode);
}
}
// == Encoder ==================================================================
function DBCSEncoder(options, codec) {
// Encoder state
this.leadSurrogate = -1;
this.seqObj = undefined;
// Static data
this.encodeTable = codec.encodeTable;
this.encodeTableSeq = codec.encodeTableSeq;
this.defaultCharSingleByte = codec.defCharSB;
this.gb18030 = codec.gb18030;
}
DBCSEncoder.prototype.write = function(str) {
var newBuf = Buffer.alloc(str.length * (this.gb18030 ? 4 : 3)),
leadSurrogate = this.leadSurrogate,
seqObj = this.seqObj, nextChar = -1,
i = 0, j = 0;
while (true) {
// 0. Get next character.
if (nextChar === -1) {
if (i == str.length) break;
var uCode = str.charCodeAt(i++);
}
else {
var uCode = nextChar;
nextChar = -1;
}
// 1. Handle surrogates.
if (0xD800 <= uCode && uCode < 0xE000) { // Char is one of surrogates.
if (uCode < 0xDC00) { // We've got lead surrogate.
if (leadSurrogate === -1) {
leadSurrogate = uCode;
continue;
} else {
leadSurrogate = uCode;
// Double lead surrogate found.
uCode = UNASSIGNED;
}
} else { // We've got trail surrogate.
if (leadSurrogate !== -1) {
uCode = 0x10000 + (leadSurrogate - 0xD800) * 0x400 + (uCode - 0xDC00);
leadSurrogate = -1;
} else {
// Incomplete surrogate pair - only trail surrogate found.
uCode = UNASSIGNED;
}
}
}
else if (leadSurrogate !== -1) {
// Incomplete surrogate pair - only lead surrogate found.
nextChar = uCode; uCode = UNASSIGNED; // Write an error, then current char.
leadSurrogate = -1;
}
// 2. Convert uCode character.
var dbcsCode = UNASSIGNED;
if (seqObj !== undefined && uCode != UNASSIGNED) { // We are in the middle of the sequence
var resCode = seqObj[uCode];
if (typeof resCode === 'object') { // Sequence continues.
seqObj = resCode;
continue;
} else if (typeof resCode == 'number') { // Sequence finished. Write it.
dbcsCode = resCode;
} else if (resCode == undefined) { // Current character is not part of the sequence.
// Try default character for this sequence
resCode = seqObj[DEF_CHAR];
if (resCode !== undefined) {
dbcsCode = resCode; // Found. Write it.
nextChar = uCode; // Current character will be written too in the next iteration.
} else {
// TODO: What if we have no default? (resCode == undefined)
// Then, we should write first char of the sequence as-is and try the rest recursively.
// Didn't do it for now because no encoding has this situation yet.
// Currently, just skip the sequence and write current char.
}
}
seqObj = undefined;
}
else if (uCode >= 0) { // Regular character
var subtable = this.encodeTable[uCode >> 8];
if (subtable !== undefined)
dbcsCode = subtable[uCode & 0xFF];
if (dbcsCode <= SEQ_START) { // Sequence start
seqObj = this.encodeTableSeq[SEQ_START-dbcsCode];
continue;
}
if (dbcsCode == UNASSIGNED && this.gb18030) {
// Use GB18030 algorithm to find character(s) to write.
var idx = findIdx(this.gb18030.uChars, uCode);
if (idx != -1) {
var dbcsCode = this.gb18030.gbChars[idx] + (uCode - this.gb18030.uChars[idx]);
newBuf[j++] = 0x81 + Math.floor(dbcsCode / 12600); dbcsCode = dbcsCode % 12600;
newBuf[j++] = 0x30 + Math.floor(dbcsCode / 1260); dbcsCode = dbcsCode % 1260;
newBuf[j++] = 0x81 + Math.floor(dbcsCode / 10); dbcsCode = dbcsCode % 10;
newBuf[j++] = 0x30 + dbcsCode;
continue;
}
}
}
// 3. Write dbcsCode character.
if (dbcsCode === UNASSIGNED)
dbcsCode = this.defaultCharSingleByte;
if (dbcsCode < 0x100) {
newBuf[j++] = dbcsCode;
}
else if (dbcsCode < 0x10000) {
newBuf[j++] = dbcsCode >> 8; // high byte
newBuf[j++] = dbcsCode & 0xFF; // low byte
}
else {
newBuf[j++] = dbcsCode >> 16;
newBuf[j++] = (dbcsCode >> 8) & 0xFF;
newBuf[j++] = dbcsCode & 0xFF;
}
}
this.seqObj = seqObj;
this.leadSurrogate = leadSurrogate;
return newBuf.slice(0, j);
}
DBCSEncoder.prototype.end = function() {
if (this.leadSurrogate === -1 && this.seqObj === undefined)
return; // All clean. Most often case.
var newBuf = Buffer.alloc(10), j = 0;
if (this.seqObj) { // We're in the sequence.
var dbcsCode = this.seqObj[DEF_CHAR];
if (dbcsCode !== undefined) { // Write beginning of the sequence.
if (dbcsCode < 0x100) {
newBuf[j++] = dbcsCode;
}
else {
newBuf[j++] = dbcsCode >> 8; // high byte
newBuf[j++] = dbcsCode & 0xFF; // low byte
}
} else {
// See todo above.
}
this.seqObj = undefined;
}
if (this.leadSurrogate !== -1) {
// Incomplete surrogate pair - only lead surrogate found.
newBuf[j++] = this.defaultCharSingleByte;
this.leadSurrogate = -1;
}
return newBuf.slice(0, j);
}
// Export for testing
DBCSEncoder.prototype.findIdx = findIdx;
// == Decoder ==================================================================
function DBCSDecoder(options, codec) {
// Decoder state
this.nodeIdx = 0;
this.prevBuf = Buffer.alloc(0);
// Static data
this.decodeTables = codec.decodeTables;
this.decodeTableSeq = codec.decodeTableSeq;
this.defaultCharUnicode = codec.defaultCharUnicode;
this.gb18030 = codec.gb18030;
}
DBCSDecoder.prototype.write = function(buf) {
var newBuf = Buffer.alloc(buf.length*2),
nodeIdx = this.nodeIdx,
prevBuf = this.prevBuf, prevBufOffset = this.prevBuf.length,
seqStart = -this.prevBuf.length, // idx of the start of current parsed sequence.
uCode;
if (prevBufOffset > 0) // Make prev buf overlap a little to make it easier to slice later.
prevBuf = Buffer.concat([prevBuf, buf.slice(0, 10)]);
for (var i = 0, j = 0; i < buf.length; i++) {
var curByte = (i >= 0) ? buf[i] : prevBuf[i + prevBufOffset];
// Lookup in current trie node.
var uCode = this.decodeTables[nodeIdx][curByte];
if (uCode >= 0) {
// Normal character, just use it.
}
else if (uCode === UNASSIGNED) { // Unknown char.
// TODO: Callback with seq.
//var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
i = seqStart; // Try to parse again, after skipping first byte of the sequence ('i' will be incremented by 'for' cycle).
uCode = this.defaultCharUnicode.charCodeAt(0);
}
else if (uCode === GB18030_CODE) {
var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
var ptr = (curSeq[0]-0x81)*12600 + (curSeq[1]-0x30)*1260 + (curSeq[2]-0x81)*10 + (curSeq[3]-0x30);
var idx = findIdx(this.gb18030.gbChars, ptr);
uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx];
}
else if (uCode <= NODE_START) { // Go to next trie node.
nodeIdx = NODE_START - uCode;
continue;
}
else if (uCode <= SEQ_START) { // Output a sequence of chars.
var seq = this.decodeTableSeq[SEQ_START - uCode];
for (var k = 0; k < seq.length - 1; k++) {
uCode = seq[k];
newBuf[j++] = uCode & 0xFF;
newBuf[j++] = uCode >> 8;
}
uCode = seq[seq.length-1];
}
else
throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte);
// Write the character to buffer, handling higher planes using surrogate pair.
if (uCode > 0xFFFF) {
uCode -= 0x10000;
var uCodeLead = 0xD800 + Math.floor(uCode / 0x400);
newBuf[j++] = uCodeLead & 0xFF;
newBuf[j++] = uCodeLead >> 8;
uCode = 0xDC00 + uCode % 0x400;
}
newBuf[j++] = uCode & 0xFF;
newBuf[j++] = uCode >> 8;
// Reset trie node.
nodeIdx = 0; seqStart = i+1;
}
this.nodeIdx = nodeIdx;
this.prevBuf = (seqStart >= 0) ? buf.slice(seqStart) : prevBuf.slice(seqStart + prevBufOffset);
return newBuf.slice(0, j).toString('ucs2');
}
DBCSDecoder.prototype.end = function() {
var ret = '';
// Try to parse all remaining chars.
while (this.prevBuf.length > 0) {
// Skip 1 character in the buffer.
ret += this.defaultCharUnicode;
var buf = this.prevBuf.slice(1);
// Parse remaining as usual.
this.prevBuf = Buffer.alloc(0);
this.nodeIdx = 0;
if (buf.length > 0)
ret += this.write(buf);
}
this.nodeIdx = 0;
return ret;
}
// Binary search for GB18030. Returns largest i such that table[i] <= val.
function findIdx(table, val) {
if (table[0] > val)
return -1;
var l = 0, r = table.length;
while (l < r-1) { // always table[l] <= val < table[r]
var mid = l + Math.floor((r-l+1)/2);
if (table[mid] <= val)
l = mid;
else
r = mid;
}
return l;
}