Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

iconv-lite

Package Overview
Dependencies
Maintainers
1
Versions
51
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

iconv-lite - npm Package Compare versions

Comparing version 0.5.2 to 0.6.0

14

Changelog.md

@@ -0,1 +1,15 @@

## 0.6.0 / 2020-06-08
* Updated 'gb18030' encoding to :2005 edition (see https://github.com/whatwg/encoding/issues/22).
* Removed `iconv.extendNodeEncodings()` mechanism. It was deprecated 5 years ago and didn't work
in recent Node versions.
* Reworked Streaming API behavior in browser environments to fix #204. Streaming API will be
excluded by default in browser packs, saving ~100Kb bundle size, unless enabled explicitly using
`iconv.enableStreamingAPI(require('stream'))`.
* Updates to development environment & tests:
* Added ./test/webpack private package to test complex new use cases that need custom environment.
It's tested as a separate job in Travis CI.
* Updated generation code for the new EUC-KR index file format from Encoding Standard.
* Removed Buffer() constructor in tests (#197 by @gabrielschulhof).
## 0.5.2 / 2020-06-08

@@ -2,0 +16,0 @@ * Added `iconv.getEncoder()` and `iconv.getDecoder()` methods to typescript definitions (#229).

96

encodings/dbcs-codec.js

@@ -52,2 +52,44 @@ "use strict";

// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 common decode nodes.
var commonThirdByteNodeIdx = this.decodeTables.length;
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
var commonFourthByteNodeIdx = this.decodeTables.length;
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
// Fill out the tree
var firstByteNode = this.decodeTables[0];
for (var i = 0x81; i <= 0xFE; i++) {
var secondByteNode = this.decodeTables[NODE_START - firstByteNode[i]];
for (var j = 0x30; j <= 0x39; j++) {
if (secondByteNode[j] === UNASSIGNED) {
secondByteNode[j] = NODE_START - commonThirdByteNodeIdx;
} else if (secondByteNode[j] > NODE_START) {
throw new Error("gb18030 decode tables conflict at byte 2");
}
var thirdByteNode = this.decodeTables[NODE_START - secondByteNode[j]];
for (var k = 0x81; k <= 0xFE; k++) {
if (thirdByteNode[k] === UNASSIGNED) {
thirdByteNode[k] = NODE_START - commonFourthByteNodeIdx;
} else if (thirdByteNode[k] === NODE_START - commonFourthByteNodeIdx) {
continue;
} else if (thirdByteNode[k] > NODE_START) {
throw new Error("gb18030 decode tables conflict at byte 3");
}
var fourthByteNode = this.decodeTables[NODE_START - thirdByteNode[k]];
for (var l = 0x30; l <= 0x39; l++) {
if (fourthByteNode[l] === UNASSIGNED)
fourthByteNode[l] = GB18030_CODE;
}
}
}
}
}
this.defaultCharUnicode = iconv.defaultCharUnicode;

@@ -96,26 +138,2 @@

if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0);
// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 decode tables.
var thirdByteNodeIdx = this.decodeTables.length;
var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0);
var fourthByteNodeIdx = this.decodeTables.length;
var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0);
for (var i = 0x81; i <= 0xFE; i++) {
var secondByteNodeIdx = NODE_START - this.decodeTables[0][i];
var secondByteNode = this.decodeTables[secondByteNodeIdx];
for (var j = 0x30; j <= 0x39; j++)
secondByteNode[j] = NODE_START - thirdByteNodeIdx;
}
for (var i = 0x81; i <= 0xFE; i++)
thirdByteNode[i] = NODE_START - fourthByteNodeIdx;
for (var i = 0x30; i <= 0x39; i++)
fourthByteNode[i] = GB18030_CODE
}
}

@@ -129,3 +147,3 @@

var bytes = [];
for (; addr > 0; addr >>= 8)
for (; addr > 0; addr >>>= 8)
bytes.push(addr & 0xFF);

@@ -255,2 +273,4 @@ if (bytes.length == 0)

var node = this.decodeTables[nodeIdx];
var hasValues = false;
var subNodeEmpty = {};
for (var i = 0; i < 0x100; i++) {

@@ -262,9 +282,20 @@ var uCode = node[i];

if (uCode >= 0)
if (uCode >= 0) {
this._setEncodeChar(uCode, mbCode);
else if (uCode <= NODE_START)
this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars);
else if (uCode <= SEQ_START)
hasValues = true;
} else if (uCode <= NODE_START) {
var subNodeIdx = NODE_START - uCode;
if (!subNodeEmpty[subNodeIdx]) { // Skip empty subtrees (they are too large in gb18030).
var newPrefix = (mbCode << 8) >>> 0; // NOTE: '>>> 0' keeps 32-bit num positive.
if (this._fillEncodeTable(subNodeIdx, newPrefix, skipEncodeChars))
hasValues = true;
else
subNodeEmpty[subNodeIdx] = true;
}
} else if (uCode <= SEQ_START) {
this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode);
hasValues = true;
}
}
return hasValues;
}

@@ -396,6 +427,11 @@

}
else {
else if (dbcsCode < 0x1000000) {
newBuf[j++] = dbcsCode >> 16;
newBuf[j++] = (dbcsCode >> 8) & 0xFF;
newBuf[j++] = dbcsCode & 0xFF;
} else {
newBuf[j++] = dbcsCode >>> 24;
newBuf[j++] = (dbcsCode >>> 16) & 0xFF;
newBuf[j++] = (dbcsCode >>> 8) & 0xFF;
newBuf[j++] = dbcsCode & 0xFF;
}

@@ -402,0 +438,0 @@ }

5

encodings/tables/gbk-added.json

@@ -30,3 +30,3 @@ [

["a896","",10],
["a8bc",""],
["a8bc","ḿ"],
["a8bf","ǹ"],

@@ -55,3 +55,4 @@ ["a8c1",""],

["fe50","⺁⺄㑳㑇⺈⺋㖞㘚㘎⺌⺗㥮㤘㧏㧟㩳㧐㭎㱮㳠⺧⺪䁖䅟⺮䌷⺳⺶⺷䎱䎬⺻䏝䓖䙡䙌"],
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93]
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93],
["8135f437",""]
]

51

lib/index.js
"use strict";
// Some environments don't have global Buffer (e.g. React Native).
// Solution would be installing npm modules "buffer" and "stream" explicitly.
var Buffer = require("safer-buffer").Buffer;

@@ -136,19 +134,48 @@

// Streaming API
// NOTE: Streaming API naturally depends on 'stream' module from Node.js. Unfortunately in browser environments this module can add
// up to 100Kb to the output bundle. To avoid unnecessary code bloat, we don't enable Streaming API in browser by default.
// If you would like to enable it explicitly, please add the following code to your app:
// > iconv.enableStreamingAPI(require('stream'));
iconv.enableStreamingAPI = function enableStreamingAPI(stream_module) {
if (iconv.supportsStreams)
return;
// Load extensions in Node. All of them are omitted in Browserify build via 'browser' field in package.json.
var nodeVer = typeof process !== 'undefined' && process.versions && process.versions.node;
if (nodeVer) {
// Dependency-inject stream module to create IconvLite stream classes.
var streams = require("./streams")(stream_module);
// Load streaming support in Node v0.10+
var nodeVerArr = nodeVer.split(".").map(Number);
if (nodeVerArr[0] > 0 || nodeVerArr[1] >= 10) {
require("./streams")(iconv);
// Not public API yet, but expose the stream classes.
iconv.IconvLiteEncoderStream = streams.IconvLiteEncoderStream;
iconv.IconvLiteDecoderStream = streams.IconvLiteDecoderStream;
// Streaming API.
iconv.encodeStream = function encodeStream(encoding, options) {
return new iconv.IconvLiteEncoderStream(iconv.getEncoder(encoding, options), options);
}
// Load Node primitive extensions.
require("./extend-node")(iconv);
iconv.decodeStream = function decodeStream(encoding, options) {
return new iconv.IconvLiteDecoderStream(iconv.getDecoder(encoding, options), options);
}
iconv.supportsStreams = true;
}
// Enable Streaming API automatically if 'stream' module is available and non-empty (the majority of environments).
var stream_module;
try {
stream_module = require("stream");
} catch (e) {}
if (stream_module && stream_module.Transform) {
iconv.enableStreamingAPI(stream_module);
} else {
// In rare cases where 'stream' module is not available by default, throw a helpful exception.
iconv.encodeStream = iconv.decodeStream = function() {
throw new Error("iconv-lite Streaming API is not enabled. Use iconv.enableStreamingAPI(require('stream')); to enable it.");
};
}
if ("Ā" != "\u0100") {
console.error("iconv-lite warning: javascript files use encoding different from utf-8. See https://github.com/ashtuchkin/iconv-lite/wiki/Javascript-source-file-encodings for more info.");
console.error("iconv-lite warning: js files use non-utf8 encoding. See https://github.com/ashtuchkin/iconv-lite/wiki/Javascript-source-file-encodings for more info.");
}

186

lib/streams.js
"use strict";
var Buffer = require("buffer").Buffer,
Transform = require("stream").Transform;
var Buffer = require("safer-buffer").Buffer;
// NOTE: Due to 'stream' module being pretty large (~100Kb, significant in browser environments),
// we opt to dependency-inject it instead of creating a hard dependency.
module.exports = function(stream_module) {
var Transform = stream_module.Transform;
// == Exports ==================================================================
module.exports = function(iconv) {
// Additional Public API.
iconv.encodeStream = function encodeStream(encoding, options) {
return new IconvLiteEncoderStream(iconv.getEncoder(encoding, options), options);
}
// == Encoder stream =======================================================
iconv.decodeStream = function decodeStream(encoding, options) {
return new IconvLiteDecoderStream(iconv.getDecoder(encoding, options), options);
function IconvLiteEncoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.decodeStrings = false; // We accept only strings, so we don't need to decode them.
Transform.call(this, options);
}
iconv.supportsStreams = true;
IconvLiteEncoderStream.prototype = Object.create(Transform.prototype, {
constructor: { value: IconvLiteEncoderStream }
});
IconvLiteEncoderStream.prototype._transform = function(chunk, encoding, done) {
if (typeof chunk != 'string')
return done(new Error("Iconv encoding stream needs strings as its input."));
try {
var res = this.conv.write(chunk);
if (res && res.length) this.push(res);
done();
}
catch (e) {
done(e);
}
}
// Not published yet.
iconv.IconvLiteEncoderStream = IconvLiteEncoderStream;
iconv.IconvLiteDecoderStream = IconvLiteDecoderStream;
iconv._collect = IconvLiteDecoderStream.prototype.collect;
};
IconvLiteEncoderStream.prototype._flush = function(done) {
try {
var res = this.conv.end();
if (res && res.length) this.push(res);
done();
}
catch (e) {
done(e);
}
}
IconvLiteEncoderStream.prototype.collect = function(cb) {
var chunks = [];
this.on('error', cb);
this.on('data', function(chunk) { chunks.push(chunk); });
this.on('end', function() {
cb(null, Buffer.concat(chunks));
});
return this;
}
// == Encoder stream =======================================================
function IconvLiteEncoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.decodeStrings = false; // We accept only strings, so we don't need to decode them.
Transform.call(this, options);
}
IconvLiteEncoderStream.prototype = Object.create(Transform.prototype, {
constructor: { value: IconvLiteEncoderStream }
});
// == Decoder stream =======================================================
IconvLiteEncoderStream.prototype._transform = function(chunk, encoding, done) {
if (typeof chunk != 'string')
return done(new Error("Iconv encoding stream needs strings as its input."));
try {
var res = this.conv.write(chunk);
if (res && res.length) this.push(res);
done();
function IconvLiteDecoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.encoding = this.encoding = 'utf8'; // We output strings.
Transform.call(this, options);
}
catch (e) {
done(e);
}
}
IconvLiteEncoderStream.prototype._flush = function(done) {
try {
var res = this.conv.end();
if (res && res.length) this.push(res);
done();
}
catch (e) {
done(e);
}
}
IconvLiteEncoderStream.prototype.collect = function(cb) {
var chunks = [];
this.on('error', cb);
this.on('data', function(chunk) { chunks.push(chunk); });
this.on('end', function() {
cb(null, Buffer.concat(chunks));
IconvLiteDecoderStream.prototype = Object.create(Transform.prototype, {
constructor: { value: IconvLiteDecoderStream }
});
return this;
}
IconvLiteDecoderStream.prototype._transform = function(chunk, encoding, done) {
if (!Buffer.isBuffer(chunk))
return done(new Error("Iconv decoding stream needs buffers as its input."));
try {
var res = this.conv.write(chunk);
if (res && res.length) this.push(res, this.encoding);
done();
}
catch (e) {
done(e);
}
}
// == Decoder stream =======================================================
function IconvLiteDecoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.encoding = this.encoding = 'utf8'; // We output strings.
Transform.call(this, options);
}
IconvLiteDecoderStream.prototype = Object.create(Transform.prototype, {
constructor: { value: IconvLiteDecoderStream }
});
IconvLiteDecoderStream.prototype._transform = function(chunk, encoding, done) {
if (!Buffer.isBuffer(chunk))
return done(new Error("Iconv decoding stream needs buffers as its input."));
try {
var res = this.conv.write(chunk);
if (res && res.length) this.push(res, this.encoding);
done();
IconvLiteDecoderStream.prototype._flush = function(done) {
try {
var res = this.conv.end();
if (res && res.length) this.push(res, this.encoding);
done();
}
catch (e) {
done(e);
}
}
catch (e) {
done(e);
}
}
IconvLiteDecoderStream.prototype._flush = function(done) {
try {
var res = this.conv.end();
if (res && res.length) this.push(res, this.encoding);
done();
IconvLiteDecoderStream.prototype.collect = function(cb) {
var res = '';
this.on('error', cb);
this.on('data', function(chunk) { res += chunk; });
this.on('end', function() {
cb(null, res);
});
return this;
}
catch (e) {
done(e);
}
}
IconvLiteDecoderStream.prototype.collect = function(cb) {
var res = '';
this.on('error', cb);
this.on('data', function(chunk) { res += chunk; });
this.on('end', function() {
cb(null, res);
});
return this;
}
return {
IconvLiteEncoderStream: IconvLiteEncoderStream,
IconvLiteDecoderStream: IconvLiteDecoderStream,
};
};

17

package.json
{
"name": "iconv-lite",
"description": "Convert character encodings in pure javascript.",
"version": "0.5.2",
"version": "0.6.0",
"license": "MIT",

@@ -29,14 +29,13 @@ "keywords": [

"browser": {
"./lib/extend-node": false,
"./lib/streams": false
"stream": false
},
"devDependencies": {
"mocha": "^3.1.0",
"request": "~2.87.0",
"unorm": "*",
"errto": "*",
"async": "*",
"c8": "*",
"semver": "6.1.2",
"iconv": "2"
"errto": "*",
"iconv": "^2.3.5",
"mocha": "^3.5.3",
"request": "~2.87.0",
"semver": "~6.1.2",
"unorm": "*"
},

@@ -43,0 +42,0 @@ "dependencies": {

54

README.md

@@ -1,13 +0,11 @@

## Pure JS character encoding conversion
## iconv-lite: Pure JS character encoding conversion
* Doesn't need native code compilation. Works on Windows and in sandboxed environments like [Cloud9](http://c9.io).
* No need for native code compilation. Quick to install, works on Windows and in sandboxed environments like [Cloud9](http://c9.io).
* Used in popular projects like [Express.js (body_parser)](https://github.com/expressjs/body-parser),
[Grunt](http://gruntjs.com/), [Nodemailer](http://www.nodemailer.com/), [Yeoman](http://yeoman.io/) and others.
* Faster than [node-iconv](https://github.com/bnoordhuis/node-iconv) (see below for performance comparison).
* Intuitive encode/decode API
* Streaming support for Node v0.10+
* [Deprecated] Can extend Node.js primitives (buffers, streams) to support all iconv-lite encodings.
* In-browser usage via [Browserify](https://github.com/substack/node-browserify) (~180k gzip compressed with Buffer shim included).
* Intuitive encode/decode API, including Streaming support.
* In-browser usage via [browserify](https://github.com/substack/node-browserify) or [webpack](https://webpack.js.org/) (~180kb gzip compressed with Buffer shim included).
* Typescript [type definition file](https://github.com/ashtuchkin/iconv-lite/blob/master/lib/index.d.ts) included.
* React Native is supported (need to explicitly `npm install` two more modules: `buffer` and `stream`).
* React Native is supported (need to install `stream` module to enable Streaming API).
* License: MIT.

@@ -26,6 +24,6 @@

// Convert from an encoded buffer to js string.
// Convert from an encoded buffer to a js string.
str = iconv.decode(Buffer.from([0x68, 0x65, 0x6c, 0x6c, 0x6f]), 'win1251');
// Convert from js string to an encoded buffer.
// Convert from a js string to an encoded buffer.
buf = iconv.encode("Sample input string", 'win1251');

@@ -37,6 +35,6 @@

### Streaming API (Node v0.10+)
### Streaming API
```javascript
// Decode stream (from binary stream to js strings)
// Decode stream (from binary data stream to js strings)
http.createServer(function(req, res) {

@@ -66,36 +64,2 @@ var converterStream = iconv.decodeStream('win1251');

### [Deprecated] Extend Node.js own encodings
> NOTE: This doesn't work on latest Node versions. See [details](https://github.com/ashtuchkin/iconv-lite/wiki/Node-v4-compatibility).
```javascript
// After this call all Node basic primitives will understand iconv-lite encodings.
iconv.extendNodeEncodings();
// Examples:
buf = new Buffer(str, 'win1251');
buf.write(str, 'gbk');
str = buf.toString('latin1');
assert(Buffer.isEncoding('iso-8859-15'));
Buffer.byteLength(str, 'us-ascii');
http.createServer(function(req, res) {
req.setEncoding('big5');
req.collect(function(err, body) {
console.log(body);
});
});
fs.createReadStream("file.txt", "shift_jis");
// External modules are also supported (if they use Node primitives, which they probably do).
request = require('request');
request({
url: "http://github.com/",
encoding: "cp932"
});
// To remove extensions
iconv.undoExtendNodeEncodings();
```
## Supported encodings

@@ -102,0 +66,0 @@ 

lib/extend-node.js
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc