Socket
Socket
Sign inDemoInstall

parse5

Package Overview
Dependencies
0
Maintainers
1
Versions
56
Alerts
File Explorer

Advanced tools

npm Scripts

Install Socket

Detect and block malicious and high-risk dependencies

Install

Comparing version 0.5.4 to 0.6.0

README.md

14

lib/default_tree_adapter.js

@@ -100,8 +100,12 @@ exports.createDocument = function () {

exports.insertText = function (parentNode, text) {
var lastChild = parentNode.childNodes[parentNode.childNodes.length - 1];
if (parentNode.childNodes.length) {
var prevNode = parentNode.childNodes[parentNode.childNodes.length - 1];
if (lastChild && lastChild.nodeName === '#text')
lastChild.value += text;
else
appendChild(parentNode, createTextNode(text));
if (prevNode.nodeName === '#text') {
prevNode.value += text;
return;
}
}
appendChild(parentNode, createTextNode(text));
};

@@ -108,0 +112,0 @@

147

lib/html.js

@@ -18,11 +18,21 @@ exports.NAMESPACES = {

ASIDE: 'aside',
B: 'b',
BASE: 'base',
BASEFONT: 'basefont',
BGSOUND: 'bgsound',
BIG: 'big',
BLOCKQUOTE: 'blockquote',
BODY: 'body',
BR: 'br',
BUTTON: 'button',
CAPTION: 'caption',
CENTER: 'center',
CODE: 'code',
COL: 'col',
COLGROUP: 'colgroup',
COMMAND: 'command',
COLGROUP: 'colgroup',
CENTER: 'center',
CODE: 'code',
DD: 'dd',
DESC: 'desc',

@@ -33,7 +43,8 @@ DETAILS: 'details',

DIV: 'div',
DD: 'dd',
DL: 'dl',
DT: 'dt',
DL: 'dl',
EM: 'em',
EMBED: 'embed',
FIELDSET: 'fieldset',

@@ -43,14 +54,42 @@ FIGCAPTION: 'figcaption',

FONT: 'font',
FORM: 'form',
FOOTER: 'footer',
FOREIGN_OBJECT: 'foreignObject',
FORM: 'form',
FRAME: 'frame',
FRAMESET: 'frameset',
H1: 'h1',
H2: 'h2',
H3: 'h3',
H4: 'h4',
H5: 'h5',
H6: 'h6',
HEAD: 'head',
HEADER: 'header',
HGROUP: 'hgroup',
HR: 'hr',
HTML: 'html',
I: 'i',
IFRAME: 'iframe',
ISINDEX: 'isindex',
IMG: 'img',
IMAGE: 'image',
INPUT: 'input',
IFRAME: 'iframe',
ISINDEX: 'isindex',
KEYGEN: 'keygen',
LABEL: 'label',
LI: 'li',
LINK: 'link',
LISTING: 'listing',
MAIN: 'main',
MALIGNMARK: 'malignmark',
MARQUEE: 'marquee',
MATH: 'math',
MENU: 'menu',
MENUITEM: 'menuitem',
META: 'meta',
MGLYPH: 'mglyph',
MI: 'mi',

@@ -61,77 +100,59 @@ MO: 'mo',

MTEXT: 'mtext',
MATH: 'math',
MAIN: 'main',
MALIGNMARK: 'malignmark',
MENU: 'menu',
MGLYPH: 'mglyph',
NAV: 'nav',
NOBR: 'nobr',
NOFRAMES: 'noframes',
NOEMBED: 'noembed',
NOSCRIPT: 'noscript',
OBJECT: 'object',
OL: 'ol',
OPTGROUP: 'optgroup',
OPTION: 'option',
P: 'p',
PARAM: 'param',
PLAINTEXT: 'plaintext',
PRE: 'pre',
SECTION: 'section',
SUMMARY: 'summary',
HTML: 'html',
HEAD: 'head',
BODY: 'body',
BR: 'br',
BASE: 'base',
BASEFONT: 'basefont',
BGSOUND: 'bgsound',
BUTTON: 'button',
META: 'meta',
MARQUEE: 'marquee',
KEYGEN: 'keygen',
LABEL: 'label',
LI: 'li',
LINK: 'link',
LISTING: 'listing',
NOBR: 'nobr',
PARAM: 'param',
RP: 'rp',
RT: 'rt',
RUBY: 'ruby',
S: 's',
SUB: 'sub',
SUP: 'sup',
SCRIPT: 'script',
SECTION: 'section',
SELECT: 'select',
SCRIPT: 'script',
SOURCE: 'source',
SMALL: 'small',
SPAN: 'span',
STYLE: 'style',
STRIKE: 'strike',
STRONG: 'strong',
SMALL: 'small',
STYLE: 'style',
SUB: 'sub',
SUMMARY: 'summary',
SUP: 'sup',
TABLE: 'table',
TBODY: 'tbody',
TEXTAREA: 'textarea',
TFOOT: 'tfoot',
TD: 'td',
TH: 'th',
THEAD: 'thead',
TITLE: 'title',
TR: 'tr',
TRACK: 'track',
TD: 'td',
TT: 'tt',
TR: 'tr',
TH: 'th',
THEAD: 'thead',
TBODY: 'tbody',
TFOOT: 'tfoot',
FRAMESET: 'frameset',
NOSCRIPT: 'noscript',
NOFRAMES: 'noframes',
MENUITEM: 'menuitem',
OL: 'ol',
OBJECT: 'object',
U: 'u',
UL: 'ul',
U: 'u',
SVG: 'svg',
VAR: 'var',
TABLE: 'table',
OPTION: 'option',
OPTGROUP: 'optgroup',
PLAINTEXT: 'plaintext',
H1: 'h1',
H2: 'h2',
H3: 'h3',
H4: 'h4',
H5: 'h5',
H6: 'h6',
HR: 'hr',
WBR: 'wbr',
XMP: 'xmp',
NOEMBED: 'noembed'
XMP: 'xmp'
};

119

lib/open_element_stack.js

@@ -7,40 +7,72 @@ var HTML = require('./html');

//Attributes
var ENCODING_ATTR = 'encoding',
TEXT_HTML_MIME_TYPE = 'text/html',
APPLICATION_XML_MIME_TYPE = 'application/xhtml+xml';
//Element utils
//Scoping elements
var SCOPING_ELEMENTS = {};
//OPTIMIZATION: Integer comparisons are low-cost, so we can use very fast tag name length filters here.
//It's faster than using dictionary.
function isImpliedEndTagRequired(tn) {
switch (tn.length) {
case 1:
return tn === $.P;
SCOPING_ELEMENTS[$.APPLET] = NS.HTML;
SCOPING_ELEMENTS[$.CAPTION] = NS.HTML;
SCOPING_ELEMENTS[$.HTML] = NS.HTML;
SCOPING_ELEMENTS[$.TABLE] = NS.HTML;
SCOPING_ELEMENTS[$.TD] = NS.HTML;
SCOPING_ELEMENTS[$.TH] = NS.HTML;
SCOPING_ELEMENTS[$.MARQUEE] = NS.HTML;
SCOPING_ELEMENTS[$.OBJECT] = NS.HTML;
SCOPING_ELEMENTS[$.MI] = NS.MATHML;
SCOPING_ELEMENTS[$.MO] = NS.MATHML;
SCOPING_ELEMENTS[$.MN] = NS.MATHML;
SCOPING_ELEMENTS[$.MS] = NS.MATHML;
SCOPING_ELEMENTS[$.MTEXT] = NS.MATHML;
SCOPING_ELEMENTS[$.ANNOTATION_XML] = NS.MATHML;
SCOPING_ELEMENTS[$.FOREIGN_OBJECT] = NS.SVG;
SCOPING_ELEMENTS[$.DESC] = NS.SVG;
SCOPING_ELEMENTS[$.TITLE] = NS.SVG;
case 2:
return tn === $.RP || tn === $.RT || tn === $.DD || tn === $.DT || tn === $.LI;
//Tag names that require implied end tag
var REQUIRES_IMPLIED_END_TAG = {};
case 6:
return tn === $.OPTION;
REQUIRES_IMPLIED_END_TAG[$.DD] = true;
REQUIRES_IMPLIED_END_TAG[$.DT] = true;
REQUIRES_IMPLIED_END_TAG[$.LI] = true;
REQUIRES_IMPLIED_END_TAG[$.OPTION] = true;
REQUIRES_IMPLIED_END_TAG[$.OPTGROUP] = true;
REQUIRES_IMPLIED_END_TAG[$.P] = true;
REQUIRES_IMPLIED_END_TAG[$.RP] = true;
REQUIRES_IMPLIED_END_TAG[$.RT] = true;
case 8:
return tn === $.OPTGROUP;
}
return false;
}
function isScopingElement(tn, ns) {
switch (tn.length) {
case 2:
if (tn === $.TD || tn === $.TH)
return ns === NS.HTML;
else if (tn === $.MI || tn === $.MO || tn == $.MN || tn === $.MS)
return ns === NS.MATHML;
break;
case 4:
if (tn === $.HTML)
return ns === NS.HTML;
else if (tn === $.DESC)
return ns === NS.SVG;
break;
case 5:
if (tn === $.TABLE)
return ns === NS.HTML;
else if (tn === $.MTEXT)
return ns === NS.MATHML;
else if (tn === $.TITLE)
return ns === NS.SVG;
break;
case 6:
return (tn === $.APPLET || tn === $.OBJECT) && ns === NS.HTML;
case 7:
return (tn === $.CAPTION || tn === $.MARQUEE) && ns === NS.HTML;
case 13:
return tn === $.FOREIGN_OBJECT && ns === NS.SVG;
case 14:
return tn === $.ANNOTATION_XML && ns === NS.MATHML;
}
return false;
}
//Stack of open elements

@@ -52,11 +84,5 @@ var OpenElementStack = exports.OpenElementStack = function (document, treeAdapter) {

this.currentTagName = null;
this.currentNamespaceURI = null;
this.treeAdapter = treeAdapter;
};
//Element in scope
OpenElementStack.prototype._isScopingElement = function (tagName, namespaceURI) {
return SCOPING_ELEMENTS[tagName] === namespaceURI;
};
//Index of element

@@ -79,3 +105,2 @@ OpenElementStack.prototype._indexOf = function (element) {

this.currentTagName = this.current && this.treeAdapter.getTagName(this.current);
this.currentNamespaceURI = this.current && this.treeAdapter.getNamespaceURI(this.current);
};

@@ -210,3 +235,3 @@

if (this._isScopingElement(tn, ns))
if (isScopingElement(tn, ns))
return false;

@@ -225,3 +250,3 @@ }

if (this._isScopingElement(tn, this.treeAdapter.getNamespaceURI(this.items[i])))
if (isScopingElement(tn, this.treeAdapter.getNamespaceURI(this.items[i])))
return false;

@@ -242,3 +267,3 @@ }

if (((tn === $.UL || tn === $.OL) && ns === NS.HTML) || this._isScopingElement(tn, ns))
if (((tn === $.UL || tn === $.OL) && ns === NS.HTML) || isScopingElement(tn, ns))
return false;

@@ -259,3 +284,3 @@ }

if ((tn === $.BUTTON && ns === NS.HTML) || this._isScopingElement(tn, ns))
if ((tn === $.BUTTON && ns === NS.HTML) || isScopingElement(tn, ns))
return false;

@@ -317,3 +342,3 @@ }

OpenElementStack.prototype.generateImpliedEndTags = function () {
while (REQUIRES_IMPLIED_END_TAG[this.currentTagName])
while (isImpliedEndTagRequired(this.currentTagName))
this.pop();

@@ -323,6 +348,4 @@ };

OpenElementStack.prototype.generateImpliedEndTagsWithExclusion = function (exclusionTagName) {
while (REQUIRES_IMPLIED_END_TAG[this.currentTagName] &&
this.currentTagName !== exclusionTagName) {
while (isImpliedEndTagRequired(this.currentTagName) && this.currentTagName !== exclusionTagName)
this.pop();
}
};

119

lib/preprocessor.js
var unicode = require('./unicode');
//Aliases
var $ = unicode.CODE_POINTS;
//Const
var CARRIAGE_RETURN_NEW_LINE_REG_EXP = /\r\n?/g;
//Utils
//OPTIMIZATION: these utility functions should not be moved out of this module. V8 Crankshaft will not inline
//this functions if they will be situated in another module due to context switch.
//Always perform inlining check before modifying this functions ('node --trace-inlining').
function isCodePointInCommonAllowedRange(cp) {
return cp > 0x001F && cp < 0x007F || cp > 0x009F && cp < 0xD800;
function isReservedCodePoint(cp) {
return cp >= 0xD800 && cp <= 0xDFFF || cp > 0x10FFFF;
}

@@ -18,67 +26,30 @@

//Aliases
var $ = unicode.CODE_POINTS;
//Preprocessor
var Preprocessor = exports.Preprocessor = function (html, errBuff) {
this.errBuff = errBuff;
var Preprocessor = exports.Preprocessor = function (html) {
//NOTE: All U+000D CARRIAGE RETURN (CR) characters must be converted to U+000A LINE FEED (LF) characters.
//Any U+000A LINE FEED (LF) characters that immediately follow a U+000D CARRIAGE RETURN (CR) character
//must be ignored.
this.html = html.replace(CARRIAGE_RETURN_NEW_LINE_REG_EXP, '\n');
this.html = html;
//NOTE: one leading U+FEFF BYTE ORDER MARK character must be ignored if any are present in the input stream.
this.pos = this.html.charCodeAt(0) === $.BOM ? 0 : -1;
this.lastCharPos = this.html.length - 1;
this.skipNextNewLine = false;
this.progressPos = this.pos;
this.gapStack = [];
this.lastGapPos = -1;
this.line = 1;
this.col = 1;
this.lineLengths = [];
};
//NOTE: surrogate pairs and CRLF's produces gaps in input sequence that should be avoided during retreat
Preprocessor.prototype._addGap = function () {
this.gapStack.push(this.lastGapPos);
this.lastGapPos = this.pos;
};
Preprocessor.prototype._jumpOverGap = function () {
this.lastGapPos = this.gapStack.pop();
this.pos--;
};
//NOTE: HTML input preprocessing
//(see: http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#preprocessing-the-input-stream)
Preprocessor.prototype._getProcessedCodePoint = function () {
while (true) {
this.pos++;
Preprocessor.prototype.advanceAndPeekCodePoint = function () {
this.pos++;
var isProgress = this.pos > this.progressPos;
if (this.pos > this.lastCharPos)
return $.EOF;
if (isProgress)
this.progressPos = this.pos;
var cp = this.html.charCodeAt(this.pos);
if (this.pos > this.lastCharPos)
return $.EOF;
var cp = this.html.charCodeAt(this.pos);
//NOTE: any U+000A LINE FEED (LF) characters that immediately follow a U+000D CARRIAGE RETURN (CR) character
//must be ignored.
if (this.skipNextNewLine && cp === $.LINE_FEED) {
this.skipNextNewLine = false;
this._addGap();
continue;
}
//NOTE: all U+000D CARRIAGE RETURN (CR) characters must be converted to U+000A LINE FEED (LF) characters
if (cp === $.CARRIAGE_RETURN) {
this.skipNextNewLine = true;
return $.LINE_FEED;
}
this.skipNextNewLine = false;
//OPTIMIZATION: first perform check if the code point in the allowed range that covers most common
//HTML input (e.g. ASCII codes) to avoid performance-cost operations for high-range code points.
if (cp >= 0xD800) {
//NOTE: try to peek a surrogate pair

@@ -91,35 +62,14 @@ if (this.pos !== this.lastCharPos) {

this.pos++;
this._addGap();
cp = getSurrogatePairCodePoint(cp, nextCp);
}
}
//OPTIMIZATION: first perform check if the code point in the allowed range
//that covers most common HTML input (e.g. ASCII codes) to avoid performance-sensitive checks.
if (isProgress && !isCodePointInCommonAllowedRange(cp)) {
if (unicode.isIllegalCodePoint(cp))
this.errBuff.push('Error');
if (unicode.isReservedCodePoint(cp)) {
this.errBuff.push('Error');
return $.REPLACEMENT_CHARACTER;
//NOTE: add gap that should be avoided during retreat
this.gapStack.push(this.lastGapPos);
this.lastGapPos = this.pos;
}
}
return cp;
if (isReservedCodePoint(cp))
cp = $.REPLACEMENT_CHARACTER;
}
};
Preprocessor.prototype.advanceAndPeekCodePoint = function () {
var cp = this._getProcessedCodePoint();
if (cp === $.LINE_FEED || cp === $.FORM_FEED) {
this.lineLengths[this.line] = this.col;
this.line++;
this.col = 1;
}
else
this.col++;
return cp;

@@ -129,13 +79,8 @@ };

Preprocessor.prototype.retreat = function () {
if (this.pos === this.lastGapPos)
this._jumpOverGap();
if (this.pos === this.lastGapPos) {
this.lastGapPos = this.gapStack.pop();
this.pos--;
}
this.pos--;
this.col--;
if (!this.col) {
this.line--;
this.col = this.lineLengths[this.line];
}
};

28

lib/unicode.js

@@ -45,2 +45,3 @@ exports.NULL_CHARACTER = '\u0000';

CDATA_END_STRING: [0x5D, 0x5D, 0x3E], //]]>
SCRIPT_STRING: [0x73, 0x63, 0x72, 0x69, 0x70, 0x74], //script
PUBLIC_STRING: [0x50, 0x55, 0x42, 0x4C, 0x49, 0x43], //PUBLIC

@@ -50,28 +51,1 @@ SYSTEM_STRING: [0x53, 0x59, 0x53, 0x54, 0x45, 0x4D] //SYSTEM

exports.NUMERIC_ENTITY_REPLACEMENTS = {
0x00: '\uFFFD', 0x0D: '\u000D', 0x80: '\u20AC', 0x81: '\u0081', 0x82: '\u201A', 0x83: '\u0192', 0x84: '\u201E',
0x85: '\u2026', 0x86: '\u2020', 0x87: '\u2021', 0x88: '\u02C6', 0x89: '\u2030', 0x8A: '\u0160', 0x8B: '\u2039',
0x8C: '\u0152', 0x8D: '\u008D', 0x8E: '\u017D', 0x8F: '\u008F', 0x90: '\u0090', 0x91: '\u2018', 0x92: '\u2019',
0x93: '\u201C', 0x94: '\u201D', 0x95: '\u2022', 0x96: '\u2013', 0x97: '\u2014', 0x98: '\u02DC', 0x99: '\u2122',
0x9A: '\u0161', 0x9B: '\u203A', 0x9C: '\u0153', 0x9D: '\u009D', 0x9E: '\u017E', 0x9F: '\u0178'
};
exports.isIllegalCodePoint = function (cp) {
//OPTIMIZATION: in most common cases HTML input characters are in BMP range. Reduce comparisons by checking only
//this range.
if (cp < 0x1FFFE) {
return cp >= 0x01 && cp <= 0x08 || cp >= 0x0E && cp <= 0x1F || cp >= 0x7F && cp <= 0x9F ||
cp >= 0xFDD0 && cp <= 0xFDEF || cp === 0x0B || cp === 0xFFFE || cp === 0xFFFF;
}
return cp === 0x1FFFE || cp === 0x1FFFF || cp === 0x2FFFE || cp === 0x2FFFF || cp === 0x3FFFE || cp === 0x3FFFF ||
cp === 0x4FFFE || cp === 0x4FFFF || cp === 0x5FFFE || cp === 0x5FFFF || cp === 0x6FFFE || cp === 0x6FFFF ||
cp === 0x7FFFE || cp === 0x7FFFF || cp === 0x8FFFE || cp === 0x8FFFF || cp === 0x9FFFE || cp === 0x9FFFF ||
cp === 0xAFFFE || cp === 0xAFFFF || cp === 0xBFFFE || cp === 0xBFFFF || cp === 0xCFFFE || cp === 0xCFFFF ||
cp === 0xDFFFE || cp === 0xDFFFF || cp === 0xEFFFE || cp === 0xEFFFF || cp === 0xFFFFE || cp === 0xFFFFF ||
cp === 0x10FFFE || cp === 0x10FFFF;
};
exports.isReservedCodePoint = function (cp) {
return cp >= 0xD800 && cp <= 0xDFFF || cp > 0x10FFFF;
};

45

package.json
{
"name": "parse5",
"description": "Fast full-featured HTML parser for Node. Based on WHATWG HTML5 specification.",
"version": "0.5.4",
"author": "Ivan Nikulin (ifaaan@gmail.com, https://github.com/inikulin)",
"keywords": ["html", "parser", "html5", "WHATWG", "specification", "fast"],
"repository": {
"type": "git",
"url": "git://github.com/inikulin/parse5.git"
},
"main": "./index.js",
"devDependencies": {
"nodeunit": "0.8.0"
},
"licenses": [{
"type": "MIT",
"url": "https://raw.github.com/inikulin/parse5/master/LICENSE"
}]
}
"name": "parse5",
"description": "Fast full-featured HTML parser for Node. Based on WHATWG HTML5 specification.",
"version": "0.6.0",
"author": "Ivan Nikulin (ifaaan@gmail.com, https://github.com/inikulin)",
"keywords": [
"html",
"parser",
"html5",
"WHATWG",
"specification",
"fast"
],
"repository": {
"type": "git",
"url": "git://github.com/inikulin/parse5.git"
},
"main": "./lib/parser.js",
"devDependencies": {
"nodeunit": "0.8.0"
},
"licenses": [
{
"type": "MIT",
"url": "https://raw.github.com/inikulin/parse5/master/LICENSE"
}
]
}

6

test/fixtures/open_element_stack_test.js

@@ -21,3 +21,2 @@ var HTML = require('../../lib/html'),

t.strictEqual(stack.currentTagName, element1.tagName);
t.strictEqual(stack.currentNamespaceURI, element1.namespaceURI);
t.strictEqual(stack.stackTop, 0);

@@ -28,3 +27,2 @@

t.strictEqual(stack.currentTagName, element2.tagName);
t.strictEqual(stack.currentNamespaceURI, element2.namespaceURI);
t.strictEqual(stack.stackTop, 1);

@@ -44,3 +42,2 @@

t.strictEqual(stack.currentTagName, element.tagName);
t.strictEqual(stack.currentNamespaceURI, element.namespaceURI);
t.strictEqual(stack.stackTop, 0);

@@ -51,3 +48,2 @@

t.ok(!stack.currentTagName);
t.ok(!stack.currentNamespaceURI);
t.strictEqual(stack.stackTop, -1);

@@ -68,3 +64,2 @@

t.strictEqual(stack.currentTagName, newElement.tagName);
t.strictEqual(stack.currentNamespaceURI, newElement.namespaceURI);
t.strictEqual(stack.stackTop, 1);

@@ -91,3 +86,2 @@

t.strictEqual(stack.currentTagName, element1.tagName);
t.strictEqual(stack.currentNamespaceURI, element1.namespaceURI);

@@ -94,0 +88,0 @@ t.done();

9

test/fixtures/parser_test.js

@@ -152,8 +152,7 @@ var fs = require('fs'),

exports[getFullTestName(test)] = function (t) {
//TODO handle errors
var parser = new Parser(test.input, test.fragmentContext),
document = parser.parse(),
serializedDocument = serializeNodeList(document.childNodes, 0);
var parser = new Parser(),
result = test.fragmentContext ? parser.parseFragment(test.input, test.fragmentContext) : parser.parse(test.input),
serializedResult = serializeNodeList(result.childNodes, 0);
t.strictEqual(serializedDocument, test.expected, getAssertionMessage(serializedDocument, test.expected));
t.strictEqual(serializedResult, test.expected, getAssertionMessage(serializedResult, test.expected));
t.done();

@@ -160,0 +159,0 @@ };

35

test/fixtures/tokenizer_test.js

@@ -9,6 +9,3 @@ var fs = require('fs'),

nextToken = null,
out = {
tokens: [],
errCount: 0
};
out = [];

@@ -28,3 +25,3 @@ tokenizer.state = initialState;

case Tokenizer.WHITESPACE_CHARACTER_TOKEN:
out.tokens.push(['Character', nextToken.ch]);
out.push(['Character', nextToken.chars]);
break;

@@ -48,15 +45,15 @@

out.tokens.push(startTagEntry);
out.push(startTagEntry);
break;
case Tokenizer.END_TAG_TOKEN:
out.tokens.push(['EndTag', nextToken.tagName]);
out.push(['EndTag', nextToken.tagName]);
break;
case Tokenizer.COMMENT_TOKEN:
out.tokens.push(['Comment', nextToken.data]);
out.push(['Comment', nextToken.data]);
break;
case Tokenizer.DOCTYPE_TOKEN:
out.tokens.push([
out.push([
'DOCTYPE',

@@ -72,6 +69,3 @@ nextToken.name,

out.errCount = tokenizer.errs.length;
out.tokens = concatCharacterTokens(out.tokens);
return out;
return concatCharacterTokens(out)
}

@@ -151,10 +145,7 @@

var expectedTokens = [],
expectedErrCount = 0;
var expected = [];
descr.output.forEach(function (tokenEntry) {
if (tokenEntry === 'ParseError')
expectedErrCount++;
else
expectedTokens.push(tokenEntry);
if (tokenEntry !== 'ParseError')
expected.push(tokenEntry);
});

@@ -168,4 +159,3 @@

input: descr.input,
expectedTokens: concatCharacterTokens(expectedTokens),
expectedErrCount: expectedErrCount,
expected: concatCharacterTokens(expected),
initialState: getTokenizerSuitableStateName(initialState),

@@ -190,4 +180,3 @@ lastStartTag: descr.lastStartTag

t.deepEqual(out.tokens, test.expectedTokens);
t.strictEqual(out.errCount, test.expectedErrCount);
t.deepEqual(out, test.expected);

@@ -194,0 +183,0 @@ t.done();

index.js
lib/named_entity_trie.js

Sorry, the diff of this file is too big to display

lib/parser.js

Sorry, the diff of this file is too big to display

lib/tokenizer.js

Sorry, the diff of this file is too big to display

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap

About

Packages

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc