js-tokens - npm Package Compare versions

Comparing version 0.3.1 to 0.4.0

changelog.md

@@ -1,3 +0,10 @@
-### Version 0.3.1 (2014-01-06) ###
+### Version 0.4.0 (2015-02-21) ###
 
+- Improved: `jsTokens.matchToToken` performance.
+- Added: Support for octal and binary number literals.
+- Added: Support for template strings.
+
+
+### Version 0.3.1 (2015-01-06) ###
+
 - Fixed: Support for unicode spaces. They used to be allowed in names (which is
@@ -4,0 +11,0 @@ very confusing), and some unicode newlines were wrongly allowed in strings and

index.js

@@ -6,16 +6,16 @@ // Copyright 2014, 2015 Simon Lydell
 // (run `npm run build`).
-module.exports = /((['"])(?:(?!\2)[^\\\r\n\u2028\u2029]|\\(?:\r\n|[\s\S]))*(\2)?)|(\/\/.*)|(\/\*(?:[^*]|\*(?!\/))*(\*\/)?)|(\/(?!\*)(?:\[(?:[^\]\\\r\n\u2028\u2029]|\\.)*\]|[^\/\]\\\r\n\u2028\u2029]|\\.)+\/(?:(?!\s*(?:\b|[\u0080-\uFFFF$\\'"~({]|[+\-!](?!=)|\.?\d))|[gmiy]{1,4}\b(?![\u0080-\uFFFF$\\]|\s*(?:[+\-*%&|^<>!=?({]|\/(?![\/*])))))|(-?(?:0[xX][\da-fA-F]+|(?:\d*\.\d+|\d+\.?)(?:[eE][+-]?\d+)?))|((?!\d)(?:(?!\s)[$\w\u0080-\uFFFF]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]{1,6}\})+)|(--|\+\+|&&|\|\||=>|\.{3}|(?:[+\-*\/%&|^]|<{1,2}|>{1,3}|!=?|={1,2})=?|[?:~])|([;,.[\](){}])|(\s+)|(^$|[\s\S])/g
+module.exports = /((['"])(?:(?!\2)[^\\\r\n\u2028\u2029]|\\(?:\r\n|[\s\S]))*(\2)?|`(?:[^`\\$]|\\[\s\S]|\$(?!\{)|\$\{(?:[^{}]|\{[^}]*\}?)*\}?)*(`)?)|(\/\/.*)|(\/\*(?:[^*]|\*(?!\/))*(\*\/)?)|(\/(?!\*)(?:\[(?:[^\]\\\r\n\u2028\u2029]|\\.)*\]|[^\/\]\\\r\n\u2028\u2029]|\\.)+\/(?:(?!\s*(?:\b|[\u0080-\uFFFF$\\'"~({]|[+\-!](?!=)|\.?\d))|[gmiy]{1,4}\b(?![\u0080-\uFFFF$\\]|\s*(?:[+\-*%&|^<>!=?({]|\/(?![\/*])))))|(-?(?:0[xX][\da-fA-F]+|0[oO][0-7]+|0[bB][01]+|(?:\d*\.\d+|\d+\.?)(?:[eE][+-]?\d+)?))|((?!\d)(?:(?!\s)[$\w\u0080-\uFFFF]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]{1,6}\})+)|(--|\+\+|&&|\|\||=>|\.{3}|(?:[+\-*\/%&|^]|<{1,2}|>{1,3}|!=?|={1,2})=?|[?:~])|([;,.[\](){}])|(\s+)|(^$|[\s\S])/g
 
 module.exports.matchToToken = function(match) {
   token = {type: "invalid", value: match[0]}
-  if (match[ 1]) token.type = "string" , token.closed = !!match[3]
-  if (match[ 4]) token.type = "comment"
-  if (match[ 5]) token.type = "comment", token.closed = !!match[6]
-  if (match[ 7]) token.type = "regex"
-  if (match[ 8]) token.type = "number"
-  if (match[ 9]) token.type = "name"
-  if (match[10]) token.type = "operator"
-  if (match[11]) token.type = "punctuation"
-  if (match[12]) token.type = "whitespace"
+       if (match[ 1]) token.type = "string" , token.closed = !!(match[3] || match[4])
+  else if (match[ 5]) token.type = "comment"
+  else if (match[ 6]) token.type = "comment", token.closed = !!match[7]
+  else if (match[ 8]) token.type = "regex"
+  else if (match[ 9]) token.type = "number"
+  else if (match[10]) token.type = "name"
+  else if (match[11]) token.type = "operator"
+  else if (match[12]) token.type = "punctuation"
+  else if (match[13]) token.type = "whitespace"
   return token
 }

package.json

 {
   "name": "js-tokens",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "author": "Simon Lydell",
@@ -5,0 +5,0 @@ "license": "MIT",

readme.md

@@ -55,4 +55,4 @@ Overview [![Build Status](https://travis-ci.org/lydell/js-tokens.png?branch=master)](https://travis-ci.org/lydell/js-tokens)
 
-Comments and strings both come in two flavors. To distinguish them, check if the
-token starts with `//`, `/*`, `'` or `"`.
+Comments and strings both come in several flavors. To distinguish them, check if
+the token starts with `//`, `/*`, `'`, `"` or `` ` ``.
 
@@ -69,6 +69,7 @@ For example usage, please see this [gist].
 contain (unescaped) newlines, so unterminated strings simply end at the end of
-the line.
+the line. Unterminated template strings can contain unescaped newlines, though,
+so they go on to the end of input.
 
 Unterminated multi-line comments are also still matched as comments. They
-simply go on to the end of the string.
+simply go on to the end of the input.
 
@@ -96,2 +97,12 @@ Unterminated regex literals are likely matched as division and whatever is
 
+### Template string interpolation ###
+
+Template strings are matched as single tokens, from the starting `` ` `` to then
+ending `` ` ``, including interpolations (whose tokens are not matched
+individually).
+
+Matching template string interpolations requires recursive balancing of `{` and
+`}`—something that JavaScript regexes cannot do. Only one level of nesting is
+supported.
+
 ### Division and regex literals collision ###
@@ -98,0 +109,0 @@

test/index.js

@@ -139,54 +139,105 @@ // Copyright 2014, 2015 Simon Lydell
     match('""')
+    match("``")
     match("'string'")
     match('"string"')
+    match("`string`")
     match("'\\''")
     match('"\\""')
+    match("`\\``")
     match("'\\\\''", "'\\\\'")
     match('"\\\\""', '"\\\\"')
+    match("`\\\\``", "`\\\\`")
     match("'\\\\\\''")
     match('"\\\\\\""')
+    match("`\\\\\\``")
     match("'\\u05aF'")
     match('"\\u05aF"')
+    match("`\\u05aF`")
     match("'invalid escape sequence is OK: \\u'")
     match('"invalid escape sequence is OK: \\u"')
+    match("`invalid escape sequence is OK: \\u`")
     match("'\\\n'")
     match('"\\\n"')
+    match("`\\\n`")
     match("'\\\r'")
     match('"\\\r"')
+    match("`\\\r`")
     match("'\\\u2028'")
     match('"\\\u2028"')
+    match("`\\\u2028`")
     match("'\\\u2029'")
     match('"\\\u2029"')
+    match("`\\\u2029`")
     match("'\\\r\n'")
     match('"\\\r\n"')
+    match("`\\\r\n`")
     match("'string'code'another string'", "'string'")
     match('"string"code"another string"', '"string"')
+    match("`string`code`another string`", "`string`")
     match("'\"'")
+    match("'`'")
     match('"\'"')
+    match('"`"')
+    match("`'`")
+    match('`"`')
 
     match("'", {closed: false})
     match('"', {closed: false})
+    match("`", {closed: false})
     match("'unclosed", {closed: false})
     match('"unclosed', {closed: false})
+    match("`unclosed", {closed: false})
     match("'\n", "'", {closed: false})
     match('"\n', '"', {closed: false})
+    match("`\n", {closed: false})
     match("'\r", "'", {closed: false})
     match('"\r', '"', {closed: false})
+    match("`\r", {closed: false})
     match("'\u2028", "'", {closed: false})
     match('"\u2028', '"', {closed: false})
+    match("`\u2028", {closed: false})
     match("'\u2029", "'", {closed: false})
     match('"\u2029', '"', {closed: false})
+    match("`\u2029", {closed: false})
     match("'\r\n", "'", {closed: false})
     match('"\r\n', '"', {closed: false})
+    match("`\r\n", {closed: false})
     match("'\\\n", {closed: false})
     match('"\\\n', {closed: false})
+    match("`\\\n", {closed: false})
     match("'\\\r", {closed: false})
     match('"\\\r', {closed: false})
+    match("`\\\r", {closed: false})
     match("'\\\u2028", {closed: false})
     match('"\\\u2028', {closed: false})
+    match("`\\\u2028", {closed: false})
     match("'\\\u2029", {closed: false})
     match('"\\\u2029', {closed: false})
+    match("`\\\u2029", {closed: false})
     match("'\\\r\n", {closed: false})
     match('"\\\r\n', {closed: false})
+    match("`\\\r\n", {closed: false})
 
+    match("'${}'")
+    match('"${}"')
+    match("`${}`")
+    match("'${a}'")
+    match('"${a}"')
+    match("`${a}`")
+    match("'a${b}c'")
+    match('"a${b}c"')
+    match("`a${b}c`")
+    match("'${'a'}'", "'${'")
+    match('"${"a"}"', '"${"')
+    match("`${`a`}`")
+    match("`${`${a}`}`")
+    match("`${fn({a: b})}`")
+    match("`${fn({a: '{'})}`")
+    match("`a${}${a}${ `${b\r}` + `${`c`}` } d $${\n(x=>{return x*2})(4)}$`")
+    match("`\\${{{}}}a`")
+
+    match("`a ${b c`.length", {closed: false})
+    match("`a ${`b${c`} d`.length", {closed: false})
+    match("`a ${ {c:d } e`.length", {closed: false})
+
   })
@@ -409,2 +460,4 @@
     match("1E0x123", "1E0")
+    match("1E0o123", "1E0")
+    match("1E0b123", "1E0")
 
@@ -453,2 +506,58 @@ match("e1", false)
 
+    match("0o1")
+    match("0oa", "0")
+    match("0o01574")
+    match("0o1e1", "0o1")
+    match("0o1E1", "0o1")
+    match("0o1g1", "0o1")
+
+    match("0O1")
+    match("0Oa", "0")
+    match("0O01574")
+    match("0O1e1", "0O1")
+    match("0O1E1", "0O1")
+    match("0O1g1", "0O1")
+
+    match("-0o1")
+    match("-0oa", "-0")
+    match("-0o01574")
+    match("-0o1e1", "-0o1")
+    match("-0o1E1", "-0o1")
+    match("-0o1g1", "-0o1")
+
+    match("0o", "0")
+    match("1o1", "1")
+    match("0o1.", "0o1")
+    match("0o1.1", "0o1")
+    match("0.0o1", "0.0")
+    match(".0o1", ".0")
+
+    match("0b1")
+    match("0ba", "0")
+    match("0b01011")
+    match("0b1e1", "0b1")
+    match("0b1E1", "0b1")
+    match("0b1g1", "0b1")
+
+    match("0B1")
+    match("0Ba", "0")
+    match("0B01011")
+    match("0B1e1", "0B1")
+    match("0B1E1", "0B1")
+    match("0B1g1", "0B1")
+
+    match("-0b1")
+    match("-0ba", "-0")
+    match("-0b01011")
+    match("-0b1e1", "-0b1")
+    match("-0b1E1", "-0b1")
+    match("-0b1g1", "-0b1")
+
+    match("0b", "0")
+    match("1b1", "1")
+    match("0b1.", "0b1")
+    match("0b1.1", "0b1")
+    match("0.0b1", "0.0")
+    match(".0b1", ".0")
+
   })
@@ -740,3 +849,2 @@
     match("#")
-    match("`")
     match("\\")
@@ -743,0 +851,0 @@ match("\\xa9", "\\")

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

44268

increased by9.17%

Lines of code

1055

increased by10.24%

Number of lines in readme file

195

increased by5.98%

No dependency changes