lex - npm Package Compare versions

Comparing version 1.5.1 to 1.6.0

component.json

@@ -5,3 +5,3 @@ {
     "description": "An elegant armor-plated JavaScript lexer modelled after flex. Easily extensible to tailor to your need for perfection.",
-    "version": "1.5.1",
+    "version": "1.6.0",
     "keywords": ["lex", "lexer", "lexical", "analysis", "scan", "scanner", "scanning", "token", "tokenize", "tokenizer", "tokenization", "flex", "jison"],
@@ -8,0 +8,0 @@ "main": "lib/lexer.js",

lib/lexer.js

@@ -10,2 +10,3 @@ if (typeof module === "object" && typeof module.exports === "object") module.exports = Lexer;
 
+    var tokens = [];
     var rules = [];
@@ -45,2 +46,4 @@ var remove = 0;
     this.lex = function () {
+        if (tokens.length) return tokens.shift();
+
         this.reject = true;
@@ -63,3 +66,9 @@
                     if (this.reject) this.index = result.index;
-                    else if (typeof token !== "undefined") {
+                    else switch (typeof token) {
+                    case "undefined":
+                        break;
+                    case "array":
+                        tokens = token.slice(1);
+                        token = token[0];
+                    default:
                         if (length) remove = 0;
@@ -77,3 +86,11 @@ return token;
                     var token = defunct.call(this, input.charAt(this.index++));
-                    if (typeof token !== "undefined") return token;
+                    switch (typeof token) {
+                    case "undefined":
+                        break;
+                    case "array":
+                        tokens = token.slice(1);
+                        return token[0];
+                    default:
+                        return token;
+                    }
                 } else {
@@ -80,0 +97,0 @@ if (this.index !== index) remove = 0;

package.json

 {
     "name": "lex",
     "description": "An elegant armor-plated JavaScript lexer modelled after flex. Easily extensible to tailor to your need for perfection.",
-    "version": "1.5.1",
+    "version": "1.6.0",
     "keywords": ["lex", "lexer", "lexical", "analysis", "scan", "scanner", "scanning", "token", "tokenize", "tokenizer", "tokenization", "flex", "jison"],
@@ -6,0 +6,0 @@ "author": "Aadit M Shah (http://aaditmshah.github.com/) <[e-mail]>",

README.md

@@ -103,1 +103,27 @@ # Lexer #
 ```
+
+Starting from v1.6.0 you can return multiple values from an action by returning an array. The elements of the array will be returned individually by the `lex` method. This allows you to implement features like [python style indentation](http://docs.python.org/release/2.5.1/ref/indentation.html "2.1.8 Indentation") as follows:
+
+```javascript
+var indent = [0];
+
+var lexer = new Lexer;
+
+lexer.addRule(/^[\t ]*/, function (lexeme) {
+    var indentation = lexeme.length;
+
+    if (indentation > indent[0]) {
+        indent.unshift(indentation);
+        return "INDENT";
+    }
+
+    var tokens = [];
+
+    while (indentation < indent[0]) {
+        tokens.push("DEDENT");
+        indent.shift();
+    }
+
+    if (tokens.length) return tokens;
+});
+```

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

10458

increased by13.98%

Lines of code

128

increased by14.29%

Number of lines in readme file

129

increased by25.24%

No dependency changes