jison - npm Package Compare versions

Comparing version 0.2.6 to 0.2.7

lib/jison/jisonlex.js

@@ -9,3 +9,3 @@ var jisonlex = require("./util/lex-parser").parser;
 
-function encodeRE (s) { return s.replace(/([.*+?^${}()|[\]\/\\])/g, '\\$1'); }
+function encodeRE (s) { return s.replace(/([.*+?^${}()|[\]\/\\])/g, '\\$1').replace(/\\\\u([a-fA-F0-9]{4})/g,'\\u$1'); }
 
@@ -12,0 +12,0 @@ jisonlex.yy = {

lib/jison/util/bnf-parser.js

@@ -56,3 +56,3 @@ /* Jison generated parser */
             if (this.$.length === 1) this.$ = this.$[0];
-
+        
 break;
@@ -102,2 +102,4 @@ case 21:this.$ = $$[$0-1]; this.$.push($$[$0])
     this.yy.lexer = this.lexer;
+    if (typeof this.lexer.yylloc == 'undefined')
+        this.lexer.yylloc = {};
     var yyloc = this.lexer.yylloc;
@@ -341,3 +343,3 @@ lstack.push(yyloc);
                                first_column: this.yylloc.last_column,
-                               last_column: lines ? lines[lines.length-1].length-1 : this.yylloc.last_column + match.length}
+                               last_column: lines ? lines[lines.length-1].length-1 : this.yylloc.last_column + match[0].length}
                 this.yytext += match[0];
@@ -358,3 +360,3 @@ this.match += match[0];
         } else {
-            this.parseError('Lexical error on line '+(this.yylineno+1)+'. Unrecognized text.\n'+this.showPosition(),
+            this.parseError('Lexical error on line '+(this.yylineno+1)+'. Unrecognized text.\n'+this.showPosition(), 
                     {text: "", token: null, line: this.yylineno});
@@ -361,0 +363,0 @@ }

lib/jison/util/lex-parser.js

@@ -66,3 +66,3 @@ /* Jison generated parser */
 case 25: this.$ = $$[$0]; 
-          if (this.$.match(/[\w\d]$/))
+          if (this.$.match(/[\w\d]$/) && !this.$.match(/\\u[a-fA-F0-9]{4}$/))
               this.$ += "\\b";
@@ -130,2 +130,4 @@
     this.yy.lexer = this.lexer;
+    if (typeof this.lexer.yylloc == 'undefined')
+        this.lexer.yylloc = {};
     var yyloc = this.lexer.yylloc;
@@ -215,3 +217,3 @@ lstack.push(yyloc);
             }
-            
+
             preErrorSymbol = symbol; // save the lookahead token
@@ -264,3 +266,3 @@ symbol = TERROR;         // insert generic error symbol as new lookahead
                     first_column: lstack[lstack.length-(len||1)].first_column,
-                    last_column: lstack[lstack.length-1].last_column,
+                    last_column: lstack[lstack.length-1].last_column
                 };
@@ -371,3 +373,3 @@ r = this.performAction.call(yyval, yytext, yyleng, yylineno, this.yy, action[1], vstack, lstack);
                                first_column: this.yylloc.last_column,
-                               last_column: lines ? lines[lines.length-1].length-1 : this.yylloc.last_column + match.length}
+                               last_column: lines ? lines[lines.length-1].length-1 : this.yylloc.last_column + match[0].length}
                 this.yytext += match[0];
@@ -374,0 +376,0 @@ this.match += match[0];

package.json

 {
   "name": "jison",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "author": "Zach Carter",
@@ -47,3 +47,6 @@ "email": "[e-mail]",
     "jison/util/typal.js": "lib/jison/util/typal.js"
-  }
+  },
+  "files": [
+    ""
+  ]
 }

tests/grammar/lex_parse.js

@@ -168,1 +168,23 @@ var assert = require("assert"),
 };
+
+exports["test unicode encoding"] = function () {
+    var lexgrammar = '%%\n"\\u03c0" return 1;';
+    var expected = {
+        rules: [
+            ["\\u03c0", "return 1;"]
+        ]
+    };
+
+    assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly");
+};
+
+exports["test unicode"] = function () {
+    var lexgrammar = '%%\n"π" return 1;';
+    var expected = {
+        rules: [
+            ["π", "return 1;"]
+        ]
+    };
+
+    assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly");
+}

tests/lexer/regexplexer.js

@@ -603,1 +603,34 @@ var RegExpLexer = require("../setup").RegExpLexer,
 
+exports["test unicode encoding"] = function() {
+    var dict = {
+        rules: [
+            ["\\u2713", "return 'CHECK';" ],
+            ["\\u03c0", "return 'PI';" ],
+            ["y", "return 'Y';" ]
+        ]
+    };
+    var input = "\u2713\u03c0y";
+
+    var lexer = new RegExpLexer(dict);
+    lexer.setInput(input);
+    
+    assert.equal(lexer.lex(), "CHECK");
+    assert.equal(lexer.lex(), "PI");
+    assert.equal(lexer.lex(), "Y");
+};
+
+exports["test unicode"] = function() {
+    var dict = {
+        rules: [
+            ["π", "return 'PI';" ],
+            ["y", "return 'Y';" ]
+        ]
+    };
+    var input = "πy";
+
+    var lexer = new RegExpLexer(dict);
+    lexer.setInput(input);
+    
+    assert.equal(lexer.lex(), "PI");
+    assert.equal(lexer.lex(), "Y");
+};

tests/parser/api.js

@@ -377,9 +377,2 @@ var Jison = require("../setup").Jison,
 
-    var lexData = {
-        rules: [
-           ["\\s", "/*ignore*/"],
-           ["x", "return 'x';"],
-           ["y", "return 'y';"]
-        ]
-    };
     var gen = new Jison.Generator(grammar);
@@ -386,0 +379,0 @@ var parser = gen.createParser();

No alert changes

Improved metrics

Total package byte prevSize

1452269

increased by4.22%

Number of package files

185

increased by1.65%

Lines of code

21508

increased by0.2%

No dependency changes