homunculus - npm Package Compare versions

Comparing version 0.2.0 to 0.2.1

dist/lexer/Lexer.js

@@ -71,2 +71,6 @@ (function(factory) {
               this.tokenList.push(token);
+              //回调可自定义处理匹配的token
+              if(match.callback) {
+                match.callback(token);
+              }
               this.index += matchLen - 1;
@@ -73,0 +77,0 @@ var n = character.count(token.val(), character.LINE);

dist/lexer/rule/EcmascriptRule.js

@@ -18,2 +18,3 @@ (function(factory) {
   var character = require('../../util/character');
+  var isProperty = false;
   var EcmascriptRule = Rule.extend(function() {
@@ -35,7 +36,15 @@ var self = this;
   
-    self.addMatch(new RegMatch(Token.ID, /^[$a-zA-Z_][$\w]*/, Lexer.SPECIAL, function() {
+    var regMatch = new RegMatch(Token.ID, /^[$a-zA-Z_][$\w]*/, Lexer.SPECIAL, function() {
       return !!(self.keyWords().hasOwnProperty(this.content()));
     }, function() {
       return ['if', 'for', 'while'].indexOf(this.content()) != -1;
-    }));
+    });
+    regMatch.callback = function(token) {
+      var s = token.content();
+      if(isProperty) {
+        token.type(Token.ID);
+      }
+      isProperty = false;
+    };
+    self.addMatch(regMatch);
   
@@ -51,3 +60,8 @@ self.addMatch(new RegMatch(Token.NUMBER, /^\.\d+(?:E[+-]?\d*)?/i, {
     });
-    self.addMatch(new CharacterSet(Token.SIGN, ':;/?.,[]{}~!^|%=-+*()~><&\\', Lexer.IS_REG));
+    var sign = new CharacterSet(Token.SIGN, ':;/?.,[]{}~!^|%=-+*()~><&\\', Lexer.IS_REG);
+    sign.callback = function(token) {
+      var s = token.content();
+      isProperty = s == '.';
+    };
+    self.addMatch(sign);
   
@@ -67,4 +81,3 @@ self.addMatch(new RegMatch(Token.NUMBER, /^0x[\da-f]*/i, {
   
-    self.addMatch(new CompleteEqual(Token.LINE, '\u2028'));
-    self.addMatch(new CompleteEqual(Token.LINE, '\u2029'));
+    self.addMatch(new CharacterSet(Token.LINE, '\u2028\u2029'));
     self.addMatch(new CharacterSet(Token.BLANK, '\f\u000b\u00A0\uFEFF\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000'));
@@ -71,0 +84,0 @@ }).statics({

dist/parser/es6/Node.js

@@ -9,75 +9,6 @@ (function(factory) {
 })(function(require, exports, module) {
-  var Class = require('../../util/Class');
-  var Node = Class(function(type, children) {
-    this.type = type;
-    if(type == Node.TOKEN) {
-      this.children = children;
-    }
-    else if(Array.isArray(children)) {
-      this.children = children;
-    }
-    else {
-      this.children = children ? [children] : [];
-    }
-    this.p = null;
-    this.pr = null;
-    this.ne = null;
+  var INode = require('../Node');
+  var Node = INode.extend(function(type, children) {
+    INode.call(this, type, children);
     return this;
-  }).methods({
-    name: function() {
-      return this.type;
-    },
-    leaves: function() {
-      return this.children;
-    },
-    leaf: function(i) {
-      return this.children[i];
-    },
-    size: function() {
-      return this.children.length;
-    },
-    first: function() {
-      return this.leaf(0);
-    },
-    last: function() {
-      return this.leaf(this.size() - 1);
-    },
-    isEmpty: function() {
-      return this.size() === 0;
-    },
-    add: function() {
-      var self = this;
-      var args = Array.prototype.slice.call(arguments, 0);
-      args.forEach(function(node) {
-        node.parent(self);
-        var last = self.children[self.children.length - 1];
-        if(last) {
-          last.next(node);
-          node.prev(last);
-        }
-        self.children.push(node);
-      });
-      return self;
-    },
-    token: function() {
-      return this.children;
-    },
-    parent: function(p) {
-      if(p) {
-        this.p = p;
-      }
-      return this.p;
-    },
-    prev: function(pr) {
-      if(pr) {
-        this.pr = pr;
-      }
-      return this.pr;
-    },
-    next: function(ne) {
-      if(ne) {
-        this.ne = ne;
-      }
-      return this.ne;
-    }
   }).statics({
@@ -92,3 +23,2 @@ SCRIPT: 'script',
     ITERSTMT: 'iterstmt',
-    TOKEN: 'token',
     FNPARAMS: 'fnparams',
@@ -95,0 +25,0 @@ BINDELEMENT: 'bindelement',

dist/parser/es6/Parser.js

@@ -9,3 +9,3 @@ (function(factory) {
 })(function(require, exports, module) {
-  var Class = require('../../util/Class');
+  var IParser = require('../Parser');
   var character = require('../../util/character');
@@ -33,4 +33,6 @@ var Lexer = require('../../lexer/Lexer');
     = true;
-  var Parser = Class(function(lexer) {
+  var Parser = IParser.extend(function(lexer) {
+    IParser.call(this, lexer);
     this.init(lexer);
+    return this;
   }).methods({
@@ -42,5 +44,2 @@ parse: function(code) {
     },
-    ast: function() {
-      return this.tree;
-    },
     init: function(lexer) {
@@ -2267,5 +2266,2 @@ this.look = null;
       throw new Error(msg + ' line ' + this.lastLine + ' col ' + this.lastCol);
-    },
-    ignore: function() {
-      return this.ignores;
     }
@@ -2272,0 +2268,0 @@ });

dist/parser/js/Node.js

@@ -9,75 +9,6 @@ (function(factory) {
 })(function(require, exports, module) {
-  var Class = require('../../util/Class');
-  var Node = Class(function(type, children) {
-    this.type = type;
-    if(type == Node.TOKEN) {
-      this.children = children;
-    }
-    else if(Array.isArray(children)) {
-      this.children = children;
-    }
-    else {
-      this.children = children ? [children] : [];
-    }
-    this.p = null;
-    this.pr = null;
-    this.ne = null;
+  var INode = require('../Node');
+  var Node = INode.extend(function(type, children) {
+    INode.call(this, type, children);
     return this;
-  }).methods({
-    name: function() {
-      return this.type;
-    },
-    leaves: function() {
-      return this.children;
-    },
-    leaf: function(i) {
-      return this.children[i];
-    },
-    size: function() {
-      return this.children.length;
-    },
-    first: function() {
-      return this.leaf(0);
-    },
-    last: function() {
-      return this.leaf(this.size() - 1);
-    },
-    isEmpty: function() {
-      return this.size() === 0;
-    },
-    add: function() {
-      var self = this;
-      var args = Array.prototype.slice.call(arguments, 0);
-      args.forEach(function(node) {
-        node.parent(self);
-        var last = self.children[self.children.length - 1];
-        if(last) {
-          last.next(node);
-          node.prev(last);
-        }
-        self.children.push(node);
-      });
-      return self;
-    },
-    token: function() {
-      return this.children;
-    },
-    parent: function(p) {
-      if(p) {
-        this.p = p;
-      }
-      return this.p;
-    },
-    prev: function(pr) {
-      if(pr) {
-        this.pr = pr;
-      }
-      return this.pr;
-    },
-    next: function(ne) {
-      if(ne) {
-        this.ne = ne;
-      }
-      return this.ne;
-    }
   }).statics({
@@ -91,3 +22,2 @@ PROGRAM: 'program',
     ITERSTMT: 'iterstmt',
-    TOKEN: 'token',
     FNPARAMS: 'fnparams',
@@ -94,0 +24,0 @@ EXPR: 'expr',

dist/parser/js/Parser.js

@@ -9,3 +9,3 @@ (function(factory) {
 })(function(require, exports, module) {
-  var Class = require('../../util/Class');
+  var IParser = require('../Parser');
   var character = require('../../util/character');
@@ -33,4 +33,6 @@ var Lexer = require('../../lexer/Lexer');
     = true;
-  var Parser = Class(function(lexer) {
+  var Parser = IParser.extend(function(lexer) {
+    IParser.call(this, lexer);
     this.init(lexer);
+    return this;
   }).methods({
@@ -42,5 +44,2 @@ parse: function(code) {
     },
-    ast: function() {
-      return this.tree;
-    },
     init: function(lexer) {
@@ -1247,5 +1246,2 @@ this.look = null;
       throw new Error(msg + ' line ' + this.lastLine + ' col ' + this.lastCol);
-    },
-    ignore: function() {
-      return this.ignores;
     }
@@ -1252,0 +1248,0 @@ });

package.json

 {
   "name": "homunculus",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "A lexer&parser by Javascript",
@@ -24,2 +24,4 @@ "maintainers": [
         "dist/parser/es6",
+        "dist/parser/Parser",
+        "dist/parser/Node",
         "dist/util"
@@ -26,0 +28,0 @@ ]

README.md

@@ -15,13 +15,103 @@ # A lexer&parser by Javascript
 * getClass(type:String, lan:String):class
- * type: lexer parser node context token
- * lan: js javascript es es5 ecmascript es6 as actionscript css
-* getLexer(lan:String):object
- * lan: js javascript es es5 ecmascript es6 as actionscript css java c++ cpp cplusplus
-* getParser(lan:String):object
- * lan: js javascript es es5 ecmascript es6 css
-* getContext(lan:String):object
- * lan: js javascript es es5 ecmascript
+  * type:
+    * lexer 
+    * parser
+    * node
+    * context
+    * token
+  * lan: 
+    * js
+    * javascript
+    * es 
+    * es5
+    * ecmascript
+    * es6
+    * as
+    * actionscript
+    * css
+* getLexer(lan:String):lexer/Lexer
+ * lan:
+   * js
+   * javascript
+   * es
+   * es5
+   * ecmascript
+   * es6
+   * as
+   * actionscript
+   * css
+   * java
+   * c++
+   * cpp
+   * cplusplus
+* getParser(lan:String):parser/Parser
+  * lan:
+    * js
+    * javascript
+    * es
+    * es5
+    * ecmascript
+    * es6
+    * css
+* getContext(lan:String):parser/Context
+  * lan:
+    * js
+    * javascript
+    * es
+    * es5
+    * ecmascript
 
+### lexer/Lexer
+#### 方法
+* constructor(Rule:lexer/rule/Rule) 传入语法规则Rule
+* parse(code:String):Array<lexer/Token> 传入代码并返回解析后的此法单元token列表
+* tokens():Array<lexer/Token> 返回已解析好的此番单元token列表
+* cache(line:init):void 设置缓冲解析行，每次最多解析几行代码，防止code过大卡死
+* finish():Boolean 设置cache有用，当前是否解析完毕
+* line():int code有多少行
+* col():int code最大列是多少
+
+#### 静态属性
+* STRICT: 0 严格模式语法错误后抛出异常
+* LOOSE: 1 宽松模式错误后忽略
+* mod(type:int):int 读取/设置模式
+
+### parser/Parser
+#### 方法
+* constructor(lexer:Lexer) 传入词法分析器
+* parse(code:String):Node 传入代码解析并返回语法树
+* ast():Node 返回已解析好的语法树
+* ignore():Node 返回解析中被忽略掉的空白注释等内容
+
+### lexer/Token
+#### 方法
+* constructor(type:int, content:String, val:String, sIndex:int) 构造函数传入token的类型、内容、字面内容和在代码中的开始字符索引
+* type(t:int):int 读取/设置类型
+* content(c:Stirng):String 读取/设置内容
+* val(v:String):String 读取/设置字面内容，字面内容不同于内容之处在于是否包含引号
+* tag(t:int):String 读取/设置类型，返回的是类型额字符串形式
+* tid(t:int):int 读取/设置token索引，默认所有token自增形式添加索引
+* sIndex(i:int):int 读取/设置token在code中的字符索引
+
+#### 静态属性
+* type(t:int):String 返回类型的字符串形式
+
+### parser/Node
+#### 方法
+* constructor(type:String, children:Node/Array<Node> = null) 传入类型和子节点
+* name():String 返回节点类型
+* leaves():Array<Node> 返回子节点列表
+* leaf(i:int):Node 返回第i个子节点
+* size():int 返回有几个子节点
+* first():Node 返回第一个子节点
+* last():Node 返回最后一个子节点
+* isEmpty():Boolean 返回是否没有子节点
+* add(...node:Node):void 添加若干个子节点
+* token():Token 实际同leaves()一样，不过当name()为Token时children存储的是终结符Token
+* parent():Node 返回父节点
+* prev():Node 返回兄弟前一个节点
+* next():Node 返回兄弟后一个节点
+
 ## AST
 当调用语法分析器解析后，会返回生成ast，这是一个树状数据结构，每个节点都是对应语法解析器目录下的Node.js的实例。<br/>
 demo目录下是一个用js的parser分析输入js代码并画出ast形状的页面。

src/lexer/Lexer.js

@@ -63,2 +63,6 @@ var Class = require('../util/Class');
             this.tokenList.push(token);
+            //回调可自定义处理匹配的token
+            if(match.callback) {
+              match.callback(token);
+            }
             this.index += matchLen - 1;
@@ -65,0 +69,0 @@ var n = character.count(token.val(), character.LINE);

src/lexer/rule/EcmascriptRule.js

@@ -10,2 +10,3 @@ var Rule = require('./Rule');
 var character = require('../../util/character');
+var isProperty = false;
 var EcmascriptRule = Rule.extend(function() {
@@ -27,7 +28,15 @@ var self = this;
 
-  self.addMatch(new RegMatch(Token.ID, /^[$a-zA-Z_][$\w]*/, Lexer.SPECIAL, function() {
+  var regMatch = new RegMatch(Token.ID, /^[$a-zA-Z_][$\w]*/, Lexer.SPECIAL, function() {
     return !!(self.keyWords().hasOwnProperty(this.content()));
   }, function() {
     return ['if', 'for', 'while'].indexOf(this.content()) != -1;
-  }));
+  });
+  regMatch.callback = function(token) {
+    var s = token.content();
+    if(isProperty) {
+      token.type(Token.ID);
+    }
+    isProperty = false;
+  };
+  self.addMatch(regMatch);
 
@@ -43,3 +52,8 @@ self.addMatch(new RegMatch(Token.NUMBER, /^\.\d+(?:E[+-]?\d*)?/i, {
   });
-  self.addMatch(new CharacterSet(Token.SIGN, ':;/?.,[]{}~!^|%=-+*()~><&\\', Lexer.IS_REG));
+  var sign = new CharacterSet(Token.SIGN, ':;/?.,[]{}~!^|%=-+*()~><&\\', Lexer.IS_REG);
+  sign.callback = function(token) {
+    var s = token.content();
+    isProperty = s == '.';
+  };
+  self.addMatch(sign);
 
@@ -59,4 +73,3 @@ self.addMatch(new RegMatch(Token.NUMBER, /^0x[\da-f]*/i, {
 
-  self.addMatch(new CompleteEqual(Token.LINE, '\u2028'));
-  self.addMatch(new CompleteEqual(Token.LINE, '\u2029'));
+  self.addMatch(new CharacterSet(Token.LINE, '\u2028\u2029'));
   self.addMatch(new CharacterSet(Token.BLANK, '\f\u000b\u00A0\uFEFF\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000'));
@@ -63,0 +76,0 @@ }).statics({

src/parser/es6/Node.js

@@ -1,74 +0,5 @@
-var Class = require('../../util/Class');
-var Node = Class(function(type, children) {
-  this.type = type;
-  if(type == Node.TOKEN) {
-    this.children = children;
-  }
-  else if(Array.isArray(children)) {
-    this.children = children;
-  }
-  else {
-    this.children = children ? [children] : [];
-  }
-  this.p = null;
-  this.pr = null;
-  this.ne = null;
+var INode = require('../Node');
+var Node = INode.extend(function(type, children) {
+  INode.call(this, type, children);
   return this;
-}).methods({
-  name: function() {
-    return this.type;
-  },
-  leaves: function() {
-    return this.children;
-  },
-  leaf: function(i) {
-    return this.children[i];
-  },
-  size: function() {
-    return this.children.length;
-  },
-  first: function() {
-    return this.leaf(0);
-  },
-  last: function() {
-    return this.leaf(this.size() - 1);
-  },
-  isEmpty: function() {
-    return this.size() === 0;
-  },
-  add: function() {
-    var self = this;
-    var args = Array.prototype.slice.call(arguments, 0);
-    args.forEach(function(node) {
-      node.parent(self);
-      var last = self.children[self.children.length - 1];
-      if(last) {
-        last.next(node);
-        node.prev(last);
-      }
-      self.children.push(node);
-    });
-    return self;
-  },
-  token: function() {
-    return this.children;
-  },
-  parent: function(p) {
-    if(p) {
-      this.p = p;
-    }
-    return this.p;
-  },
-  prev: function(pr) {
-    if(pr) {
-      this.pr = pr;
-    }
-    return this.pr;
-  },
-  next: function(ne) {
-    if(ne) {
-      this.ne = ne;
-    }
-    return this.ne;
-  }
 }).statics({
@@ -83,3 +14,2 @@ SCRIPT: 'script',
   ITERSTMT: 'iterstmt',
-  TOKEN: 'token',
   FNPARAMS: 'fnparams',
@@ -86,0 +16,0 @@ BINDELEMENT: 'bindelement',

src/parser/es6/Parser.js

@@ -1,2 +0,2 @@
-var Class = require('../../util/Class');
+var IParser = require('../Parser');
 var character = require('../../util/character');
@@ -24,4 +24,6 @@ var Lexer = require('../../lexer/Lexer');
   = true;
-var Parser = Class(function(lexer) {
+var Parser = IParser.extend(function(lexer) {
+  IParser.call(this, lexer);
   this.init(lexer);
+  return this;
 }).methods({
@@ -33,5 +35,2 @@ parse: function(code) {
   },
-  ast: function() {
-    return this.tree;
-  },
   init: function(lexer) {
@@ -2258,7 +2257,4 @@ this.look = null;
     throw new Error(msg + ' line ' + this.lastLine + ' col ' + this.lastCol);
-  },
-  ignore: function() {
-    return this.ignores;
   }
 });
 module.exports = Parser;

src/parser/js/Node.js

@@ -1,74 +0,5 @@
-var Class = require('../../util/Class');
-var Node = Class(function(type, children) {
-  this.type = type;
-  if(type == Node.TOKEN) {
-    this.children = children;
-  }
-  else if(Array.isArray(children)) {
-    this.children = children;
-  }
-  else {
-    this.children = children ? [children] : [];
-  }
-  this.p = null;
-  this.pr = null;
-  this.ne = null;
+var INode = require('../Node');
+var Node = INode.extend(function(type, children) {
+  INode.call(this, type, children);
   return this;
-}).methods({
-  name: function() {
-    return this.type;
-  },
-  leaves: function() {
-    return this.children;
-  },
-  leaf: function(i) {
-    return this.children[i];
-  },
-  size: function() {
-    return this.children.length;
-  },
-  first: function() {
-    return this.leaf(0);
-  },
-  last: function() {
-    return this.leaf(this.size() - 1);
-  },
-  isEmpty: function() {
-    return this.size() === 0;
-  },
-  add: function() {
-    var self = this;
-    var args = Array.prototype.slice.call(arguments, 0);
-    args.forEach(function(node) {
-      node.parent(self);
-      var last = self.children[self.children.length - 1];
-      if(last) {
-        last.next(node);
-        node.prev(last);
-      }
-      self.children.push(node);
-    });
-    return self;
-  },
-  token: function() {
-    return this.children;
-  },
-  parent: function(p) {
-    if(p) {
-      this.p = p;
-    }
-    return this.p;
-  },
-  prev: function(pr) {
-    if(pr) {
-      this.pr = pr;
-    }
-    return this.pr;
-  },
-  next: function(ne) {
-    if(ne) {
-      this.ne = ne;
-    }
-    return this.ne;
-  }
 }).statics({
@@ -82,3 +13,2 @@ PROGRAM: 'program',
   ITERSTMT: 'iterstmt',
-  TOKEN: 'token',
   FNPARAMS: 'fnparams',
@@ -85,0 +15,0 @@ EXPR: 'expr',

src/parser/js/Parser.js

@@ -1,2 +0,2 @@
-var Class = require('../../util/Class');
+var IParser = require('../Parser');
 var character = require('../../util/character');
@@ -24,4 +24,6 @@ var Lexer = require('../../lexer/Lexer');
   = true;
-var Parser = Class(function(lexer) {
+var Parser = IParser.extend(function(lexer) {
+  IParser.call(this, lexer);
   this.init(lexer);
+  return this;
 }).methods({
@@ -33,5 +35,2 @@ parse: function(code) {
   },
-  ast: function() {
-    return this.tree;
-  },
   init: function(lexer) {
@@ -1238,7 +1237,4 @@ this.look = null;
     throw new Error(msg + ' line ' + this.lastLine + ' col ' + this.lastCol);
-  },
-  ignore: function() {
-    return this.ignores;
   }
 });
 module.exports = Parser;

tests/jslexer.js

@@ -145,2 +145,13 @@ var homunculus = require('../');
     });
+    it('super', function() {
+      var lexer = homunculus.getLexer('js');
+      var tokens = lexer.parse('super.super');
+      expect(tokens[0].type()).to.eql(Token.KEYWORD);
+      expect(tokens[2].type()).to.eql(Token.ID);
+    });
+    it('other keyword can be property name', function() {
+      var lexer = homunculus.getLexer('js');
+      var tokens = lexer.parse('a.var');
+      expect(tokens[2].type()).to.eql(Token.ID);
+    });
     it('unknow token', function() {
@@ -147,0 +158,0 @@ var lexer = homunculus.getLexer('js');

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1282099

increased by0.21%

Number of package files

88

increased by4.76%

Number of lines in readme file

116

increased by346.15%

Worsened metrics

Lines of code

32883

decreased by-0.13%

No dependency changes