homunculus - npm Package Compare versions

Comparing version 0.0.7 to 0.0.8

gulpfile.js

 var gulp = require('gulp');
-var clean = require('gulp-clean');
-var transform = require('./transform');
 
+var fs = require('fs');
+var path = require('path');
+
+function clean(dir, includeSelf) {
+  if(!fs.existsSync(dir)) {
+    return;
+  }
+  fs.readdirSync(dir).forEach(function(f) {
+    f = dir + path.sep + f;
+    if(!fs.existsSync(f)) {
+      return;
+    }
+    var stat = fs.statSync(f);
+    if(stat.isDirectory()) {
+      clean(f, true);
+    }
+    else if(stat.isFile()) {
+      fs.unlinkSync(f);
+    }
+  });
+  if(includeSelf) {
+    fs.rmdirSync(dir);
+  }
+}
+function web2src(dir) {
+  fs.readdirSync(dir).forEach(function(f) {
+    f = dir + path.sep + f;
+    var stat = fs.statSync(f);
+    var target = f.replace('web', 'src');
+    if(stat.isDirectory()) {
+      if(!fs.existsSync(target)) {
+        fs.mkdirSync(target);
+      }
+      web2src(f);
+    }
+    else if(stat.isFile()) {
+      console.log(f, '->', target);
+      var s = fs.readFileSync(f, { encoding: 'utf-8' });
+      s = s.replace(/^define[^\n]*\n\t?/, '');
+      s = s.replace(/\n\t/g, '\n');
+      s = s.replace(/\t/g, '  ');
+      s = s.replace(/[\s]*\}\)\;[\s]*$/, '');
+      fs.writeFileSync(target, s, { encoding: 'utf-8' });
+    }
+  });
+}
+function src2web(dir) {
+  fs.readdirSync(dir).forEach(function(f) {
+    f = dir + path.sep + f;
+    var stat = fs.statSync(f);
+    var target = f.replace('src', 'web');
+    if(stat.isDirectory()) {
+      if(!fs.existsSync(target)) {
+        fs.mkdirSync(target);
+      }
+      src2web(f);
+    }
+    else if(stat.isFile()) {
+      console.log(f, '->', target);
+      var s = fs.readFileSync(f, { encoding: 'utf-8' });
+      s = 'define(function(require, exports, module) {\n  ' + s.replace(/\n/g, '\n  ') + '\n});';
+      fs.writeFileSync(target, s, { encoding: 'utf-8' });
+    }
+  });
+}
+
 gulp.task('src2web', ['clean-web'], function() {
-  transform.src2web();
+  src2web('./src');
 });
 gulp.task('web2src', ['clean-src'], function() {
-  transform.web2src();
+  web2src('./web');
 });
 gulp.task('clean-web', function() {
-  gulp.src('./web/*', { read: false })
-    .pipe(clean());
+  clean('./web');
 });
 gulp.task('clean-src', function() {
-  gulp.src('./src/*')
-    .pipe(clean());
+  clean('./src');
 });
 
 gulp.task('default', ['src2web']);

package.json

 {
   "name": "homunculus",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "description": "A lexer&parser by Javascript",
@@ -24,8 +24,3 @@ "maintainers": [
   "author": "army8735",
-  "license": [
-    {
-      "type": "MIT",
-      "url": "https://github.com/addyosmani/es6-module-loader/blob/master/LICENSE-MIT"
-    }
-  ],
+  "license": "MIT",
   "dependencies": {},
@@ -40,5 +35,4 @@ "main": "./homunculus",
     "gulp": "^3.5.5",
-    "gulp-clean": "^0.2.4",
     "expect.js": "^0.3.1"
   }
 }

src/lexer/Lexer.js

@@ -45,5 +45,5 @@ var Class = require('../util/Class');
           if(match.match(this.peek, this.code, this.index)) {
-            var token = new Token(match.tokenType(), match.content(), match.val()),
-              error = match.error(),
-              matchLen = match.content().length;
+            var token = new Token(match.tokenType(), match.content(), match.val(), this.sIndex++, this.index - 1);
+            var error = match.error();
+            var matchLen = match.content().length;
             if(token.type() == Token.ID && this.rule.keyWords().hasOwnProperty(token.content())) {
@@ -59,6 +59,6 @@ token.type(Token.KEYWORD);
             if(n) {
-              var i = match.content().indexOf(character.LINE),
-                j = match.content().lastIndexOf(character.LINE);
-              this.colMax = Math.max(this.colMax, this.colNum + i);
-              this.colNum = match.content().length - j;
+              var j = match.content().indexOf(character.LINE);
+              var k = match.content().lastIndexOf(character.LINE);
+              this.colMax = Math.max(this.colMax, this.colNum + j);
+              this.colNum = match.content().length - k;
             }
@@ -157,3 +157,3 @@ else {
     }
-    var token = new Token(Token.REG, this.code.slice(lastIndex, --this.index));
+    var token = new Token(Token.REG, this.code.slice(lastIndex, --this.index), this.sIndex++, lastIndex);
     temp.push(token);
@@ -212,2 +212,3 @@ this.tokenList.push(token);
     this.colMax = 0; //最大列数
+    this.sIndex = 0; //token的索引
   }
@@ -214,0 +215,0 @@ }).statics({

src/lexer/Token.js

 var Class = require('../util/Class');
 var character = require('../util/character');
 var types;
-var Token = Class(function(type, content, val) {
-    this.t = type;
-    this.c = content;
-    if(character.isUndefined(val)) {
-      val = content;
+var Token = Class(function(type, content, val, index, sIndex) {
+  this.t = type; //token类型
+  this.c = content; //token的字面内容，string包括头尾的引号
+  if(character.isNumber(val)) {
+    sIndex = index;
+    index = val;
+    val = content;
+  }
+  else if(character.isUndefined(val)) {
+    val = content;
+    index = sIndex = -1;
+  }
+  this.v = val; //token的值，一般情况下等于content，特殊如string情况下值是不加头尾的引号
+  this.i = index; //token的索引
+  this.si = sIndex; //token在源码字符串中的索引
+}).methods({
+  type: function(t) {
+    if(!character.isUndefined(t)) {
+      this.t = t;
     }
-    this.v = val;
-  }).methods({
-    type: function(t) {
-      if(!character.isUndefined(t)) {
-        this.t = t;
-      }
-      return this.t;
-    },
-    content: function(c) {
-      if(!character.isUndefined(c)) {
-        this.c = c;
-      }
-      return this.c;
-    },
-    val: function(v) {
-      if(!character.isUndefined(v)) {
-        this.v = v;
-      }
-      return this.v;
-    },
-    tag: function(t) {
-      if(!character.isUndefined(t)) {
-        this.t = t;
-      }
-      return Token.type(this.t);
+    return this.t;
+  },
+  content: function(c) {
+    if(!character.isUndefined(c)) {
+      this.c = c;
     }
-  }).statics({
-    IGNORE: -2,
-    VIRTUAL: -1,
-    OTHER: 0,
-    BLANK: 1,
-    TAB: 2,
-    LINE: 3,
-    NUMBER: 4,
-    ID: 5,
-    COMMENT: 6,
-    STRING: 7,
-    SIGN: 8,
-    REG: 9,
-    KEYWORD: 10,
-    ANNOT: 11,
-    HEAD: 12,
-    TEMPLATE: 13,
-    ENTER: 14,
-    PROPERTY: 15,
-    VARS: 16,
-    HACK: 17,
-    IMPORTANT: 18,
-    type: function(tag) {
-      if(character.isUndefined(types)) {
-        types = [];
-        Object.keys(Token).forEach(function(o) {
-          if(typeof Token[o] == 'number') {
-            types[Token[o]] = o;
-          }
-        });
-      }
-      return types[tag];
+    return this.c;
+  },
+  val: function(v) {
+    if(!character.isUndefined(v)) {
+      this.v = v;
     }
-  });
+    return this.v;
+  },
+  tag: function(t) {
+    if(!character.isUndefined(t)) {
+      this.t = t;
+    }
+    return Token.type(this.t);
+  },
+  index: function(i) {
+    if(!character.isUndefined(i)) {
+      this.i = i;
+    }
+    return this.i;
+  },
+  sIndex: function(si) {
+    if(!character.isUndefined(si)) {
+      this.si = si;
+    }
+    return this.si;
+  }
+}).statics({
+  IGNORE: -2,
+  VIRTUAL: -1,
+  OTHER: 0,
+  BLANK: 1,
+  TAB: 2,
+  LINE: 3,
+  NUMBER: 4,
+  ID: 5,
+  COMMENT: 6,
+  STRING: 7,
+  SIGN: 8,
+  REG: 9,
+  KEYWORD: 10,
+  ANNOT: 11,
+  HEAD: 12,
+  TEMPLATE: 13,
+  ENTER: 14,
+  PROPERTY: 15,
+  VARS: 16,
+  HACK: 17,
+  IMPORTANT: 18,
+  type: function(tag) {
+    if(character.isUndefined(types)) {
+      types = [];
+      Object.keys(Token).forEach(function(o) {
+        if(typeof Token[o] == 'number') {
+          types[Token[o]] = o;
+        }
+      });
+    }
+    return types[tag];
+  }
+});
 module.exports = Token;

src/util/character.js

@@ -47,2 +47,8 @@ exports.LINE = '\n';
 };
+exports.isString = function(s) {
+  return Object.prototype.toString.call(s) == "[object String]";
+};
+exports.isNumber = function(s) {
+  return Object.prototype.toString.call(s) == "[object Number]";
+};
 exports.escapeHTML = function(str) {
@@ -49,0 +55,0 @@ var xmlchar = {

tests/jslexer.js

@@ -76,2 +76,3 @@ var homunculus = require('../homunculus');
       var code = fs.readFileSync(path.join(__dirname, './jslexer/1.js'), { encoding: 'utf-8' });
+      code = code.replace(/\r\n/g, '\n');
       var tokens = lexer.parse(code);
@@ -85,2 +86,3 @@ var res = fs.readFileSync(path.join(__dirname, './jslexer/1.txt'), { encoding: 'utf-8' });
       var code = fs.readFileSync(path.join(__dirname, './jslexer/2.js'), { encoding: 'utf-8' });
+      code = code.replace(/\r\n/g, '\n');
       var tokens = lexer.parse(code);
@@ -94,2 +96,3 @@ var res = fs.readFileSync(path.join(__dirname, './jslexer/2.txt'), { encoding: 'utf-8' });
       var code = fs.readFileSync(path.join(__dirname, './jslexer/3.js'), { encoding: 'utf-8' });
+      code = code.replace(/\r\n/g, '\n');
       var tokens = lexer.parse(code);
@@ -101,2 +104,33 @@ var res = fs.readFileSync(path.join(__dirname, './jslexer/3.txt'), { encoding: 'utf-8' });
   });
+  describe('index test', function() {
+    var lexer = homunculus.getLexer('js');
+    var tokens = lexer.parse('var a = /reg/;var b=/**/2;');
+    it('tokens length', function() {
+      expect(tokens.length).to.eql(15);
+    });
+    it('tokens index 1', function() {
+      expect(tokens[0].index()).to.eql(0);
+    });
+    it('tokens index 2', function() {
+      expect(tokens[1].index()).to.eql(1);
+    });
+    it('tokens index 3', function() {
+      expect(tokens[6].index()).to.eql(6);
+    });
+    it('tokens index 4', function() {
+      expect(tokens[13].index()).to.eql(13);
+    });
+    it('tokens sIndex 1', function() {
+      expect(tokens[0].sIndex()).to.eql(0);
+    });
+    it('tokens sIndex 2', function() {
+      expect(tokens[1].sIndex()).to.eql(3);
+    });
+    it('tokens sIndex 3', function() {
+      expect(tokens[6].sIndex()).to.eql(8);
+    });
+    it('tokens sIndex 4', function() {
+      expect(tokens[13].sIndex()).to.eql(24);
+    });
+  });
   describe('js lib exec test', function() {
@@ -103,0 +137,0 @@ it('jquery 1.11.0', function() {

tests/jsparser.js

@@ -79,2 +79,37 @@ var homunculus = require('../homunculus');
     });
+    it('newexpr 1', function() {
+      var parser = homunculus.getParser('js');
+      var node = parser.parse('new A');
+      expect(tree(node)).to.eql([JsNode.PROGRAM, [JsNode.STMT, [JsNode.NEWEXPR, ['new', JsNode.PRMREXPR, ['A']]]]]);
+    });
+    it('newexpr 2', function() {
+      var parser = homunculus.getParser('js');
+      var node = parser.parse('new A()');
+      expect(tree(node)).to.eql([JsNode.PROGRAM, [JsNode.STMT, [JsNode.NEWEXPR, ['new', JsNode.PRMREXPR, ['A'], JsNode.ARGS, ['(', ')']]]]]);
+    });
+    it('newexpr 3', function() {
+      var parser = homunculus.getParser('js');
+      var node = parser.parse('new A().f');
+      expect(tree(node)).to.eql([JsNode.PROGRAM, [JsNode.STMT, [JsNode.MMBEXPR, [JsNode.NEWEXPR, ['new', JsNode.PRMREXPR, ['A'], JsNode.ARGS, ['(', ')']], '.', 'f']]]]);
+    });
+    it('newexpr 4', function() {
+      var parser = homunculus.getParser('js');
+      var node = parser.parse('new A().f()');
+      expect(tree(node)).to.eql([JsNode.PROGRAM, [JsNode.STMT, [JsNode.CALLEXPR, [JsNode.MMBEXPR, [JsNode.NEWEXPR, ['new', JsNode.PRMREXPR, ['A'], JsNode.ARGS, ['(', ')']], '.', 'f'], JsNode.ARGS, ['(', ')']]]]]);
+    });
+    it('newexpr 5', function() {
+      var parser = homunculus.getParser('js');
+      var node = parser.parse('new new A().f()');
+      expect(tree(node)).to.eql([JsNode.PROGRAM, [JsNode.STMT, [JsNode.NEWEXPR, ['new', JsNode.MMBEXPR, [JsNode.NEWEXPR, ['new', JsNode.PRMREXPR, ['A'], JsNode.ARGS, ['(', ')']], '.', 'f'], JsNode.ARGS, ['(', ')']]]]]);
+    });
+    it('mmbexpr 1', function() {
+      var parser = homunculus.getParser('js');
+      var node = parser.parse('a.b.c');
+      expect(tree(node)).to.eql([JsNode.PROGRAM, [JsNode.STMT, [JsNode.MMBEXPR, [JsNode.PRMREXPR, ['a'], '.', 'b', '.', 'c']]]]);
+    });
+    it('mmbexpr 2', function() {
+      var parser = homunculus.getParser('js');
+      var node = parser.parse('a.b[c]');
+      expect(tree(node)).to.eql([JsNode.PROGRAM, [JsNode.STMT, [JsNode.MMBEXPR, [JsNode.PRMREXPR, ['a'], '.', 'b', '[', JsNode.PRMREXPR, ['c'], ']']]]]);
+    });
     it('node parent,prev,next', function() {
@@ -81,0 +116,0 @@ var parser = homunculus.getParser('js');

web/lexer/Lexer.js

@@ -46,5 +46,5 @@ define(function(require, exports, module) {
             if(match.match(this.peek, this.code, this.index)) {
-              var token = new Token(match.tokenType(), match.content(), match.val()),
-                error = match.error(),
-                matchLen = match.content().length;
+              var token = new Token(match.tokenType(), match.content(), match.val(), this.sIndex++, this.index - 1);
+              var error = match.error();
+              var matchLen = match.content().length;
               if(token.type() == Token.ID && this.rule.keyWords().hasOwnProperty(token.content())) {
@@ -60,6 +60,6 @@ token.type(Token.KEYWORD);
               if(n) {
-                var i = match.content().indexOf(character.LINE),
-                  j = match.content().lastIndexOf(character.LINE);
-                this.colMax = Math.max(this.colMax, this.colNum + i);
-                this.colNum = match.content().length - j;
+                var j = match.content().indexOf(character.LINE);
+                var k = match.content().lastIndexOf(character.LINE);
+                this.colMax = Math.max(this.colMax, this.colNum + j);
+                this.colNum = match.content().length - k;
               }
@@ -158,3 +158,3 @@ else {
       }
-      var token = new Token(Token.REG, this.code.slice(lastIndex, --this.index));
+      var token = new Token(Token.REG, this.code.slice(lastIndex, --this.index), this.sIndex++, lastIndex);
       temp.push(token);
@@ -213,2 +213,3 @@ this.tokenList.push(token);
       this.colMax = 0; //最大列数
+      this.sIndex = 0; //token的索引
     }
@@ -215,0 +216,0 @@ }).statics({

web/lexer/Token.js

@@ -5,69 +5,89 @@ define(function(require, exports, module) {
   var types;
-  var Token = Class(function(type, content, val) {
-      this.t = type;
-      this.c = content;
-      if(character.isUndefined(val)) {
-        val = content;
+  var Token = Class(function(type, content, val, index, sIndex) {
+    this.t = type; //token类型
+    this.c = content; //token的字面内容，string包括头尾的引号
+    if(character.isNumber(val)) {
+      sIndex = index;
+      index = val;
+      val = content;
+    }
+    else if(character.isUndefined(val)) {
+      val = content;
+      index = sIndex = -1;
+    }
+    this.v = val; //token的值，一般情况下等于content，特殊如string情况下值是不加头尾的引号
+    this.i = index; //token的索引
+    this.si = sIndex; //token在源码字符串中的索引
+  }).methods({
+    type: function(t) {
+      if(!character.isUndefined(t)) {
+        this.t = t;
       }
-      this.v = val;
-    }).methods({
-      type: function(t) {
-        if(!character.isUndefined(t)) {
-          this.t = t;
-        }
-        return this.t;
-      },
-      content: function(c) {
-        if(!character.isUndefined(c)) {
-          this.c = c;
-        }
-        return this.c;
-      },
-      val: function(v) {
-        if(!character.isUndefined(v)) {
-          this.v = v;
-        }
-        return this.v;
-      },
-      tag: function(t) {
-        if(!character.isUndefined(t)) {
-          this.t = t;
-        }
-        return Token.type(this.t);
+      return this.t;
+    },
+    content: function(c) {
+      if(!character.isUndefined(c)) {
+        this.c = c;
       }
-    }).statics({
-      IGNORE: -2,
-      VIRTUAL: -1,
-      OTHER: 0,
-      BLANK: 1,
-      TAB: 2,
-      LINE: 3,
-      NUMBER: 4,
-      ID: 5,
-      COMMENT: 6,
-      STRING: 7,
-      SIGN: 8,
-      REG: 9,
-      KEYWORD: 10,
-      ANNOT: 11,
-      HEAD: 12,
-      TEMPLATE: 13,
-      ENTER: 14,
-      PROPERTY: 15,
-      VARS: 16,
-      HACK: 17,
-      IMPORTANT: 18,
-      type: function(tag) {
-        if(character.isUndefined(types)) {
-          types = [];
-          Object.keys(Token).forEach(function(o) {
-            if(typeof Token[o] == 'number') {
-              types[Token[o]] = o;
-            }
-          });
-        }
-        return types[tag];
+      return this.c;
+    },
+    val: function(v) {
+      if(!character.isUndefined(v)) {
+        this.v = v;
       }
-    });
+      return this.v;
+    },
+    tag: function(t) {
+      if(!character.isUndefined(t)) {
+        this.t = t;
+      }
+      return Token.type(this.t);
+    },
+    index: function(i) {
+      if(!character.isUndefined(i)) {
+        this.i = i;
+      }
+      return this.i;
+    },
+    sIndex: function(si) {
+      if(!character.isUndefined(si)) {
+        this.si = si;
+      }
+      return this.si;
+    }
+  }).statics({
+    IGNORE: -2,
+    VIRTUAL: -1,
+    OTHER: 0,
+    BLANK: 1,
+    TAB: 2,
+    LINE: 3,
+    NUMBER: 4,
+    ID: 5,
+    COMMENT: 6,
+    STRING: 7,
+    SIGN: 8,
+    REG: 9,
+    KEYWORD: 10,
+    ANNOT: 11,
+    HEAD: 12,
+    TEMPLATE: 13,
+    ENTER: 14,
+    PROPERTY: 15,
+    VARS: 16,
+    HACK: 17,
+    IMPORTANT: 18,
+    type: function(tag) {
+      if(character.isUndefined(types)) {
+        types = [];
+        Object.keys(Token).forEach(function(o) {
+          if(typeof Token[o] == 'number') {
+            types[Token[o]] = o;
+          }
+        });
+      }
+      return types[tag];
+    }
+  });
   module.exports = Token;
 });

web/util/character.js

@@ -48,2 +48,8 @@ define(function(require, exports, module) {
   };
+  exports.isString = function(s) {
+    return Object.prototype.toString.call(s) == "[object String]";
+  };
+  exports.isNumber = function(s) {
+    return Object.prototype.toString.call(s) == "[object Number]";
+  };
   exports.escapeHTML = function(str) {
@@ -50,0 +56,0 @@ var xmlchar = {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Dev dependency count

3

decreased by-25%

Worsened metrics

Total package byte prevSize

874289

decreased by-7.06%

Number of package files

70

decreased by-2.78%

Lines of code

23548

decreased by-7.16%

No dependency changes