tokenizer - npm Package Compare versions

Comparing version 0.0.1-2-g5805b9e to 0.1.0

lib/Tokenizer.js

@@ -12,2 +12,4 @@ var EventEmitter = require('events').EventEmitter;
                         // with regex[RegExp] and type[String]
+    this._ignored = {}  // a hash of ignored token types
+                        // these will be parsed but not emitted
     this._checkToken = check_token_cb || noop;
@@ -81,2 +83,3 @@ }
     var type = this._checkToken(str, rule) || rule.type;
+    if(this._ignored[type]) return;
     this.emit('data', str, type); // act like a readable stream
@@ -87,6 +90,31 @@ this.emit('token', str, type);
 Tokenizer.prototype.addRule = function addRule(regex, type) {
-    // TODO: check types
+    // this is useful for built-in rules
+    if(!type) {
+        return this.addRule(regex[0], regex[1]);
+    }
+    assert.equal(typeof regex, 'function');
+    //assert.ok(regex instanceof RegExp);
+    assert.equal(typeof type, 'string');
     this._regexes.push({regex:regex,type:type});
 };
 
+/**
+ * set some tokens to be ignored. these won't be emitted
+ */
+Tokenizer.prototype.ignore = function ignore(ignored) {
+    if(typeof ignore === 'array') {
+        for (var i = 0; i < ignored.length; ++i) {
+            this.ignore(ignored[i]);
+        };
+        return;
+    }
+    this._ignored[ignored] = true;
+};
+
+
 module.exports = Tokenizer;
+
+// built-in rules
+Tokenizer.whitespace    = [/^(\s)+$/, 'whitespace'];
+Tokenizer.word          = [/^\w+$/, 'word'];
+Tokenizer.number        = [/^\d+$/, 'number'];

package.json

 {
   "name": "tokenizer",
   "description": "A wide purpose tokenizer for node.js which looks like a stream",
-  "version": "0.0.1-2-g5805b9e",
+  "version": "0.1.0",
   "homepage": "http://github.com/floby/node-tokenizer",
@@ -18,2 +18,2 @@ "repository": {
   }
-}
+}

README.md

@@ -5,26 +5,38 @@ # Synopsis
 
-node-tokenizer is published on npm to you can install it with
+node-tokenizer is published on npm so you can install it with
     npm install tokenizer
 
 ## How to
+
 * require the Tokenizer constructor
-        var Tokenizer = require('tokenizer');
+
+            var Tokenizer = require('tokenizer');
+
 * construct one (we'll see what the callback is used for)
-        var t = new Tokenizer(mycallback);
+
+            var t = new Tokenizer(mycallback);
+
 * add rules
-        t.addRule(/myawesome regex/, 'type');
+
+            t.addRule(/^my regex$/, 'type');
+
 * write or pump to it
-        t.write(data);
-        // or
-        stream.pipe(t);
+
+            t.write(data);
+            // or
+            stream.pipe(t);
+
 * listen for new tokens
-        t.on('token', function(token, type) {
-            // do something useful
-            // type is the type of the token (specified with addRule)
-            // token is the actual matching string
-        })
-        // alternatively you can listen on the 'data' event
+
+            t.on('token', function(token, type) {
+                // do something useful
+                // type is the type of the token (specified with addRule)
+                // token is the actual matching string
+            })
+            // alternatively you can listen on the 'data' event
+
 * look out for the end
-        t.on('end', callback);
 
+            t.on('end', callback);
+
 the optional callback argument for the constructor is a function that will
@@ -34,2 +46,3 @@ be called for each token in order to specify a different type by returning
 and match, an object like this 
+
     {
@@ -54,5 +67,5 @@ regex: /whatever/ // the regex that matched the token
 * a lot of optimisation
-* being able to share rules across several tokenizer 
-    (although this can be achieved with inheritance)
+* being able to share rules across several tokenizers
+    (although this can be achieved through inheritance)
 * probably more hooks
 * more checking

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Native code

Supply chain risk

Contains native code (e.g., compiled binaries or shared libraries). Including native code can obscure malicious behavior.

Found 2 instances in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

36615

increased by420.32%

Number of package files

8

increased by33.33%

Lines of code

155

increased by19.23%

Number of lines in readme file

69

increased by23.21%

Worsened metrics

Number of medium supply chain risk alerts

2

increased byInfinity%

No dependency changes