tokenizer - npm Package Compare versions

Comparing version 0.1.0 to 0.1.1

examples/simple.js

@@ -9,3 +9,6 @@ var Tokenizer = require('../lib/Tokenizer');
 t.addRule(/^"[^"]*"$/, 'citation');
-t.addRule(/^"[^"]*$/, 'maybe citation')
+t.addRule(/^"[^"]*$/, 'maybe citation');
+// the 'maybe citation' rule is here to continue matching until
+// the closing quote is found
+
 t.addRule(/^salut$/i, 'salut');
@@ -12,0 +15,0 @@ t.addRule(/^[',;.:!?-]$/, 'ponctuation');

lib/Tokenizer.js

 var EventEmitter = require('events').EventEmitter;
-var sys = require('sys');
+var util = require('util');
 var assert = require('assert');
@@ -16,3 +16,3 @@
 }
-sys.inherits(Tokenizer, EventEmitter);
+util.inherits(Tokenizer, EventEmitter);
 
@@ -93,4 +93,3 @@ Tokenizer.prototype.write = function write(data, nobuffer) {
     }
-    assert.equal(typeof regex, 'function');
-    //assert.ok(regex instanceof RegExp);
+    assert.ok((regex instanceof RegExp) || (typeof regex === 'function'))
     assert.equal(typeof type, 'string');
@@ -97,0 +96,0 @@ this._regexes.push({regex:regex,type:type});

package.json

 {
   "name": "tokenizer",
   "description": "A wide purpose tokenizer for node.js which looks like a stream",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "homepage": "http://github.com/floby/node-tokenizer",
@@ -16,4 +16,4 @@ "repository": {
   "engines": {
-    "node": "*"
+    "node": "~0.4 ~0.5 ~0.6 ~0.7 ~0.8 ~0.9"
   }
 }

README.md

@@ -5,4 +5,3 @@ # Synopsis
 
-node-tokenizer is published on npm so you can install it with
-    npm install tokenizer
+node-tokenizer is published on npm so you can install it with `npm install tokenizer`
 
@@ -13,30 +12,42 @@ ## How to
 
-            var Tokenizer = require('tokenizer');
+``` javascript
+var Tokenizer = require('tokenizer');
+```
 
 * construct one (we'll see what the callback is used for)
 
-            var t = new Tokenizer(mycallback);
+``` javascript
+var t = new Tokenizer(mycallback);
+``` 
 
 * add rules
 
-            t.addRule(/^my regex$/, 'type');
+``` javascript
+t.addRule(/^my regex$/, 'type');
+```
 
 * write or pump to it
 
-            t.write(data);
-            // or
-            stream.pipe(t);
+``` javascript
+t.write(data);
+// or
+stream.pipe(t);
+```
 
 * listen for new tokens
 
-            t.on('token', function(token, type) {
-                // do something useful
-                // type is the type of the token (specified with addRule)
-                // token is the actual matching string
-            })
-            // alternatively you can listen on the 'data' event
+``` javascript
+t.on('token', function(token, type) {
+    // do something useful
+    // type is the type of the token (specified with addRule)
+    // token is the actual matching string
+})
+// alternatively you can listen on the 'data' event
+```
 
 * look out for the end
 
-            t.on('end', callback);
+``` javascript
+t.on('end', callback);
+```
 
@@ -48,6 +59,8 @@ the optional callback argument for the constructor is a function that will
 
-    {
-        regex: /whatever/ // the regex that matched the token
-        type: 'type' // the type of the token
-    }
+``` javascript
+{
+    regex: /whatever/ // the regex that matched the token
+    type: 'type' // the type of the token
+}
+```
 
@@ -54,0 +67,0 @@ Have a look in the example folder

No alert changes

Improved metrics

Lines of code

156

increased by0.65%

Number of lines in readme file

82

increased by18.84%

Worsened metrics

Total package byte prevSize

8018

decreased by-78.1%

Number of package files

6

decreased by-25%

No dependency changes