floodesh - npm Package Compare versions

Comparing version 0.3.2 to 0.4.0

lib/cli-init.js

@@ -65,3 +65,2 @@
 program
-    .version('0.1.0')
     .arguments('[name]')
@@ -68,0 +67,0 @@ .action(name => appName  = name)

lib/cli.js

@@ -11,3 +11,3 @@
 program
-    .version('0.1.0')
+    .version('0.4.0')
     .command('init','generate scaffold')
@@ -14,0 +14,0 @@ .command('start [mode]','start in worker or client mode.')

lib/context.js

@@ -40,2 +40,3 @@
     .method('get')
+    .method('is')
     .getter('status')
@@ -49,4 +50,2 @@ .getter('message')
     .getter('header')
-    .getter('href')
-    .getter('uri')
     .getter('contentType');
@@ -64,3 +63,2 @@
     //.method('set')
-    .method('is')
     .access('querystring')
@@ -67,0 +65,0 @@ .access('idempotent')

lib/request.js

@@ -48,3 +48,3 @@
   get url() {
-    return this.req.href;
+    return this.href;
   },
@@ -71,5 +71,3 @@
   get href() {
-    // support: `GET http://example.com/foo`
-    if (/^https?:\/\//i.test(this.originalUrl)) return this.originalUrl;
-    return this.origin + this.originalUrl;
+      return this.req.uri.href;
   },
@@ -139,4 +137,2 @@
    * Parse the "Host" header field host
-   * and support X-Forwarded-Host when a
-   * proxy is enabled.
    *
@@ -148,7 +144,3 @@ * @return {String} hostname:port
   get host() {
-    const proxy = this.app.proxy;
-    let host = proxy && this.get('X-Forwarded-Host');
-    host = host || this.get('Host');
-    if (!host) return '';
-    return host.split(/\s*,\s*/)[0];
+      return this.req.host;
   },
@@ -166,5 +158,3 @@
   get hostname() {
-    const host = this.host;
-    if (!host) return '';
-    return host.split(':')[0];
+    return this.req.uri.hostname;
   },
@@ -213,7 +203,2 @@
    * Return the protocol string "http" or "https"
-   * when requested with TLS. When the proxy setting
-   * is enabled the "X-Forwarded-Proto" header
-   * field will be trusted. If you're running behind
-   * a reverse proxy that supplies https for you this
-   * may be enabled.
    *
@@ -225,20 +210,5 @@ * @return {String}
   get protocol() {
-    const proxy = this.app.proxy;
-    if (this.socket.encrypted) return 'https';
-    if (!proxy) return 'http';
-    const proto = this.get('X-Forwarded-Proto') || 'http';
-    return proto.split(/\s*,\s*/)[0];
+      return this.req.uri.protocol;
   },
 
-    /**
-     * Return the request socket.
-     *
-     * @return {Connection}
-     * @api public
-     */
-
-    get socket() {
-	return this.req.socket;
-    },
-    
   /**
@@ -245,0 +215,0 @@ * Short-hand for:

lib/response.js

@@ -16,25 +16,2 @@
 module.exports = {
-    get $(){
-	return this._$;
-    },
-
-    set $(value){
-	this._$ = value;
-    },
-    
-    get content(){
-	return this._content || "";
-    },
-
-    get href(){
-	return this.res.request.href;
-    },
-
-    get uri(){
-	return this.href;
-    },
-    
-    set content(value){
-	this._content = value;
-    },
   /**
@@ -41,0 +18,0 @@ * Return response header.

lib/worker.js

@@ -28,3 +28,3 @@
 	    if(err) {
-		console.error(err.stack);
+		//console.error(err.stack); // TODO: may lead to crash `uncaughtException: EIO: i/o error, write`
 		ctx.done();// tell scheduler to release a resource
@@ -45,7 +45,7 @@
 	    ctx.response.res = res;
-	    ctx.request.req = res.req; // attach node built-in request while got response
+	    ctx.request.req = res.request; // attach an instance of `Request` while got response, need see if it is necessary.
 	    self.emit("responding",ctx);
 	});
 	
-	ctx.request.req = req; // req is a wrapper of node built-in request
+	ctx.request.req = req; // req is an instance of `Request`, which is an Object in `request` module.
     }
@@ -156,5 +156,5 @@
 	if(e instanceof Error){
-	    console.error(e.stack);
+	    //console.error(e.stack);
 	}else{
-	    console.error(e);
+	    //console.error(e);
 	}
@@ -161,0 +161,0 @@

package.json

 {
     "name": "floodesh",
-    "version": "0.3.2",
+    "version": "0.4.0",
     "description": "Floodesh is a distributed web spider/crawler written with Nodejs.",
@@ -5,0 +5,0 @@ "bin": "./bin/floodesh",

README.md

@@ -0,1 +1,3 @@
+# V0.4.x API is different from earlier version, please do not upgrade your previous dependence.
+
 # Floodesh
@@ -9,5 +11,5 @@ Floodesh is middleware based web spider written with Nodejs. "Floodesh" is a combination of two words, `flood` and `mesh`.
 ## Gearman Server Installation
-Make sure `libboost-all-dev`, `gperf`, `libevent-dev` and `uuid-dev` have been installed in you system.
+Make sure `g++`, `make`, `libboost-all-dev`, `gperf`, `libevent-dev` and `uuid-dev` have been installed.
 
-	wget https://launchpad.net/gearmand/1.2/1.1.12/+download/gearmand-1.1.12.tar.gz | tar zxf
+	wget https://launchpad.net/gearmand/1.2/1.1.12/+download/gearmand-1.1.12.tar.gz | tar xvf
 	cd gearmand-1.1.12
@@ -43,8 +45,2 @@ ./configure
 
-### ctx.is(types)
-  *  `type`s String|Array
-  *  Return: String|false|null
-
-Check if the incoming request contains the "Content-Type" header field, and it contains any of the give mime `type`s.If there is no request body, `null` is returned.If there is no content type, `false` is returned.Otherwise, it returns the first `type` that matches.
-
 ### ctx.querystring
@@ -93,3 +89,3 @@ *  String
   
-Return the protocol string "http" or "https"
+Return the protocol string "http" or "https".
 
@@ -153,8 +149,2 @@ ### ctx.host
 
-### ctx.href
-  *  String
-
-### ctx.uri
-  *  String
-
 ### ctx.contentType
@@ -166,6 +156,11 @@ *  String
   *  Return: String
-  
+
 Get value by key in response headers
 
+### ctx.is(types)
+  *  `type`s String|Array
+  *  Return: String|false|null
 
+Check if the incoming response contains the "Content-Type" header field, and it contains any of the give mime `type`s.If there is no response body, `null` is returned.If there is no content type, `false` is returned.Otherwise, it returns the first `type` that matches.
+
 # Middlewares
@@ -172,0 +167,0 @@ * [mof-cheerio](https://www.npmjs.com/package/mof-cheerio). A simple wrapper of `Cheerio`

templates/test/test-spider.js

@@ -6,7 +6,10 @@
 const sinon = require('sinon')
+const cheerio = require('cheerio')
+const Spider = require('../lib/spider.js')
+
 require('should-sinon')
 
 describe('Spider',function(){
+    let ctx={dataSet:new Map(), tasks:[]};
     beforeEach( () => {//do something before each test case.
-	
     });
@@ -13,0 +16,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

51714

decreased by-1.93%

Lines of code

1641

decreased by-2.73%

Number of lines in readme file

173

decreased by-2.81%

No dependency changes