node-ral - npm Package Compare versions

Comparing version 0.0.6 to 0.0.7

index.js

@@ -16,2 +16,4 @@ /*
 module.exports.Protocol = require('./lib/protocol.js');
-module.exports.Logger = require('./lib/logger.js');
+module.exports.Logger = require('./lib/logger.js');
+module.exports.ConfigNormalizer = require('./lib/config/configNormalizer.js');
+module.exports.NormalizerManager = require('./lib/config/normalizerManager.js');

lib/config.js

@@ -10,3 +10,4 @@ /*
 var _ = require('underscore');
-var util = require('./util.js');
+var ralUtil = require('./util.js');
+var util = require('util');
 var expect = require('chai').expect;
@@ -18,12 +19,6 @@ var iconv = require('iconv-lite');
 var RalModule = require('./ralmodule.js');
+var NormalizerManager = require('./config/normalizerManager.js');
+var configUpdater = require('./config/configUpdater.js');
+var ctx = require('./ctx.js');
 
-var defaultServiceInfo = {
-    retry: 0,
-    timeout: 3000,
-    encoding: 'utf8',
-    hybird: false,
-    pack: 'string',
-    unpack: 'string',
-    path: '/'
-};
 
@@ -35,22 +30,36 @@ /**
 var config = {};
+var rawConf = {};
+var updateNeededRawConf = {};
 var contextCache = {};
+var normalizerManager = new NormalizerManager();
+var updateInterval = null;
 
-function parse(config) {
-
-    /**
-     * normalize config and inherit default config
-     * @param serviceInfo
-     * @returns {*}
-     */
-    function normalize(serviceInfo) {
-        serviceInfo.encoding = serviceInfo.encoding && serviceInfo.encoding.toString().toLowerCase();
-        //inherit default config
-        _.map(defaultServiceInfo, function(value, key){
-            if (serviceInfo.hasOwnProperty(key) === false){
-                serviceInfo[key] = value;
+/**
+ * regularly update config with config nomalizer
+ * @param interval
+ * @param all
+ */
+var enableUpdate = function(interval, all){
+    interval = interval || ctx.CONFIG_UPDATE_INTERVAL;
+    if (updateInterval){
+        clearInterval(updateInterval);
+    }
+    updateInterval = setInterval(function(){
+        configUpdater.update(function(err, confs){
+            if (!err){
+                _.map(confs, function(conf){
+                    config[conf.serviceID] = conf;
+                    contextCache[conf.serviceID] = {};
+                });
             }
-        });
-        serviceInfo.timeout = Math.ceil(serviceInfo.timeout);
-    }
+        }, all);
+    }, interval);
+};
 
+/**
+ * parse rawConf to valid conf
+ * @param rawConf
+ * @returns {*}
+ */
+function loadRawConf(rawConf) {
     /**
@@ -96,10 +105,80 @@ * validate
 
-    _.map(config, function (serviceInfo, serviceID) {
-        normalize(serviceInfo);
-        serviceInfo.serviceID = serviceID;
+    var clone = ralUtil.merge({}, rawConf);
+    updateNeededRawConf = {};
+    var needUpdate = false;
+    _.map(clone, function (serviceInfo, serviceID) {
+        //normalize
+        normalizerManager.apply(serviceInfo);
+        if (normalizerManager.needUpdate(serviceInfo)){
+            updateNeededRawConf[serviceID] = serviceInfo;
+            needUpdate = true;
+        }
         validate(serviceID, serviceInfo);
     });
+    if (needUpdate){
+        logger.notice('config auto update was started');
+        enableUpdate();
+    }else{
+        logger.notice('config autp update wase disabled');
+    }
+    config = clone;
+    return clone;
+}
+
+/**
+ * load config by config folder
+ * @param confPath
+ * @returns {{}}
+ */
+function load(confPath){
+    function _load(confPath) {
+        var confFromFile = {};
+
+        function _loadByFile(confPath){
+            //load js or json as config
+            var ext = path.extname(confPath);
+            if (ext === '.js') {
+                logger.trace('load config from ' +confPath);
+                _.extend(confFromFile, require(confPath));
+            } else if (ext === '.json') {
+                logger.trace('load config from ' +confPath);
+                var content = fs.readFileSync(confPath);
+                _.extend(confFromFile, JSON.parse(content.toString()));
+            }
+        }
+
+        function _loadByFolder(confPath){
+            var files = ralUtil.readdirSync(confPath);
+            files.map(_loadByFile);
+        }
+
+        var stats = fs.statSync(confPath);
+        if (stats.isFile()) {
+            _loadByFile(confPath);
+        } else if (stats.isDirectory()) {
+            _loadByFolder(confPath);
+        }
+        return confFromFile;
+    }
+
+    confPath = path.normalize(confPath);
+    try{
+        //load raw conf
+        rawConf = _load(confPath);
+        _.map(rawConf, function(value, key){
+            value.serviceID = key;
+        });
+        //parse conf to normalized config
+        loadRawConf(rawConf);
+    }catch(err){
+        logger.fatal('config [' + confPath + '] load failed');
+        throw err;
+    }
     return config;
 }
 
+/**
+ * get runtime config context
+ * @param serviceID
+ */
 function parseContext(serviceID){
@@ -120,44 +199,7 @@ var serviceInfo =  config[serviceID];
 
-function _load(confPath) {
 
-    function _loadByFile(confPath){
-        //load js or json as config
-        var ext = path.extname(confPath);
-        if (ext === '.js') {
-            logger.trace('load config from ' +confPath);
-            _.extend(config, require(confPath));
-        } else if (ext === '.json') {
-            logger.trace('load config from ' +confPath);
-            var content = fs.readFileSync(confPath);
-            _.extend(config, JSON.parse(content.toString()));
-        }
-    }
-
-    function _loadByFolder(confPath){
-        var files = util.readdirSync(confPath);
-        files.map(_loadByFile);
-    }
-
-    var stats = fs.statSync(confPath);
-    if (stats.isFile()) {
-        _loadByFile(confPath);
-    } else if (stats.isDirectory()) {
-        _loadByFolder(confPath);
-    }
-}
-
-function load(confPath){
-    confPath = path.normalize(confPath);
-    try{
-        _load(confPath);
-        config = parse(config);
-    }catch(err){
-        logger.fatal('config [' + confPath + '] load failed');
-        throw err;
-    }
-    return config;
-}
-
-module.exports.parse = parse;
+module.exports.loadRawConf = loadRawConf;
 module.exports.load = load;
+module.exports.normalizerManager = normalizerManager;
+module.exports.normalize = normalizerManager.apply.bind(normalizerManager);
 
@@ -167,3 +209,3 @@ module.exports.getConf = function(name){
     if (config[name]){
-        var conf =  util.merge({}, config[name]);
+        var conf =  ralUtil.merge({}, config[name]);
         conf.__defineGetter__('context', function(){
@@ -185,2 +227,13 @@ if (!contextCache[name]){
     return Object.keys(config);
-};
+};
+
+module.exports.getRawConf = function(){
+    return ralUtil.merge({}, rawConf);
+};
+
+module.exports.getUpdateNeededRawConf = function(){
+    return ralUtil.merge({}, updateNeededRawConf);
+};
+
+module.exports.enableUpdate = enableUpdate;
+

lib/ctx.js

@@ -11,1 +11,3 @@ /*
 module.exports.currentIDC = module.exports.IDC_ALL;
+//default config update time is 300s
+module.exports.CONFIG_UPDATE_INTERVAL = 300000;

lib/ext/converter/formDataConverter.js

@@ -12,3 +12,3 @@ /*
 var util = require('util');
-var FormData = require('form-data');
+var FormData;
 var iconv = require('iconv-lite');
@@ -29,2 +29,3 @@ var _ = require('underscore');
     data = data || {};
+    FormData = FormData || require('form-data');
     var formData = new FormData();
@@ -31,0 +32,0 @@ //only handle fisrt level key-value

lib/ext/protocol/httpProtocol.js

@@ -15,3 +15,3 @@ /*
 var urlencode = require('urlencode');
-var request = require('request');
+var http = require('http');
 
@@ -28,10 +28,6 @@ function HttpProtocol(){
 
-HttpProtocol.prototype._getUri = function(options){
-    return url.resolve(['http://', options.server.host, ':', options.server.port].join(''), options.path);
-};
-
 HttpProtocol.prototype.normalizeConfig = HttpProtocol.normalizeConfig = function(config){
     config = Protocol.normalizeConfig(config);
     if (typeof config.query !== 'object'){
-        config.query = urlencode.parse(config.query, config.encoding);
+        config.query = urlencode.parse(config.query, {charset: config.encoding});
     }
@@ -42,18 +38,25 @@ return config;
 HttpProtocol.prototype._request = function(config, callback){
+    var response = new ResponseStream(),
+        query = urlencode.stringify(config.query, {charset: config.encoding}),
+        path,
+        piped = false;
+    if (query){
+        path = config.path + '?' + query;
+    }else{
+        path = config.path;
+    }
     var opt = {
-        uri: this._getUri(config),
-        qs: config.query,
+        host: config.server.host,
+        port: config.server.port,
+        path: path,
         method: config.method,
         headers: config.headers,
-        gzip: config.gzip || false,
         //disable http pool to avoid connect problem https://github.com/mikeal/request/issues/465
-        pool: false
+        agent: false
     };
     logger.trace('request start ' + JSON.stringify(opt));
-    var req = request(opt);
-    req.on('response', function(res) {
+    var req = http.request(opt, function(res){
         if (res.statusCode >= 400) {
             req.emit('error', new Error('Server Status Error: ' + res.statusCode));
         }
-        var response = new ResponseStream();
         res.pipe(response);
@@ -64,3 +67,12 @@ callback && callback(response);
         req.write(config.payload);
+        req.end();
+    }else{
+        //auto end if no pipe
+        process.nextTick(function(){
+            piped || req.end();
+        });
     }
+    req.on('pipe', function(){
+        piped = true;
+    });
     return req;
@@ -67,0 +79,0 @@ };

lib/logger.js

@@ -22,5 +22,7 @@ /*
         notice: function (msg) {
+            if (options.disable) return;
             logger.getLogger(options).notice('[yog-ral] ' + prefix + Array.prototype.slice.call(arguments, 0).join(''));
         },
         warning: function (msg) {
+            if (options.disable) return;
             logger.getLogger(options).warning('[yog-ral] ' + prefix + Array.prototype.slice.call(arguments, 0).join(''));
@@ -30,2 +32,3 @@ },
         fatal: function (msg) {
+            if (options.disable) return;
             logger.getLogger(options).fatal('[yog-ral] ' + prefix + Array.prototype.slice.call(arguments, 0).join(''));
@@ -35,2 +38,3 @@ },
         trace: function () {
+            if (options.disable) return;
             logger.getLogger(options).trace('[yog-ral] ' + prefix + Array.prototype.slice.call(arguments, 0).join(''));
@@ -40,2 +44,3 @@ },
         debug: function (msg) {
+            if (options.disable) return;
             logger.getLogger(options).debug('[yog-ral] ' + prefix + Array.prototype.slice.call(arguments, 0).join(''));
@@ -42,0 +47,0 @@ }

lib/ral.js

@@ -65,15 +65,24 @@ /*
     var timer = this.timer, context = this.conf.context,
-        conf = this.conf, unpack, payload, response, abort = false,
-        onError = this.onError.bind(this),
-        callRetry = this.callRetry.bind(this),
-
+        conf = this.conf, unpack, payload, request, response, abort = false,
         me = this;
 
+    function onError (err){
+        if (abort){
+            return;
+        }
+        endRequest();
+        me.onError(err);
+    }
+
+    function callRetry (err){
+        if (abort){
+            return;
+        }
+        endRequest();
+        me.callRetry(err);
+    }
+
     function onEnd (){
         //store request time when response end
-        clearTimeout(me.timeout);
-        timer.end('talk');
-        timer.end('request');
         me.emit('end');
-        logger.notice('request end ' + ralUtil.qs(me.getLogInfo()));
     }
@@ -84,7 +93,11 @@
         if (!abort){
+            clearTimeout(me.timeout);
+            timer.end('talk');
+            timer.end('request');
             me.emit('data', data);
+            logger.notice('request end ' + ralUtil.qs(me.getLogInfo()));
         }
     }
 
-    function onTimeout (){
+    function endRequest(){
         abort = true;
@@ -95,4 +108,2 @@ //end timer
         timer.end('request');
-        //end stream
-        request.abort();
         //remove event listen
@@ -104,5 +115,14 @@ if (unpack){
         }
+        if (request){
+            //end stream
+            request.abort();
+            request.removeAllListeners('error');
+            request.on('error', function(){});
+        }
         if (response){
             response.removeAllListeners();
         }
+    }
+
+    function onTimeout (){
         logger.trace('request timeout');
@@ -127,7 +147,8 @@ callRetry(new Error('request time out'));
                 try{
+                    timer.start('unpack');
                     unpack = context.unpack(conf, data);
+                    timer.end('unpack');
                     onData(unpack);
-                    timer.start('unpack');
                 }catch(ex){
-                    me.callRetry(ex);
+                    callRetry(ex);
                 }
@@ -138,2 +159,23 @@ });
 
+    function onResp(resp){
+        timer.end('write');
+        if (abort){
+            return;
+        }
+        response = resp;
+        timer.start('read');
+        //pipe the response stream to unpack stream
+        unpackResponse();
+        response.on('data', function() {
+            timer.end('read');
+            if (context.unpackConverter.isStreamify) {
+                timer.start('unpack');
+            }
+        });
+        //transport error event from unpack
+        response.on('error', callRetry);
+        //store request time when response end
+        response.once('end', onEnd);
+    }
+
     timer.start('talk');
@@ -166,18 +208,11 @@ //need pack data first to make sure the context which handled by converter can be passed into protocol
     timer.start('write');
-    var request = this.request = context.protocol.talk(conf, function(resp){
-        response = resp;
-        timer.end('write');
-        timer.start('read');
-        //create a unpack converter stream
-        unpackResponse();
-        response.on('data', function() {
-            timer.end('read');
-            timer.start('unpack');
+    try{
+        request = this.request = context.protocol.talk(conf,onResp);
+    }catch(err){
+        //delay error trigger to let user caught error
+        process.nextTick(function(){
+            onError(err);
         });
-        //pipe the response stream to unpack stream
-        //transport error event from unpack
-        response.on('error', callRetry);
-        //store request time when response end
-        response.once('end', onEnd);
-    });
+        return;
+    }
     request.on('error', callRetry);
@@ -184,0 +219,0 @@

package.json

 {
   "name": "node-ral",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "a rpc client for node",
@@ -37,8 +37,9 @@ "main": "index.js",
   "devDependencies": {
+    "async": "0.9.0",
     "combined-stream": "0.0.5",
+    "formidable": "1.0.15",
     "istanbul": "0.3.0",
     "mocha": "1.21.3",
-    "should": "4.0.4",
-    "formidable": "1.0.15"
+    "should": "4.0.4"
   }
 }

README.md

@@ -130,4 +130,4 @@ node-ral
 - Issue
-    - [ ] Form unpack support GBK when use form-urlencoded
+    - [X] Form unpack support GBK when use form-urlencoded
     - [X] Use Stream in Protocol
     - [X] Use Stream in Converter

test/config.js

@@ -6,48 +6,51 @@ 'use strict';
 var path = require('path');
+var RalModule = require('../lib/ralmodule.js');
 
-describe('config parser', function() {
+RalModule.load(__dirname + path.sep + '../lib/ext');
 
-    it('parse right config', function() {
+describe('config loadRawConfr', function() {
+
+    it('loadRawConf right config', function() {
         var right_conf = require('./config/right_config.js');
-        (function(){config.parse(right_conf);}).should.not.throw();
+        (function(){config.loadRawConf(right_conf);}).should.not.throw();
     });
 
-    it('parse config without balance', function() {
+    it('loadRawConf config without balance', function() {
         var wrong_conf = require('./config/wrong_config.js').without_balance;
-        (function(){config.parse(wrong_conf);}).should.throw(/balance/);
+        (function(){config.loadRawConf(wrong_conf);}).should.throw(/balance/);
     });
 
-//    it('parse config without unpack', function() {
+//    it('loadRawConf config without unpack', function() {
 //        var wrong_conf = require('./config/wrong_config.js').without_unpack;
-//        (function(){config.parse(wrong_conf);}).should.throw(/unpack/);
+//        (function(){config.loadRawConf(wrong_conf);}).should.throw(/unpack/);
 //    });
 //
-//    it('parse config without pack', function() {
+//    it('loadRawConf config without pack', function() {
 //        var wrong_conf = require('./config/wrong_config.js').without_pack;
-//        (function(){config.parse(wrong_conf);}).should.throw(/pack/);
+//        (function(){config.loadRawConf(wrong_conf);}).should.throw(/pack/);
 //    });
 
-    it('parse config without protocol', function() {
+    it('loadRawConf config without protocol', function() {
         var wrong_conf = require('./config/wrong_config.js').without_protocol;
-        (function(){config.parse(wrong_conf);}).should.throw(/protocol/);
+        (function(){config.loadRawConf(wrong_conf);}).should.throw(/protocol/);
     });
 
-    it('parse config with invalid encoding', function() {
+    it('loadRawConf config with invalid encoding', function() {
         var wrong_conf = require('./config/wrong_config.js').with_invalid_encoding;
-        (function(){config.parse(wrong_conf);}).should.throw(/encoding is valid/);
+        (function(){config.loadRawConf(wrong_conf);}).should.throw(/encoding is valid/);
     });
 
-    it('parse config with out server', function() {
+    it('loadRawConf config with out server', function() {
         var wrong_conf = require('./config/wrong_config.js').without_server;
-        (function(){config.parse(wrong_conf);}).should.throw(/server/);
+        (function(){config.loadRawConf(wrong_conf);}).should.throw(/server/);
     });
 
-    it('parse config with out server info', function() {
+    it('loadRawConf config with out server info', function() {
         var wrong_conf = require('./config/wrong_config.js').without_server_info;
-        (function(){config.parse(wrong_conf);}).should.throw(/server/);
+        (function(){config.loadRawConf(wrong_conf);}).should.throw(/server/);
     });
 
-    it('parse config with out port', function() {
+    it('loadRawConf config with out port', function() {
         var wrong_conf = require('./config/wrong_config.js').without_port;
-        (function(){config.parse(wrong_conf);}).should.throw(/port/);
+        (function(){config.loadRawConf(wrong_conf);}).should.throw(/port/);
     });
@@ -54,0 +57,0 @@ });

test/converter.js

@@ -195,12 +195,9 @@ /*
         });
-        var request = httpProtocol.talk(options);
-        var response = '';
-        request.on('data', function (data) {
-            response += data.toString();
+        var request = httpProtocol.talk(options, function(res){
+            res.on('data', function (data) {
+                server.close();
+                data.toString().should.be.equal('hear you 张三李四');
+                done();
+            });
         });
-        request.on('end', function () {
-            server.close();
-            response.toString().should.be.equal('hear you 张三李四');
-            done();
-        });
         pack.pipe(request);
@@ -223,12 +220,9 @@ });
         var pack = converter.pack(options, data);
-        var request = httpProtocol.talk(options);
-        var response = '';
-        request.on('data', function (data) {
-            response += data.toString();
+        var request = httpProtocol.talk(options, function (res) {
+            res.on('data', function(data){
+                data.toString().should.be.equal('hear you 张三李四');
+                server.close();
+                done();
+            });
         });
-        request.on('end', function () {
-            response.toString().should.be.equal('hear you 张三李四');
-            server.close();
-            done();
-        });
         pack.pipe(request);
@@ -311,12 +305,9 @@ });
         options.payload = pack;
-        var request = httpProtocol.talk(options);
-        var response = '';
-        request.on('data', function (data) {
-            response += data.toString();
+        var request = httpProtocol.talk(options, function (res) {
+            res.on('data', function(data){
+                data.toString().should.be.equal('hear you 张三李四');
+                server.close();
+                done();
+            });
         });
-        request.on('end', function () {
-            server.close();
-            response.toString().should.be.equal('hear you 张三李四');
-            done();
-        });
     });
@@ -323,0 +314,0 @@ });

test/ral.js

@@ -106,7 +106,12 @@ /*
         var req = RAL('GET_QS_SERV', {
-            path : '/404'
+            path : '/404',
+            tag: 404
         });
         req.on('error', function(err){
             err.toString().should.be.match(/404/);
-            var req_normal = RAL('GET_QS_SERV');
+            var req_normal = RAL('GET_QS_SERV',{
+                query: {
+                    normal: true
+                }
+            });
             req_normal.on('data', function(data){
@@ -113,0 +118,0 @@ data.query.from.should.eql('ral');

New alerts

Shell access

Supply chain risk

This module accesses the system shell. Accessing the system shell increases the risk of executing arbitrary code.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

241331

increased by8.41%

Number of package files

57

increased by32.56%

Lines of code

6528

increased by8.76%

Worsened metrics

Dev dependency count

6

increased by20%

Number of low supply chain risk alerts

8

increased by14.29%

Number of medium supply chain risk alerts

8

increased by100%

No dependency changes