ali-oss - npm Package Compare versions

Comparing version 2.8.0 to 3.0.0

History.md

 
+3.0.0 / 2015-09-30 
+==================
+
+  * refactor: change signatureUrl to normal function
+  * feat: add available checking
+
 2.8.0 / 2015-09-29
@@ -3,0 +9,0 @@ ==================

lib/cluster.js

@@ -1,5 +0,8 @@
-/*!
- * ali-oss - lib/cluster.js
+/**!
  * Copyright(c) ali-sdk and other contributors.
- * Author: dead_horse <[e-mail]>
+ * MIT Licensed
+ *
+ * Authors:
+ *   dead_horse <[e-mail]>
+ *   fengmk2 <[e-mail]> (http://fengmk2.com)
  */
@@ -13,5 +16,8 @@
 
+const co = require('co');
+const defer = require('co-defer');
 const Base = require('sdk-base');
 const util = require('util');
-
+const ready = require('get-ready');
+const currentIP = require('address').ip();
 const RR = 'roundRobin';
@@ -21,3 +27,3 @@ const MS = 'masterSlave';
 module.exports = function (oss) {
-  const Client = function (options) {
+  function Client(options) {
     if (!(this instanceof Client)) {
@@ -30,6 +36,14 @@ return new Client(options);
     }
-    this.clients = options.cluster.map(function (opt) {
+
+    Base.call(this);
+
+    this.clients = [];
+    this.availables = {};
+
+    for (let i = 0; i < options.cluster.length; i++) {
+      let opt = options.cluster[i];
       opt.timeout = opt.timeout || options.timeout;
-      return oss(opt);
-    });
+      this.clients.push(oss(opt));
+      this.availables[i] = true;
+    }
 
@@ -39,27 +53,12 @@ this.schedule = options.schedule || RR;
 
-    Base.call(this);
-  };
+    const heartbeatInterval = options.heartbeatInterval || 10000;
+    this._checkAvailableLock = false;
+    this._timerId = defer.setInterval(this._checkAvailable.bind(this), heartbeatInterval);
+    this._init();
+  }
 
   util.inherits(Client, Base);
-  let proto = Client.prototype;
+  const proto = Client.prototype;
+  ready.mixin(proto);
 
-  proto._choose = function (index) {
-    if (this.schedule === MS) {
-      return this.clients[index];
-    }
-
-    let chosen = this.clients[this.index++];
-    if (this.index >= this.clients.length) {
-      this.index = 0;
-    }
-    return chosen;
-  };
-
-  proto.onerror = function (err) {
-    if (err.status && err.status > 200 && err.status < 500) {
-      throw err;
-    }
-    this.emit('error', err);
-  };
-
   const GET_METHODS = [
@@ -83,23 +82,39 @@ 'head',
     proto[method] = function* () {
-      let index = 0;
-      let max = this.clients.length;
-      let res;
-      let client;
+      const args = Array.prototype.slice.call(arguments);
+      let client = this._chooseAvailable();
+      let lastError;
+      try {
+        return yield client[method].apply(client, args);
+      } catch (err) {
+        if (err.status && err.status >= 200 && err.status < 500) {
+          // 200 ~ 499 belong to normal response, don't try again
+          throw err;
+        }
+        // < 200 || >= 500 need to retry from other cluser node
+        lastError = err;
+      }
 
-      while (index < max) {
-        client = this._choose(index++);
-        try {
-          res = yield client[method].apply(client, arguments);
-        } catch (err) {
-          this.onerror(err);
+      for (let i = 0; i < this.clients.length; i++) {
+        let c = this.clients[i];
+        if (c === client) {
           continue;
         }
-        return res;
+        try {
+          return yield c[method].apply(client, args);
+        } catch (err){
+          if (err.status && err.status >= 200 && err.status < 500) {
+            // 200 ~ 499 belong to normal response, don't try again
+            throw err;
+          }
+          // < 200 || >= 500 need to retry from other cluser node
+          lastError = err;
+        }
       }
-      let err = new Error('all clients are down');
-      err.name = 'AllServerDownError';
-      throw err;
+
+      lastError.message += ' (all clients are down)';
+      throw lastError;
     };
   });
 
+  // must cluster node write success
   PUT_METHODS.forEach(function (method) {
@@ -115,25 +130,107 @@ proto[method] = function* () {
 
-  proto.signatureUrl = function* (name) {
-    let index = 0;
-    let max = this.clients.length;
-    let client;
+  proto.signatureUrl = function (/* name */) {
+    const args = Array.prototype.slice.call(arguments);
+    let client = this._chooseAvailable();
+    return client.signatureUrl.apply(client, args);
+  };
 
-    while (index < max - 1) {
-      client = this._choose(index++);
-      try {
-        yield client.head(name);
-      } catch (err) {
-        if (!err.status || err.status > 500 || err.status < 200) {
-          this.emit('error', err);
-          continue;
+  proto._init = function() {
+    const that = this;
+    co(function*() {
+      yield that._checkAvailable(true);
+      that.ready(true);
+    }).catch(function(err) {
+      that.emit('error', err);
+    });
+  };
+
+  proto._checkAvailable = function*(first) {
+    const name = '._ali-oss.check.status.' + currentIP + '.txt';
+    if (first) {
+      // only start will try to write the file
+      yield this.put(name, new Buffer('check available started at ' + Date()));
+    }
+
+    if (this._checkAvailableLock) {
+      return;
+    }
+    this._checkAvailableLock = true;
+    let downStatusFiles = [];
+    for (let i = 0; i < this.clients.length; i++) {
+      let client = this.clients[i];
+      // check 3 times
+      let available = yield this._checkStatus(client, name);
+      if (!available) {
+        // check again
+        available = yield this._checkStatus(client, name);
+      }
+      if (!available) {
+        // check again
+        available = yield this._checkStatus(client, name);
+        if (!available) {
+          downStatusFiles.push(client._objectUrl(name));
         }
       }
-      return client.signatureUrl.apply(client, arguments);
+      this.availables[i] = available;
     }
+    this._checkAvailableLock = false;
 
-    // don't check last one
-    return this._choose(index).signatureUrl.apply(client, arguments);
+    if (downStatusFiles.length > 0) {
+      const err = new Error(downStatusFiles.length + ' data node down, please check status file: ' + downStatusFiles.join(', '));
+      err.name = 'CheckAvailableError';
+      this.emit('error', err);
+    }
   };
 
+  proto._checkStatus = function*(client, name) {
+    let available = true;
+    try {
+      yield client.head(name);
+    } catch (err) {
+      // 404 will be available too
+      if (!err.status || err.status >= 500 || err.status < 200) {
+        available = false;
+      }
+    }
+    return available;
+  };
+
+  proto._chooseAvailable = function() {
+    if (this.schedule === MS) {
+      for (let i = 0; i < this.clients.length; i++) {
+        if (this.availables[i]) {
+          return this.clients[i];
+        }
+      }
+      // all down, try to use this first one
+      return this.clients[0];
+    }
+
+    // RR
+    let n = this.clients.length;
+    while (n > 0) {
+      let i = this._nextRRIndex();
+      if (this.availables[i]) {
+        return this.clients[i];
+      }
+      n--;
+    }
+    // all down, try to use this first one
+    return this.clients[0];
+  };
+
+  proto._nextRRIndex = function() {
+    let index = this.index++;
+    if (this.index >= this.clients.length) {
+      this.index = 0;
+    }
+    return index;
+  };
+
+  proto.close = function() {
+    clearInterval(this._timerId);
+    this._timerId = null;
+  };
+
   return Client;
 };

package.json

 {
   "name": "ali-oss",
-  "version": "2.8.0",
+  "version": "3.0.0",
   "description": "aliyun oss(open storage service) node client",
@@ -10,8 +10,6 @@ "main": "lib/client.js",
   "scripts": {
-    "test": "mocha --harmony --check-leaks -t 30000 -r co-mocha -r should test/*.test.js",
-    "test-cov": "node --harmony node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -R dot --check-leaks -t 60000 -r co-mocha -r should test/*.test.js",
+    "test": "mocha --check-leaks -t 30000 -r thunk-mocha -r should test/*.test.js",
+    "test-cov": "istanbul cover node_modules/.bin/_mocha -- -R dot --check-leaks -t 60000 -r thunk-mocha -r should test/*.test.js",
     "jshint": "jshint .",
-    "autod": "autod -w --prefix '~' -D mocha,autod,should,co-mocha,istanbul-harmony",
-    "cnpm": "npm install --registry=https://registry.npm.taobao.org",
-    "contributors": "contributors -f plain -o AUTHORS"
+    "autod": "autod -w --prefix '~' -D mocha,autod,should,thunk-mocha,istanbul"
   },
@@ -35,13 +33,15 @@ "repository": {
   "devDependencies": {
-    "autod": "~2.1.3",
-    "co": "*",
-    "co-fs": "~1.2.0",
-    "co-mocha": "~1.1.2",
-    "istanbul-harmony": "~0.3.16",
-    "mm": "~1.3.4",
-    "mocha": "~2.3.3",
-    "should": "~7.1.0"
+    "autod": "2",
+    "co-fs": "1",
+    "istanbul": "*",
+    "mm": "1",
+    "mocha": "2",
+    "should": "7",
+    "thunk-mocha": "0"
   },
   "dependencies": {
+    "address": "~1.0.0",
     "agentkeepalive": "~2.0.3",
+    "co": "~4.6.0",
+    "co-defer": "~1.0.0",
     "copy-to": "~2.0.1",
@@ -51,2 +51,3 @@ "debug": "~2.2.0",
     "end-or-error": "~1.0.1",
+    "get-ready": "~1.0.0",
     "humanize-ms": "~1.0.1",
@@ -53,0 +54,0 @@ "is-type-of": "~0.3.1",

README.md

@@ -1684,2 +1684,12 @@ ali-oss
 });
+
+// listen error event to logging error
+client.on('error', function(err) {
+  console.error(err.stack);
+});
+
+// client init ready
+client.ready(function() {
+  console.log('cluster client init ready, go ahead!');
+});
 ```
@@ -1697,4 +1707,2 @@
 
-___Notice: client.signatureUrl now is a generatorFunction.___
-
 ### Put Methods
@@ -1701,0 +1709,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

99025

increased by3.27%

Dev dependency count

7

decreased by-12.5%

Lines of code

1342

increased by7.19%

Number of lines in readme file

1773

increased by0.45%

Worsened metrics

Dependency count

17

increased by30.77%

Dependency changes

• + Addedaddress@~1.0.0

• + Addedco@~4.6.0

• + Addedco-defer@~1.0.0

• + Addedget-ready@~1.0.0

• + Addedaddress@1.0.3(transitive)

• + Addedco@4.6.0(transitive)

• + Addedco-defer@1.0.0(transitive)

• + Addedget-ready@1.0.0(transitive)

• - Removedaddress@2.0.3(transitive)