ali-oss - npm Package Compare versions

Comparing version 4.5.0 to 4.6.0

History.md

 
+4.6.0 / 2016-10-27
+==================
+
+  * feat: implement parallel uploading for multipartUpload (#149)
+
 4.5.0 / 2016-10-13
@@ -3,0 +8,0 @@ ==================

lib/multipart.js

@@ -11,2 +11,3 @@ 'use strict';
 var mime = require('mime');
+var gather = require('co-gather');
 
@@ -64,4 +65,3 @@ var proto = exports;
     uploadId: uploadId,
-    doneParts: [],
-    nextPart: 0
+    doneParts: []
   };
@@ -84,3 +84,2 @@
   var doneParts = checkpoint.doneParts;
-  var nextPart = checkpoint.nextPart;
   var name = checkpoint.name;
@@ -90,10 +89,12 @@
   var numParts = partOffs.length;
-  for (var i = nextPart; i < numParts; i++) {
-    var partNo = i + 1;
-    var pi = partOffs[i];
+
+  var uploadPartJob = function* (self, partNo) {
+    var pi = partOffs[partNo - 1];
     var data = {
-      stream: this._createStream(file, pi.start, pi.end),
+      stream: self._createStream(file, pi.start, pi.end),
       size: pi.end - pi.start
     };
-    var result = yield this._uploadPart(name, uploadId, partNo, data);
+
+    var result = yield self._uploadPart(name, uploadId, partNo, data);
+
     doneParts.push({
@@ -103,9 +104,32 @@ number: partNo,
     });
-    checkpoint.nextPart = i + 1;
+    checkpoint.doneParts = doneParts;
 
     if (options && options.progress) {
-      yield options.progress(partNo / numParts, checkpoint);
+      yield options.progress(doneParts.length / numParts, checkpoint);
     }
+  };
+
+  var all = Array.from(new Array(numParts), (x, i) => i + 1);
+  var done = doneParts.map(p => p.number);
+  var todo = all.filter(p => done.indexOf(p) < 0);
+
+  var jobs = [];
+  for (var i = 0; i < todo.length; i++) {
+    jobs.push(uploadPartJob(this, todo[i]));
   }
 
+  const defaultParallel = 5;
+  var parallel = options.parallel || defaultParallel;
+
+  // yield in parallel
+  var results = yield gather(jobs, parallel);
+
+  // check errors after all jobs are completed
+  for (var i = 0; i < results.length; i++) {
+    if (results[i].isError) {
+      throw new Error(
+        'Failed to upload some parts with error: ' + results[i].error.toString());
+    }
+  }
+
   return yield this._completeMultipartUpload(name, uploadId, doneParts, options);
@@ -236,2 +260,3 @@ };
 proto._completeMultipartUpload = function* _completeMultipartUpload(name, uploadId, parts, options) {
+  parts.sort((a, b) => a.number - b.number);
   var xml = '<?xml version="1.0" encoding="UTF-8"?>\n<CompleteMultipartUpload>\n';
@@ -238,0 +263,0 @@ for (var i = 0; i < parts.length; i++) {

package.json

 {
   "name": "ali-oss",
-  "version": "4.5.0",
+  "version": "4.6.0",
   "description": "aliyun oss(open storage service) node client",
@@ -70,2 +70,3 @@ "main": "lib/client.js",
     "co-defer": "^1.0.0",
+    "co-gather": "^0.0.1",
     "copy-to": "^2.0.1",
@@ -72,0 +73,0 @@ "dateformat": "^1.0.12",

README.md

@@ -1367,2 +1367,3 @@ oss-nodejs-sdk
 - [options] {Object} optional args
+  - [parallel] {Number} the number of parts to be uploaded in parallel
   - [partSize] {Number} the suggested size for each part
@@ -1391,2 +1392,3 @@ - [progress] {Function} the progress callback called after each
 var result = yield store.multipartUpload('object', '/tmp/file', {
+  parallel: 4,
   partSize: 1024 * 1024,
@@ -1393,0 +1395,0 @@ progress: function* (p, cpt) {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

135379

increased by0.68%

Lines of code

1975

increased by0.87%

Number of lines in readme file

2088

increased by0.1%

Worsened metrics

Dependency count

20

increased by5.26%

Dependency changes

• + Addedco-gather@^0.0.1

• + Addedco-gather@0.0.1(transitive)

• + Addedco-thread@0.0.1(transitive)