s3 - npm Package Compare versions

Comparing version 4.2.0 to 4.3.0

CHANGELOG.md

@@ -0,1 +1,10 @@
+### 4.3.0
+
+ * fix open file descriptor leak. Thanks
+   [Ross Wilson](https://github.com/wilsonwc)
+ * add downloadBuffer API
+ * uploadDir: add 'fileUploadStart', 'fileUploadEnd' events
+ * downloadDir: add 'fileDownloadStart', 'fileDownloadEnd' events
+ * update aws-sdk to 2.0.19
+
 ### 4.2.0
@@ -2,0 +11,0 @@

lib/index.js

@@ -15,2 +15,3 @@ var AWS = require('aws-sdk');
 var mime = require('mime');
+var StreamSink = require('streamsink');
 
@@ -154,2 +155,5 @@ var MAX_PUTOBJECT_SIZE = 5 * 1024 * 1024 * 1024;
       localFileSlicer.on('error', handleError);
+      localFileSlicer.on('close', function() {
+        uploader.emit('fileClosed');
+      });
 
@@ -192,2 +196,6 @@ // keep an extra reference alive until we decide that we're completely
       if (err) return handleError(err);
+      if (localFileSlicer) {
+        localFileSlicer.unref();
+        localFileSlicer = null;
+      }
       uploader.emit('end', data);
@@ -739,2 +747,95 @@ }
 
+Client.prototype.downloadBuffer = function(s3Params) {
+  var self = this;
+  var downloader = new EventEmitter();
+  s3Params = extend({}, s3Params);
+
+  downloader.progressAmount = 0;
+
+  doWithRetry(doDownloadWithPend, self.s3RetryCount, self.s3RetryDelay, function(err, buffer) {
+    if (err) {
+      downloader.emit('error', err);
+      return;
+    }
+    downloader.emit('end', buffer);
+  });
+
+  return downloader;
+
+  function doDownloadWithPend(cb) {
+    self.s3Pend.go(function(pendCb) {
+      doTheDownload(function(err, buffer) {
+        pendCb();
+        cb(err, buffer);
+      });
+    });
+  }
+
+  function doTheDownload(cb) {
+    var errorOccurred = false;
+    var request = self.s3.getObject(s3Params);
+    var hashCheckPend = new Pend();
+    request.on('build', function() {
+      request.httpRequest.headers.Expect = '100-continue';
+    });
+    request.on('httpHeaders', function(statusCode, headers, resp) {
+      if (statusCode >= 300) {
+        handleError(new Error("http status code " + statusCode));
+        return;
+      }
+      var contentLength = parseInt(headers['content-length'], 10);
+      downloader.progressTotal = contentLength;
+      downloader.progressAmount = 0;
+      downloader.emit('progress');
+      downloader.emit('httpHeaders', statusCode, headers, resp);
+      var eTag = cleanETag(headers.etag);
+      var eTagCount = getETagCount(eTag);
+
+      var outStream = new StreamSink();
+      var multipartETag = new MultipartETag({size: contentLength, count: eTagCount});
+      var httpStream = resp.httpResponse.createUnbufferedStream();
+
+      httpStream.on('error', handleError);
+      outStream.on('error', handleError);
+
+      hashCheckPend.go(function(cb) {
+        multipartETag.on('end', function() {
+          if (multipartETag.bytes !== contentLength) {
+            handleError(new Error("Downloaded size does not match Content-Length"));
+            return;
+          }
+          if (eTagCount === 1 && !multipartETag.anyMatch(eTag)) {
+            handleError(new Error("ETag does not match MD5 checksum"));
+            return;
+          }
+          cb();
+        });
+      });
+      multipartETag.on('progress', function() {
+        downloader.progressAmount = multipartETag.bytes;
+        downloader.emit('progress');
+      });
+      outStream.on('finish', function() {
+        if (errorOccurred) return;
+        hashCheckPend.wait(function() {
+          cb(null, outStream.toBuffer());
+        });
+      });
+
+      httpStream.pipe(multipartETag);
+      httpStream.pipe(outStream);
+      multipartETag.resume();
+    });
+
+    request.send(handleError);
+
+    function handleError(err) {
+      if (!err) return;
+      if (errorOccurred) return;
+      errorOccurred = true;
+      cb(err);
+    }
+  }
+};
+
 function syncDir(self, params, directionIsToS3) {
@@ -948,6 +1049,8 @@ var ee = new EventEmitter();
         ee.progressTotal += s3Object.Size;
-        upDownFileParams.s3Params.Key = s3Object.Key;
+        var fullKey = s3Object.Key;
+        upDownFileParams.s3Params.Key = fullKey;
         upDownFileParams.localFile = fullPath;
         var downloader = self.downloadFile(upDownFileParams);
         var prevAmountDone = 0;
+        ee.emit('fileDownloadStart', fullPath, fullKey);
         downloader.on('error', handleError);
@@ -961,3 +1064,6 @@ downloader.on('progress', function() {
         });
-        downloader.on('end', checkDoMoreWork);
+        downloader.on('end', function() {
+          ee.emit('fileDownloadEnd', fullPath, fullKey);
+          checkDoMoreWork();
+        });
       }
@@ -999,3 +1105,4 @@ }
         ee.progressTotal += localFileStat.size;
-        upDownFileParams.s3Params.Key = prefix + localFileStat.s3Path;
+        var fullKey = prefix + localFileStat.s3Path;
+        upDownFileParams.s3Params.Key = fullKey;
         upDownFileParams.localFile = fullPath;
@@ -1005,2 +1112,3 @@ var uploader = self.uploadFile(upDownFileParams);
         var prevAmountTotal = localFileStat.size;
+        ee.emit('fileUploadStart', fullPath, fullKey);
         uploader.on('error', handleError);
@@ -1019,3 +1127,6 @@ uploader.on('progress', function() {
         });
-        uploader.on('end', checkDoMoreWork);
+        uploader.on('end', function() {
+          ee.emit('fileUploadEnd', fullPath, fullKey);
+          checkDoMoreWork();
+        });
       }
@@ -1022,0 +1133,0 @@ }

package.json

 {
   "name": "s3",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "description": "high level amazon s3 client. upload and download files and directories",
@@ -37,3 +37,3 @@ "main": "lib/index.js",
   "dependencies": {
-    "aws-sdk": "~2.0.17",
+    "aws-sdk": "~2.0.19",
     "findit": "~2.0.0",
@@ -45,3 +45,4 @@ "graceful-fs": "~3.0.2",
     "fd-slicer": "~0.1.0",
-    "mime": "~1.2.11"
+    "mime": "~1.2.11",
+    "streamsink": "~1.2.0"
   },
@@ -48,0 +49,0 @@ "bugs": {

README.md

@@ -246,2 +246,3 @@ # High Level Amazon S3 Client
    `createReadStream(options)`. See the fd-slicer README for more information.
+ * `'fileClosed'` - emitted when `localFile` has been closed.
 
@@ -276,6 +277,32 @@ And these methods:
  * `'error' (err)`
- * `'end'` - emitted when the file is uploaded successfully
+ * `'end'` - emitted when the file is downloaded successfully
  * `'progress'` - emitted when `progressAmount` and `progressTotal`
    properties change.
 
+### client.downloadBuffer(s3Params)
+
+http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property
+
+ * `s3Params`: params to pass to AWS SDK `getObject`.
+
+The difference between using AWS SDK `getObject` and this one:
+
+ * This works with a buffer only.
+ * If the reported MD5 upon download completion does not match, it retries.
+ * Retry based on the client's retry settings.
+ * Progress reporting.
+
+Returns an `EventEmitter` with these properties:
+
+ * `progressAmount`
+ * `progressTotal`
+
+And these events:
+
+ * `'error' (err)`
+ * `'end' (buffer)` - emitted when the file is downloaded successfully.
+   `buffer` is a `Buffer` containing the object data.
+ * `'progress'` - emitted when `progressAmount` and `progressTotal`
+   properties change.
+
 ### client.listObjects(params)
@@ -401,2 +428,6 @@
  * `'progress'` - emitted when any of the above progress properties change.
+ * `'fileUploadStart' (localFilePath, s3Key)` - emitted when a file begins
+   uploading.
+ * `'fileUploadEnd' (localFilePath, s3Key)` - emitted when a file successfully
+   finishes uploading.
 
@@ -465,4 +496,8 @@ `uploadDir` works like this:
  * `'error' (err)`
- * `'end'` - emitted when all files are uploaded
+ * `'end'` - emitted when all files are downloaded
  * `'progress'` - emitted when any of the progress properties above change
+ * `'fileDownloadStart' (localFilePath, s3Key)` - emitted when a file begins
+   downloading.
+ * `'fileDownloadEnd' (localFilePath, s3Key)` - emitted when a file successfully
+   finishes downloading.
 
@@ -469,0 +504,0 @@ `downloadDir` works like this:

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

67497

increased by7.76%

Lines of code

1347

increased by7.93%

Number of lines in readme file

581

increased by6.41%

Worsened metrics

Dependency count

9

increased by12.5%

Dependency changes

• + Addedstreamsink@~1.2.0

• + Addedstreamsink@1.2.0(transitive)

• Updatedaws-sdk@~2.0.19