s3 - npm Package Compare versions

Comparing version 4.3.1 to 4.4.0

CHANGELOG.md

@@ -0,1 +1,9 @@
+### 4.4.0
+
+ * Export aws-sdk dependency
+ * Update dependencies
+ * Ability to handle symlinks in uploadDir and downloadDir
+ * Expose S3 constants
+ * Add downloadStream API
+
 ### 4.3.1
@@ -2,0 +10,0 @@

lib/index.js

@@ -6,3 +6,3 @@ var AWS = require('aws-sdk');
 var rimraf = require('rimraf');
-var findit = require('findit');
+var findit = require('findit2');
 var Pend = require('pend');
@@ -14,5 +14,6 @@ var path = require('path');
 var MultipartETag = require('./multipart_etag');
-var FdSlicer = require('fd-slicer');
+var fd_slicer = require('fd-slicer');
 var mime = require('mime');
 var StreamSink = require('streamsink');
+var PassThrough = require('stream').PassThrough;
 
@@ -35,3 +36,9 @@ var MAX_PUTOBJECT_SIZE = 5 * 1024 * 1024 * 1024;
 exports.MultipartETag = MultipartETag;
+exports.AWS = AWS;
 
+exports.MAX_PUTOBJECT_SIZE = MAX_PUTOBJECT_SIZE;
+exports.MAX_DELETE_COUNT = MAX_DELETE_COUNT;
+exports.MAX_MULTIPART_COUNT = MAX_MULTIPART_COUNT;
+exports.MIN_MULTIPART_SIZE = MIN_MULTIPART_SIZE;
+
 function Client(options) {
@@ -155,3 +162,3 @@ options = options || {};
       if (err) return handleError(err);
-      localFileSlicer = new FdSlicer(fd, {autoClose: true});
+      localFileSlicer = fd_slicer.createFromFd(fd, {autoClose: true});
       localFileSlicer.on('error', handleError);
@@ -270,3 +277,2 @@ localFileSlicer.on('close', function() {
         });
-        var pend = new Pend();
         s3Params.Body = inStream;
@@ -280,2 +286,4 @@ s3Params.ContentLength = end - start;
         var overallDelta = 0;
+        var pend = new Pend();
+        var haveETag = pend.hold();
         multipartETag.on('progress', function() {
@@ -289,11 +297,9 @@ if (fatalError || errorOccurred) return;
         });
-        pend.go(function(cb) {
-          multipartETag.on('end', function() {
-            if (fatalError || errorOccurred) return;
-            var delta = multipartETag.bytes - prevBytes;
-            uploader.progressAmount += delta;
-            uploader.progressTotal += (end - start) - multipartETag.bytes;
-            uploader.emit('progress');
-            cb();
-          });
+        multipartETag.on('end', function() {
+          if (fatalError || errorOccurred) return;
+          var delta = multipartETag.bytes - prevBytes;
+          uploader.progressAmount += delta;
+          uploader.progressTotal += (end - start) - multipartETag.bytes;
+          uploader.emit('progress');
+          haveETag();
         });
@@ -457,6 +463,2 @@ inStream.pipe(multipartETag);
     var request = self.s3.getObject(s3Params);
-    request.on('build', function() {
-      request.httpRequest.headers.Expect = '100-continue';
-    });
-
     var errorOccurred = false;
@@ -782,5 +784,2 @@ var hashCheckPend = new Pend();
     var hashCheckPend = new Pend();
-    request.on('build', function() {
-      request.httpRequest.headers.Expect = '100-continue';
-    });
     request.on('httpHeaders', function(statusCode, headers, resp) {
@@ -846,2 +845,54 @@ if (statusCode >= 300) {
 
+Client.prototype.downloadStream = function(s3Params) {
+  var self = this;
+  var downloadStream = new PassThrough();
+  s3Params = extend({}, s3Params);
+
+  doDownloadWithPend(function(err) {
+    if (err) downloadStream.emit('error', err);
+  });
+  return downloadStream;
+
+  function doDownloadWithPend(cb) {
+    self.s3Pend.go(function(pendCb) {
+      doTheDownload(function(err) {
+        pendCb();
+        cb(err);
+      });
+    });
+  }
+
+  function doTheDownload(cb) {
+    var errorOccurred = false;
+    var request = self.s3.getObject(s3Params);
+    var hashCheckPend = new Pend();
+    request.on('httpHeaders', function(statusCode, headers, resp) {
+      if (statusCode >= 300) {
+        handleError(new Error("http status code " + statusCode));
+        return;
+      }
+      downloadStream.emit('httpHeaders', statusCode, headers, resp);
+      var httpStream = resp.httpResponse.createUnbufferedStream();
+
+      httpStream.on('error', handleError);
+
+      downloadStream.on('finish', function() {
+        if (errorOccurred) return;
+        cb();
+      });
+
+      httpStream.pipe(downloadStream);
+    });
+
+    request.send(handleError);
+
+    function handleError(err) {
+      if (!err) return;
+      if (errorOccurred) return;
+      errorOccurred = true;
+      cb(err);
+    }
+  }
+};
+
 function syncDir(self, params, directionIsToS3) {
@@ -851,3 +902,3 @@ var ee = new EventEmitter();
     fs: fs,
-    followSymlinks: !!params.followSymlinks,
+    followSymlinks: (params.followSymlinks == null) ? true : !!params.followSymlinks,
   };
@@ -1253,3 +1304,3 @@ var localDir = params.localDir;
     });
-    walker.on('directory', function(dir, stat) {
+    walker.on('directory', function(dir, stat, stop, linkPath) {
       if (fatalError) return walker.stop();
@@ -1259,3 +1310,3 @@ // we only need to save directories when deleteRemoved is true
       if (!deleteRemoved || directionIsToS3) return;
-      var relPath = path.relative(localDir, dir);
+      var relPath = path.relative(localDir, linkPath || dir);
       if (relPath === '') return;
@@ -1267,5 +1318,5 @@ stat.path = relPath;
     });
-    walker.on('file', function(file, stat) {
+    walker.on('file', function(file, stat, linkPath) {
       if (fatalError) return walker.stop();
-      var relPath = path.relative(localDir, file);
+      var relPath = path.relative(localDir, linkPath || file);
       stat.path = relPath;
@@ -1272,0 +1323,0 @@ stat.s3Path = toUnixSep(relPath);

package.json

 {
   "name": "s3",
-  "version": "4.3.1",
+  "version": "4.4.0",
   "description": "high level amazon s3 client. upload and download files and directories",
@@ -33,14 +33,14 @@ "main": "lib/index.js",
   "devDependencies": {
-    "mocha": "~1.21.4",
+    "mocha": "~2.0.1",
     "ncp": "~1.0.1"
   },
   "dependencies": {
-    "aws-sdk": "~2.0.19",
-    "findit": "~2.0.0",
-    "graceful-fs": "~3.0.3",
+    "aws-sdk": "~2.0.31",
+    "fd-slicer": "~1.0.0",
+    "findit2": "~2.2.3",
+    "graceful-fs": "~3.0.5",
+    "mime": "~1.2.11",
     "mkdirp": "~0.5.0",
-    "pend": "~1.1.3",
+    "pend": "~1.2.0",
     "rimraf": "~2.2.8",
-    "fd-slicer": "~0.2.1",
-    "mime": "~1.2.11",
     "streamsink": "~1.2.0"
@@ -47,0 +47,0 @@ },

README.md

@@ -144,2 +144,7 @@ # High Level Amazon S3 Client
 
+### s3.AWS
+
+This contains a reference to the aws-sdk module. It is a valid use case to use
+both this module and the lower level aws-sdk module in tandem.
+
 ### s3.createClient(options)
@@ -307,2 +312,19 @@
 
+### client.downloadStream(s3Params)
+
+http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property
+
+ * `s3Params`: params to pass to AWS SDK `getObject`.
+
+The difference between using AWS SDK `getObject` and this one:
+
+ * This works with a stream only.
+
+If you want retries, progress, or MD5 checking, you must code it yourself.
+
+Returns a `ReadableStream` with these additional events:
+
+ * `'httpHeaders' (statusCode, headers)` - contains the HTTP response
+   headers and status code.
+
 ### client.listObjects(params)
@@ -390,3 +412,3 @@
  * (optional) `getS3Params` - function which will be called for every file that
-   needs to be uploaded. See below.
+   needs to be uploaded. You can use this to skip some files. See below.
  * (optional) `defaultContentType`: Unless you explicitly set the `ContentType`
@@ -397,2 +419,4 @@ parameter in `s3Params`, it will be automatically set for you based on the
    `application/octet-stream`.
+ * (optional) `followSymlinks` - Set this to `false` to ignore symlinks.
+   Defaults to `true`.
 
@@ -462,3 +486,6 @@ ```js
  * (optional) `getS3Params` - function which will be called for every object that
-   needs to be downloaded. See below.
+   needs to be downloaded. You can use this to skip downloading some objects.
+   See below.
+ * (optional) `followSymlinks` - Set this to `false` to ignore symlinks.
+   Defaults to `true`.
 
@@ -465,0 +492,0 @@ ```js

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

70074

increased by3.76%

Lines of code

1389

increased by3.12%

Number of lines in readme file

608

increased by4.65%

Dependency changes

• + Addedfindit2@~2.2.3

• + Addedfd-slicer@1.0.1(transitive)

• + Addedfindit2@2.2.3(transitive)

• + Addedpend@1.2.0(transitive)

• - Removedfindit@~2.0.0

• - Removedfd-slicer@0.2.1(transitive)

• - Removedfindit@2.0.0(transitive)

• - Removedpend@1.1.3(transitive)

• Updatedaws-sdk@~2.0.31

• Updatedfd-slicer@~1.0.0

• Updatedgraceful-fs@~3.0.5

• Updatedpend@~1.2.0