s3 - npm Package Compare versions

Comparing version 1.2.0 to 1.2.1

index.js

@@ -351,3 +351,4 @@ var AWS = require('aws-sdk');
       function findNext1000(cb) {
-        findAllS3Objects(data.NextMarker, prefix, cb);
+        var nextMarker = data.NextMarker || data.Contents[data.Contents.length - 1].Key;
+        findAllS3Objects(nextMarker, prefix, cb);
       }
@@ -485,2 +486,3 @@
   ee.progressAmount = 0;
+  ee.objectsFound = 0;
 
@@ -524,3 +526,3 @@ var pend = new Pend();
       if (!localFileStat || !compareETag(s3Object.ETag, localFileStat.md5sum)) {
-        downloadOneFile(relPath);
+        downloadOneFile(relPath, s3Object);
       }
@@ -530,23 +532,44 @@ }
 
-    function downloadOneFile(relPath) {
+    function downloadOneFile(relPath, s3Object) {
       var fullPath = path.join(localDir, toNativeSep(relPath));
       pend.go(function(cb) {
-        ee.progressTotal += s3Object.Size;
-        upDownFileParams.s3Params.Key = prefix + relPath;
-        upDownFileParams.localFile = fullPath;
-        upDownFileParams.localFileStat = null;
-        var downloader = self.downloadFile(upDownFileParams);
-        var prevAmountDone = 0;
-        downloader.on('error', function(err) {
-          cb(err);
-        });
-        downloader.on('progress', function() {
-          var delta = downloader.progressAmount - prevAmountDone;
-          prevAmountDone = downloader.progressAmount;
-          ee.progressAmount += delta;
-          ee.emit('progress');
-        });
-        downloader.on('end', function() {
-          cb();
-        });
+        if (getS3Params) {
+          getS3Params(fullPath, s3Object, haveS3Params);
+        } else {
+          startDownload();
+        }
+
+        function haveS3Params(err, s3Params) {
+          if (err) return cb(err);
+
+          if (!s3Params) {
+            //user has decided to skip this file
+            cb();
+            return;
+          }
+
+          upDownFileParams.s3Params = extend(extend({}, baseUpDownS3Params), s3Params);
+          startDownload();
+        }
+
+        function startDownload() {
+          ee.progressTotal += s3Object.Size;
+          upDownFileParams.s3Params.Key = prefix + relPath;
+          upDownFileParams.localFile = fullPath;
+          upDownFileParams.localFileStat = null;
+          var downloader = self.downloadFile(upDownFileParams);
+          var prevAmountDone = 0;
+          downloader.on('error', function(err) {
+            cb(err);
+          });
+          downloader.on('progress', function() {
+            var delta = downloader.progressAmount - prevAmountDone;
+            prevAmountDone = downloader.progressAmount;
+            ee.progressAmount += delta;
+            ee.emit('progress');
+          });
+          downloader.on('end', function() {
+            cb();
+          });
+        }
       });
@@ -688,2 +711,4 @@ }
     finder.on('data', function(data) {
+      ee.objectsFound += data.Contents.length;
+      ee.emit('progress');
       data.Contents.forEach(function(object) {
@@ -690,0 +715,0 @@ var key = object.Key.substring(prefix.length);

package.json

 {
   "name": "s3",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "high level amazon s3 client. upload and download files and directories",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

@@ -288,5 +288,5 @@ # High Level Amazon S3 Client
 
+ * `localDir` - source path on local file system to sync to S3
  * `deleteRemoved` - delete s3 objects with no corresponding local file.
    default false
- * `localDir` - source path on local file system to sync to S3
  * `getS3Params` - optional function which will be called for every file that
@@ -327,4 +327,6 @@ needs to be uploaded. See below.
 
+ * `localDir` - destination directory on local file system to sync to
  * `deleteRemoved` - delete local files with no corresponding s3 object. default `false`
- * `localDir` - destination directory on local file system to sync to
+ * `getS3Params` - optional function which will be called for every object that
+   needs to be downloaded. See below.
  * `s3Params`
@@ -334,2 +336,18 @@ - `Prefix` (required)
 
+```js
+function getS3Params(localFile, s3Object, callback) {
+  // localFile is the destination path where the object will be written to
+  // s3Object is same as one element in the `Contents` array from here:
+  // http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjects-property
+
+  // call callback like this:
+  var err = new Error(...); // only if there is an error
+  var s3Params = { // if there is no error
+    VersionId: "abcd", // just an example
+  };
+  // pass `null` for `s3Params` if you want to skip dowlnoading this object.
+  callback(err, s3Params);
+}
+```
+
 Returns an `EventEmitter` with these properties:
@@ -373,2 +391,8 @@
 
+### 1.2.1
+
+ * fix `listObjects` for greater than 1000 objects
+ * `downloadDir` supports `getS3Params` parameter
+ * `uploadDir` and `downloadDir` expose `objectsFound` progress
+
 ### 1.2.0
@@ -375,0 +399,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

35748

increased by4.8%

Lines of code

786

increased by2.75%

Number of lines in readme file

417

increased by6.11%

No dependency changes