metalsmith-downloader - npm Package Compare versions

Comparing version 0.1.2 to 0.2.0

index.js

@@ -0,1 +1,22 @@
+/**
+ * Copyright (c) 2016 Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+ * OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+ * IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
 var debug = require('debug')('metalsmith-downloader');
@@ -5,2 +26,3 @@ var fs = require('fs-extra');
 var request = require('request');
+var queue = require('queue');
 
@@ -10,6 +32,3 @@ function checkFileExists(filename) {
     fs.stat(filename, function(err, stats) {
-      if (err)
-        resolve(false);
-      else
-        resolve(stats.isFile());
+      resolve(err ? false : stats.isFile());
     });
@@ -38,3 +57,3 @@ });
           fs.unlink(filename, function(err) {
-            if (err)
+            if (err)  // FIXME: Consider this as fatal error
               debug('Error deleting file ' + filename + ': ' + err);
@@ -76,3 +95,4 @@ reject(pipeError);
 
-function chmodFile(filename, mode) {
+function chmodFile (filename, mode) {
+  debug('Changing mode of file ' + filename);
   return new Promise(function(resolve, reject) {
@@ -102,69 +122,103 @@ fs.chmod(filename, mode, function(err) {
 
-module.exports = function downloader(options) {
-  var incremental = options && options.incremental;
-  var cacheDir = options && options.cache;
+function createProcessFile(options) {
+  var incremental = options.incremental;
+  var cacheDir = options.cache;
+  var dest = options.dest;
 
-  return function(files, metalsmith, done) {
-    var dest = metalsmith.destination();
+  return function processFile(filename, file) {
+    var filepath = path.resolve(cacheDir || dest, filename);
+    var contentsUrl = file.contentsUrl;
 
-    var downloadableFiles = {};
-    Object.keys(files).forEach(function(filename) {
-      var file = files[filename];
-      if (!file || !file.contentsUrl)
-        return;
+    return checkFileExists(filepath)
+      .then(function(exists) {
+        // NOTE: This won't work with retries unless we successfully remove the failed file
+        if (cacheDir && exists) {
+          debug('File ' + filename + ' found in cache, not downloading');
+          return Promise.resolve();
+        }
 
-      debug('Removing file ' + filename + ' from Metalsmith');
-      delete files[filename];
+        if (incremental && exists) {
+          debug('File ' + filename + ' already exists, not downloading');
+          return Promise.resolve();
+        }
 
-      downloadableFiles[filename] = file;
-    });
+        debug('Downloading file ' + filename + ' from ' + contentsUrl);
+        return downloadFile(filepath, contentsUrl)
+          .then(function() {
+            if (file.mode) {
+              // FIXME: Shouldn't this affect filepath and not filename?
+              return chmodFile(filename, file.mode);
+            }
 
-    Promise.all(
-      Object.keys(downloadableFiles).map(function(filename) {
-        var filepath = path.resolve(cacheDir || dest, filename);
-        var file = downloadableFiles[filename];
-        var contentsUrl = file.contentsUrl;
+            return Promise.resolve();
+          })
+          .then(function() {
+            debug('File ' + filename + ' downloaded successfully');
+          });
+      }).then(function() {
+        if (!cacheDir)
+          return Promise.resolve();
 
-        return checkFileExists(filepath)
-          .then(function(exists) {
-            if (cacheDir && exists) {
-              debug('File ' + filename + ' found in cache, not downloading');
-              return Promise.resolve();
-            }
+        var destpath = path.resolve(dest, filename);
+        debug('Copying ' + filepath + ' to ' + destpath);
+        // FIXME: Don't copy if destpath exists && incremental?
+        return copyFile(filepath, destpath);
+      }).catch(function(err) {
+        debug('Error downloading file ' + filename + ': ' + err);
+      });
+  }
+}
 
-            if (incremental && exists) {
-              debug('File ' + filename + ' already exists, not downloading');
-              return Promise.resolve();
-            }
+function createProcessFileWrapper(q, options) {
+  var processFile = createProcessFile(options);
+  var maxRetries = options.retries || 0;
 
-            debug('Downloading file ' + filename + ' from ' + contentsUrl);
-            return downloadFile(filepath, contentsUrl)
-              .then(function() {
-                if (file.mode) {
-                  debug('Changing mode of file ' + filename);
-                  return chmodFile(filename, file.mode);
-                }
-                return Promise.resolve();
-              })
-              .then(function() {
-                debug('File ' + filename + ' downloaded successfully');
-              });
-          }).then(function() {
-            if (!cacheDir)
-              return Promise.resolve();
+  return function processFileWrapper(filename, file, retries) {
+    retries = retries || 0;
 
-            var destpath = path.resolve(dest, filename);
-            debug('Copying ' + filepath + ' to ' + destpath);
-            // FIXME: Don't copy if destpath exists && incremental?
-            return copyFile(filepath, destpath);
-          }).catch(function(err) {
-            debug('Error downloading file ' + filename + ': ' + err);
-          });
+    return function processFileWrapperInner(cb) {
+      if (retries > maxRetries) {
+        return setTimeout(function() {
+          cb(Error(`Number of retries exceeds ${maxRetries} on ${filename}`));
+        }, 0);
+      }
+
+      debug('Processing ' + filename);
+
+      return processFile(filename, file)
+        .then(function() {
+          cb();
+        })
+        .catch(function() {
+          debug('Retrying ' + filename);
+          q.push(processFileWrapper(filename, file, retries + 1));
+          cb();
+        })
+    }
+  }
+}
+
+module.exports = function downloader(options) {
+
+  return function(files, metalsmith, done) {
+    var _options = Object.assign(options || {}, {
+      dest: metalsmith.destination()
+    });
+    var q = queue({concurrency: _options.concurrency || Infinity});
+
+    var processFile = createProcessFileWrapper(q, _options);
+
+    Object.keys(files)
+      .filter(function(filename) {
+        var file = files[filename];
+        return file && file.contentsUrl;
       })
-    ).then(function() {
-      done();
-    }).catch(function(err) {
-      done(err);
-    });
+      .forEach(function(filename) {
+        var file = files[filename];
+        delete files[filename];
+        q.push(processFile(filename, file));
+      });
+
+    q.start(done);
   };
 };

package.json

 {
   "name": "metalsmith-downloader",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Download assets dynamically in your Metalsmith build",
@@ -12,2 +12,3 @@ "main": "index.js",
     "fs-extra": "^0.30.0",
+    "queue": "^4.0.0",
     "request": "^2.74.0"
@@ -14,0 +15,0 @@ },

No alert changes

Improved metrics

Total package byte prevSize

7792

increased by34.23%

Lines of code

193

increased by29.53%

Worsened metrics

Dependency count

4

increased by33.33%

Dependency changes

• + Addedqueue@^4.0.0

• + Addedqueue@4.5.1(transitive)