ceph-s3 - npm Package Compare versions

Comparing version 1.0.1 to 1.1.0

lib/client.js

@@ -6,2 +6,4 @@ 'use strict';
 var knox = require('knox');
+var Promise = require('bluebird');
+var Backoff = require('backoff').FibonacciStrategy;
 
@@ -38,19 +40,19 @@ /**
 
-    this.s3 = knox.createClient(params);
+    var s3 = this.s3 = knox.createClient(params);
 
-    this.s3
-        .put('/', {})
-        .on('response', function s3BucketCreated(response) {
-            // 200 indicates successful bucket creation
-            if (response.statusCode !== 200) {
-                return done(new Error('Couldn\'t create bucket'));
-            }
+    return new Promise(function initConnectionPromise(resolve, reject) {
 
-            done();
-        })
-        .on('error', function s3Error(err) {
-            done(err);
-        })
-        .end();
+        s3.put('/', {})
+            .on('response', function s3BucketCreated(response) {
+                // 200 indicates successful bucket creation
+                if (response.statusCode !== 200) {
+                    return reject(new Error('Couldn\'t create bucket'));
+                }
 
+                resolve();
+            })
+            .on('error', reject)
+            .end();
+
+    }).nodeify(done || _.noop);
 }
@@ -79,3 +81,7 @@
 
-    this.s3.getFile(filename, headers, callback);
+    var s3 = this.s3;
+
+    return Promise.fromNode(function getFilePromise(done) {
+        s3.getFile(filename, headers, done);
+    }).nodeify(callback || _.noop);
 };
@@ -94,10 +100,56 @@
         filename = options.filename,
-        headers  = options.headers || {};
+        headers  = options.headers || {},
+        s3 = this.s3;
 
-    this.s3.putBuffer(fileBuffer, filename, headers, function s3PutBufferCallback(err, res) {
-        if (err) {
-            return callback(err);
+    return new Promise(function storeFilePromise(resolve, reject) {
+        s3.putBuffer(fileBuffer, filename, headers, function s3PutBufferCallback(err, res) {
+            if (err) {
+                return reject(err);
+            }
+
+            if (res.statusCode !== 200) {
+                return reject(new Error('response status code is ' + res.statusCode));
+            }
+
+            resolve(filename);
+        });
+    }).nodeify(callback || _.noop);
+};
+
+/**
+ * Tries to convert any incoming files into objects suitable for .storeFiles
+ * @param {any} files              incoming files
+ * @param {object} headers         headers appended to each file
+ * @param {string} prefix          prefix for saved files, ex.: user_id:121212:dump.html => prefix will be "user_id:121212"
+ * @return {array}                 array of suitable objects with .filename, .headers and .buffer properties
+ */
+S3Client.prototype.file2obj = function (files, headers, prefix) {
+
+    // default variables
+    headers = headers || {};
+    prefix = prefix || '';
+    files = _.isArray(files) ? files : [files];
+
+    var iterator = 0;
+    return files.map(function remapFiles(file) {
+
+        // transform content to buffer
+        if (Buffer.isBuffer(file)) {
+            file = { buffer: file };
+        } else if (typeof file === 'string') {
+            file = { buffer: new Buffer(file, 'utf-8') };
+        } else if (file && typeof file === 'object' && typeof file.buffer === 'string') {
+            file.buffer = new Buffer(file.buffer, 'utf-8');
         }
 
-        callback(null, res.statusCode === 200 ? filename : false);
+        // make sure we conform to format and we have unique filenames
+        if (Buffer.isBuffer(file.buffer)) {
+            file.headers = _.defaults(file.headers || {}, headers);
+            file.filename = prefix + (file.filename || ++iterator);
+        } else {
+            console.error('Input file object: ', file);
+            throw new Error('you have passed malformed file object');
+        }
+
+        return file;
     });
@@ -108,2 +160,44 @@
 /**
+ * Store raw crawling result into local cerph private cloud
+ *
+ * @param {Array[Object]} files     - files to store: [{ buffer, headers, filename}, {...}]
+ * @param {Object}        retryOpts - specify what to do on failure { initialDelay, randomisationFactor, maxDelay, retryCount }
+ * @param {function}      callback  - <err>
+ */
+S3Client.prototype.storeFilesWithRetry = function (files, retryOpts, callback) {
+
+    _.defaults(retryOpts, {
+        initialDelay: 1000,
+        randomisationFactor: 0.3,
+        maxDelay: 600000
+    });
+
+    var maxRetryCount = retryOpts.retryCount || 5;
+    var storeFile = this.storeFile;
+
+    return Promise
+        .resolve(files)
+        .map(function uploadFile(file, currentTry, backoffInstance) {
+
+            return storeFile(file).catch(function uploadToS3Failed(err) {
+                // default the current try to 0
+                currentTry = currentTry || 0;
+
+                if (currentTry >= maxRetryCount) {
+                    return Promise.reject(err);
+                }
+
+                backoffInstance = backoffInstance || new Backoff(retryOpts);
+                return Promise
+                    .delay(backoffInstance.next())
+                    .then(function () {
+                        return uploadFile(file, ++currentTry, backoffInstance);
+                    });
+            });
+        })
+        .nodeify(callback || _.noop);
+
+};
+
+/**
  * Public API
@@ -110,0 +204,0 @@ * @type {S3Client}

package.json

 {
   "name": "ceph-s3",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Knox wrapper with logic to simply pass set of users via config file and be able to work with your custom internal S3 compatible API",
@@ -28,5 +28,7 @@ "main": "index.js",
   "dependencies": {
+    "backoff": "^2.4.1",
+    "bluebird": "^2.9.25",
     "knox": "^0.9.2",
-    "lodash": "^3.6.0"
+    "lodash": "^3.8.0"
   }
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

10958

increased by47.28%

Lines of code

231

increased by50%

Worsened metrics

Dependency count

4

increased by100%

Dependency changes

• + Addedbackoff@^2.4.1

• + Addedbluebird@^2.9.25

• + Addedbackoff@2.5.0(transitive)

• + Addedbluebird@2.11.0(transitive)

• + Addedprecond@0.2.3(transitive)

• Updatedlodash@^3.8.0