Socket
Socket
Sign inDemoInstall

grunt-s3

Package Overview
Dependencies
Maintainers
1
Versions
14
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

grunt-s3 - npm Package Compare versions

Comparing version 0.0.9 to 0.1.0

Gruntfile.js
tasks/lib/common.js
tasks/lib/s3.js
test/download.js
test/files/a.txt
test/files/b.txt
test/files/c.txt
test/files/subdir/c.txt
test/files/subdir/d.txt
test/upload.js

17

package.json
{
"name": "grunt-s3",
"description": "A grunt task to automate moving files to/from Amazon S3.",
"version": "0.0.9",
"version": "0.1.0",
"author": "Aaron Forsander (https://github.com/pifantastic)",

@@ -12,3 +12,3 @@ "homepage": "https://github.com/pifantastic/grunt-s3",

"bugs": {
"url" : "https://github.com/pifantastic/grunt-s3/issues"
"url": "https://github.com/pifantastic/grunt-s3/issues"
},

@@ -21,3 +21,3 @@ "licenses": [

],
"main": "grunt.js",
"main": "Gruntfile.js",
"bin": "bin/grunt-s3",

@@ -28,3 +28,3 @@ "scripts": {

"engines": {
"node": ">= 0.6.0"
"node": ">= 0.8.0"
},

@@ -37,5 +37,3 @@ "keywords": [

"dependencies": {
"grunt": "~0.3.9",
"async": "~0.1.18",
"underscore": "~1.3.3",
"grunt": "0.4.x",
"underscore.deferred": "~0.1.4",

@@ -46,4 +44,7 @@ "knox": "0.4.1",

"devDependencies": {
"nodeunit": "~0.7.4"
"nodeunit": "~0.7.4",
"grunt-contrib-jshint": "~0.1.0",
"grunt-contrib-nodeunit": "~0.1.1",
"libyaml": "~0.2.1"
}
}

22

README.md

@@ -0,3 +1,4 @@

[![Build Status](https://secure.travis-ci.org/pifantastic/grunt-s3.png?branch=master)](https://travis-ci.org/pifantastic/grunt-s3)
# Grunt + Amazon S3
# Grunt 0.4.x + Amazon S3

@@ -20,5 +21,12 @@ ## About

1. `npm install grunt-s3`
2. Add `grunt.loadNpmTasks('grunt-s3');` to your project's `grunt.js` file.
```sh
npm install grunt-s3 --save-dev
```
Then add this line to your project's `Gruntfile.js`:
```javascript
grunt.loadNpmTasks('grunt-s3');
```
## Configuration

@@ -29,2 +37,4 @@

* **bucket** - (*string*) An Amazon S3 bucket
* **maxOperations** - (*number*) max number of concurrent transfers - if not specified or set to 0, will be unlimited.
* **encodePaths** - (*boolean*) if set to true, will encode the uris of destinations to prevent 505 errors. Default: false
* **headers** - (*object*) An object containing any headers you would like to send along with the

@@ -41,2 +51,3 @@ transfers i.e. `{ 'X-Awesomeness': 'Out-Of-This-World', 'X-Stuff': 'And Things!' }`

the above values may also be overriden.
* **debug** - (*boolean*) If true, no transfers with S3 will occur, will print all actions for review by user

@@ -172,3 +183,2 @@ ### Example

### grunt.helper('s3.pull', src, dest, options)
Download a file from s3. Returns a Promises/J-style Deferred object.

@@ -228,2 +238,6 @@

#### v0.1.0
* Update to be compatible with `grunt` version `0.4.x`.
#### v0.0.9

@@ -230,0 +244,0 @@

430

tasks/s3.js

@@ -10,3 +10,3 @@ /*jshint esnext:true*/

* Description: Move files to and from s3
* Dependencies: knox, async, underscore.deferred
* Dependencies: knox, underscore.deferred
*

@@ -17,90 +17,13 @@ */

/**
* Module dependencies.
*/
// Core.
const util = require('util');
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const url = require('url');
const zlib = require('zlib');
const s3 = require('./lib/s3').init(grunt);
// Npm.
const knox = require('knox');
const mime = require('mime');
const async = require('async');
const _ = require('underscore');
const deferred = require('underscore.deferred');
_.mixin(deferred);
const existsSync = ('existsSync' in fs) ? fs.existsSync : path.existsSync;
/**
* Grunt aliases.
*/
var log = grunt.log;
const _ = grunt.util._;
const async = grunt.util.async;
const log = grunt.log;
/**
* Success/error messages.
*/
const MSG_UPLOAD_SUCCESS = '↗'.blue + ' Uploaded: %s (%s)';
const MSG_DOWNLOAD_SUCCESS = '↙'.yellow + ' Downloaded: %s (%s)';
const MSG_DELETE_SUCCESS = '✗'.red + ' Deleted: %s';
const MSG_COPY_SUCCESS = '→'.cyan + ' Copied: %s to %s';
const MSG_ERR_NOT_FOUND = '¯\\_(ツ)_/¯ File not found: %s';
const MSG_ERR_UPLOAD = 'Upload error: %s (%s)';
const MSG_ERR_DOWNLOAD = 'Download error: %s (%s)';
const MSG_ERR_DELETE = 'Delete error: %s (%s)';
const MSG_ERR_COPY = 'Copy error: %s to %s';
const MSG_ERR_CHECKSUM = 'Expected hash: %s but found %s for %s';
/**
* Create an Error object based off of a formatted message. Arguments
* are identical to those of util.format.
*
* @param {String} Format.
* @param {...string|number} Values to insert into Format.
* @returns {Error}
*/
function makeError () {
var msg = util.format.apply(util, _.toArray(arguments));
return new Error(msg);
}
/**
* Get the grunt s3 configuration options, filling in options from
* environment variables if present. Also supports grunt template strings.
*
* @returns {Object} The s3 configuration.
*/
function getConfig () {
var config = grunt.config('s3') || {};
// Look for and process grunt template stings
var keys = ['key', 'secret', 'bucket'];
keys.forEach(function(key) {
if (config.hasOwnProperty(key)) {
config[key] = grunt.template.process(config[key]);
}
});
return _.defaults(config, {
key : process.env.AWS_ACCESS_KEY_ID,
secret : process.env.AWS_SECRET_ACCESS_KEY
});
}
/**
* Clone an object.
*
* @returns {Object} A clone of the original object.
*/
function clone (obj) {
return JSON.parse(JSON.stringify(obj));
}
/**
* Transfer files to/from s3.

@@ -112,3 +35,3 @@ *

var done = this.async();
var config = _.defaults(getConfig(), {
var config = _.defaults(s3.getConfig(), {
upload: [],

@@ -122,5 +45,11 @@ download: [],

if (config.debug) {
log.writeln("Running in debug mode, no transfers will be made".yellow);
log.writeln();
}
config.upload.forEach(function(upload) {
// Expand list of files to upload.
var files = grunt.file.expandFiles(upload.src);
var files = grunt.file.expand({ filter: "isFile" }, upload.src),
destPath = grunt.template.process(upload.dest);

@@ -134,7 +63,17 @@ files.forEach(function(file) {

// the destination.
var dest = (files.length === 1 && file === upload.src) ?
upload.dest :
path.join(upload.dest, path.basename(file));
var dest;
if (files.length === 1 && file === upload.src) {
dest = destPath;
}
else {
if (upload.rel) {
dest = path.join(destPath, path.relative(grunt.file.expand({ filter: "isDirectory" }, upload.rel)[0], file));
}
else {
dest = path.join(destPath, path.basename(file));
}
}
if(config.encodePaths === true) dest = encodeURIComponent(dest)
transfers.push(grunt.helper('s3.put', file, dest, upload));
transfers.push(s3.upload.bind(s3,file, dest, upload));
});

@@ -144,313 +83,38 @@ });

config.download.forEach(function(download) {
transfers.push(grunt.helper('s3.pull', download.src, download.dest, download));
transfers.push(s3.download.bind(s3,download.src, download.dest, download));
});
config.del.forEach(function(del) {
transfers.push(grunt.helper('s3.delete', del.src, del));
transfers.push(s3.del.bind(s3,del.src, del));
});
config.copy.forEach(function(copy) {
transfers.push(grunt.helper('s3.copy', copy.src, copy.dest, copy));
transfers.push(s3.copy.bind(s3,copy.src, copy.dest, copy));
});
var total = transfers.length;
var errors = 0;
// Keep a running total of errors/completions as the transfers complete.
transfers.forEach(function(transfer) {
var eachTransfer = config.maxOperations > 0
? async.forEachLimit.bind(async,transfers,config.maxOperations)
: async.forEach.bind(async,transfers)
eachTransfer(function(transferFn,completed){
var transfer = transferFn()
transfer.done(function(msg) {
log.ok(msg);
});
completed()
})
transfer.fail(function(msg) {
log.error(msg);
++errors;
});
transfer.always(function() {
// If this was the last transfer to complete, we're all done.
if (--total === 0) {
done(!errors);
}
});
});
completed()
})
},function(){
// we're all done.
done(!errors);
})
});
/**
* Publishes the local file at src to the s3 dest.
*
* Verifies that the upload was successful by comparing an md5 checksum of
* the local and remote versions.
*
* @param {String} src The local path to the file to upload.
* @param {String} dest The s3 path, relative to the bucket, to which the src
* is uploaded.
* @param {Object} [options] An object containing options which override any
* option declared in the global s3 config.
*/
grunt.registerHelper('s3.put', function (src, dest, opts) {
var dfd = new _.Deferred();
var options = clone(opts || {});
// Make sure the local file exists.
if (!existsSync(src)) {
return dfd.reject(makeError(MSG_ERR_NOT_FOUND, src));
}
var config = _.defaults(options || {}, getConfig());
var headers = options.headers || {};
if (options.access) {
headers['x-amz-acl'] = options.access;
}
// Pick out the configuration options we need for the client.
var client = knox.createClient(_(config).pick([
'endpoint', 'port', 'key', 'secret', 'access', 'bucket'
]));
// Encapsulate this logic to make it easier to gzip the file first if
// necesssary.
function upload(cb) {
cb = cb || function () {};
// Upload the file to s3.
client.putFile(src, dest, headers, function (err, res) {
// If there was an upload error or any status other than a 200, we
// can assume something went wrong.
if (err || res.statusCode !== 200) {
cb(makeError(MSG_ERR_UPLOAD, src, err || res.statusCode));
}
else {
// Read the local file so we can get its md5 hash.
fs.readFile(src, function (err, data) {
if (err) {
cb(makeError(MSG_ERR_UPLOAD, src, err));
}
else {
// The etag head in the response from s3 has double quotes around
// it. Strip them out.
var remoteHash = res.headers.etag.replace(/"/g, '');
// Get an md5 of the local file so we can verify the upload.
var localHash = crypto.createHash('md5').update(data).digest('hex');
if (remoteHash === localHash) {
var msg = util.format(MSG_UPLOAD_SUCCESS, src, localHash);
cb(null, msg);
}
else {
cb(makeError(MSG_ERR_CHECKSUM, localHash, remoteHash, src));
}
}
});
}
});
}
// If gzip is enabled, gzip the file into a temp file and then perform the
// upload.
if (options.gzip) {
headers['Content-Encoding'] = 'gzip';
headers['Content-Type'] = mime.lookup(src);
var charset = mime.charsets.lookup(headers['Content-Type'], null);
if (charset) {
headers['Content-Type'] += '; charset=' + charset;
}
// Determine a unique temp file name.
var tmp = src + '.gz';
var incr = 0;
while (existsSync(tmp)) {
tmp = src + '.' + (incr++) + '.gz';
}
var input = fs.createReadStream(src);
var output = fs.createWriteStream(tmp);
// Gzip the file and upload when done.
input.pipe(zlib.createGzip()).pipe(output)
.on('error', function (err) {
dfd.reject(makeError(MSG_ERR_UPLOAD, src, err));
})
.on('close', function () {
// Update the src to point to the newly created .gz file.
src = tmp;
upload(function (err, msg) {
// Clean up the temp file.
fs.unlinkSync(tmp);
if (err) {
dfd.reject(err);
}
else {
dfd.resolve(msg);
}
});
});
}
else {
// No need to gzip so go ahead and upload the file.
upload(function (err, msg) {
if (err) {
dfd.reject(err);
}
else {
dfd.resolve(msg);
}
});
}
return dfd;
});
/**
* Download a file from s3.
*
* Verifies that the download was successful by downloading the file and
* comparing an md5 checksum of the local and remote versions.
*
* @param {String} src The s3 path, relative to the bucket, of the file being
* downloaded.
* @param {String} dest The local path where the download will be saved.
* @param {Object} [options] An object containing options which override any
* option declared in the global s3 config.
*/
grunt.registerHelper('s3.pull', function (src, dest, opts) {
var dfd = new _.Deferred();
var options = clone(opts);
var config = _.defaults(options || {}, getConfig());
// Create a local stream we can write the downloaded file to.
var file = fs.createWriteStream(dest);
// Pick out the configuration options we need for the client.
var client = knox.createClient(_(config).pick([
'endpoint', 'port', 'key', 'secret', 'access', 'bucket'
]));
// Upload the file to s3.
client.getFile(src, function (err, res) {
// If there was an upload error or any status other than a 200, we
// can assume something went wrong.
if (err || res.statusCode !== 200) {
return dfd.reject(makeError(MSG_ERR_DOWNLOAD, src, err || res.statusCode));
}
res
.on('data', function (chunk) {
file.write(chunk);
})
.on('error', function (err) {
return dfd.reject(makeError(MSG_ERR_DOWNLOAD, src, err));
})
.on('end', function () {
file.end();
// Read the local file so we can get its md5 hash.
fs.readFile(dest, function (err, data) {
if (err) {
return dfd.reject(makeError(MSG_ERR_DOWNLOAD, src, err));
}
else {
// The etag head in the response from s3 has double quotes around
// it. Strip them out.
var remoteHash = res.headers.etag.replace(/"/g, '');
// Get an md5 of the local file so we can verify the download.
var localHash = crypto.createHash('md5').update(data).digest('hex');
if (remoteHash === localHash) {
var msg = util.format(MSG_DOWNLOAD_SUCCESS, src, localHash);
dfd.resolve(msg);
}
else {
dfd.reject(makeError(MSG_ERR_CHECKSUM, localHash, remoteHash, src));
}
}
});
});
});
return dfd;
});
/**
* Copy a file from s3 to s3.
*
* @param {String} src The s3 path, including the bucket, to the file to
* copy.
* @param {String} dest The s3 path, relative to the bucket, to the file to
* create.
* @param {Object} [options] An object containing options which override any
* option declared in the global s3 config.
*/
grunt.registerHelper('s3.copy', function (src, dest, opts) {
var dfd = new _.Deferred();
var options = clone(opts);
var config = _.defaults(options || {}, getConfig());
// Pick out the configuration options we need for the client.
var client = knox.createClient(_(config).pick([
'endpoint', 'port', 'key', 'secret', 'access', 'bucket'
]));
var headers = {
'Content-Length': 0,
'x-amz-copy-source' : src
};
if (options.headers) {
_(headers).extend(options.headers);
headers['x-amz-metadata-directive'] = 'REPLACE';
}
// Copy the src file to dest.
var req = client.put(dest, headers);
req.on('response', function (res) {
if (res.statusCode !== 200) {
dfd.reject(makeError(MSG_ERR_COPY, src, dest));
}
else {
dfd.resolve(util.format(MSG_COPY_SUCCESS, src, dest));
}
});
return dfd;
});
/**
* Delete a file from s3.
*
* @param {String} src The s3 path, relative to the bucket, to the file to
* delete.
* @param {Object} [options] An object containing options which override any
* option declared in the global s3 config.
*/
grunt.registerHelper('s3.delete', function (src, opts) {
var dfd = new _.Deferred();
var options = clone(opts);
var config = _.defaults(options || {}, getConfig());
// Pick out the configuration options we need for the client.
var client = knox.createClient(_(config).pick([
'endpoint', 'port', 'key', 'secret', 'access', 'bucket'
]));
// Upload the file to this endpoint.
client.deleteFile(src, function (err, res) {
if (err || res.statusCode !== 204) {
dfd.reject(makeError(MSG_ERR_DELETE, src, err || res.statusCode));
}
else {
dfd.resolve(util.format(MSG_DELETE_SUCCESS, src));
}
});
return dfd;
});
};
grunt.js
test/file.txt
test/put.js
.travis.yml

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc