skipper - npm Package Compare versions

Comparing version 0.1.0 to 0.1.1

examples/express/app.js

 var Express = require('express')
 	, path = require('path')
-	, Receiver = require('./example_receiver')
+	, Receiver = require('../exampleReceiver')
 	, FileParser = require('../..');
@@ -5,0 +5,0 @@ require('colors');

examples/sails/api/controllers/FileController.js

 /**
+ * Module dependencies
+ */
+
+var Receiver = require('../../../exampleReceiver');
+
+
+
+
+/**
  * FileController.js
@@ -11,95 +20,19 @@ *
 
-	// Note: Because of the way the file parser works, the the file upload will not work
-	// if the request stream closes before the process executes the call to the blob adapter above (`File.write`)
-	// This is really only a problem for local testing, since the database calls before File.write will always
-	// finish before the end of the request stream, since the request is sending lots of binary data 
-	// Still, we should make sure and handle this case-- so... TODO (Mike): patch file-parser
+	// NOTE:
+	// This example demonstrates skipper (aka file-parser) in its
+	// completely raw usage.  When the receiver implementation is complete,
+	// this will become much simpler.
 	upload: function(req, res) {
-		sails.log('Request reached the controller.  `req.body` ===', req.body);
 
-		var PARAM_TO_INSPECT_FOR_FILES = 'hm';
-		var uploadStream = req.file(PARAM_TO_INSPECT_FOR_FILES);
+		req.file('avatar').upload( Receiver() , function (err, files) {
+			if (err) return res.serverError(err);
+		
+			res.json({
+				message: files.length + ' file(s) uploaded successfully!',
+				files: files
+			});
 
-		// Create File of type `binary`
-		var stream = File.write(uploadStream, {
+		});
 
-			// Cumulative bytes allowed per request on this uploadstream
-			maxBytes: 20 * 1000 * 1000 * 1000,
-
-			// Optional map function for generating the name of the file when it is stored in the adapter
-			// (nonsense-ified mutation of the original filename, i.e. `downloadName`)
-			// saveAs: function (origFilename) { return 'newFilename.foo'; }
-
-		}, function allUploadsComplete(err, files) {
-
-			if (!err && typeof files === 'object') {
-				sails.log(
-					require('util').format(
-						'File adapter triggered callback with %s and %s.',
-						Object.keys(files).length + ' files',
-						err ? 'an error: '+err : 'no error.'
-					)
-				);
-			}
-			else {
-				sails.log.error(
-					require('util').format(
-						'File adapter triggered callback with an error: '+err
-					)
-				);
-				return res.serverError(err);
-			}
-
-			var megabytes = _.reduce(files, function (b, file) {
-				return b+file.size;
-			}, 0);
-			megabytes /= 1000000;
-			sails.log('Uploaded ~'+megabytes+' MB across '+Object.keys(files).length+' different files...');
-
-			console.log();
-			console.log();
-			console.log();
-			sails.log('Now waiting for 2000ms on purpose...');
-
-			setTimeout(function waitAWhileToBeEvenMoreOrnery () {
-				if (err) return res.serverError(err);
-
-				if (!files || !_.keys(files).length) {
-					return res.badRequest([{
-						message: 'No files were uploaded to the `'+PARAM_TO_INSPECT_FOR_FILES+'` parameter.',
-						files: files
-					}]);
-				}
-
-				sails.log('Done!');
-				res.json({
-					message: _.keys(files).length + ' files uploaded!',
-					files: files
-				});
-
-			}, 2000);
-
-		});
 	}
 };
-
-
-
-/**
- * Example `generateBlobName` fn that adds a nonse
- * to both sides of the filename.
- *
- * @param {String} original filename
- * @returns {String} name to save the file as in the blob store
- */
-function generateBlobName(filename) {
-	var ext, name, filenameParts = filename.split(/(.+)\.([^.]+)$/);
-	if (filenameParts.length < 3) {
-		name = filename;
-		ext = '';
-	} else {
-		name = filenameParts[1];
-		ext = '.' + filenameParts[2];
-	}
-	return nonce() + name + nonce() + ext;
-}

examples/sails/app.js

@@ -1,40 +0,19 @@
-/**
- * app.js
- *
- * Use `app.js` to run your app without `sails lift`.
- * To start the server, run: `node app.js`.
- * 
- * This is handy in situations where the sails CLI is not relevant or useful.
- *
- * For example:
- *   => `node app.js`
- *   => `forever start app.js`
- *   => `node debug app.js`
- *   => `modulus deploy`
- *   => `heroku scale`
- * 
- *
- * The same command-line arguments are supported, e.g.:
- * `node app.js --silent --port=80 --prod`
- */
+require('colors');
 
-// Ensure a "sails" can be located:
-var sails;
-try {
-	var Sails = require('sails/lib/app');
-}
-catch (e) {
-	console.error('To run an app using `node app.js`, you usually need to have a version of `sails` installed in the same directory as your app.');
-	console.error('To do that, run `npm install sails`');
-	console.error('');
-	console.error('Alternatively, if you have sails installed globally (i.e. you did `npm install -g sails`), you can use `sails lift`.');
-	console.error('When you run `sails lift`, your app will still use a local `./node_modules/sails` dependency if it exists,');
-	console.error('but if it doesn\'t, the app will run with the global sails instead!');
-	return;
-}
+require('sails').lift({
+	hooks: {
+		grunt: false
+	},
+	express: {
+		bodyParser: require('../../'),
+		silenceMultipartWarning: true
+	}
+}, function sailsIsReady (err, sails) {
+	if (err) throw err;
 
-// Instantiate sails
-sails = Sails();
-
-// Start server
-sails.lift();
+	sails.log.blank();
+	sails.log.info(
+		'Send a multipart form upload with a file in the `avatar` field to '+
+		'http://localhost:1337/file/upload'.underline
+	);
+});

examples/sails/README.md

@@ -1,2 +0,2 @@
-# example
+# skipper example
 
@@ -7,11 +7,7 @@ a [Sails](http://sailsjs.org) application
 
-Lift this example app, then send a `form-data` (e.g. multipart) POST request to [http://localhost:1337/file/upload](http://localhost:1337/file/upload).  By default, uploaded file(s) will be uploaded to the app's `.tmp` folder.
+Start this example by running
+```sh
+$ node app
+```
 
-To test the S3 (or other blob) adapters, just change `config/connections.js`.
-
-
-
-> **Do you use POSTman?**
->
-> Here is a POSTman collection with an example upload request you can send to test this example:
-> https://www.getpostman.com/collections/7ca6b5331838b5320c5e
+Then send a multipart form-data POST request to [http://localhost:1337/file/upload](http://localhost:1337/file/upload).  By default, uploaded file(s) will be uploaded to the app's `.tmp` folder.  We recommend using a tool like [POSTman](https://www.getpostman.com).

index.js

@@ -86,2 +86,6 @@ /**
 
+					// If the content type is explicitly set to "multipart/form-data",
+					// we should not try to rerun the JSON bodyparser- it may hang forever.
+					if (req.is('multipart/form-data')) return next();
+
 					// Otherwise, set an explicit JSON content-type
@@ -88,0 +92,0 @@ // and try parsing the request body again.

lib/Parser/onTextParam.js

@@ -80,2 +80,5 @@ /**
 			self.emit('warning', '`'+field+'` param already exists in req.body, ignoring new value.');
+
+			// Consider this text parameter complete, since we won't wait for its bytes any longer.
+			textParamMetadata.done = true;
 			return;
@@ -82,0 +85,0 @@ }

lib/Upstream.js

@@ -33,3 +33,5 @@ /**
 		// (no buffering is happening, so it's ok for this to be longer)
-		maxTimeToWaitForFirstFile: 4000,
+		// This needs to be long enough to allow any policies/middleware to run.
+		// Should not need to exceed 500ms in most cases.
+		maxTimeToWaitForFirstFile: 500,
 
@@ -36,0 +38,0 @@ // The max # of ms this Upstream will buffer bytes and wait to be plugged

package.json

 {
   "name": "skipper",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Bodyparser for Express/Sails. Exposes simple API for streaming multiple files to disk, S3, etc. without buffering to a .tmp directory.",
@@ -24,5 +24,7 @@ "main": "index.js",
   "author": "Mike McNeil",
-  "contributors": [{
-    "name": "Scott Gress"
-  }],
+  "contributors": [
+    {
+      "name": "Scott Gress"
+    }
+  ],
   "license": "MIT",
@@ -42,4 +44,5 @@ "dependencies": {
     "request": "~2.33.0",
-    "concat-stream": "~1.4.1"
+    "concat-stream": "~1.4.1",
+    "sails": "~0.10.0"
   }
 }

README.md

@@ -1,13 +0,192 @@
-skipper ![](http://i.imgur.com/RfxxWzT.png)
-===========
+# [![Skipper](http://i.imgur.com/jwWrBji.png)](https://github.com/balderdashy/skipper)
 
-##### streaming multipart file upload parser
+### Streaming Multipart File Upload Parsing 
 
+Skipper is an opinionated variant of Connect's body parser designed to support streaming upload of monolithic files to a compatible blob receiver, while still allowing application code to run in a timely manner; without writing .tmp files to disk.
 
-Opinionated variant of Connect's body parser designed to support streaming upload of monolithic files to a compatible Waterline adapter, while still allowing application code to run in a timely manner; without writing .tmp files to disk.
 
-
 This module may or may not be included as a part of the stable release of Sails v0.10-- need more documentation, examples, and "receivers" (currently receivers for S3 and local disk exist.)
 
+#### Usage
 
+##### With Sails
+
+Install it with npm using the `--save` flag.  This will automatically add it to your app's package.json file. 
+
+```sh
+
+npm install --save
+
+```
+
+Skipper intends to be a drop-in replacement for the Connect Body Parser which Sails uses by default (via Express.js).  
+
+Therefore, the next step is to disable it and hook up Skipper.
+
+
+###### Change Config
+
+Do this by adding the line `bodyParser: require('skipper')` to the `express` object in `/myApp/config/express.js`.  It should look something like this.
+
+```javascript
+
+/**
+ * Configure advanced options for the Express server inside of Sails.
+ *
+ * For more information on configuration, check out:
+ * http://sailsjs.org/#documentation
+ */
+
+module.exports.express = {
+
+ // ... much comment.  so amaze. wow
+ 
+	// Defaults to a slightly modified version of `express.bodyParser`, i.e.:
+	// If the Connect `bodyParser` doesn't understand the HTTP body request 
+	// data, Sails runs it again with an artificial header, forcing it to try
+	// and parse the request body as JSON.  (this allows JSON to be used as your
+	// request data without the need to specify a 'Content-type: application/json'
+	// header)
+	// 
+	// If you want to change any of that, you can override the bodyParser with
+	// your own custom middleware:
+	bodyParser: require('skipper')
+
+};
+
+
+/**
+ * HTTP Flat-File Cache
+ * 
+ * These settings are for Express' static middleware- the part that serves
+ * flat-files like images, css, client-side templates, favicons, etc.
+ *
+ * ... more comments ...
+ */
+module.exports.cache = {
+
+	// The number of seconds to cache files being served from disk
+	// (only works in production mode)
+	maxAge: 31557600000
+};
+
+
+```
+
+###### Create an API
+
+Now that it's hooked up, we need to generate a new `api` for serving/storing the files.  Do this using the sails command line tool.
+
+```sh
+
+dude@littleDude:~/node/myApp$ sails generate api file
+
+debug: Generated a new controller `file` at api/controllers/FileController.js!
+debug: Generated a new model `File` at api/models/File.js!
+
+info: REST API generated @ http://localhost:1337/file
+info: and will be available the next time you run `sails lift`.
+
+dude@littleDude:~/node/myApp$ 
+
+```
+
+###### Write Controller Actions
+
+Refer to `myApp/api/controllers/FileController.js` below and modify it as you see fit.
+
+```javascript 
+// myApp/api/controllers/FileController.js
+
+/**
+ * FilesController.js 
+ *
+ * @description ::
+ * @docs        :: http://sailsjs.org/#!documentation/controllers
+ */
+
+module.exports = {
+
+  index: function (req,res){
+
+    res.writeHead(200, {'content-type': 'text/html'});
+    res.end(
+      '<form action="http://localhost:1337/files/upload" enctype="multipart/form-data" method="post">'+
+      '<input type="text" name="title"><br>'+
+      '<input type="file" name="avatar" multiple="multiple"><br>'+
+      '<input type="submit" value="Upload">'+
+      '</form>'
+    )
+  },
+  upload: function (req,res){
+
+var Writable = require('stream').Writable;
+var fs = require('fs');
+
+    function newReceiverStream (options) {
+        options = options || {};
+
+        // Default the output path for files to `/dev/null` if no `id` option
+        // is passed in (for testing purposes only)
+        var filePath = options.id || '/dev/null';
+
+        var receiver__ = Writable({objectMode: true});
+
+        // This `_write` method is invoked each time a new file is received
+        // from the Readable stream (Upstream) which is pumping filestreams
+        // into this receiver.  (filename === `__newFile.filename`).
+        receiver__._write = function onFile (__newFile, encoding, done) {
+        var outs = fs.createWriteStream(filePath, encoding);
+        __newFile.pipe(outs);
+
+        // Garbage-collect the bytes that were already written for this file.
+        // (called when a read or write error occurs)
+        function gc (err) {
+          console.log('************** Garbage collecting file `'+__newFile.filename+'` located @ '+filePath+'...');
+          fs.unlink(filePath, function (gcErr) {
+            if (gcErr) return done([err].concat([gcErr]));
+            return done(err);
+          });
+        }
+
+
+        __newFile.on('error', function (err) {
+          console.log('***** READ error on file '+__newFile.filename, '::',err);
+        });
+        outs.on('error', function failedToWriteFile (err) {
+        	console.log('OH THE BITS',__newFile,'enc',encoding,'dun',done);
+          gc(err);
+        });
+
+        outs.on('finish', function successfullyWroteFile () {
+          done();
+        });
+
+        };
+
+        return receiver__;
+    }
+    
+var streamOptions = {id:'/home/dude/node/fileUploadExample/assets/images/shitBird.jpeg'};
+    req.file('avatar').upload( newReceiverStream(streamOptions) , function (err, files) {
+          if (err) return res.serverError(err);
+        
+          res.json({
+            message: files.length + ' file(s) uploaded successfully!',
+            files: files
+          });
+
+        });
+  }
+
+};
+
+
+```
+
+##### With Express
+
+You're on your own, homie... nah, it's comin'
+
+
 #### Status
@@ -14,0 +193,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Number of lines in readme file

222

increased by416.28%

Worsened metrics

Total package byte prevSize

63166

decreased by-12.18%

Dev dependency count

5

increased by25%

Number of package files

34

decreased by-19.05%

Lines of code

1521

decreased by-16.43%

No dependency changes