Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
skipper-disk
Advanced tools
skipper-disk - npm Package Compare versions
Comparing version 0.3.1 to 0.4.0
308
index.js
@@ -5,11 +5,6 @@ /** | ||
| var WritableStream = require('stream').Writable; | ||
| var TransformStream = require('stream').Transform; | ||
| var fsx = require('fs-extra'); | ||
| var path = require('path'); | ||
| var _ = require('lodash'); | ||
| var UUIDGenerator = require('node-uuid'); | ||
| var r_buildDiskReceiverStream = require('./standalone/build-disk-receiver-stream'); | ||
| /** | ||
@@ -38,5 +33,7 @@ * skipper-disk | ||
| }, | ||
| ls: function(dirpath, cb) { | ||
| return fsx.readdir(dirpath, cb); | ||
| }, | ||
| read: function(fd, cb) { | ||
@@ -50,158 +47,8 @@ if (cb) { | ||
| receive: DiskReceiver, | ||
| receiver: DiskReceiver // (synonym for `.receive()`) | ||
| receive: function (options){ | ||
| return r_buildDiskReceiverStream(options); | ||
| } | ||
| }; | ||
| return adapter; | ||
| /** | ||
| * A simple receiver for Skipper that writes Upstreams to | ||
| * disk at the configured path. | ||
| * | ||
| * Includes a garbage-collection mechanism for failed | ||
| * uploads. | ||
| * | ||
| * @param {Object} options | ||
| * @return {Stream.Writable} | ||
| */ | ||
| function DiskReceiver(options) { | ||
| options = options || {}; | ||
| _.defaults(options, { | ||
| // The default `saveAs` implements a unique filename by combining: | ||
| // • a generated UUID (like "4d5f444-38b4-4dc3-b9c3-74cb7fbbc932") | ||
| // • the uploaded file's original extension (like ".jpg") | ||
| saveAs: function(__newFile, cb) { | ||
| return cb(null, UUIDGenerator.v4() + path.extname(__newFile.filename)); | ||
| }, | ||
| // Bind a progress event handler, e.g.: | ||
| // function (milestone) { | ||
| // milestone.id; | ||
| // milestone.name; | ||
| // milestone.written; | ||
| // milestone.total; | ||
| // milestone.percent; | ||
| // }, | ||
| onProgress: undefined, | ||
| // Upload limit (in bytes) | ||
| // defaults to ~15MB | ||
| maxBytes: 15000000, | ||
| // By default, upload files to `./.tmp/uploads` (relative to cwd) | ||
| dirname: '.tmp/uploads' | ||
| }); | ||
| var receiver__ = WritableStream({ objectMode: true }); | ||
| // if onProgress handler was provided, bind an event automatically: | ||
| if (_.isFunction(options.onProgress)) { | ||
| receiver__.on('progress', options.onProgress); | ||
| } | ||
| // Track the progress of all file uploads that pass through this receiver | ||
| // through one or more attached Upstream(s). | ||
| receiver__._files = []; | ||
| // Keep track of the number total bytes written so that maxBytes can | ||
| // be enforced. | ||
| var totalBytesWritten = 0; | ||
| // This `_write` method is invoked each time a new file is received | ||
| // from the Readable stream (Upstream) which is pumping filestreams | ||
| // into this receiver. (filename === `__newFile.filename`). | ||
| receiver__._write = function onFile(__newFile, encoding, done) { | ||
| // ------------------------------------------------------- | ||
| // ------------------------------------------------------- | ||
| // ------------------------------------------------------- | ||
| // | ||
| // Determine the file descriptor-- the unique identifier. | ||
| // Often represents the location where file should be written. | ||
| var fd; | ||
| var dirPath; | ||
| if (options.dirname) { | ||
| dirPath = path.resolve(options.dirname); | ||
| } | ||
| else dirPath = process.cwd(); | ||
| // Run `saveAs` to get the desired name for the file | ||
| options.saveAs(__newFile, function (err, filename){ | ||
| if (err) return done(err); | ||
| if (options.fd) { | ||
| fd = path.resolve(options.fd); | ||
| } | ||
| else fd = path.join(dirPath, filename); | ||
| // Attach fd as metadata to the file stream for use back in skipper core | ||
| __newFile._skipperFD = fd; | ||
| // | ||
| // ------------------------------------------------------- | ||
| // ------------------------------------------------------- | ||
| // ------------------------------------------------------- | ||
| // Ensure necessary parent directories exist: | ||
| fsx.mkdirs(dirPath, function(mkdirsErr) { | ||
| // If we get an error here, it's probably because the Node | ||
| // user doesn't have write permissions at the designated | ||
| // path. | ||
| if (mkdirsErr) { | ||
| return done(mkdirsErr); | ||
| } | ||
| // Error reading from the file stream | ||
| __newFile.on('error', function(err) { | ||
| log('***** READ error on file ' + __newFile.filename, '::', err); | ||
| }); | ||
| // Create a new write stream to write to disk | ||
| var outs__ = fsx.createWriteStream(fd, encoding); | ||
| // When the file is done writing, call the callback | ||
| outs__.on('finish', function successfullyWroteFile() { | ||
| log('finished file: ' + __newFile.filename); | ||
| done(); | ||
| }); | ||
| outs__.on('E_EXCEEDS_UPLOAD_LIMIT', function (err) { | ||
| done(err); | ||
| }); | ||
| var __progress__ = buildProgressStream(options, __newFile, receiver__, outs__); | ||
| // Finally pipe the progress THROUGH the progress stream | ||
| // and out to disk. | ||
| __newFile | ||
| .pipe(__progress__) | ||
| .pipe(outs__); | ||
| }); | ||
| }); | ||
| }; | ||
| return receiver__; | ||
| } // </DiskReceiver> | ||
| }; | ||
@@ -212,144 +59,1 @@ | ||
| function buildProgressStream (options, __newFile, receiver__, outs__) { | ||
| var log = options.log || function noOpLog(){}; | ||
| // Generate a progress stream and unique id for this file | ||
| // then pipe the bytes down to the outs___ stream | ||
| // We will pipe the incoming file stream to this, which will | ||
| var localID = _.uniqueId(); | ||
| var guessedTotal = 0; | ||
| var writtenSoFar = 0; | ||
| var __progress__ = new TransformStream(); | ||
| __progress__._transform = function(chunk, enctype, next) { | ||
| // Update the guessedTotal to make % estimate | ||
| // more accurate: | ||
| guessedTotal += chunk.length; | ||
| writtenSoFar += chunk.length; | ||
| // Do the actual "writing", which in our case will pipe | ||
| // the bytes to the outs___ stream that writes to disk | ||
| this.push(chunk); | ||
| // Emit an event that will calculate our total upload | ||
| // progress and determine whether we're within quota | ||
| this.emit('progress', { | ||
| id: localID, | ||
| fd: __newFile._skipperFD, | ||
| name: __newFile.name, | ||
| written: writtenSoFar, | ||
| total: guessedTotal, | ||
| percent: (writtenSoFar / guessedTotal) * 100 | 0 | ||
| }); | ||
| next(); | ||
| }; | ||
| // This event is fired when a single file stream emits a progress event. | ||
| // Each time we receive a file, we must recalculate the TOTAL progress | ||
| // for the aggregate file upload. | ||
| // | ||
| // events emitted look like: | ||
| /* | ||
| { | ||
| percentage: 9.05, | ||
| transferred: 949624, | ||
| length: 10485760, | ||
| remaining: 9536136, | ||
| eta: 10, | ||
| runtime: 0, | ||
| delta: 295396, | ||
| speed: 949624 | ||
| } | ||
| */ | ||
| __progress__.on('progress', function singleFileProgress(milestone) { | ||
| // Lookup or create new object to track file progress | ||
| var currentFileProgress = _.find(receiver__._files, { | ||
| id: localID | ||
| }); | ||
| if (currentFileProgress) { | ||
| currentFileProgress.written = milestone.written; | ||
| currentFileProgress.total = milestone.total; | ||
| currentFileProgress.percent = milestone.percent; | ||
| currentFileProgress.stream = __newFile; | ||
| } else { | ||
| currentFileProgress = { | ||
| id: localID, | ||
| fd: __newFile._skipperFD, | ||
| name: __newFile.filename, | ||
| written: milestone.written, | ||
| total: milestone.total, | ||
| percent: milestone.percent, | ||
| stream: __newFile | ||
| }; | ||
| receiver__._files.push(currentFileProgress); | ||
| } | ||
| //////////////////////////////////////////////////////////////// | ||
| // Recalculate `totalBytesWritten` so far for this receiver instance | ||
| // (across ALL OF ITS FILES) | ||
| // using the sum of all bytes written to each file in `receiver__._files` | ||
| totalBytesWritten = _.reduce(receiver__._files, function(memo, status) { | ||
| memo += status.written; | ||
| return memo; | ||
| }, 0); | ||
| log(currentFileProgress.percent, '::', currentFileProgress.written, '/', currentFileProgress.total, ' (file #' + currentFileProgress.id + ' :: ' + /*'update#'+counter*/ '' + ')'); //receiver__._files.length+' files)'); | ||
| // Emit an event on the receiver. Someone using Skipper may listen for this to show | ||
| // a progress bar, for example. | ||
| receiver__.emit('progress', currentFileProgress); | ||
| // and then enforce its `maxBytes`. | ||
| if (options.maxBytes && totalBytesWritten >= options.maxBytes) { | ||
| var err = new Error(); | ||
| err.code = 'E_EXCEEDS_UPLOAD_LIMIT'; | ||
| err.name = 'Upload Error'; | ||
| err.maxBytes = options.maxBytes; | ||
| err.written = totalBytesWritten; | ||
| err.message = 'Upload limit of ' + err.maxBytes + ' bytes exceeded (' + err.written + ' bytes written)'; | ||
| // Stop listening for progress events | ||
| __progress__.removeAllListeners('progress'); | ||
| // Unpipe the progress stream, which feeds the disk stream, so we don't keep dumping to disk | ||
| __progress__.unpipe(); | ||
| // Clean up any files we've already written | ||
| (function gc(err) { | ||
| // Garbage-collects the bytes that were already written for this file. | ||
| // (called when a read or write error occurs) | ||
| log('************** Garbage collecting file `' + __newFile.filename + '` located @ ' + fd + '...'); | ||
| adapter.rm(fd, function(gcErr) { | ||
| if (gcErr) return outs__.emit('E_EXCEEDS_UPLOAD_LIMIT',[err].concat([gcErr])); | ||
| return outs__.emit('E_EXCEEDS_UPLOAD_LIMIT',err); | ||
| }); | ||
| })(err); | ||
| return; | ||
| // Don't do this--it releases the underlying pipes, which confuses node when it's in the middle | ||
| // of a write operation. | ||
| // outs__.emit('error', err); | ||
| // | ||
| // | ||
| } | ||
| }); | ||
| return __progress__; | ||
| } | ||
| { | ||
| "name": "skipper-disk", | ||
| "version": "0.3.1", | ||
| "version": "0.4.0", | ||
| "description": "Receive Skipper's file uploads on your local filesystem", | ||
@@ -5,0 +5,0 @@ "main": "index.js", |
113
README.md
@@ -8,3 +8,2 @@ # [<img title="skipper-disk - Local disk adapter for Skipper" src="http://i.imgur.com/P6gptnI.png" width="200px" alt="skipper emblem - face of a ship's captain"/>](https://github.com/balderdashy/skipper-disk) Disk Blob Adapter | ||
| > This module is bundled as the default blob adapter in Skipper, and consequently [Sails](https://github.com/balderdashy/sails). | ||
@@ -19,2 +18,6 @@ ======================================== | ||
| Also make sure you have skipper [installed as your body parser](http://beta.sailsjs.org/#/documentation/concepts/Middleware?q=adding-or-overriding-http-middleware). | ||
| > Skipper is installed by defaut in [Sails](https://github.com/balderdashy/sails) v0.10. | ||
| ======================================== | ||
@@ -24,28 +27,63 @@ | ||
| First instantiate a blob adapter (`blobAdapter`): | ||
| > This module is bundled as the default file upload adapter in Skipper, so the following usage is slightly simpler than it is with the other Skipper file upload adapters. | ||
| ```js | ||
| var blobAdapter = require('skipper-disk')(); | ||
| In the route(s) / controller action(s) where you want to accept file uploads, do something like: | ||
| ```javascript | ||
| function (req, res) { | ||
| req.file('avatar') | ||
| .upload(function whenDone(err, uploadedFiles) { | ||
| if (err) return res.negotiate(err); | ||
| else return res.json({ | ||
| files: uploadedFiles, | ||
| textParams: req.params.all() | ||
| }); | ||
| }); | ||
| } | ||
| ``` | ||
| Build a receiver (`receiving`): | ||
| ```js | ||
| var receiving = blobAdapter.receive(); | ||
| ``` | ||
| ======================================== | ||
| Then stream file(s) from a particular field (`req.file('foo')`): | ||
| ## Details | ||
| ```js | ||
| req.file('foo').upload(receiving, function (err, filesUploaded) { | ||
| // ... | ||
| }); | ||
| ```javascript | ||
| function (req, res) { | ||
| req.file('avatar') | ||
| .upload({ | ||
| // Specify skipper-disk explicitly (you don't need to do this b/c skipper-disk is the default) | ||
| adapter: require('skipper-disk'), | ||
| // You can apply a file upload limit (in bytes) | ||
| maxBytes: 1000000 | ||
| }, function whenDone(err, uploadedFiles) { | ||
| if (err) return res.negotiate(err); | ||
| else return res.json({ | ||
| files: uploadedFiles, | ||
| textParams: req.params.all() | ||
| }); | ||
| }); | ||
| } | ||
| ``` | ||
| | Option | Type | Details | | ||
| |-----------|:----------:|---------| | ||
| | `dirname` | ((string)) | The path to the directory on disk where file uploads should be streamed. May be specified as an absolute path (e.g. `/Users/mikermcneil/foo`) or a relative path from the current working directory. Defaults to `".tmp/uploads/"`. `dirname` can be used with `saveAs`- the filename from saveAs will be relative to dirname. | ||
| | `saveAs()` | ((function)) -or- ((string)) | Optional. By default, Skipper decides an "at-rest" filename for your uploaded files (called the `fd`) by generating a UUID and combining it with the file's original file extension when it was uploaded ("e.g. 24d5f444-38b4-4dc3-b9c3-74cb7fbbc932.jpg"). <br/> If `saveAs` is specified as a string, any uploaded file(s) will be saved to that particular path instead (useful for simple single-file uploads).<br/> If `saveAs` is specified as a function, that function will be called each time a file is received, passing it the raw stream and a callback. When ready, your `saveAs` function should call the callback, passing the appropriate at-rest filename (`fd`) as the second argument to the callback (and passing an error as the first argument if something went wrong). For example: <br/> `function (__newFileStream,cb) { cb(null, 'theUploadedFile.foo'); }` | | ||
| ======================================== | ||
| ## Options | ||
| ## Specifying Options | ||
| All options may be passed either into the blob adapter's factory method: | ||
| All options may be passed in using any of the following approaches, in ascending priority order (e.g. the 3rd appraoch overrides the 1st) | ||
| ##### 1. In the blob adapter's factory method: | ||
| ```js | ||
@@ -57,3 +95,3 @@ var blobAdapter = require('skipper-disk')({ | ||
| Or directly into a receiver: | ||
| ##### 2. In a call to the `.receive()` factory method: | ||
@@ -66,13 +104,42 @@ ```js | ||
| ##### 3. Directly into the `.upload()` method of the Upstream returned by `req.file()`: | ||
| | Option | Type | Details | | ||
| |-----------|:----------:|---------| | ||
| | `dirname` | ((string)) | The path to the directory on disk where file uploads should be streamed. May be specified as an absolute path (e.g. `/Users/mikermcneil/foo`) or a relative path from the current working directory. Defaults to `".tmp/uploads/"` | ||
| | `saveAs()` | ((function)) | An optional function that can be used to define the logic for naming files (with callback optional). For example: <br/> `function (file) {return Math.random()+file.name;}` or `function (filename,cb) {foo.asyncall(function(err,result){ options.filename = result[0]; cb(null)}); }`<br/> By default, Skipper-disk generate a random-Number for filename on your disk (e.g. 24d5f444-38b4-4dc3-b9c3-74cb7fbbc932.jpg) - that is given as "id" in upload()-callback. <br/> | | ||
| ```js | ||
| var upstream = req.file('foo').upload({ | ||
| // These options will be applied unless overridden. | ||
| }); | ||
| ``` | ||
| ======================================== | ||
| ## Advanced Usage | ||
| ## Low-Level Usage | ||
| > **Warning:** | ||
| > You probably shouldn't try doing anything in this section unless you've implemented streams before, and in particular _streams2_ (i.e. "suck", not "spew" streams). | ||
| #### File adapter instances, receivers, upstreams, and binary streams | ||
| First instantiate a blob adapter (`blobAdapter`): | ||
| ```js | ||
| var blobAdapter = require('skipper-disk')(); | ||
| ``` | ||
| Build a receiver (`receiving`): | ||
| ```js | ||
| var receiving = blobAdapter.receive(); | ||
| ``` | ||
| Then you can stream file(s) from a particular field (`req.file('foo')`): | ||
| ```js | ||
| req.file('foo').upload(receiving, function (err, filesUploaded) { | ||
| // ... | ||
| }); | ||
| ``` | ||
| #### `upstream.pipe(receiving)` | ||
@@ -95,2 +162,4 @@ | ||
| ======================================== | ||
@@ -97,0 +166,0 @@ |
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Improved metrics
- Total package byte prevSize
- increased by19.63%
21727
- Number of package files
- increased by37.5%
11
- Lines of code
- increased by0.76%
264
- Number of lines in readme file
- increased by59.48%
185
Worsened metrics
- Number of low supply chain risk alerts
- increased by100%
2
No dependency changes