formstream - npm Package Compare versions

Comparing version 1.4.0 to 1.5.0

lib/formstream.js

@@ -5,2 +5,3 @@ /**
 
+```txt
 --FormStreamBoundary1349886663601\r\n
@@ -19,3 +20,6 @@ Content-Disposition: form-data; name="foo"\r\n
 \r\n
-<FILE-CONTENT>\r\n
+<FILE-CONTENT-CHUNK-1>
+...
+<FILE-CONTENT-CHUNK-N>
+\r\n
 --FormStreamBoundary1349886663601\r\n
@@ -27,2 +31,3 @@ Content-Disposition: form-data; name="pic"; filename="fawave.png"\r\n
 --FormStreamBoundary1349886663601--
+```
 
@@ -34,2 +39,3 @@ *
 
+var debug = require('util').debuglog('formstream');
 var Stream = require('stream');
@@ -42,2 +48,3 @@ var parseStream = require('pause-stream');
 var destroy = require('destroy');
+var hex = require('node-hex');
 
@@ -48,5 +55,5 @@ var PADDING = '--';
 
-function FormStream() {
+function FormStream(options) {
   if (!(this instanceof FormStream)) {
-    return new FormStream();
+    return new FormStream(options);
   }
@@ -63,4 +70,6 @@
   this._knownStreamSize = 0;
+  this._minChunkSize = options && options.minChunkSize || 0;
 
   this.isFormStream = true;
+  debug('start boundary\n%s', this._boundary);
 }
@@ -93,3 +102,3 @@
     size += this._streams[i][0].length;
-    size += NEW_LINE_BUFFER.length; // stream field end pedding size
+    size += NEW_LINE_BUFFER.length; // stream field end padding size
   }
@@ -99,3 +108,3 @@
   this._isAllStreamSizeKnown = true;
-
+  debug('set total size: %s', size);
   return this;
@@ -111,3 +120,2 @@ };
   this._contentLength += this._knownStreamSize;
-
   // calculate length of end padding
@@ -126,2 +134,3 @@ this._contentLength += this._endData.length;
 
+  debug('headers: %j', headers);
   return headers;
@@ -131,11 +140,11 @@ };
 FormStream.prototype.file = function (name, filepath, filename, filesize) {
-  var mimeType = mime.getType(filepath);
-
   if (typeof filename === 'number' && !filesize) {
     filesize = filename;
     filename = path.basename(filepath);
-  } else if (!filename) {
+  }
+  if (!filename) {
     filename = path.basename(filepath);
   }
 
+  var mimeType = mime.getType(filename);
   var stream = fs.createReadStream(filepath);
@@ -213,11 +222,12 @@
 
-  this._buffers.push([leading, buffer]);
-
   // plus buffer length to total content-length
-  this._contentLength += leading.length;
-  this._contentLength += buffer.length;
-  this._contentLength += NEW_LINE_BUFFER.length;
+  var bufferSize = leading.length + buffer.length + NEW_LINE_BUFFER.length;
+  this._buffers.push(Buffer.concat([leading, buffer, NEW_LINE_BUFFER], bufferSize));  
+  this._contentLength += bufferSize;
 
   process.nextTick(this.resume.bind(this));
-
+  if (debug.enabled) {
+    debug('new buffer field, content size: %d\n%s%s',
+      buffer.length, leading.toString(), hex(buffer));
+  }
   return this;
@@ -229,12 +239,11 @@ };
 
-  var disps = [];
+  var dispositions = [];
 
   if (disposition) {
     for (var k in disposition) {
-      disps.push(k + '="' + disposition[k] + '"');
+      dispositions.push(k + '="' + disposition[k] + '"');
     }
   }
 
-  leading.push('Content-Disposition: form-data; ' + disps.join('; '));
-
+  leading.push('Content-Disposition: form-data; ' + dispositions.join('; '));
   if (type) {
@@ -246,3 +255,2 @@ leading.push('Content-Type: ' + type);
   leading.push('');
-
   return Buffer.from(leading.join(NEW_LINE));
@@ -257,8 +265,4 @@ };
   for (var i = 0; i < this._buffers.length; i++) {
-    var item = this._buffers[i];
-    this.emit('data', item[0]); // part leading
-    this.emit('data', item[1]); // part content
-    this.emit('data', NEW_LINE_BUFFER);
+    this.emit('data', this._buffers[i]);
   }
-
   this._buffers = [];
@@ -269,4 +273,11 @@ };
   var self = this;
-  // item: [ fieldData, stream ]
-  self.emit('data', item[0]);
+  // item: [ leading, stream ]
+  var streamSize = 0;
+  var chunkCount = 0;
+  const leading = item[0];
+  self.emit('data', leading);
+  chunkCount++;
+  if (debug.enabled) {
+    debug('new stream, chunk index %d\n%s', chunkCount, leading.toString());
+  }
 
@@ -276,5 +287,18 @@ var stream = item[1];
     self.emit('data', data);
+    streamSize += leading.length;
+    chunkCount++;
+    if (debug.enabled) {
+      if (data.length > 512) {
+        debug('stream chunk, size %d, chunk index %d, stream size %d\n%s......   only show 512 bytes   ......',
+          data.length, chunkCount, streamSize, hex(data.slice(0, 512)));
+      } else {
+        debug('stream chunk, size %d, chunk index %d, stream size %d\n%s',
+          data.length, chunkCount, streamSize, hex(data));
+      }
+    }
   });
   stream.on('end', function () {
     self.emit('data', NEW_LINE_BUFFER);
+    chunkCount++;
+    debug('stream end, chunk index %d, stream size %d', chunkCount, streamSize);
     return process.nextTick(self.drain.bind(self));
@@ -285,2 +309,61 @@ });
 
+FormStream.prototype._emitStreamWithChunkSize = function (item, minChunkSize) {
+  var self = this;
+  // item: [ leading, stream ]
+  var streamSize = 0;
+  var chunkCount = 0;
+  var bufferSize = 0;
+  var buffers = [];
+  const leading = item[0];
+  buffers.push(leading);
+  bufferSize += leading.length;
+  if (debug.enabled) {
+    debug('new stream, with min chunk size: %d\n%s', minChunkSize, leading.toString());
+  }
+
+  var stream = item[1];
+  stream.on('data', function (data) {
+    if (typeof data === 'string') {
+      data = Buffer.from(data, 'utf-8');
+    }
+    buffers.push(data);
+    bufferSize += data.length;
+    streamSize += data.length;
+    debug('got stream data size %d, buffer size %d, stream size %d',
+      data.length, bufferSize, streamSize);
+    if (bufferSize >= minChunkSize) {
+      const chunk = Buffer.concat(buffers, bufferSize);
+      buffers = [];
+      bufferSize = 0;
+      self.emit('data', chunk);
+      chunkCount++;
+      if (debug.enabled) {
+        if (chunk.length > 512) {
+          debug('stream chunk, size %d, chunk index %d, stream size %d\n%s......   only show 512 bytes   ......',
+            chunk.length, chunkCount, streamSize, hex(chunk.slice(0, 512)));
+        } else {
+          debug('stream chunk, size %d, chunk index %d, stream size %d\n%s',
+            chunk.length, chunkCount, streamSize, hex(chunk));
+        }
+      }
+    }
+  });
+  stream.on('end', function () {
+    buffers.push(NEW_LINE_BUFFER);
+    bufferSize += NEW_LINE_BUFFER.length;
+    const chunk = Buffer.concat(buffers, bufferSize);
+    self.emit('data', chunk);
+    chunkCount++;
+    if (chunk.length > 512) {
+      debug('stream end, size %d, chunk index %d, stream size %d\n%s......   only show 512 bytes   ......',
+        chunk.length, chunkCount, streamSize, hex(chunk.slice(0, 512)));
+    } else {
+      debug('stream end, size %d, chunk index %d, stream size %d\n%s',
+        chunk.length, chunkCount, streamSize, hex(chunk));
+    }
+    return process.nextTick(self.drain.bind(self));
+  });
+  stream.resume();
+};
+
 FormStream.prototype._emitEnd = function () {
@@ -292,5 +375,9 @@ // ending format:
   this.emit('end');
+  if (debug.enabled) {
+    debug('end boundary\n%s', this._endData.toString());
+  }
 };
 
 FormStream.prototype.drain = function () {
+  // debug('drain');
   this._emitBuffers();
@@ -300,3 +387,7 @@
   if (item) {
-    this._emitStream(item);
+    if (this._minChunkSize && this._minChunkSize > 0) {
+      this._emitStreamWithChunkSize(item, this._minChunkSize);
+    } else {
+      this._emitStream(item);
+    }
   } else {
@@ -310,2 +401,3 @@ this._emitEnd();
 FormStream.prototype.resume = function () {
+  // debug('resume');
   this.paused = false;
@@ -323,2 +415,3 @@
   this.emit('destroy');
+  // debug('destroy or close');
 };

package.json

 {
   "name": "formstream",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "A multipart/form-data encoded stream, helper for file upload.",
@@ -14,3 +14,3 @@ "main": "lib/formstream.js",
     "cov": "egg-bin cov",
-    "ci": "npm run lint && npm run tsd && npm run cov",
+    "ci": "npm run lint && npm run tsd && npm run cov && NODE_DEBUG=formstream npm run cov",
     "lint": "jshint .",
@@ -36,2 +36,3 @@ "tsd": "tsd",
     "mime": "^2.5.2",
+    "node-hex": "^1.0.1",
     "pause-stream": "~0.0.11"
@@ -38,0 +39,0 @@ },

README.md

 # formstream
 
 [![NPM version][npm-image]][npm-url]
+[![CI](https://github.com/node-modules/formstream/actions/workflows/ci.yml/badge.svg)](https://github.com/node-modules/formstream/actions/workflows/ci.yml)
+[![Test coverage][codecov-image]][codecov-url]
 [![npm download][download-image]][download-url]
@@ -8,2 +10,4 @@
 [npm-url]: https://npmjs.org/package/formstream
+[codecov-image]: https://codecov.io/github/node-modules/formstream/coverage.svg?branch=master
+[codecov-url]: https://codecov.io/github/node-modules/formstream?branch=master
 [download-image]: https://img.shields.io/npm/dm/formstream.svg?style=flat-square
@@ -17,3 +21,3 @@ [download-url]: https://npmjs.org/package/formstream
 ```bash
-$ npm install formstream
+npm install formstream
 ```
@@ -63,15 +67,40 @@
 fs.stat(filepath, function (err, stat) {
-  formstream().field('status', 'share picture')
-      .field('access_token', 'your access token')
-      .file('pic', filepath, 'logo.png', stat.size)
-      .pipe(process.stdout); // your request stream
+  formstream()
+    .field('status', 'share picture')
+    .field('access_token', 'your access token')
+    .file('pic', filepath, 'logo.png', stat.size)
+    .pipe(process.stdout); // your request stream
 });
 ```
 
+### Set min chunk buffer size
+
+Some web servers have a limit on the number of chunks, and you can set `minChunkSize` to ensure the size of chunk sent to the server.
+
+```js
+var fs = require('fs');
+var FormStream = require('formstream');
+
+var filepath = './big-file.zip';
+fs.stat(filepath, function (err, stat) {
+  new FormStream({
+    // send >= 2MB chunk buffer size to the server
+    minChunkSize: 1024 * 1024 * 2,
+  }).field('status', 'share file')
+    .field('access_token', 'your access token')
+    .file('file', filepath, 'big-file.zip', stat.size)
+    .pipe(process.stdout); // your request stream
+});
+```
+
 ## API Doc
 
-### formstream()
+### formstream([options])
 
 Create a form instance.
 
+#### Arguments
+
+- **options.minChunkSize** Number - min chunk size to emit data event
+
 #### Returns
@@ -146,3 +175,3 @@
 
-- ***headers*** Object - Additional headers
+- **headers** Object - Additional headers
 
@@ -180,25 +209,4 @@ #### Example
 
-(The MIT License)
+[MIT](LICENSE)
 
-Copyright (c) 2012 - 2014 fengmk2 <[e-mail]>
-Copyright(c) node-modules and other contributors.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 <!-- GITCONTRIBUTOR_START -->
@@ -214,2 +222,2 @@
 
-<!-- GITCONTRIBUTOR_END -->
+<!-- GITCONTRIBUTOR_END -->

types/formstream.d.ts

@@ -67,5 +67,10 @@ import { Readable, Stream } from 'stream'
 
+interface FormStreamOptions {
+  /** min chunk size to emit data event */
+  minChunkSize?: number;
+}
+
 declare const formStream: {
-  new (): FormStream
-  (): FormStream
+  new (options?: FormStreamOptions): FormStream
+  (options?: FormStreamOptions): FormStream
 }
@@ -72,0 +77,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

21947

increased by27.81%

Number of package files

5

increased by25%

Lines of code

407

increased by33.88%

Number of lines in readme file

218

increased by4.31%

Worsened metrics

Dependency count

4

increased by33.33%

Dependency changes

• + Addednode-hex@^1.0.1

• + Addednode-hex@1.0.1(transitive)