@chainsafe/snappy-stream - npm Package Compare versions

Comparing version 5.0.0 to 5.0.3

.github/.dependabot.yaml

+version: 2
+updates:
+  - package-ecosystem: "yarn"
+    allow:
+      # Allow both direct and indirect updates for all packages
+      - dependency-type: "production"
+    commit-message:
+      prefix: "chore: "

.github/workflows/semantic-pr.yaml

+name: "Semantic PR"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  main:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          types: |
+            fix
+            feat
+            chore
+          validateSingleCommit: true
+          

CHANGELOG.md

+# Changelog
+
+### [5.0.3](https://github.com/ChainSafe/node-snappy-stream/compare/v5.0.2...v5.0.3) (2022-04-27)
+
+
+### Bug Fixes
+
+* compressor didn't chunkify big payload ([#3](https://github.com/ChainSafe/node-snappy-stream/issues/3)) ([a658af0](https://github.com/ChainSafe/node-snappy-stream/commit/a658af0c165d2b7885f99aa7b6e297f93d530701))
+
+
+### Miscellaneous
+
+* automatic release ([#4](https://github.com/ChainSafe/node-snappy-stream/issues/4)) ([2051ada](https://github.com/ChainSafe/node-snappy-stream/commit/2051ada25695c0f8f381b638435d47a2c759bde0))
+* **master:** release 5.0.1 ([#5](https://github.com/ChainSafe/node-snappy-stream/issues/5)) ([f646c2f](https://github.com/ChainSafe/node-snappy-stream/commit/f646c2f1be28e03ffca60e2633614a188dd39c17))
+* **master:** release 5.0.2 ([#8](https://github.com/ChainSafe/node-snappy-stream/issues/8)) ([7967349](https://github.com/ChainSafe/node-snappy-stream/commit/7967349142fce27eb67750c67957039858cee2a7))
+
+### [5.0.2](https://github.com/ChainSafe/node-snappy-stream/compare/v5.0.1...v5.0.2) (2022-04-27)
+
+
+### Bug Fixes
+
+* compressor didn't chunkify big payload ([#3](https://github.com/ChainSafe/node-snappy-stream/issues/3)) ([a658af0](https://github.com/ChainSafe/node-snappy-stream/commit/a658af0c165d2b7885f99aa7b6e297f93d530701))
+
+
+### Miscellaneous
+
+* automatic release ([#4](https://github.com/ChainSafe/node-snappy-stream/issues/4)) ([2051ada](https://github.com/ChainSafe/node-snappy-stream/commit/2051ada25695c0f8f381b638435d47a2c759bde0))
+* **master:** release 5.0.1 ([#5](https://github.com/ChainSafe/node-snappy-stream/issues/5)) ([f646c2f](https://github.com/ChainSafe/node-snappy-stream/commit/f646c2f1be28e03ffca60e2633614a188dd39c17))
+
+### [5.0.1](https://github.com/ChainSafe/node-snappy-stream/compare/v5.0.0...v5.0.1) (2022-04-26)
+
+
+### Bug Fixes
+
+* compressor didn't chunkify big payload ([#3](https://github.com/ChainSafe/node-snappy-stream/issues/3)) ([a658af0](https://github.com/ChainSafe/node-snappy-stream/commit/a658af0c165d2b7885f99aa7b6e297f93d530701))
+
+
+### Miscellaneous
+
+* automatic release ([#4](https://github.com/ChainSafe/node-snappy-stream/issues/4)) ([2051ada](https://github.com/ChainSafe/node-snappy-stream/commit/2051ada25695c0f8f381b638435d47a2c759bde0))

.github/workflows/main.yaml

@@ -1,2 +0,2 @@
-name: Node.js CI
+name: Node.js CI/CD
 
@@ -10,3 +10,3 @@ on:
 jobs:
-  build:
+  test:
 
@@ -20,3 +20,3 @@ runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - run: sudo apt-get update -qq
@@ -26,6 +26,39 @@ - run: sudo apt-get install libsnappy-dev
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
-      - run: yarn --frozen-lockfile
-      - run: yarn test
+      - run: yarn
+      - run: yarn test
+  
+  maybe-release:
+    name: release
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    steps:
+        # you should probably do this after your regular CI checks passes
+      - uses: google-github-actions/release-please-action@v3 # it will analyze commits and create PR with new version and updated CHANGELOG:md file. On merging it will create github release page with changelog
+        id: release
+        with:
+          release-type: node
+          package-name: release-please-action
+          changelog-types: '[{"type":"feat","section":"Features","hidden":false},{"type":"fix","section":"Bug Fixes","hidden":false},{"type":"chore","section":"Miscellaneous","hidden":false}]'
+      
+      - uses: actions/checkout@v3
+        # these if statements ensure that a publication only occurs when
+        # a new release is created:
+        if: ${{ steps.release.outputs.release_created }}
+        
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+        if: ${{ steps.release.outputs.release_created }}
+      
+      - run: yarn install
+        if: ${{ steps.release.outputs.release_created }}
+
+      - run: yarn publish
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} # use npm "Automation" token and put in Github repository secrets under "NPM_TOKEN"
+        if: ${{ steps.release.outputs.release_created }}

lib/compress-stream.js

@@ -0,1 +1,9 @@
+/**
+ * As per the snappy framing format for streams, the size of any uncompressed chunk can be
+ * no longer than 65536 bytes.
+ *
+ * From: https://github.com/google/snappy/blob/main/framing_format.txt#L90:L92
+ */
+const UNCOMPRESSED_CHUNK_SIZE = 65536;
+
 var Transform = require('stream').Transform
@@ -53,18 +61,37 @@ , util = require('util')
 
-CompressStream.prototype._transform = function (chunk, enc, callback) {
-  var self = this
+/**
+ * Some compression benchmarks : 
+ * 
+ *   i) Chunking in transform with snappy.compressSync (the new implementation)
+ *   ii) Chunking from outside with compressStream.write (using original snappy.compress)
+ *   iii) No chunking (Original)
+ * 
+ *  | Size              | in transform | compressStream.write | orginal (no chunking) |
+ *  |-------------------|--------------|----------------------|-----------------------|
+ *  | 10kb   (1 chunk)  | 0.0229 ms    | 0.0385 ms            | 0.0388 ms             |
+ *  | 100kb  (2 chunks) | 0.0562 ms    | 0.1051 ms            | 0.0844 ms             |
+ *  | 1000kb (16 chunks)| 0.382  ms    | 0.7971 ms            | 0.1998 ms             |
+ * 
+ */
 
-  snappy.compress(chunk, function (err, compressed) {
-    if (err)
-      return callback(err)
+CompressStream.prototype._transform = function(chunk, enc, callback) {
+  new Promise(() => {
+    try {
+      for (let startFrom = 0; startFrom < chunk.length; startFrom += UNCOMPRESSED_CHUNK_SIZE) {
+        const endAt = startFrom + Math.min(chunk.length - startFrom, UNCOMPRESSED_CHUNK_SIZE);
+        const bytesChunk = chunk.slice(startFrom, endAt);
+        const compressed = snappy.compressSync(bytesChunk)
+        if (compressed.length < bytesChunk.length)
+          this._compressed(bytesChunk, compressed)
+        else
+          this._uncompressed(bytesChunk)
 
-    if (compressed.length < chunk.length)
-      self._compressed(chunk, compressed)
-    else
-      self._uncompressed(chunk)
-
-    callback()
-  })
+      }
+      callback();
+    } catch (err) {
+      return callback(err);
+    }
+  }).catch(e => console.log(e))
 }
 
-module.exports = CompressStream
+module.exports = CompressStream

package.json

 {
   "name": "@chainsafe/snappy-stream",
-  "version": "5.0.0",
+  "version": "5.0.3",
   "description": "Compress data over a Stream using the snappy framing format",
@@ -14,3 +14,6 @@ "main": "index.js",
   ],
-  "author": "David Björklund <[e-mail]>",
+  "contributors": [
+    "David Björklund <[e-mail]>",
+    "ChainSafe <[e-mail]>"
+  ],
   "license": "MIT",
@@ -34,8 +37,8 @@ "devDependencies": {
     "type": "git",
-    "url": "https://github.com/kesla/node-snappy-stream.git"
+    "url": "https://github.com/chainsafe/node-snappy-stream.git"
   },
   "bugs": {
-    "url": "https://github.com/kesla/node-snappy-stream/issues"
+    "url": "https://github.com/chainsafe/node-snappy-stream/issues"
   },
-  "homepage": "https://github.com/kesla/node-snappy-stream"
+  "homepage": "https://github.com/chainsafe/node-snappy-stream"
 }

test/compress-test.js

 var spawn = require('child_process').spawn
-
   , createCompressStream = require('../').createCompressStream
@@ -8,2 +7,9 @@ , test = require('tap').test
 
+const UNCOMPRESSED_CHUNK_SIZE = 65536
+let superLargeInput = largerInput;
+for (let i = largerInput.length; i <= UNCOMPRESSED_CHUNK_SIZE; i += largerInput.length) {
+  superLargeInput = Buffer.concat([superLargeInput, largerInput]);
+}
+const superLargeInputString = superLargeInput.toString();
+
 test('compress small string', function (t) {
@@ -52,1 +58,24 @@ var child = spawn('python', [ '-m', 'snappy', '-d' ])
 })
+
+
+test('compress very very large string', function (t) {
+  var child = spawn('python', [ '-m', 'snappy', '-d' ])
+    , compressStream = createCompressStream()
+    , data = ''
+
+  child.stdout.on('data', function (chunk) {
+    data = data + chunk.toString()
+  })
+
+  child.stdout.on('end', function () {
+    t.equal(data, superLargeInputString)
+    t.end()
+  })
+
+  child.stderr.pipe(process.stderr)
+
+  compressStream.pipe(child.stdin)
+
+  compressStream.write(superLargeInputString)
+  compressStream.end()
+})

No alert changes

Improved metrics

Total package byte prevSize

47010

17.44%

Number of package files

18

28.57%

Lines of code

571

9.6%

No dependency changes