@ronomon/deduplication - npm Package Compare versions

Comparing version 1.3.0 to 1.4.0

demo.js

@@ -8,3 +8,3 @@ var assert = require('assert');
 if (!file) {
-  console.error('usage: node demo.js file');
+  console.error('usage: node demo.js <file>');
   return;
@@ -14,3 +14,3 @@ }
 var fileOffset = 0;
-var fileSize = fs.fstatSync(fd).size;
+var chunkOffset = 0;
 
@@ -28,18 +28,15 @@ // Recommended average, minimum and maximum chunk size constants:
   if (error) throw error;
+  assert(chunkOffset === fileOffset);
 }
 
-function read() {
-  var length = Math.min(source.length, fileSize - fileOffset);
-  assert(length >= 0);
-  if (length === 0) return close();
-  var sourceSize = fs.readSync(fd, source, 0, length, fileOffset);
-  if (sourceSize === 0) return close();
-  write(sourceSize);
+function read(sourceStart) {
+  var length = source.length - sourceStart;
+  assert(length > 0);
+  var bytesRead = fs.readSync(fd, source, sourceStart, length, fileOffset);
+  fileOffset += bytesRead;
+  var flags = (bytesRead < length) ? 1 : 0;
+  write(sourceStart + bytesRead, flags);
 }
 
-function write(sourceSize) {
-  assert(fileOffset + sourceSize <= fileSize);
-  var flags = 0;
-  // We set flags = 1 to indicate if this is the last source buffer:
-  if (fileOffset + sourceSize === fileSize) flags |= 1;
+function write(sourceSize, flags) {
   Deduplication.deduplicate(
@@ -66,8 +63,23 @@ average,
         offset += 4;
-        console.log('hash=' + hash + ' offset=' + fileOffset + ' size=' + size);
-        fileOffset += size;
+        console.log(
+          'hash=' + hash + ' offset=' + chunkOffset + ' size=' + size
+        );
+        chunkOffset += size;
       }
       assert(offset === targetOffset);
-      if (flags === 1) assert(fileOffset === fileSize);
-      read();
+      // Anything remaining in the source buffer should be moved to the
+      // beginning of the source buffer, and become the sourceStart for the
+      // next read so that we do not read data we have already read:
+      var remaining = sourceSize - sourceOffset;
+      if (remaining > 0) {
+        // Assert that we can copy directly within the source buffer:
+        assert(remaining < sourceOffset);
+        source.copy(source, 0, sourceOffset, sourceOffset + remaining);
+      }
+      if (flags === 1) {
+        assert(remaining === 0);
+        close();
+      } else {
+        read(remaining);
+      }
     }
@@ -77,2 +89,2 @@ );
 
-read();
+read(0);

package.json

 {
   "name": "@ronomon/deduplication",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Fast multi-threaded content-dependent chunking deduplication for Buffers in C++ with a reference implementation in Javascript. Ships with extensive tests, a fuzz test and a benchmark.",
@@ -5,0 +5,0 @@ "main": "binding.node",

README.md

@@ -120,3 +120,3 @@ # deduplication
 if (!file) {
-  console.error('usage: node demo.js file');
+  console.error('usage: node demo.js <file>');
   return;
@@ -126,3 +126,3 @@ }
 var fileOffset = 0;
-var fileSize = fs.fstatSync(fd).size;
+var chunkOffset = 0;
 
@@ -140,18 +140,15 @@ // Recommended average, minimum and maximum chunk size constants:
   if (error) throw error;
+  assert(chunkOffset === fileOffset);
 }
 
-function read() {
-  var length = Math.min(source.length, fileSize - fileOffset);
-  assert(length >= 0);
-  if (length === 0) return close();
-  var sourceSize = fs.readSync(fd, source, 0, length, fileOffset);
-  if (sourceSize === 0) return close();
-  write(sourceSize);
+function read(sourceStart) {
+  var length = source.length - sourceStart;
+  assert(length > 0);
+  var bytesRead = fs.readSync(fd, source, sourceStart, length, fileOffset);
+  fileOffset += bytesRead;
+  var flags = (bytesRead < length) ? 1 : 0;
+  write(sourceStart + bytesRead, flags);
 }
 
-function write(sourceSize) {
-  assert(fileOffset + sourceSize <= fileSize);
-  var flags = 0;
-  // We set flags = 1 to indicate if this is the last source buffer:
-  if (fileOffset + sourceSize === fileSize) flags |= 1;
+function write(sourceSize, flags) {
   Deduplication.deduplicate(
@@ -178,8 +175,23 @@ average,
         offset += 4;
-        console.log('hash=' + hash + ' offset=' + fileOffset + ' size=' + size);
-        fileOffset += size;
+        console.log(
+          'hash=' + hash + ' offset=' + chunkOffset + ' size=' + size
+        );
+        chunkOffset += size;
       }
       assert(offset === targetOffset);
-      if (flags === 1) assert(fileOffset === fileSize);
-      read();
+      // Anything remaining in the source buffer should be moved to the
+      // beginning of the source buffer, and become the sourceStart for the
+      // next read so that we do not read data we have already read:
+      var remaining = sourceSize - sourceOffset;
+      if (remaining > 0) {
+        // Assert that we can copy directly within the source buffer:
+        assert(remaining < sourceOffset);
+        source.copy(source, 0, sourceOffset, sourceOffset + remaining);
+      }
+      if (flags === 1) {
+        assert(remaining === 0);
+        close();
+      } else {
+        read(remaining);
+      }
     }
@@ -189,3 +201,3 @@ );
 
-read();
+read(0);
 ```
@@ -192,0 +204,0 @@

test.js

@@ -433,2 +433,3 @@ var crypto = require('crypto');
     function verify(sourceOffset, targetOffset) {
+      Test.equal(hash(test.source), test.sourceHash, namespace, 'source');
       var sourceSize = sourceOffset - test.sourceOffset;
@@ -435,0 +436,0 @@ Test.equal(isInt(sourceSize), test.sourceSize, namespace, 'sourceSize');

No alert changes

Improved metrics

Total package byte prevSize

91296

increased by1.01%

Lines of code

1160

increased by1.13%

Number of lines in readme file

215

increased by5.91%

No dependency changes