unzip-stream - npm Package Compare versions

Comparing version 0.2.3 to 0.3.0

lib/unzip-stream.js

@@ -20,4 +20,8 @@ 'use strict';
     CENTRAL_DIRECTORY_FILE_HEADER_SUFFIX: 8,
-    CENTRAL_DIRECTORY_END:                9,
-    CENTRAL_DIRECTORY_END_COMMENT:        10,
+    CDIR64_END:                           9,
+    CDIR64_END_DATA_SECTOR:               10,
+    CDIR64_LOCATOR:                       11,
+    CENTRAL_DIRECTORY_END:                12,
+    CENTRAL_DIRECTORY_END_COMMENT:        13,
+    TRAILING_JUNK:                        14,
 
@@ -29,6 +33,8 @@ ERROR: 99
 
-const LOCAL_FILE_HEADER_SIG = 0x04034b50;
-const DATA_DESCRIPTOR_SIG = 0x08074b50;
-const CENTRAL_DIRECTORY_SIG = 0x02014b50;
-const CENTRAL_DIRECTORY_END_SIG = 0x06054b50;
+const SIG_LOCAL_FILE_HEADER  = 0x04034b50;
+const SIG_DATA_DESCRIPTOR    = 0x08074b50;
+const SIG_CDIR_RECORD        = 0x02014b50;
+const SIG_CDIR64_RECORD_END  = 0x06064b50;
+const SIG_CDIR64_LOCATOR_END = 0x07064b50;
+const SIG_CDIR_RECORD_END    = 0x06054b50;
 
@@ -45,2 +51,3 @@ function UnzipStream(options) {
     this.state = states.STREAM_START;
+    this.skippedBytes = 0;
     this.parsedEntity = null;
@@ -75,5 +82,17 @@ this.outStreamInfo = {};
             break;
+        case states.CDIR64_END:
+            requiredLength = 52;
+            break;
+        case states.CDIR64_END_DATA_SECTOR:
+            requiredLength = this.parsedEntity.centralDirectoryRecordSize - 44;
+            break;
+        case states.CDIR64_LOCATOR:
+            requiredLength = 16;
+            break;
         case states.CENTRAL_DIRECTORY_END:
             requiredLength = 18;
             break;
+        case states.CENTRAL_DIRECTORY_END_COMMENT:
+            requiredLength = this.parsedEntity.commentLength;
+            break;
         case states.FILE_DATA:
@@ -83,2 +102,5 @@ return 0;
             return 0;
+        case states.TRAILING_JUNK:
+            if (this.options.debug) console.log("found", chunk.length, "bytes of TRAILING_JUNK");
+            return chunk.length;
         default:
@@ -96,10 +118,17 @@ return chunk.length;
         case states.START:
-            switch (chunk.readUInt32LE(0)) {
-                case LOCAL_FILE_HEADER_SIG:
+            var signature = chunk.readUInt32LE(0);
+            switch (signature) {
+                case SIG_LOCAL_FILE_HEADER:
                     this.state = states.LOCAL_FILE_HEADER;
                     break;
-                case CENTRAL_DIRECTORY_SIG:
+                case SIG_CDIR_RECORD:
                     this.state = states.CENTRAL_DIRECTORY_FILE_HEADER;
                     break;
-                case CENTRAL_DIRECTORY_END_SIG:
+                case SIG_CDIR64_RECORD_END:
+                    this.state = states.CDIR64_END;
+                    break;
+                case SIG_CDIR64_LOCATOR_END:
+                    this.state = states.CDIR64_LOCATOR;
+                    break;
+                case SIG_CDIR_RECORD_END:
                     this.state = states.CENTRAL_DIRECTORY_END;
@@ -109,2 +138,17 @@ break;
                     var isStreamStart = this.state === states.STREAM_START;
+                    if (!isStreamStart && (signature & 0xffff) !== 0x4b50 && this.skippedBytes < 26) {
+                        // we'll allow a padding of max 28 bytes
+                        var remaining = signature;
+                        var toSkip = 4;
+                        for (var i = 1; i < 4 && remaining !== 0; i++) {
+                            remaining = remaining >>> 8;
+                            if ((remaining & 0xff) === 0x50) {
+                                toSkip = i;
+                                break;
+                            }
+                        }
+                        this.skippedBytes += toSkip;
+                        if (this.options.debug) console.log('Skipped', this.skippedBytes, 'bytes');
+                        return toSkip;
+                    }
                     this.state = states.ERROR;
@@ -116,3 +160,3 @@ var errMsg = isStreamStart ? "Not a valid zip file" : "Invalid signature in zip file";
                         try { asString = chunk.slice(0, 4).toString(); } catch (e) {}
-                        console.log("Unexpected signature in zip file: 0x" + sig.toString(16), '"' + asString + '"');
+                        console.log("Unexpected signature in zip file: 0x" + sig.toString(16), '"' + asString + '", skipped', this.skippedBytes, 'bytes');
                     }
@@ -122,2 +166,3 @@ this.emit("error", new Error(errMsg));
             }
+            this.skippedBytes = 0;
             return requiredLength;
@@ -137,6 +182,14 @@
             var extra = this._readExtraFields(extraDataBuffer);
-            if (extra && extra.parsed && extra.parsed.path && !isUtf8) {
-                entry.path = extra.parsed.path;
+            if (extra && extra.parsed) {
+                if (extra.parsed.path && !isUtf8) {
+                    entry.path = extra.parsed.path;
+                }
+                if (Number.isFinite(extra.parsed.uncompressedSize) && this.parsedEntity.uncompressedSize === FOUR_GIGS-1) {
+                    this.parsedEntity.uncompressedSize = extra.parsed.uncompressedSize;
+                }
+                if (Number.isFinite(extra.parsed.compressedSize) && this.parsedEntity.compressedSize === FOUR_GIGS-1) {
+                    this.parsedEntity.compressedSize = extra.parsed.compressedSize;
+                }
             }
-            this.parsedEntity.extra = extra.parsed;
+            this.parsedEntity.extra = extra.parsed || {};
 
@@ -149,3 +202,3 @@ if (this.options.debug) {
                 });
-                console.log("decoded local file entry:", JSON.stringify(debugObj, null, 2));
+                console.log("decoded LOCAL_FILE_HEADER:", JSON.stringify(debugObj, null, 2));
             }
@@ -192,3 +245,3 @@ this._prepareOutStream(this.parsedEntity, entry);
                 });
-                console.log("decoded central directory file entry:", JSON.stringify(debugObj, null, 2));
+                console.log("decoded CENTRAL_DIRECTORY_FILE_HEADER:", JSON.stringify(debugObj, null, 2));
             }
@@ -199,3 +252,27 @@ this.state = states.START;
 
+        case states.CDIR64_END:
+            this.parsedEntity = this._readEndOfCentralDirectory64(chunk);
+            if (this.options.debug) {
+                console.log("decoded CDIR64_END_RECORD:", this.parsedEntity);
+            }
+            this.state = states.CDIR64_END_DATA_SECTOR;
+
+            return requiredLength;
+
+        case states.CDIR64_END_DATA_SECTOR:
+            this.state = states.START;
+
+            return requiredLength;
+
+        case states.CDIR64_LOCATOR:
+            // ignore, nothing interesting
+            this.state = states.START;
+
+            return requiredLength;
+
         case states.CENTRAL_DIRECTORY_END:
+            this.parsedEntity = this._readEndOfCentralDirectory(chunk);
+            if (this.options.debug) {
+                console.log("decoded CENTRAL_DIRECTORY_END:", this.parsedEntity);
+            }
             this.state = states.CENTRAL_DIRECTORY_END_COMMENT;
@@ -206,4 +283,9 @@
         case states.CENTRAL_DIRECTORY_END_COMMENT:
-            return chunk.length;
+            if (this.options.debug) {
+                console.log("decoded CENTRAL_DIRECTORY_END_COMMENT:", chunk.slice(0, requiredLength).toString());
+            }
+            this.state = states.TRAILING_JUNK;
 
+            return requiredLength;
+
         case states.ERROR:
@@ -221,3 +303,3 @@ return chunk.length; // discard
 
-    var isDirectory = vars.compressedSize === 0 && /[\/\\]$/.test(entry.path);
+    var isDirectory = vars.uncompressedSize === 0 && /[\/\\]$/.test(entry.path);
     // protect against malicious zip files which want to extract to parent dirs
@@ -233,3 +315,3 @@ entry.path = entry.path.replace(/^([/\\]*[.]+[/\\]+)*[/\\]*/, "");
 
-    var isVersionSupported = vars.versionsNeededToExtract <= 21;
+    var isVersionSupported = vars.versionsNeededToExtract <= 45;
 
@@ -244,14 +326,16 @@ this.outStreamInfo = {
         var pattern = new Buffer(4);
-        pattern.writeUInt32LE(DATA_DESCRIPTOR_SIG, 0);
+        pattern.writeUInt32LE(SIG_DATA_DESCRIPTOR, 0);
+        var zip64Mode = vars.extra.zip64Mode;
+        var extraSize = zip64Mode ? 20 : 12;
         var searchPattern = {
             pattern: pattern,
-            requiredExtraSize: 12
+            requiredExtraSize: extraSize
         }
 
         var matcherStream = new MatcherStream(searchPattern, function (matchedChunk, sizeSoFar) {
-            var vars = self._readDataDescriptor(matchedChunk);
+            var vars = self._readDataDescriptor(matchedChunk, zip64Mode);
 
             var compressedSizeMatches = vars.compressedSize === sizeSoFar;
             // let's also deal with archives with 4GiB+ files without zip64
-            if (!compressedSizeMatches && sizeSoFar >= FOUR_GIGS) {
+            if (!zip64Mode && !compressedSizeMatches && sizeSoFar >= FOUR_GIGS) {
                 var overflown = sizeSoFar - FOUR_GIGS;
@@ -267,6 +351,7 @@ while (overflown >= 0) {
             self.state = states.FILE_DATA_END;
+            var sliceOffset = zip64Mode ? 24 : 16;
             if (self.data.length > 0) {
-                self.data = Buffer.concat([matchedChunk.slice(16), self.data]);
+                self.data = Buffer.concat([matchedChunk.slice(sliceOffset), self.data]);
             } else {
-                self.data = matchedChunk.slice(16);
+                self.data = matchedChunk.slice(sliceOffset);
             }
@@ -348,2 +433,18 @@
         switch (vars.extraId) {
+            case 0x0001:
+                fieldType = "Zip64 extended information extra field";
+                var z64vars = binary.parse(data.slice(index, index+vars.extraSize))
+                    .word64lu('uncompressedSize')
+                    .word64lu('compressedSize')
+                    .word64lu('offsetToLocalHeader')
+                    .word32lu('diskStartNumber')
+                    .vars;
+                if (z64vars.uncompressedSize !== null) {
+                    extra.uncompressedSize = z64vars.uncompressedSize;
+                }
+                if (z64vars.compressedSize !== null) {
+                    extra.compressedSize = z64vars.compressedSize;
+                }
+                extra.zip64Mode = true;
+                break;
             case 0x000a:
@@ -475,3 +576,14 @@ fieldType = "NTFS extra field";
 
-UnzipStream.prototype._readDataDescriptor = function (data) {
+UnzipStream.prototype._readDataDescriptor = function (data, zip64Mode) {
+    if (zip64Mode) {
+        var vars = binary.parse(data)
+            .word32lu('dataDescriptorSignature')
+            .word32lu('crc32')
+            .word64lu('compressedSize')
+            .word64lu('uncompressedSize')
+            .vars;
+
+        return vars;
+    }
+
     var vars = binary.parse(data)
@@ -510,2 +622,18 @@ .word32lu('dataDescriptorSignature')
 
+UnzipStream.prototype._readEndOfCentralDirectory64 = function (data) {
+    var vars = binary.parse(data)
+        .word64lu('centralDirectoryRecordSize')
+        .word16lu('versionMadeBy')
+        .word16lu('versionsNeededToExtract')
+        .word32lu('diskNumber')
+        .word32lu('diskNumberWithCentralDirectoryStart')
+        .word64lu('centralDirectoryEntries')
+        .word64lu('totalCentralDirectoryEntries')
+        .word64lu('sizeOfCentralDirectory')
+        .word64lu('offsetToStartOfCentralDirectory')
+        .vars;
+
+    return vars;
+}
+
 UnzipStream.prototype._readEndOfCentralDirectory = function (data) {
@@ -515,4 +643,4 @@ var vars = binary.parse(data)
         .word16lu('diskStart')
-        .word16lu('numberOfRecordsOnDisk')
-        .word16lu('numberOfRecords')
+        .word16lu('centralDirectoryEntries')
+        .word16lu('totalCentralDirectoryEntries')
         .word32lu('sizeOfCentralDirectory')
@@ -519,0 +647,0 @@ .word32lu('offsetToStartOfCentralDirectory')

package.json

 {
   "name": "unzip-stream",
-  "version": "0.2.3",
+  "version": "0.3.0",
   "description": "Process zip files using streaming API",
@@ -22,6 +22,6 @@ "author": "Michal Hruby <[e-mail]>",
   "devDependencies": {
-    "tap": ">= 0.3.0 < 1",
-    "temp": ">= 0.4.0 < 1",
     "dirdiff": ">= 0.0.1 < 1",
-    "stream-buffers": ">= 0.2.5 < 1"
+    "stream-buffers": ">= 0.2.5 < 1",
+    "tap": "^11.0.1",
+    "temp": ">= 0.4.0 < 1"
   },
@@ -43,4 +43,5 @@ "directories": {
   "scripts": {
-    "test": "tap ./test/*.js"
+    "test": "tap ./test/*.js",
+    "coverage": "tap ./test/*.js --cov --coverage-report=html"
   }
 }

README.md

@@ -83,2 +83,2 @@ # unzip-stream
 
-Currently only ZIP files up to version 2.1 are supported - which means no Zip64 support. There's also no support for encrypted (password protected) zips, or symlinks.
+Currently ZIP files up to version 4.5 are supported (which includes Zip64 support - archives with 4GB+ files). There's no support for encrypted (password protected) zips, or symlinks.

test.js

@@ -6,3 +6,3 @@ const unzip = require('./unzip');
 
-//console.log('Trying to open', process.argv[2]);
+console.log('Trying to open', process.argv[2]);
 
@@ -12,2 +12,3 @@ let parser = unzip.Parse({debug: true});
 
+/*
 let req = http.get('https://www.colorado.edu/conflict/peace/download/peace_example.ZIP');
@@ -18,3 +19,4 @@
 });
-//fs.createReadStream(process.argv[2]).pipe(parser);
+*/
+fs.createReadStream(process.argv[2]).pipe(parser);
 

No alert changes

Improved metrics

Total package byte prevSize

782895

increased by1344.32%

Number of package files

76

increased by181.48%

Lines of code

1226

increased by32.4%

Worsened metrics

Number of low supply chain risk alerts

11

increased by57.14%

No dependency changes