simple-xlsx-reader - npm Package Compare versions

Comparing version 1.0.2 to 1.0.3

package.json

 {
   "name": "simple-xlsx-reader",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "",
@@ -5,0 +5,0 @@ "main": "src/index.js",

src/index.js

 const unzipper = require('unzipper');
 const pipe = require('multipipe')
-const { Writable, Duplex } = require('stream');
+const { Writable, Duplex, Transform } = require('stream');
 const xmlParser = require('simple-xml-reader');
@@ -12,14 +12,34 @@ const fs = require('fs');
 
-const emptyChildrenOfSheetData = (elem) => {
-  if (elem.name === 'row') {
-    elem.parent.children = [];
+const throttle = (pauseInMilis, countToPause) => {
+  let count = 0;
+  return new Transform({
+    objectMode: true,
+    transform(chunk, _, cb) {
+      count++;
+      if (countToPause !== 0 && count == countToPause) {
+        count = 0;
+        setTimeout(cb, pauseInMilis, null, chunk);
+      } else {
+        cb(null, chunk);
+      }
+    }
+  })
+}
+
+const emptyChildrenOfParent = (node) => {
+  return {
+    mapElement: (e) => {
+      if (e.name === node) {
+        e.parent.children = [];
+      }
+      return e;
+    }
   }
-
-  return elem;
 }
 
-const loadSheet = (push) => (entry) => {
+const loadSheet = (push, throttleConfig) => (entry) => {
   return new Promise((resolve, reject) => {
     entry
-      .pipe(xmlParser({ mapElement: emptyChildrenOfSheetData }))
+      .pipe(xmlParser(emptyChildrenOfParent('row')))
+      .pipe(throttle(throttleConfig.pauseInMilis, throttleConfig.countToPause))
       .pipe(new Writable({
@@ -43,4 +63,4 @@ objectMode: true,
 
-const loadSheets = (push, entries) => {
-  const loadSheetsWithConfig = loadSheet(push);
+const loadSheets = (push, entries, throttle) => {
+  const loadSheetsWithConfig = loadSheet(push, throttle);
 
@@ -59,3 +79,3 @@ return chainPromises(entries.map(entry => loadSheetsWithConfig.bind(null, entry)));
   if (col.attrs.t === 's') {
-    return sharedStrings[parseInt(value)].children[0].text;
+    return sharedStrings[parseInt(value)];
   }
@@ -104,5 +124,5 @@
 
-const load = (fn) => (entry, cb) => {
+const load = (fn, xmlOpts = {}) => (entry, cb) => {
   entry
-    .pipe(xmlParser())
+    .pipe(xmlParser(xmlOpts))
     .pipe(new Writable({
@@ -130,5 +150,5 @@ objectMode: true,
 
-const copyStream = (entry, path) => {
+const copyStream = (fromStream, toPath) => {
   return new Promise((resolve, reject) => {
-    entry.pipe(fs.createWriteStream(path))
+    fromStream.pipe(fs.createWriteStream(toPath))
       .on('finish', resolve)
@@ -164,13 +184,23 @@ .on('error', reject);
 
-  const { mapCol, mapRow, filterSheets } = Object.assign({
-    mapCol: (c) => c,
-    mapRow: (r) => r,
-    filterSheets: () => true
-  }, opts)
+  const { 
+    mapCol, 
+    mapRow, 
+    filterSheets, 
+    throttleEnabled, 
+    pauseInMilis, 
+    countToPause 
+  } = Object.assign({
+      mapCol: (c) => c,
+      mapRow: (r) => r,
+      filterSheets: () => true,
+      throttleEnabled: true,
+      pauseInMilis: 3,
+      countToPause: 2000
+    }, opts)
 
   const loadSharedStrings = load((chunk) => {
     if (chunk.name === 'si') {
-      sharedStrings.push(chunk)
+      sharedStrings.push(chunk.children[0].text)
     }
-  });
+  }, emptyChildrenOfParent('si'));
 
@@ -227,3 +257,3 @@ const loadStyles = load((chunk) => {
 
-    loadSheets(push, entriesToProcess)
+    loadSheets(push, entriesToProcess, { pauseInMilis, countToPause: throttleEnabled ? countToPause : 0 })
       .then(() => {
@@ -230,0 +260,0 @@ sheetEntries.splice(0);

teste.js

@@ -6,4 +6,4 @@ const fs = require('fs')
 let count = 0;
-fs.createReadStream('/home/tarcisio/Downloads/rawdata1M.xlsx')
-// fs.createReadStream('/home/tarcisio/Temp/big.xlsx')
+// fs.createReadStream('/home/tarcisio/Downloads/rawdata1M.xlsx')
+fs.createReadStream('/home/tarcisio/Temp/big.xlsx')
     // .pipe(new unzipper.ParseOne(/xl\/sharedStrings\.xml/g))
@@ -22,2 +22,7 @@ .pipe(xlsxParser())
         console.log('Total', count)
-    })
+    })
+
+// const x = new Array(500000);
+// x.fill('xxxxxxxxxxxx');
+
+// setTimeout(() => console.log('x'), 10000)

No alert changes

Improved metrics

Total package byte prevSize

31282

increased by3.01%

Lines of code

427

increased by8.38%

No dependency changes