fs-jetpack - npm Package Compare versions

Comparing version 4.0.0 to 4.0.1

CHANGELOG.md

@@ -0,3 +1,6 @@
+# 4.0.1 (2020-10-27)
+- `inspectTree()` behaves better in concurrency terms (opens only few files at once) 
+
 # 4.0.0 (2020-10-22)
-- Package published to npm registry now contains only the essential files (e.g. no tests as shipped), to make the smallest footprint possible
+- Package published to npm registry now contains only the essential files (e.g. no tests are shipped), to make the smallest footprint possible
 
@@ -4,0 +7,0 @@ # 3.2.0 (2020-10-15)

lib/inspect_tree.js

@@ -44,11 +44,11 @@ "use strict";
 
-const generateTreeNodeRelativePath = (parent, path) => {
-  if (!parent) {
+const relativePathInTree = (parentInspectObj, inspectObj) => {
+  if (parentInspectObj === undefined) {
     return ".";
   }
-  return `${parent.relativePath}/${pathUtil.basename(path)}`;
+  return parentInspectObj.relativePath + "/" + inspectObj.name;
 };
 
 // Creates checksum of a directory by using
-// checksums and names of all its children inside.
+// checksums and names of all its children.
 const checksumOfDir = (inspectList, algo) => {
@@ -62,2 +62,32 @@ const hash = crypto.createHash(algo);
 
+const calculateTreeDependentProperties = (
+  parentInspectObj,
+  inspectObj,
+  options
+) => {
+  if (options.relativePath) {
+    inspectObj.relativePath = relativePathInTree(parentInspectObj, inspectObj);
+  }
+
+  if (inspectObj.type === "dir") {
+    inspectObj.children.forEach(childInspectObj => {
+      calculateTreeDependentProperties(inspectObj, childInspectObj, options);
+    });
+
+    inspectObj.size = 0;
+    inspectObj.children.forEach(child => {
+      inspectObj.size += child.size || 0;
+    });
+
+    if (options.checksum) {
+      inspectObj[options.checksum] = checksumOfDir(
+        inspectObj.children,
+        options.checksum
+      );
+    }
+  }
+};
+
+const maxConcurrentOperations = 5;
+
 // ---------------------------------------------------------
@@ -67,41 +97,53 @@ // Sync
 
-const inspectTreeNodeSync = (path, options, parent) => {
-  const treeBranch = inspect.sync(path, options);
+const inspectTreeSync = (path, opts) => {
+  const options = opts || {};
+  const concurrentOperationsQueue = [];
+  let nowDoingConcurrentOperations = 0;
 
-  if (treeBranch) {
-    if (options.relativePath) {
-      treeBranch.relativePath = generateTreeNodeRelativePath(parent, path);
+  const checkConcurrentOperations = () => {
+    if (
+      concurrentOperationsQueue.length > 0 &&
+      nowDoingConcurrentOperations < maxConcurrentOperations
+    ) {
+      const callback = concurrentOperationsQueue.pop();
+      nowDoingConcurrentOperations += 1;
+      callback();
+      nowDoingConcurrentOperations -= 1;
+      checkConcurrentOperations();
     }
+  };
 
-    if (treeBranch.type === "dir") {
-      treeBranch.size = 0;
-      treeBranch.children = list.sync(path).map(filename => {
-        const subBranchPath = pathUtil.join(path, filename);
-        const treeSubBranch = inspectTreeNodeSync(
-          subBranchPath,
-          options,
-          treeBranch
-        );
-        // Add together all childrens' size to get directory combined size.
-        treeBranch.size += treeSubBranch.size || 0;
-        return treeSubBranch;
+  const whenConcurrencySpotIsFree = callback => {
+    concurrentOperationsQueue.push(callback);
+    checkConcurrentOperations();
+  };
+
+  const inspectDirSync = (path, inspectObj) => {
+    const checkChild = (filename, index) => {
+      whenConcurrencySpotIsFree(() => {
+        const childPath = pathUtil.join(path, filename);
+        const childInspectObj = inspect.sync(childPath, options);
+        inspectObj.children[index] = childInspectObj;
+        if (childInspectObj.type === "dir") {
+          inspectDirSync(childPath, childInspectObj);
+        }
       });
+    };
 
-      if (options.checksum) {
-        treeBranch[options.checksum] = checksumOfDir(
-          treeBranch.children,
-          options.checksum
-        );
-      }
-    }
+    whenConcurrencySpotIsFree(() => {
+      inspectObj.children = list.sync(path);
+      inspectObj.children.forEach((filename, index) => {
+        checkChild(filename, index);
+      });
+    });
+  };
+
+  const tree = inspect.sync(path, options);
+  if (tree && tree.type === "dir") {
+    inspectDirSync(path, tree);
+    calculateTreeDependentProperties(undefined, tree, options);
   }
-
-  return treeBranch;
+  return tree;
 };
 
-const inspectTreeSync = (path, options) => {
-  const opts = options || {};
-  return inspectTreeNodeSync(path, opts, undefined);
-};
-
 // ---------------------------------------------------------
@@ -111,34 +153,68 @@ // Async
 
-const inspectTreeNodeAsync = (path, options, parent) => {
+const inspectTreeAsync = (path, opts) => {
+  const options = opts || {};
+  const concurrentOperationsQueue = [];
+  let nowDoingConcurrentOperations = 0;
+  let tree;
+
   return new Promise((resolve, reject) => {
-    const inspectAllChildren = treeBranch => {
-      return new Promise((resolve2, reject2) => {
-        list.async(path).then(children => {
-          const doNext = index => {
-            if (index === children.length) {
-              if (options.checksum) {
-                // We are done, but still have to calculate checksum of whole directory.
-                treeBranch[options.checksum] = checksumOfDir(
-                  treeBranch.children,
-                  options.checksum
-                );
-              }
-              resolve2();
-            } else {
-              const subPath = pathUtil.join(path, children[index]);
-              inspectTreeNodeAsync(subPath, options, treeBranch)
-                .then(treeSubBranch => {
-                  children[index] = treeSubBranch;
-                  treeBranch.size += treeSubBranch.size || 0;
-                  doNext(index + 1);
-                })
-                .catch(reject2);
-            }
-          };
+    const doneWithReadingTree = () => {
+      calculateTreeDependentProperties(undefined, tree, options);
+      resolve(tree);
+    };
 
-          treeBranch.children = children;
-          treeBranch.size = 0;
+    const checkConcurrentOperations = () => {
+      if (
+        concurrentOperationsQueue.length === 0 &&
+        nowDoingConcurrentOperations === 0
+      ) {
+        doneWithReadingTree();
+      } else if (
+        concurrentOperationsQueue.length > 0 &&
+        nowDoingConcurrentOperations < maxConcurrentOperations
+      ) {
+        const callback = concurrentOperationsQueue.pop();
+        nowDoingConcurrentOperations += 1;
+        callback();
+      }
+    };
 
-          doNext(0);
+    const whenConcurrencySpotIsFree = callback => {
+      concurrentOperationsQueue.push(callback);
+      checkConcurrentOperations();
+    };
+
+    const concurrentJobDone = () => {
+      nowDoingConcurrentOperations -= 1;
+      checkConcurrentOperations();
+    };
+
+    const inspectDirAsync = (path, inspectObj) => {
+      const checkChild = (filename, index) => {
+        whenConcurrencySpotIsFree(() => {
+          const childPath = pathUtil.join(path, filename);
+          inspect
+            .async(childPath, options)
+            .then(childInspectObj => {
+              inspectObj.children[index] = childInspectObj;
+              if (childInspectObj.type === "dir") {
+                inspectDirAsync(childPath, childInspectObj);
+              }
+              concurrentJobDone();
+            })
+            .catch(reject);
         });
+      };
+
+      whenConcurrencySpotIsFree(() => {
+        list
+          .async(path)
+          .then(children => {
+            inspectObj.children = children;
+            inspectObj.children.forEach((filename, index) => {
+              checkChild(filename, index);
+            });
+            concurrentJobDone();
+          })
+          .catch(reject);
       });
@@ -149,23 +225,10 @@ };
       .async(path, options)
-      .then(treeBranch => {
-        if (!treeBranch) {
-          // Given path doesn't exist. We are done.
-          resolve(treeBranch);
+      .then(treeRoot => {
+        if (treeRoot === undefined) {
+          resolve(treeRoot);
+        } else if (treeRoot.type !== "dir") {
+          resolve(treeRoot);
         } else {
-          if (options.relativePath) {
-            treeBranch.relativePath = generateTreeNodeRelativePath(
-              parent,
-              path
-            );
-          }
-
-          if (treeBranch.type !== "dir") {
-            resolve(treeBranch);
-          } else {
-            inspectAllChildren(treeBranch)
-              .then(() => {
-                resolve(treeBranch);
-              })
-              .catch(reject);
-          }
+          tree = treeRoot;
+          inspectDirAsync(path, treeRoot);
         }
@@ -177,7 +240,2 @@ })
 
-const inspectTreeAsync = (path, options) => {
-  const opts = options || {};
-  return inspectTreeNodeAsync(path, opts);
-};
-
 // ---------------------------------------------------------
@@ -184,0 +242,0 @@ // API

lib/symlink.js

@@ -23,3 +23,3 @@ "use strict";
     if (err.code === "ENOENT") {
-      // Parent directories don't exist. Just create them and rety.
+      // Parent directories don't exist. Just create them and retry.
       dir.createSync(pathUtil.dirname(path));
@@ -43,3 +43,3 @@ fs.symlinkSync(symlinkValue, path);
         if (err.code === "ENOENT") {
-          // Parent directories don't exist. Just create them and rety.
+          // Parent directories don't exist. Just create them and retry.
           dir
@@ -46,0 +46,0 @@ .createAsync(pathUtil.dirname(path))

package.json

 {
   "name": "fs-jetpack",
   "description": "Better file system API",
-  "version": "4.0.0",
+  "version": "4.0.1",
   "author": "Jakub Szwacz <[e-mail]>",
@@ -6,0 +6,0 @@ "dependencies": {

README.md

@@ -1,2 +0,2 @@
-fs-jetpack [![Build Status](https://travis-ci.org/szwacz/fs-jetpack.svg?branch=master)](https://travis-ci.org/szwacz/fs-jetpack) [![Build status](https://ci.appveyor.com/api/projects/status/er206e91fpuuqf58?svg=true)](https://ci.appveyor.com/project/szwacz/fs-jetpack) [![codecov](https://codecov.io/gh/szwacz/fs-jetpack/branch/master/graph/badge.svg)](https://codecov.io/gh/szwacz/fs-jetpack)
+fs-jetpack [![Build Status](https://travis-ci.com/szwacz/fs-jetpack.svg?branch=master)](https://travis-ci.com/szwacz/fs-jetpack) [![Build status](https://ci.appveyor.com/api/projects/status/er206e91fpuuqf58?svg=true)](https://ci.appveyor.com/project/szwacz/fs-jetpack) [![codecov](https://codecov.io/gh/szwacz/fs-jetpack/branch/master/graph/badge.svg)](https://codecov.io/gh/szwacz/fs-jetpack)
 ==========
@@ -3,0 +3,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

131746

increased by1.26%

Lines of code

2973

increased by1.75%

Number of medium supply chain risk alerts

0

decreased by-100%

No dependency changes