Socket
Socket
Sign inDemoInstall

fs-jetpack

Package Overview
Dependencies
Maintainers
1
Versions
61
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

fs-jetpack - npm Package Compare versions

Comparing version 4.0.1 to 4.1.0

7

CHANGELOG.md

@@ -0,3 +1,8 @@

# 4.1.0 (2020-11-13)
- `findAsync()` is now 5 times faster, and `find()` 2 times faster
- `inspectTree()` now sorts results alphabetically, directories firsts, files second
- Refactored internals of methods `find()`, `copy()` and `inspectTree()`
# 4.0.1 (2020-10-27)
- `inspectTree()` behaves better in concurrency terms (opens only few files at once)
- `inspectTree()` behaves better in concurrency terms (opens only few files at once)

@@ -4,0 +9,0 @@ # 4.0.0 (2020-10-22)

32

lib/copy.js

@@ -301,12 +301,12 @@ "use strict";

const stream = treeWalker
.stream(from, { inspectOptions })
.on("readable", () => {
const item = stream.read();
treeWalker.async(
from,
{ inspectOptions },
(srcPath, item) => {
if (item) {
const rel = pathUtil.relative(from, item.path);
const rel = pathUtil.relative(from, srcPath);
const destPath = pathUtil.resolve(to, rel);
if (opts.allowedToCopy(item.path, item.item, destPath)) {
if (opts.allowedToCopy(srcPath, item, destPath)) {
filesInProgress += 1;
copyItemAsync(item.path, item.item, destPath, opts)
copyItemAsync(srcPath, item, destPath, opts)
.then(() => {

@@ -321,10 +321,14 @@ filesInProgress -= 1;

}
})
.on("error", reject)
.on("end", () => {
allFilesDelivered = true;
if (allFilesDelivered && filesInProgress === 0) {
resolve();
},
err => {
if (err) {
reject(err);
} else {
allFilesDelivered = true;
if (allFilesDelivered && filesInProgress === 0) {
resolve();
}
}
});
}
);
})

@@ -331,0 +335,0 @@ .catch(reject);

60

lib/find.js

@@ -39,5 +39,5 @@ "use strict";

const processFoundObjects = (foundObjects, cwd) => {
return foundObjects.map(inspectObj => {
return pathUtil.relative(cwd, inspectObj.absolutePath);
const processFoundPaths = (foundPaths, cwd) => {
return foundPaths.map(path => {
return pathUtil.relative(cwd, path);
});

@@ -65,3 +65,3 @@ };

const findSync = (path, options) => {
const foundInspectObjects = [];
const foundAbsolutePaths = [];
const matchesAnyOfGlobs = matcher.create(

@@ -80,9 +80,3 @@ path,

path,
{
maxLevelsDeep,
inspectOptions: {
absolutePath: true,
symlinks: "follow"
}
},
{ maxLevelsDeep, symlinks: "follow" },
(itemPath, item) => {

@@ -94,3 +88,3 @@ if (item && itemPath !== path && matchesAnyOfGlobs(itemPath)) {

) {
foundInspectObjects.push(item);
foundAbsolutePaths.push(itemPath);
}

@@ -101,3 +95,3 @@ }

return processFoundObjects(foundInspectObjects, options.cwd);
return processFoundPaths(foundAbsolutePaths, options.cwd);
};

@@ -122,3 +116,3 @@

return new Promise((resolve, reject) => {
const foundInspectObjects = [];
const foundAbsolutePaths = [];
const matchesAnyOfGlobs = matcher.create(

@@ -135,19 +129,7 @@ path,

const walker = treeWalker
.stream(path, {
maxLevelsDeep,
inspectOptions: {
absolutePath: true,
symlinks: "follow"
}
})
.on("readable", () => {
const data = walker.read();
if (
data &&
data.item &&
data.path !== path &&
matchesAnyOfGlobs(data.path)
) {
const item = data.item;
treeWalker.async(
path,
{ maxLevelsDeep, symlinks: "follow" },
(itemPath, item) => {
if (item && itemPath !== path && matchesAnyOfGlobs(itemPath)) {
if (

@@ -157,10 +139,14 @@ (item.type === "file" && options.files === true) ||

) {
foundInspectObjects.push(item);
foundAbsolutePaths.push(itemPath);
}
}
})
.on("error", reject)
.on("end", () => {
resolve(processFoundObjects(foundInspectObjects, options.cwd));
});
},
err => {
if (err) {
reject(err);
} else {
resolve(processFoundPaths(foundAbsolutePaths, options.cwd));
}
}
);
});

@@ -167,0 +153,0 @@ };

187

lib/inspect_tree.js

@@ -8,2 +8,3 @@ "use strict";

const validate = require("./utils/validate");
const treeWalker = require("./utils/tree_walker");

@@ -77,2 +78,11 @@ const validateInput = (methodName, path, options) => {

inspectObj.size = 0;
inspectObj.children.sort((a, b) => {
if (a.type === "dir" && b.type === "file") {
return -1;
}
if (a.type === "file" && b.type === "dir") {
return 1;
}
return a.name.localeCompare(b.name);
});
inspectObj.children.forEach(child => {

@@ -91,3 +101,12 @@ inspectObj.size += child.size || 0;

const maxConcurrentOperations = 5;
const findParentInTree = (treeNode, pathChain, item) => {
const name = pathChain[0];
if (pathChain.length > 1) {
const itemInTreeForPathChain = treeNode.children.find(child => {
return child.name === name;
});
return findParentInTree(itemInTreeForPathChain, pathChain.slice(1), item);
}
return treeNode;
};

@@ -100,48 +119,27 @@ // ---------------------------------------------------------

const options = opts || {};
const concurrentOperationsQueue = [];
let nowDoingConcurrentOperations = 0;
let tree;
const checkConcurrentOperations = () => {
if (
concurrentOperationsQueue.length > 0 &&
nowDoingConcurrentOperations < maxConcurrentOperations
) {
const callback = concurrentOperationsQueue.pop();
nowDoingConcurrentOperations += 1;
callback();
nowDoingConcurrentOperations -= 1;
checkConcurrentOperations();
treeWalker.sync(path, { inspectOptions: options }, (itemPath, item) => {
if (item) {
if (item.type === "dir") {
item.children = [];
}
const relativePath = pathUtil.relative(path, itemPath);
if (relativePath === "") {
tree = item;
} else {
const parentItem = findParentInTree(
tree,
relativePath.split(pathUtil.sep),
item
);
parentItem.children.push(item);
}
}
};
});
const whenConcurrencySpotIsFree = callback => {
concurrentOperationsQueue.push(callback);
checkConcurrentOperations();
};
const inspectDirSync = (path, inspectObj) => {
const checkChild = (filename, index) => {
whenConcurrencySpotIsFree(() => {
const childPath = pathUtil.join(path, filename);
const childInspectObj = inspect.sync(childPath, options);
inspectObj.children[index] = childInspectObj;
if (childInspectObj.type === "dir") {
inspectDirSync(childPath, childInspectObj);
}
});
};
whenConcurrencySpotIsFree(() => {
inspectObj.children = list.sync(path);
inspectObj.children.forEach((filename, index) => {
checkChild(filename, index);
});
});
};
const tree = inspect.sync(path, options);
if (tree && tree.type === "dir") {
inspectDirSync(path, tree);
if (tree) {
calculateTreeDependentProperties(undefined, tree, options);
}
return tree;

@@ -156,82 +154,37 @@ };

const options = opts || {};
const concurrentOperationsQueue = [];
let nowDoingConcurrentOperations = 0;
let tree;
return new Promise((resolve, reject) => {
const doneWithReadingTree = () => {
calculateTreeDependentProperties(undefined, tree, options);
resolve(tree);
};
const checkConcurrentOperations = () => {
if (
concurrentOperationsQueue.length === 0 &&
nowDoingConcurrentOperations === 0
) {
doneWithReadingTree();
} else if (
concurrentOperationsQueue.length > 0 &&
nowDoingConcurrentOperations < maxConcurrentOperations
) {
const callback = concurrentOperationsQueue.pop();
nowDoingConcurrentOperations += 1;
callback();
}
};
const whenConcurrencySpotIsFree = callback => {
concurrentOperationsQueue.push(callback);
checkConcurrentOperations();
};
const concurrentJobDone = () => {
nowDoingConcurrentOperations -= 1;
checkConcurrentOperations();
};
const inspectDirAsync = (path, inspectObj) => {
const checkChild = (filename, index) => {
whenConcurrencySpotIsFree(() => {
const childPath = pathUtil.join(path, filename);
inspect
.async(childPath, options)
.then(childInspectObj => {
inspectObj.children[index] = childInspectObj;
if (childInspectObj.type === "dir") {
inspectDirAsync(childPath, childInspectObj);
}
concurrentJobDone();
})
.catch(reject);
});
};
whenConcurrencySpotIsFree(() => {
list
.async(path)
.then(children => {
inspectObj.children = children;
inspectObj.children.forEach((filename, index) => {
checkChild(filename, index);
});
concurrentJobDone();
})
.catch(reject);
});
};
inspect
.async(path, options)
.then(treeRoot => {
if (treeRoot === undefined) {
resolve(treeRoot);
} else if (treeRoot.type !== "dir") {
resolve(treeRoot);
treeWalker.async(
path,
{ inspectOptions: options },
(itemPath, item) => {
if (item) {
if (item.type === "dir") {
item.children = [];
}
const relativePath = pathUtil.relative(path, itemPath);
if (relativePath === "") {
tree = item;
} else {
const parentItem = findParentInTree(
tree,
relativePath.split(pathUtil.sep),
item
);
parentItem.children.push(item);
}
}
},
err => {
if (err) {
reject(err);
} else {
tree = treeRoot;
inspectDirAsync(path, treeRoot);
if (tree) {
calculateTreeDependentProperties(undefined, tree, options);
}
resolve(tree);
}
})
.catch(reject);
}
);
});

@@ -238,0 +191,0 @@ };

4

lib/remove.js
"use strict";
const rimraf = require('rimraf');
const promisify = require('./utils/promisify');
const rimraf = require("rimraf");
const promisify = require("./utils/promisify");
const promisifiedRimraf = promisify(rimraf);

@@ -6,0 +6,0 @@ const validate = require("./utils/validate");

293

lib/utils/tree_walker.js
"use strict";
const Readable = require("stream").Readable;
const fs = require("fs");
const pathUtil = require("path");

@@ -8,2 +8,15 @@ const inspect = require("../inspect");

const fileType = dirent => {
if (dirent.isDirectory()) {
return "dir";
}
if (dirent.isFile()) {
return "file";
}
if (dirent.isSymbolicLink()) {
return "symlink";
}
return "other";
};
// ---------------------------------------------------------

@@ -13,116 +26,214 @@ // SYNC

const walkSync = (path, options, callback, currentLevel) => {
const item = inspect.sync(path, options.inspectOptions);
const initialWalkSync = (path, options, callback) => {
if (options.maxLevelsDeep === undefined) {
options.maxLevelsDeep = Infinity;
}
const performInspectOnEachNode = options.inspectOptions !== undefined;
if (options.symlinks) {
if (options.inspectOptions === undefined) {
options.inspectOptions = { symlinks: options.symlinks };
} else {
options.inspectOptions.symlinks = options.symlinks;
}
}
callback(path, item);
if (item && item.type === "dir" && currentLevel < options.maxLevelsDeep) {
list.sync(path).forEach(child => {
walkSync(
path + pathUtil.sep + child,
options,
callback,
currentLevel + 1
);
const walkSync = (path, currentLevel) => {
fs.readdirSync(path, { withFileTypes: true }).forEach(direntItem => {
const withFileTypesNotSupported = typeof direntItem === "string";
let fileItemPath;
if (withFileTypesNotSupported) {
fileItemPath = pathUtil.join(path, direntItem);
} else {
fileItemPath = pathUtil.join(path, direntItem.name);
}
let fileItem;
if (performInspectOnEachNode) {
fileItem = inspect.sync(fileItemPath, options.inspectOptions);
} else if (withFileTypesNotSupported) {
// New "withFileTypes" API not supported, need to do extra inspect
// on each node, to know if this is a directory or a file.
const inspectObject = inspect.sync(
fileItemPath,
options.inspectOptions
);
fileItem = { name: inspectObject.name, type: inspectObject.type };
} else {
const type = fileType(direntItem);
if (type === "symlink" && options.symlinks === "follow") {
const symlinkPointsTo = fs.statSync(fileItemPath);
fileItem = { name: direntItem.name, type: fileType(symlinkPointsTo) };
} else {
fileItem = { name: direntItem.name, type };
}
}
callback(fileItemPath, fileItem);
if (fileItem.type === "dir" && currentLevel < options.maxLevelsDeep) {
walkSync(fileItemPath, currentLevel + 1);
}
});
};
const item = inspect.sync(path, options.inspectOptions);
if (item) {
if (performInspectOnEachNode) {
callback(path, item);
} else {
// Return simplified object, not full inspect object
callback(path, { name: item.name, type: item.type });
}
if (item.type === "dir") {
walkSync(path, 1);
}
} else {
callback(path, undefined);
}
};
const initialWalkSync = (path, options, callback) => {
walkSync(path, options, callback, 0);
};
// ---------------------------------------------------------
// STREAM
// ASYNC
// ---------------------------------------------------------
const walkStream = (path, options) => {
const rs = new Readable({ objectMode: true });
let nextTreeNode = {
path,
parent: undefined,
level: 0
};
let running = false;
let readSome;
const maxConcurrentOperations = 5;
const error = function(err) {
rs.emit("error", err);
};
const initialWalkAsync = (path, options, callback, doneCallback) => {
if (options.maxLevelsDeep === undefined) {
options.maxLevelsDeep = Infinity;
}
const performInspectOnEachNode = options.inspectOptions !== undefined;
if (options.symlinks) {
if (options.inspectOptions === undefined) {
options.inspectOptions = { symlinks: options.symlinks };
} else {
options.inspectOptions.symlinks = options.symlinks;
}
}
const findNextUnprocessedNode = node => {
if (node.nextSibling) {
return node.nextSibling;
} else if (node.parent) {
return findNextUnprocessedNode(node.parent);
const concurrentOperationsQueue = [];
let nowDoingConcurrentOperations = 0;
const checkConcurrentOperations = () => {
if (
concurrentOperationsQueue.length === 0 &&
nowDoingConcurrentOperations === 0
) {
doneCallback();
} else if (
concurrentOperationsQueue.length > 0 &&
nowDoingConcurrentOperations < maxConcurrentOperations
) {
const operation = concurrentOperationsQueue.pop();
nowDoingConcurrentOperations += 1;
operation();
}
return undefined;
};
const pushAndContinueMaybe = data => {
const theyWantMore = rs.push(data);
running = false;
if (!nextTreeNode) {
// Previous was the last node. The job is done.
rs.push(null);
} else if (theyWantMore) {
readSome();
}
const whenConcurrencySlotAvailable = operation => {
concurrentOperationsQueue.push(operation);
checkConcurrentOperations();
};
if (options.maxLevelsDeep === undefined) {
options.maxLevelsDeep = Infinity;
}
const concurrentOperationDone = () => {
nowDoingConcurrentOperations -= 1;
checkConcurrentOperations();
};
readSome = () => {
const theNode = nextTreeNode;
const walkAsync = (path, currentLevel) => {
const goDeeperIfDir = (fileItemPath, fileItem) => {
if (fileItem.type === "dir" && currentLevel < options.maxLevelsDeep) {
walkAsync(fileItemPath, currentLevel + 1);
}
};
running = true;
whenConcurrencySlotAvailable(() => {
fs.readdir(path, { withFileTypes: true }, (err, files) => {
if (err) {
doneCallback(err);
} else {
files.forEach(direntItem => {
const withFileTypesNotSupported = typeof direntItem === "string";
inspect
.async(theNode.path, options.inspectOptions)
.then(inspected => {
theNode.inspected = inspected;
if (
inspected &&
inspected.type === "dir" &&
theNode.level < options.maxLevelsDeep
) {
list
.async(theNode.path)
.then(childrenNames => {
const children = childrenNames.map(name => {
return {
name,
path: theNode.path + pathUtil.sep + name,
parent: theNode,
level: theNode.level + 1
};
let fileItemPath;
if (withFileTypesNotSupported) {
fileItemPath = pathUtil.join(path, direntItem);
} else {
fileItemPath = pathUtil.join(path, direntItem.name);
}
if (performInspectOnEachNode || withFileTypesNotSupported) {
whenConcurrencySlotAvailable(() => {
inspect
.async(fileItemPath, options.inspectOptions)
.then(fileItem => {
if (performInspectOnEachNode) {
callback(fileItemPath, fileItem);
} else {
callback(fileItemPath, {
name: fileItem.name,
type: fileItem.type
});
}
goDeeperIfDir(fileItemPath, fileItem);
concurrentOperationDone();
})
.catch(err => {
doneCallback(err);
});
});
children.forEach((child, index) => {
child.nextSibling = children[index + 1];
});
} else {
const type = fileType(direntItem);
if (type === "symlink" && options.symlinks === "follow") {
whenConcurrencySlotAvailable(() => {
fs.stat(fileItemPath, (err, symlinkPointsTo) => {
if (err) {
doneCallback(err);
} else {
const fileItem = {
name: direntItem.name,
type: fileType(symlinkPointsTo)
};
callback(fileItemPath, fileItem);
goDeeperIfDir(fileItemPath, fileItem);
concurrentOperationDone();
}
});
});
} else {
const fileItem = { name: direntItem.name, type };
callback(fileItemPath, fileItem);
goDeeperIfDir(fileItemPath, fileItem);
}
}
});
concurrentOperationDone();
}
});
});
};
nextTreeNode = children[0] || findNextUnprocessedNode(theNode);
pushAndContinueMaybe({ path: theNode.path, item: inspected });
})
.catch(error);
inspect
.async(path, options.inspectOptions)
.then(item => {
if (item) {
if (performInspectOnEachNode) {
callback(path, item);
} else {
nextTreeNode = findNextUnprocessedNode(theNode);
pushAndContinueMaybe({ path: theNode.path, item: inspected });
// Return simplified object, not full inspect object
callback(path, { name: item.name, type: item.type });
}
})
.catch(error);
};
rs._read = function() {
if (!running) {
readSome();
}
};
return rs;
if (item.type === "dir") {
walkAsync(path, 1);
} else {
doneCallback();
}
} else {
callback(path, undefined);
doneCallback();
}
})
.catch(err => {
doneCallback(err);
});
};

@@ -135,2 +246,2 @@

exports.sync = initialWalkSync;
exports.stream = walkStream;
exports.async = initialWalkAsync;

2

package.json
{
"name": "fs-jetpack",
"description": "Better file system API",
"version": "4.0.1",
"version": "4.1.0",
"author": "Jakub Szwacz <jakub@szwacz.com>",

@@ -6,0 +6,0 @@ "dependencies": {

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc