fortune-fs - npm Package Compare versions

Comparing version 1.1.0 to 1.2.0

index.js

@@ -6,5 +6,10 @@ 'use strict'
 var msgpack = require('msgpack-lite')
-var mkdirp = require('mkdirp')
+var { mkdirp } = require('mkdirp')
 var lockFile = require('lockfile')
 
+// in benchmarking tests with 124732 records,
+// a concurrency limit of 128 was the sweet spot.
+// lower limits took longer.
+// higher limits caused performance degradation.
+let concurrentReads = 128
 
@@ -29,2 +34,9 @@ /**
 
+    if(Number.isInteger(this.options.concurrentReads)) {
+      if(this.options.concurrentReads < 1) {
+        throw new RangeError("concurrentReads must be > 0")
+      }
+      concurrentReads = this.options.concurrentReads
+    }
+
     primaryKey = properties.common.constants.primary
@@ -50,6 +62,3 @@ map = properties.common.map
             var typeDir = path.join(dbPath, type)
-
-            mkdirp(typeDir, function (error) {
-              return error ? reject(error) : resolve()
-            })
+            mkdirp(typeDir).then(resolve,reject)
           })
@@ -91,23 +100,39 @@ }))
       })
-    ).then(function (files) {
-      return Promise.all(map(files, function (file) {
-        return new Promise(function (resolve, reject) {
-          var filePath = path.join(dbPath, type, '' + file)
+    ).then(async function (files) {
+      const allThePromises = []
+      let readsInFlight = 0
+      const iterator = files[Symbol.iterator]()
 
-          fs.readFile(filePath, function (error, buffer) {
-            var record
+      while (allThePromises.length < files.length) {
+        if(readsInFlight >= concurrentReads) {
+          // back off
+          await pause(0)
+        }
+        else {
+          allThePromises.push(
+            new Promise(function (resolve, reject) {
+              var filePath = path.join(dbPath, type, '' + iterator.next().value)
+              readsInFlight +=1
 
-            if (error)
-              return error.code === 'ENOENT' ? resolve() : reject(error)
+              fs.readFile(filePath, function (error, buffer) {
+                var record
 
-            record = msgpack.decode(buffer)
+                if (error)
+                  return error.code === 'ENOENT' ? resolve() : reject(error)
 
-            if (!(type in self.db)) self.db[type] = {}
+                record = msgpack.decode(buffer)
 
-            self.db[type][record[primaryKey]] = record
+                if (!(type in self.db)) self.db[type] = {}
 
-            return resolve()
-          })
-        })
-      }))
+                self.db[type][record[primaryKey]] = record
+
+                return resolve()
+              })
+            }).finally(()=>{
+              readsInFlight -=1
+            }))
+        }
+      }
+
+      return Promise.all(allThePromises)
     }).then(function () {
@@ -210,1 +235,13 @@ return DefaultAdapter.prototype.find.call(self, type, ids, options)
 }
+
+/**
+ * Pause for an amount of time
+ * @param {number} ms milliseconds to pause
+ * @returns {promise}
+ * @instance
+ * @example
+ * await pause(1 * 1000); // pause for 1 second
+*/
+function pause(ms) {
+  return new Promise(resolve => { return setTimeout(resolve, ms) })
+}

package.json

 {
   "name": "fortune-fs",
   "description": "File system adapter for Fortune.",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "license": "MIT",
@@ -18,14 +18,11 @@ "repository": {
     "lockfile": "^1.0.4",
-    "mkdirp": "^0.5.1",
+    "mkdirp": "^3.0.1",
     "msgpack-lite": "^0.1.26"
   },
   "devDependencies": {
-    "eslint": "^5.9.0",
-    "eslint-config-boss": "^1.0.6",
-    "fortune": "^5.5.14",
+    "@stylistic/eslint-plugin-js": "^1.7.2",
+    "eslint": "^9.1.1",
+    "fortune": "^5.5.19",
     "tapdance": "^5.1.1"
   },
-  "eslintConfig": {
-    "extends": "boss/es5"
-  },
   "keywords": [
@@ -32,0 +29,0 @@ "fs",

README.md

 # Fortune File System Adapter
 
-[![Build Status](https://img.shields.io/travis/fortunejs/fortune-fs/master.svg?style=flat-square)](https://travis-ci.org/fortunejs/fortune-fs)
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/fortunejs/fortune-fs/test.yml)
 [![npm Version](https://img.shields.io/npm/v/fortune-fs.svg?style=flat-square)](https://www.npmjs.com/package/fortune-fs)
-[![License](https://img.shields.io/npm/l/fortune-fs.svg?style=flat-square)](https://raw.githubusercontent.com/fortunejs/fortune-fs/master/LICENSE)
+[![License](https://img.shields.io/npm/l/fortune.svg?style=flat-square)](https://raw.githubusercontent.com/fortunejs/fortune-fs/master/LICENSE)
 
@@ -30,4 +30,33 @@ This is a file system adapter for Fortune.js.
 
+## Options
+
+
+| Option | Default |  |
+| --- | --- | ---|
+| `concurrentReads`| `128` | limits how many files can be read concurrently by `Adapter.find()` |
+
+### Options Example
+
+```js
+const path = require('path')
+const fortune = require('fortune')
+const fsAdapter = require('fortune-fs')
+
+const store = fortune(recordTypes, {
+  adapter: [ fsAdapter, {
+    // Absolute path to database directory.
+    path: path.join(__dirname, 'db'),
+    concurrentReads: 32
+  } ]
+})
+```
+
+
+## Troubleshooting
+
+If you have a large number of records (files), you may encounter `Error: EMFILE: too many open files`. Set the `concurrentReads` [option](#options) to a lower value to resolve this.
+
+
 ## License
 
 This software is licensed under the [MIT license](https://raw.githubusercontent.com/fortunejs/fortune-fs/master/LICENSE).

test.js

 'use strict'
 
-var path = require('path')
 var testAdapter = require('fortune/test/adapter')
 var fsAdapter = require('./index')
+const fortune = require('fortune')
+const assert = require('node:assert/strict')
 
 testAdapter(fsAdapter)
+
+assert.doesNotThrow(() => {
+  const concurrentReads = 1
+  const store = fortune({}, {
+    adapter: [fsAdapter, { concurrentReads }],
+  })
+  assert.equal(store.adapter.options.concurrentReads, concurrentReads)
+})
+
+assert.throws(() => {
+  fortune({}, {
+    adapter: [fsAdapter, { concurrentReads: 0 }],
+  })
+})

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

11667

increased by31.08%

Number of package files

7

increased by16.67%

Lines of code

240

increased by42.01%

Number of lines in readme file

62

increased by87.88%

Worsened metrics

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedmkdirp@3.0.1(transitive)

• - Removedminimist@1.2.8(transitive)

• - Removedmkdirp@0.5.6(transitive)

• Updatedmkdirp@^3.0.1