fd - npm Package Compare versions

Comparing version 0.0.0 to 0.0.1

index.js

 const fs = require('fs')
 
-var cleanupFd    = function (path) {
+var totalOpenFds = 0 // across all instances
+
+    // the reason we use a combination of path+fd to store references
+    // in this._usingfd is that it is possible to have multiple fds
+    // for the same file simultaneously. Particularly in the situation
+    // where an fd is pending for a close but hasn't been checked back
+    // in by all clients
+  , key          = function (path, fd) {
+      return fd + ':' + path
+    }
+
+  , cleanupFd    = function (path, fd) {
       // clear the use counter & close if pending
-      if (this._pendingClose[path]) {
-        fs.close(this._pendingClose[path], function () {})
-        module.exports._totalOpenFds--
-        delete this._pendingClose[path]
+
+      delete this._usingfd[key(path, fd)]
+      delete this._usingfd[path]
+
+      if (this._pendingClose[key(path, fd)]) {
+        console.log('close', path, fd)
+        fs.close(fd, function () {})
+        totalOpenFds--
+        delete this._pendingClose[key(path, fd)]
       }
     }
 
-  , close       = function (path) {
+  , close       = function (path, fd) {
       // dispose of this fd when possible
-      var fdr = this._fds[path]
-      if (!fdr) return
 
-      this._pendingClose[path] = fdr.fd
+      this._pendingClose[key(path, fd)] = fd
       // nextTick needed to match the nextTick in an async-cache otherwise we may
       // close it in the tick prior to it being actually needed
       process.nextTick(function () {
-        if (fdr.co === 0)
-          cleanupFd.call(this, path)
+        if (!this._usingfd[key(path, fd)])
+          cleanupFd.call(this, path, fd)
       }.bind(this))
@@ -31,12 +45,11 @@ }
       // then just use that.
-      if (this._fds[path] && !this._pendingClose[path])
-        return cb(null, this._fds[path].fd)
+      if (this._usingfd[path] && !this._pendingClose[key(path, this._usingfd[path])])
+        return cb(null, this._usingfd[path])
 
       fs.open(path, 'r', function (er, fd) {
         if (!er) {
-          module.exports._totalOpenFds++
-          this._fds[path] = {
-              fd : fd
-            , co : 0
-          }
+        console.log('open', path, fd)
+          totalOpenFds++
+          this._usingfd[path] = fd
+          this._usingfd[fd + path] = 0
         }
@@ -48,27 +61,34 @@
 
-  , checkout     = function (path) {
+  , checkout     = function (path, fd) {
       // call whenever you *may* be about to use the fd, to ensure it's not cleaned up
 
-      var fdr = this._fds[path]
-      if (!fdr) throw new Error('no fd for path: ' + path)
-
-      fdr.co++
-      return fdr.fd
+      this._usingfd[path] = fd
+      this._usingfd[key(path, fd)] = (this._usingfd[key(path, fd)] || 0) + 1
     }
 
-  , checkin     = function (path) {
+  , checkin     = function (path, fd) {
       // call sometime after checkout() when you know you're not using it
 
-      var fdr = this._fds[path]
-      if (!fdr) throw new Error('no fd for path: ' + path)
+      if (this._usingfd[path] && --this._usingfd[key(path, fd)] === 0)
+        cleanupFd.call(this, path, fd)
+    }
 
-      if ((fdr.co = Math.max(fdr.co - 1, 0)) === 0)
-        cleanupFd.call(this, path)
+  , checkinfn   = function (path, fd) {
+      // make a checkin function that can be safely called multiple times
+
+      var called = false
+      return function () {
+        if (!called) {
+          this.checkin(path, fd)
+          called = true
+        }
+      }.bind(this)
     }
 
   , FDManager   = {
-        open     : open
-      , close    : close
-      , checkout : checkout
-      , checkin  : checkin
+        open      : open
+      , close     : close
+      , checkout  : checkout
+      , checkin   : checkin
+      , checkinfn : checkinfn
     }
@@ -78,3 +98,3 @@
       return Object.create(FDManager, {
-          _fds      : { value: Object.create(null) }
+          _usingfd      : { value: Object.create(null) }
         , _pendingClose : { value: Object.create(null) }
@@ -85,2 +105,2 @@ })
 module.exports               = create
-module.exports._totalOpenFds = 0
+module.exports._totalOpenFds = totalOpenFds

package.json

 {
     "name"            : "fd"
   , "description"     : "File descriptor manager"
-  , "version"         : "0.0.0"
+  , "version"         : "0.0.1"
   , "homepage"        : "https://github.com/rvagg/node-fd"
@@ -6,0 +6,0 @@ , "authors"         : [

README.md

@@ -5,21 +5,122 @@ # fd [![Build Status](https://secure.travis-ci.org/rvagg/node-fd.png)](http://travis-ci.org/rvagg/node-fd)
 
+**fd** manages `fs.open()` and `fs.close()` calls safely for you where there may be timing issues related to multiple-use of the same descriptor.
+
+**fd** provides `checkin()` and `checkout()` functions so your application can register its intent to use a file descriptor after it's been opened and then register that it has finished with the descriptor so that any pending `fs.close()` operations may be performed.
+
+**fd** naturally couples with [async-cache](https://github.com/isaacs/async-cache) to provide a safe pool of file descriptors.
+
 ## Example
 
-*(this example is a bit contrived, something more realistic coming soon)*
+Lets make a static resource web server! This example can be found in the *example/* directory of this repository.
 
+We use [async-cache](https://github.com/isaacs/async-cache) to cache both `fs.sync()` calls and `fd`s, but we hook it up to **fd** so we can safely manage opens and closes.
+
 ```js
-var fdman = require('fd')()
+const fdman = require('fd')()
+    , http  = require('http')
+    , fs    = require('fs')
+    , path  = require('path')
+    , AC    = require('async-cache')
+    , mime  = require('mime')
 
-fdman.open('/foo/bar/baz.txt', function (err, fd) {
-  fdman.checkout('/foo/bar/baz.txt')
-  // do something with `fd`
-  fdman.checkin('/foo/bar/baz.txt')
-  fdman.close('/foo/bar/baz.txt')
-})
-```
+    , ROOT  = path.join(__dirname, 'public')
 
+      // an async cache for fs.stat calls, fresh for 10s
+    , statCache = AC({
+          max    : 10
+        , maxAge : 1000
+        , load   : function (path, callback) {
+                     fs.stat(path, callback)
+                   }
+      })
+
+      // an async cache for fds, fresh for 10s
+    , fdCache = AC({
+          max     : 2
+        , maxAge  : 1000
+          // use fdman to open & close
+        , load    : fdman.open.bind(fdman)
+        , dispose : fdman.close.bind(fdman)
+      })
+
+    , serveError = function (res) {
+        res.statusCode = 404
+        res.setHeader('content-type', 'text/plain')
+        res.end(http.STATUS_CODES[res.statusCode] + '\n')
+      }
+
+http.createServer(function (req, res) {
+  var p = path.join(ROOT, req.url)
+
+  // get a fs.stat for this file
+  statCache.get(p, function (err, stat) {
+    if (err || !stat.isFile())
+      return serveError(res)
+
+    // get an fd for this file
+    fdCache.get(p, function (err, fd) {
+      var mimeType = mime.lookup(path.extname(p))
+          // get a safe checkin function from fdman that
+          // we could safely all multiple times for this single
+          // checkout
+        , checkin = fdman.checkinfn(p, fd)
+
+      // check out the fd for use
+      fdman.checkout(p, fd)
+
+      res.setHeader(
+          'content-type'
+          // don't force download, just show it
+        , mimeType != 'application/octet-stream' ? mimeType : 'text/plain'
+      )
+
+      // stream from the fd to the response
+      var st = fs.createReadStream(p, { fd: fd, start: 0, end: stat.size })
+        .on('end', function () {
+          checkin(p, fd)
+        })
+        .on('error', function () {
+          checkin(p, fd)
+        })
+
+      // override destroy so we don't close the fd
+      st.destroy = function () {}
+
+      st.pipe(res)
+
+    })
+  })
+}).listen(8080)```
+
 ## API
 
+### fd()
+Create a new instance of **fd**. Typically called with `var fdman = require('fd')()`. You can have multiple, separate instances of **fd** operating at the same time, hence the need to instantiate.
+
+### fdman.open(path, callback)
+Equivalent to `fs.open(path, callback)`, you'll get back an `err` and `fd` parameters but the descriptor will go into the managed pool.
+
+### fdman.close(path, fd)
+Will call `fs.close(fd)` *only when the `fd` is no longer in use*. i.e. it will wait till all current uses have been checked in (see below).
+
+### fdman.checkout(path, fd)
+Called when your application may need to use the `fd`. This should be called as early as possible, even if your application may not end up using it.
+
+It is important to perform a `checkout()` as soon as you have a reference to the file descriptor if you may be using it, otherwise an asynchronous call may interrupt and call `close()` before you use it. You *don't have to use the `fd`* to register your intent to use it, as long as you eventually call `checkin()`.
+
+### fdman.checkin(path, fd)
+Register with **fd** that you have finished using the descriptor and it can be safely closed if need be.
+
+The descriptor may not need to be closed or there may be other uses of the descriptor currently checked out so a `checkin()` won't automatically lead to a `close()`.
+
+### fdman.checkinfn(path, fd)
+Returns a function that, when called, will safely perform a `checkin()` for you on the given path and descriptor. An important property of the function is that it will only perform a single `checkin()` regardless of how many times it is called.
+
+This returned function is helpful for calling `checkin()` from multiple points in your application, such as in case of error, and you don't need to worry about whether it's been previously called for the current `checkout()`.
+
+See the example above how this can be used.
+
+
 ## Licence
 
 fd is Copyright (c) 2012 Rod Vagg [@rvagg](https://twitter.com/rvagg) and licenced under the MIT licence. All rights not explicitly granted in the MIT license are reserved. See the included LICENSE file for more details.

test.js

@@ -41,6 +41,6 @@ var tap        = require('tap')
     // check it out
-    t.equal(fdman.checkout('/foo/bar/baz.txt'), 101)
+    fdman.checkout('/foo/bar/baz.txt', 101)
 
     // close it
-    fdman.close('/foo/bar/baz.txt')
+    fdman.close('/foo/bar/baz.txt', fd)
 
@@ -52,3 +52,3 @@ process.nextTick(function () {
       // check it in
-      fdman.checkin('/foo/bar/baz.txt')
+      fdman.checkin('/foo/bar/baz.txt', 101)
       process.nextTick(function () {
@@ -64,2 +64,38 @@ // file should now be closed
 
+tap.test('open / checkout / close with checkinfn', function (t) {
+  var fsMock = sinon.mock(fs)
+    , fdman  = FDManager()
+    , closeSpy
+
+  fsMock.expects('open').once().withArgs('/foo/bar/baz.txt', 'r').callsArgWith(2, null, 101)
+  closeSpy = fsMock.expects('close').once().withArgs(101)
+
+  // open a file
+  fdman.open('/foo/bar/baz.txt', function (err, fd) {
+    t.notOk(err)
+    t.equal(fd, 101)
+
+    // check it out
+    fdman.checkout('/foo/bar/baz.txt', 101)
+    var checkin = fdman.checkinfn('/foo/bar/baz.txt', 101)
+
+    // close it
+    fdman.close('/foo/bar/baz.txt', fd)
+
+    process.nextTick(function () {
+      // file should NOT be closed
+      t.equal(closeSpy.callCount, 0)
+
+      // check it in
+      checkin('/foo/bar/baz.txt', 101)
+      process.nextTick(function () {
+        // file should now be closed
+        fsMock.verify()
+        t.equal(0, FDManager._totalOpenFds)
+        t.end()
+      })
+    })
+  })
+})
+
 tap.test('many open files', function (t) {
@@ -116,3 +152,3 @@ var filetree = {}
                     // checkout the file, `j` ms later
-                    t.equal(fdman.checkout(f), fd)
+                    fdman.checkout(f, fd)
                     if (j === i - 1) {
@@ -138,3 +174,3 @@ // on the last checkout, read from the fd to make sure it's
                       // the read doesn't cause problems)
-                      fdman.checkin(f)
+                      fdman.checkin(f, fd)
                       callback()
@@ -152,3 +188,3 @@ }, j + (Math.random() * 10))
                 // closeable
-                fdman.close(f)
+                fdman.close(f, fd)
                 callback()
@@ -155,0 +191,0 @@ }, Math.floor(10 + Math.random() * i))

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

111221

increased by878.11%

Number of package files

28

increased by250%

Lines of code

1017

increased by386.6%

Number of lines in readme file

125

increased by420.83%

Worsened metrics

Number of low supply chain risk alerts

5

increased by150%

Number of medium supply chain risk alerts

2

increased byInfinity%

No dependency changes