mongodb-queue
Advanced tools
Comparing version 3.1.0 to 4.0.0
@@ -28,10 +28,10 @@ /** | ||
module.exports = function(mongoDbClient, name, opts) { | ||
return new Queue(mongoDbClient, name, opts) | ||
module.exports = function(db, name, opts) { | ||
return new Queue(db, name, opts) | ||
} | ||
// the Queue object itself | ||
function Queue(mongoDbClient, name, opts) { | ||
if ( !mongoDbClient ) { | ||
throw new Error("mongodb-queue: provide a mongodb.MongoClient") | ||
function Queue(db, name, opts) { | ||
if ( !db ) { | ||
throw new Error("mongodb-queue: provide a mongodb.MongoClient.db") | ||
} | ||
@@ -43,4 +43,5 @@ if ( !name ) { | ||
this.db = db | ||
this.name = name | ||
this.col = mongoDbClient.collection(name) | ||
this.col = db.collection(name) | ||
this.visibility = opts.visibility || 30 | ||
@@ -223,3 +224,3 @@ this.delay = opts.delay || 0 | ||
self.col.count(function(err, count) { | ||
self.col.countDocuments(function(err, count) { | ||
if (err) return callback(err) | ||
@@ -238,3 +239,3 @@ callback(null, count) | ||
self.col.count(query, function(err, count) { | ||
self.col.countDocuments(query, function(err, count) { | ||
if (err) return callback(err) | ||
@@ -254,3 +255,3 @@ callback(null, count) | ||
self.col.count(query, function(err, count) { | ||
self.col.countDocuments(query, function(err, count) { | ||
if (err) return callback(err) | ||
@@ -268,3 +269,3 @@ callback(null, count) | ||
self.col.count(query, function(err, count) { | ||
self.col.countDocuments(query, function(err, count) { | ||
if (err) return callback(err) | ||
@@ -271,0 +272,0 @@ callback(null, count) |
{ | ||
"name": "mongodb-queue", | ||
"version": "3.1.0", | ||
"version": "4.0.0", | ||
"description": "Message queues which uses MongoDB.", | ||
"main": "mongodb-queue.js", | ||
"scripts": { | ||
"test": "set -e; for FILE in test/*.js; do node $FILE; done" | ||
"test": "set -e; for FILE in test/*.js; do echo --- $FILE ---; node $FILE; done" | ||
}, | ||
"dependencies": {}, | ||
"devDependencies": { | ||
"tape": "^4.2.2", | ||
"mongodb": "^2.0.48", | ||
"async": "^1.5.0" | ||
"async": "^2.6.2", | ||
"mongodb": "^3.1.13", | ||
"tape": "^4.10.1" | ||
}, | ||
@@ -15,0 +15,0 @@ "homepage": "https://github.com/chilts/mongodb-queue", |
116
README.md
@@ -8,2 +8,10 @@ # mongodb-queue # | ||
Now compatible with the MongoDB v3 driver. | ||
For MongoDB v2 driver use mongodb-queue@3. | ||
**NOTE**: This package is considered feature complete and **STABLE** hence there is not a whole lot of development on | ||
it though it is being used extensively. Use it with all your might and let us know of any problems - it should be | ||
bullet-proof. | ||
## Synopsis ## | ||
@@ -17,6 +25,11 @@ | ||
var con = 'mongodb://localhost:27017/test' | ||
const url = 'mongodb://localhost:27017/' | ||
const client = new mongodb.MongoClient(url, { useNewUrlParser: true }) | ||
mongodb.MongoClient.connect(con, function(err, db) { | ||
var queue = mongoDbQueue(db, 'my-queue') | ||
client.connect(err => { | ||
const db = client.db('test') | ||
const queue = mongoDbQueue(db, 'my-queue') | ||
// ... | ||
}) | ||
@@ -28,3 +41,3 @@ ``` | ||
```js | ||
queue.add('Hello, World!', function(err, id) { | ||
queue.add('Hello, World!', (err, id) => { | ||
// Message with payload 'Hello, World!' added. | ||
@@ -38,3 +51,3 @@ // 'id' is returned, useful for logging. | ||
```js | ||
queue.get(function(err, msg) { | ||
queue.get((err, msg) => { | ||
console.log('msg.id=' + msg.id) | ||
@@ -50,3 +63,3 @@ console.log('msg.ack=' + msg.ack) | ||
```js | ||
queue.ping(msg.ack, function(err, id) { | ||
queue.ping(msg.ack, (err, id) => { | ||
// Visibility window now increased for this message id. | ||
@@ -60,3 +73,3 @@ // 'id' is returned, useful for logging. | ||
```js | ||
queue.ack(msg.ack, function(err, id) { | ||
queue.ack(msg.ack, (err, id) => { | ||
// This msg removed from queue for this ack. | ||
@@ -72,3 +85,3 @@ // The 'id' of the message is returned, useful for logging. | ||
```js | ||
queue.clean(function(err) { | ||
queue.clean((err) => { | ||
// All processed (ie. acked) messages have been deleted | ||
@@ -84,3 +97,3 @@ }) | ||
```js | ||
queue.createIndexes(function(err, indexName) { | ||
queue.createIndexes((err, indexName) => { | ||
// The indexes needed have been added to MongoDB. | ||
@@ -184,3 +197,3 @@ }) | ||
If you pop a message off the `queue` over `maxRetries` times and have still not acked it, | ||
If you pop a message off the `queue` over `maxRetries` times and still have not acked it, | ||
it will be pushed onto the `deadQueue` for you. This happens when you `.get()` (not when | ||
@@ -212,3 +225,3 @@ you miss acking a message in it's visibility window). By doing it when you call `.get()`, | ||
payload: 'Hello, World!', | ||
tries: 1 } | ||
tries: 1 | ||
} | ||
@@ -228,3 +241,3 @@ ``` | ||
payload: 'Hello, World!', | ||
tries: 5 } | ||
tries: 5 | ||
}, | ||
@@ -245,3 +258,3 @@ tries: 1 | ||
```js | ||
queue.add('Hello, World!', function(err, id) { | ||
queue.add('Hello, World!', (err, id) => { | ||
// Message with payload 'Hello, World!' added. | ||
@@ -255,3 +268,3 @@ // 'id' is returned, useful for logging. | ||
```js | ||
queue.add({ err: 'E_BORKED', msg: 'Broken' }, function(err, id) { | ||
queue.add({ err: 'E_BORKED', msg: 'Broken' }, (err, id) => { | ||
// Message with payload { err: 'E_BORKED', msg: 'Broken' } added. | ||
@@ -265,3 +278,3 @@ // 'id' is returned, useful for logging. | ||
```js | ||
queue.add(['msg1', 'msg2', 'msg3'], function(err, ids) { | ||
queue.add(['msg1', 'msg2', 'msg3'], (err, ids) => { | ||
// Messages with payloads 'msg1', 'msg2' & 'msg3' added. | ||
@@ -275,3 +288,3 @@ // All 'id's are returned as an array, useful for logging. | ||
```js | ||
queue.add('Later', { delay: 120 }, function(err, id) { | ||
queue.add('Later', { delay: 120 }, (err, id) => { | ||
// Message with payload 'Later' added. | ||
@@ -288,3 +301,3 @@ // 'id' is returned, useful for logging. | ||
```js | ||
queue.get(function(err, msg) { | ||
queue.get((err, msg) => { | ||
// You can now process the message | ||
@@ -298,3 +311,3 @@ // IMPORTANT: The callback will not wait for an message if the queue is empty. The message will be undefined if the queue is empty. | ||
```js | ||
queue.get({ visibility: 10 }, function(err, msg) { | ||
queue.get({ visibility: 10 }, (err, msg) => { | ||
// You can now process the message for 10s before it goes back into the queue if not ack'd instead of the duration that is set on the queue in general | ||
@@ -321,4 +334,4 @@ }) | ||
```js | ||
queue.get(function(err, msg) { | ||
queue.ack(msg.ack, function(err, id) { | ||
queue.get((err, msg) => { | ||
queue.ack(msg.ack, (err, id) => { | ||
// this message has now been removed from the queue | ||
@@ -336,4 +349,4 @@ }) | ||
```js | ||
queue.get(function(err, msg) { | ||
queue.ping(msg.ack, function(err, id) { | ||
queue.get((err, msg) => { | ||
queue.ping(msg.ack, (err, id) => { | ||
// this message has had it's visibility window extended | ||
@@ -347,4 +360,4 @@ }) | ||
```js | ||
queue.get(function(err, msg) { | ||
queue.ping(msg.ack, { visibility: 10 }, function(err, id) { | ||
queue.get((err, msg) => { | ||
queue.ping(msg.ack, { visibility: 10 }, (err, id) => { | ||
// this message has had it's visibility window extended by 10s instead of the visibilty set on the queue in general | ||
@@ -361,3 +374,3 @@ }) | ||
```js | ||
queue.total(function(err, count) { | ||
queue.total((err, count) => { | ||
console.log('This queue has seen %d messages', count) | ||
@@ -372,3 +385,3 @@ }) | ||
```js | ||
queue.size(function(err, count) { | ||
queue.size((err, count) => { | ||
console.log('This queue has %d current messages', count) | ||
@@ -384,3 +397,3 @@ }) | ||
```js | ||
queue.inFlight(function(err, count) { | ||
queue.inFlight((err, count) => { | ||
console.log('A total of %d messages are currently being processed', count) | ||
@@ -396,3 +409,3 @@ }) | ||
```js | ||
queue.done(function(err, count) { | ||
queue.done((err, count) => { | ||
console.log('This queue has processed %d messages', count) | ||
@@ -409,3 +422,3 @@ }) | ||
```js | ||
queue.clean(function(err) { | ||
queue.clean((err) => { | ||
console.log('The processed messages have been deleted from the queue') | ||
@@ -438,14 +451,13 @@ }) | ||
## Note on MongoDB Version ## | ||
## Roadmap ## | ||
When using MongoDB v2.6 and the v1.3.23 version of the mongodb driver from npm, I was getting | ||
a weird error similar to "key $exists must not start with '$'". Yes, very strange. Anyway, the fix | ||
is to install a later version of the driver. I have tried this with v1.4.9 and it seems ok. | ||
We may add the ability for each function to return a promise in the future so it can be used as such, or with | ||
async/await. | ||
## Releases ## | ||
Yay! We made it to v1.0. This means that development may slow down but to be honest, I have pretty | ||
much all of the functionality I want in this thing done. Thanks to everyone for feedback, reports | ||
and pull requests. | ||
### 4.0.0 (2019-02-20) ### | ||
* [NEW] Updated entire codebase to be compatible with the mongodb driver v3 | ||
### 2.1.0 (2016-04-21) ### | ||
@@ -539,5 +551,33 @@ | ||
Written by [Andrew Chilton](http://chilts.org/) - | ||
[Twitter](https://twitter.com/andychilton). | ||
``` | ||
$ npx chilts | ||
╒════════════════════════════════════════════════════╕ | ||
│ │ | ||
│ Andrew Chilton (Personal) │ | ||
│ ------------------------- │ | ||
│ │ | ||
│ Email : andychilton@gmail.com │ | ||
│ Web : https://chilts.org │ | ||
│ Twitter : https://twitter.com/andychilton │ | ||
│ GitHub : https://github.com/chilts │ | ||
│ GitLab : https://gitlab.org/chilts │ | ||
│ │ | ||
│ Apps Attic Ltd (My Company) │ | ||
│ --------------------------- │ | ||
│ │ | ||
│ Email : chilts@appsattic.com │ | ||
│ Web : https://appsattic.com │ | ||
│ Twitter : https://twitter.com/AppsAttic │ | ||
│ GitLab : https://gitlab.com/appsattic │ | ||
│ │ | ||
│ Node.js / npm │ | ||
│ ------------- │ | ||
│ │ | ||
│ Profile : https://www.npmjs.com/~chilts │ | ||
│ Card : $ npx chilts │ | ||
│ │ | ||
╘════════════════════════════════════════════════════╛ | ||
``` | ||
## License ## | ||
@@ -544,0 +584,0 @@ |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -167,5 +167,5 @@ test('clean: check deleted messages are deleted', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -172,0 +172,0 @@ }) |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -176,5 +176,5 @@ test('first test', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -181,0 +181,0 @@ }) |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -108,5 +108,5 @@ test('first test', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -113,0 +113,0 @@ }) |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -102,5 +102,5 @@ test('delay: check messages on this queue are returned after the delay', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -107,0 +107,0 @@ }) |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -22,5 +22,5 @@ test('visibility: check message is back in queue after 3s', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -27,0 +27,0 @@ }) |
@@ -9,3 +9,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -78,5 +78,5 @@ test('many: add ' + total + ' messages, get ' + total + ' back', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -83,0 +83,0 @@ }) |
@@ -9,3 +9,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -69,5 +69,5 @@ test('multi: add ' + total + ' messages, get ' + total + ' back', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -74,0 +74,0 @@ }) |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -178,5 +178,5 @@ test('ping: check a retrieved message with a ping can still be acked', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -183,0 +183,0 @@ }) |
@@ -1,24 +0,42 @@ | ||
var mongodb = require('mongodb') | ||
const mongodb = require('mongodb') | ||
var conStr = 'mongodb://localhost:27017/mongodb-queue' | ||
const url = 'mongodb://localhost:27017/' | ||
const dbName = 'mongodb-queue' | ||
const collections = [ | ||
'default', | ||
'delay', | ||
'multi', | ||
'visibility', | ||
'clean', | ||
'ping', | ||
'stats1', | ||
'stats2', | ||
'queue', | ||
'dead-queue', | ||
'queue-2', | ||
'dead-queue-2', | ||
] | ||
module.exports = function(callback) { | ||
mongodb.MongoClient.connect(conStr, function(err, db) { | ||
if (err) throw err | ||
var done = 0 | ||
// let's empty out some collections to make sure there are no messages | ||
var collections = [ | ||
'default', 'delay', 'multi', 'visibility', 'clean', 'ping', | ||
'stats1', 'stats2', | ||
'queue', 'dead-queue', 'queue-2', 'dead-queue-2' | ||
] | ||
collections.forEach(function(col) { | ||
db.collection(col).remove(function() { | ||
done += 1 | ||
if ( done === collections.length ) { | ||
callback(db) | ||
} | ||
}) | ||
}) | ||
const client = new mongodb.MongoClient(url, { useNewUrlParser: true }) | ||
client.connect(err => { | ||
// we can throw since this is test-only | ||
if (err) throw err | ||
const db = client.db(dbName) | ||
// empty out some collections to make sure there are no messages | ||
let done = 0 | ||
collections.forEach((col) => { | ||
db.collection(col).deleteMany(() => { | ||
done += 1 | ||
if ( done === collections.length ) { | ||
callback(client, db) | ||
} | ||
}) | ||
}) | ||
}) | ||
} |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -202,5 +202,5 @@ test('first test', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -207,0 +207,0 @@ }) |
@@ -7,3 +7,3 @@ var async = require('async') | ||
setup(function(db) { | ||
setup(function(client, db) { | ||
@@ -169,5 +169,5 @@ test('visibility: check message is back in queue after 3s', function(t) { | ||
test('db.close()', function(t) { | ||
t.pass('db.close()') | ||
db.close() | ||
test('client.close()', function(t) { | ||
t.pass('client.close()') | ||
client.close() | ||
t.end() | ||
@@ -174,0 +174,0 @@ }) |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
75191
1499
561
15