bull - npm Package Compare versions

Comparing version 3.4.7 to 3.4.8

CHANGELOG.md

@@ -1,7 +0,12 @@
-Changelog
-=========
+# Changelog
 
-v.3.4.7
--------
+## v.3.4.8
 
+- emit waiting event when waking up sleep jobs fixing #792
+- throw error if missing processor file fixing #954
+
+[Changes](https://github.com/OptimalBits/bull/compare/v3.4.7...v3.4.8)
+
+## v.3.4.7
+
 - Fixes to deal with removing correctly in priority queues #984
@@ -11,4 +16,3 @@
 
-v.3.4.6
--------
+## v.3.4.6
 
@@ -19,4 +23,4 @@ - Reverted use of arrow function that was incompatible with older versions of node.
 
-v.3.4.5
--------
+## v.3.4.5
+
 - Fixed Unhandled promise rejections #1012.
@@ -26,4 +30,3 @@
 
-v.3.4.4
--------
+## v.3.4.4
 
@@ -37,4 +40,3 @@ - Partially fixed #845. When call queue.close() bull throws Error: Connection is closed.
 
-v.3.4.3
--------
+## v.3.4.3
 
@@ -51,4 +53,3 @@ - Fixed #994 queue.getJobs() race condition.
 
-v.3.4.2
--------
+## v.3.4.2
 
@@ -62,4 +63,3 @@ - Fixed #903 Globally paused queue cannot receive job (or not shown in Arena untill queue is globally resumed).
 
-v.3.4.1
--------
+## v.3.4.1
 
@@ -70,4 +70,3 @@ - Better check for closing in moveUnlockedJobsToWait, possibly fixing #806.
 
-v.3.4.0
--------
+## v.3.4.0
 
@@ -80,4 +79,3 @@ - Added support for prioritized delayed jobs.
 
-v.3.3.10
--------
+## v.3.3.10
 
@@ -90,6 +88,4 @@ - Faster next job fetch #868
 
+## v.3.3.9
 
-v.3.3.9
--------
-
 - Support custom backoff strategies.
@@ -101,6 +97,4 @@ - Fixed #786. Handling of ES6 default export.
 
+## v.3.3.8
 
-v.3.3.8
--------
-
 - Fixed #812. External process doesn't terminate on `queue.close()`.
@@ -112,6 +106,4 @@ - Fixed #830. Named Process Sent to Wrong Processor.
 
+## v.3.3.7
 
-v.3.3.7
--------
-
 - Fixed #807.
@@ -122,4 +114,3 @@ - Adding ability to limit by stacktrace. #798.
 
-v.3.3.6
--------
+## v.3.3.6
 
@@ -132,4 +123,3 @@ - Fixed #766, #781, #716.
 
-v.3.3.5
--------
+## v.3.3.5
 
@@ -140,6 +130,4 @@ - Fixed #764, #762, #759.
 
+## v.3.3.4
 
-v.3.3.4
--------
-
 - Fixed #748.
@@ -149,4 +137,3 @@
 
-v.3.3.3
--------
+## v.3.3.3
 
@@ -159,4 +146,3 @@ - Re-fixed #739.
 
-v.3.3.2
--------
+## v.3.3.2
 
@@ -169,4 +155,3 @@ - Fixed #721. SyntaxError: Unexpected token u in JSON at position 0.
 
-v.3.3.1
--------
+## v.3.3.1
 
@@ -177,6 +162,5 @@ - Fixed #714
 
-v.3.3.0
--------
+## v.3.3.0
 
-- Added a method ```Queue##removeRepeatable``` to remove repeatable jobs.
+- Added a method `Queue##removeRepeatable` to remove repeatable jobs.
 - Now also emits drained as a global event.
@@ -187,4 +171,3 @@ - Fixed #518, #624
 
-v.3.2.0
--------
+## v.3.2.0
 
@@ -195,4 +178,3 @@ - Added support for running jobs in child processes #488
 
-v.3.1.0
--------
+## v.3.1.0
 
@@ -205,9 +187,7 @@ - Added rate limiter support.
 
-v.3.0.0
--------
+## v.3.0.0
 
 - No changes.
 
-v.3.0.0-rc.10
--------------
+## v.3.0.0-rc.10
 
@@ -219,6 +199,4 @@ - Fixed #666.
 
+## v.3.0.0-rc.9
 
-v.3.0.0-rc.9
-------------
-
 - Fixed #672.
@@ -229,6 +207,4 @@ - Fixed #670
 
+## v.3.0.0-rc.8
 
-v.3.0.0-rc.8
-------------
-
 - Enhanced job fetching #651 (faster and more reliable).
@@ -238,4 +214,3 @@
 
-v.3.0.0-rc.7
-------------
+## v.3.0.0-rc.7
 
@@ -246,6 +221,4 @@ - Fixed #659
 
+## v.3.0.0-rc.6
 
-v.3.0.0-rc.6
-------------
-
 - Fixed #645.
@@ -255,4 +228,3 @@
 
-v.3.0.0-rc.5
-------------
+## v.3.0.0-rc.5
 
@@ -267,4 +239,3 @@ - Improved performance, specially when having many concurrent workers.
 
-v.3.0.0-rc.4
-------------
+## v.3.0.0-rc.4
 
@@ -280,4 +251,3 @@ - Added support for naming workers in redis connections #530.
 
-v.3.0.0-rc.3
-------------
+## v.3.0.0-rc.3
 
@@ -290,4 +260,3 @@ - Fixed #579.
 
-v.3.0.0-rc.2
-------------
+## v.3.0.0-rc.2
 
@@ -299,4 +268,3 @@ - Improved performance of moveToActive #550.
 
-v.3.0.0-rc.1
-------------
+## v.3.0.0-rc.1
 
@@ -309,4 +277,3 @@ - Improved error and lock handling for failed jobs #499, #539.
 
-v.3.0.0-alpha.4
----------------
+## v.3.0.0-alpha.4
 
@@ -317,4 +284,3 @@ - Implemented repeatable jobs. #252.
 
-v.3.0.0-alpha.3
----------------
+## v.3.0.0-alpha.3
 
@@ -325,4 +291,3 @@ - Simplified global events #501.
 
-v.3.0.0-alpha.2
----------------
+## v.3.0.0-alpha.2
 
@@ -333,6 +298,4 @@ - Eliminated possible memory leak #503
 
+## v.3.0.0-alpha.1
 
-v.3.0.0-alpha.1
----------------
-
 - improved job fetch mechanism. #480.
@@ -347,4 +310,3 @@ - job.jobId changed to job.id.
 
-v.2.2.6
--------
+## v.2.2.6
 
@@ -356,6 +318,4 @@ - Persisted failedReason when storing job data.
 
+## v.2.2.5
 
-v.2.2.5
--------
-
 - Fixed so that redis key prefix works properly.
@@ -365,6 +325,4 @@
 
+## v.2.2.4
 
-v.2.2.4
--------
-
 - Allow reusing certain redis connections.
@@ -374,4 +332,3 @@
 
-v.2.2.3
--------
+## v.2.2.3
 
@@ -383,4 +340,3 @@ - Added getJobCounts.
 
-v.2.2.2
--------
+## v.2.2.2
 
@@ -391,4 +347,3 @@ - Fixed redis script cache gets bloated after update to bull 2.0 #426
 
-v.2.2.1
--------
+## v.2.2.1
 
@@ -400,4 +355,3 @@ - Re-added createClient option that was removed by mistake.
 
-v.2.2.0
--------
+## v.2.2.0
 
@@ -411,6 +365,4 @@ - Much improved priority queues, simpler, faster and more reliable.
 
+## v.2.1.2
 
-v.2.1.2
--------
-
 - Fixed Error renewing lock LockError: Exceeded 0 attempts to lock the resource #437
@@ -421,4 +373,3 @@ - Fixed Unable to renew nonexisting lock on job fail #441
 
-v.2.1.1
--------
+## v.2.1.1
 
@@ -429,4 +380,3 @@ - Catch errors produced in timers. Related to #441
 
-v.2.1.0
--------
+## v.2.1.0
 
@@ -441,4 +391,3 @@ - Fixed #397, Error: Unable to renew nonexisting lock
 
-v.2.0.0
--------
+## v.2.0.0
 
@@ -449,4 +398,3 @@ - Changed redis module to ioredis fixing many issues along the way, see changes.
 
-v.1.1.3
--------
+## v.1.1.3
 
@@ -458,4 +406,3 @@ - fixed "Broken constructor pattern from recent commit" #384
 
-v1.1.2
--------
+## v1.1.2
 
@@ -466,4 +413,3 @@ - regained backwards compatibility in events by using disturbed 1.0.6
 
-v1.1.1
--------
+## v1.1.1
 
@@ -475,4 +421,3 @@ - Returned this in queue##on and queue##once for backwards compatibility.
 
-v1.1.0
--------
+## v1.1.0
 
@@ -487,4 +432,3 @@ - Fixed [job corruption issue](https://github.com/OptimalBits/bull/pull/359)
 
-v1.0.0
--------
+## v1.0.0
 
@@ -495,4 +439,3 @@ - improvements in clean (fixes and performance).
 
-v1.0.0-rc4
-----------
+## v1.0.0-rc4
 
@@ -504,4 +447,3 @@ - fixed lock renew logic.
 
-v1.0.0-rc3
-----------
+## v1.0.0-rc3
 
@@ -512,4 +454,3 @@ - smaller fixes.
 
-v1.0.0-rc2
-----------
+## v1.0.0-rc2
 
@@ -526,7 +467,6 @@ - Improved locking when removing and processing stalled jobs.
 
-v1.0.0-rc1
-----------
+## v1.0.0-rc1
 
 - Removed all potential dangerous hazards by atomizing many operations using
-cached LUA scripts.
+  cached LUA scripts.
 - Improved performance around 400% compared to previous version.
@@ -538,4 +478,3 @@ - Better pause/resume (#266), and added pause for local workers.
 
-v0.7.2
-------
+## v0.7.2
 
@@ -548,4 +487,3 @@ - Added local pause/resume functionality
 
-v0.7.1
-------
+## v0.7.1
 
@@ -556,4 +494,3 @@ - fixed storing of stacktraces
 
-v0.7.0
-------
+## v0.7.0
 
@@ -566,4 +503,3 @@ - store the return value from the job handlers.
 
-v0.4.0
-------
+## v0.4.0
 
@@ -574,4 +510,3 @@ - added a Queue##clean method
 
-v0.3.0
-------
+## v0.3.0
 
@@ -587,15 +522,12 @@ - added support for custom clients.
 
-v0.2.7
-------
+## v0.2.7
 
 [Changes](https://github.com/OptimalBits/bull/compare/v0.2.6...v0.2.7)
 
-v0.2.6
-------
+## v0.2.6
 
 - [Fix] #103 When a queue start it do not process delayed job.
-[Changes](https://github.com/OptimalBits/bull/compare/v0.2.5...v0.2.6)
+  [Changes](https://github.com/OptimalBits/bull/compare/v0.2.5...v0.2.6)
 
-v0.2.5
-------
+## v0.2.5
 
@@ -608,24 +540,19 @@ - [upgrade] Upgraded node redis to version 0.12.x
 
-v0.2.4
-------
+## v0.2.4
 
 [Changes](https://github.com/OptimalBits/bull/compare/v0.2.3...v0.2.4)
 
-v0.2.3
-------
+## v0.2.3
 
 [Changes](https://github.com/OptimalBits/bull/compare/v0.1.9...v0.2.3)
 
-v0.1.9
-------
+## v0.1.9
 
 - [Improvement] Faster job removal. (manast)
 
-v0.1.8
-------
+## v0.1.8
 
 - [Improvement] Better promisification of redis methods. (manast)
 
-v0.1.7
-------
+## v0.1.7
 
@@ -636,4 +563,3 @@ - [Feature] Added a convenience method for getting a job. (loginx)
 
-v0.1.6
-------
+## v0.1.6
 
@@ -640,0 +566,0 @@ - [Fix] Improved and corrected job's getters.

lib/commands/index.js

@@ -23,6 +23,2 @@ /**
 
-//
-// for some very strange reason, defining scripts with this code results in this error
-// when executing the scripts: ERR value is not an integer or out of range
-//
 module.exports = (function() {
@@ -29,0 +25,0 @@ var scripts;

lib/job.js

@@ -182,2 +182,9 @@ /*eslint-env node */
 
+/**
+ * Moves a job to the completed queue.
+ * Returned job to be used with Queue.prototype.nextJobFromJobData.
+ * @param returnValue {string} The jobs success message.
+ * @param ignoreLock {boolean} True when wanting to ignore the redis lock on this job.
+ * @returns {Promise} Returns the jobData of the next job in the waiting queue.
+ */
 Job.prototype.moveToCompleted = function(returnValue, ignoreLock) {
@@ -204,2 +211,8 @@ this.returnvalue = returnValue || 0;
 
+/**
+ * Moves a job to the failed queue.
+ * @param err {string} The jobs error message.
+ * @param ignoreLock {boolean} True when wanting to ignore the redis lock on this job.
+ * @returns void
+ */
 Job.prototype.moveToFailed = function(err, ignoreLock) {
@@ -206,0 +219,0 @@ var _this = this;

lib/process/child-pool.js

@@ -20,27 +20,25 @@ 'use strict';
   var _this = this;
-  return checkProcessorFile(processFile).then(function() {
-    var child = _this.getFree(processFile).pop();
+  var child = _this.getFree(processFile).pop();
 
-    if (child) {
-      _this.retained[child.pid] = child;
-      return child;
-    }
+  if (child) {
+    _this.retained[child.pid] = child;
+    return Promise.resolve(child);
+  }
 
-    // if node process is running with --inspect, don't include that option
-    // when spawning the children
-    var execArgv = _.filter(process.execArgv, function(arg) {
-      return arg.indexOf('--inspect') === -1;
-    });
+  // if node process is running with --inspect, don't include that option
+  // when spawning the children
+  var execArgv = _.filter(process.execArgv, function(arg) {
+    return arg.indexOf('--inspect') === -1;
+  });
 
-    child = fork(path.join(__dirname, './master.js'), {
-      execArgv: execArgv
-    });
-    child.processFile = processFile;
+  child = fork(path.join(__dirname, './master.js'), {
+    execArgv: execArgv
+  });
+  child.processFile = processFile;
 
-    _this.retained[child.pid] = child;
+  _this.retained[child.pid] = child;
 
-    child.on('exit', _this.remove.bind(_this, child));
+  child.on('exit', _this.remove.bind(_this, child));
 
-    return initChild(child, processFile).return(child);
-  });
+  return initChild(child, processFile).return(child);
 };
@@ -95,18 +93,2 @@
 
-var checkProcessorFile = function(processorFile) {
-  return new Promise(function(resolve, reject) {
-    fs.exists(processorFile, function(stats, err) {
-      if (err) {
-        reject(err);
-      } else {
-        if (stats) {
-          resolve();
-        } else {
-          reject(new Error('File does not exists'));
-        }
-      }
-    });
-  });
-};
-
 module.exports = ChildPool;

lib/queue.js

@@ -9,2 +9,4 @@ /*eslint-env node */
 
+var fs = require('fs');
+var path = require('path');
 var util = require('util');
@@ -41,3 +43,3 @@ var url = require('url');
         v    |             -- > failed
-        delayed
+        delayed 
 */
@@ -665,3 +667,11 @@
   if (typeof handler === 'string') {
+    var processorFile =
+      handler + (path.extname(handler) === '.js' ? '' : '.js');
+
+    if (!fs.existsSync(processorFile)) {
+      throw new Error('File ' + processorFile + ' does not exist');
+    }
+
     this.childPool = this.childPool || require('./process/child-pool')();
+
     var sandbox = require('./process/sandbox');
@@ -668,0 +678,0 @@ this.handlers[name] = sandbox(handler, this.childPool).bind(this);

lib/scripts.js

@@ -333,3 +333,3 @@ /**
     ];
-    var args = [queue.toKey(''), delayedTimestamp];
+    var args = [queue.toKey(''), delayedTimestamp, queue.token];
     return queue.client.updateDelaySet(keys.concat(args));
@@ -336,0 +336,0 @@ },

package.json

 {
   "name": "bull",
-  "version": "3.4.7",
+  "version": "3.4.8",
   "description": "Job manager",
@@ -5,0 +5,0 @@ "main": "./index.js",

PATTERNS.md

@@ -13,2 +13,3 @@
 - [Custom backoff strategy](#custom-backoff-strategy)
+- [Async multiple-repo job processor](#async-multiple-repo-job-processor)
 
@@ -201,1 +202,50 @@ If you have any other common patterns you want to add, pull request them!
 ```
+
+Manually fetching jobs
+----------------------------------
+
+If you want the actual job processing to be done in a seperate repo/service than where `bull` is running, this pattern may be for you.
+
+Manually transitioning states for jobs can be done with a few simple methods.
+
+1. Adding a job to the 'waiting' queue. Grab the queue and call `add`.
+
+```typescript
+import Queue from "bull";
+
+const queue = new Queue(description, queueOptions);
+queue.add({ random_attr: "random_value" });
+```
+
+2. Pulling a job from 'waiting' and moving it to 'active'.
+
+```typescript
+const job: Job = await queue.getNextJob();
+```
+
+3. Move the job to the 'failed' queue if something goes wrong.
+
+```typescript
+const (nextJobData, nextJobId) = await job.moveToFailed(
+  {
+    message: "Call to external service failed!",
+  },
+  true,
+);
+```
+
+3. Move the job to the 'completed' queue.
+
+```typescript
+const (nextJobData, nextJobId) = await job.moveToCompleted("succeeded", true);
+```
+
+4. Return the next job if one is returned.
+
+```typescript
+if (nextJobdata) {
+  return Job.fromJSON(queue, nextJobData, nextJobId);
+}
+```
+
+Then you can easily wrap `bull` in an API for use with external systems.

README.md

@@ -70,3 +70,3 @@
 
-Do you want to sponsor bull development? Please, let us now!
+Do you want to sponsor bull development? Please, let us know!
 
@@ -73,0 +73,0 @@

REFERENCE.md

@@ -40,2 +40,4 @@
   - [Job#finished](#jobfinished)
+  - [Job#moveToCompleted](#jobMoveToCompleted)
+  - [Job#moveToFailed](#moveToFailed)
 
@@ -119,3 +121,3 @@ - [Events](#events)
 
-`drainDelay`: A timeout for when the queue is in `drained` state (empty waiting for jobs). It is used when calling `queue.getNextJob()`, which will pass itto `.brpoplpush` on the Redis client.
+`drainDelay`: A timeout for when the queue is in `drained` state (empty waiting for jobs). It is used when calling `queue.getNextJob()`, which will pass it to `.brpoplpush` on the Redis client.
 
@@ -699,3 +701,23 @@ ```js
 
+### Job#moveToCompleted
 
+```ts
+moveToCompleted(returnValue, ignoreLock): Promise<string[Jobdata, JobId] | null>
+```
+
+Moves a job to the `completed` queue. Pulls a job from 'waiting' to 'active' and returns a tuple containing the next jobs data and id. If no job is in the `waiting` queue, returns null.
+
+---
+
+### Job#moveToFailed
+
+```ts
+moveToFailed(errorInfo, ignoreLock): Promise<string[Jobdata, JobId] | null>
+```
+
+Moves a job to the `failed` queue. Pulls a job from 'waiting' to 'active' and returns a tuple containing the next jobs data and id. If no job is in the `waiting` queue, returns null.
+
+---
+
+
 Events
@@ -711,2 +733,6 @@ ------
 
+.on('waiting', function(jobId){
+  // A Job is waiting to be processed as soon as a worker is idling.
+});
+
 .on('active', function(job, jobPromise){
@@ -713,0 +739,0 @@ // A job has started. You can use `jobPromise.cancel()`` to abort it.

test/test_child-pool.js

@@ -19,13 +19,2 @@ /*eslint-env node */
 
-  it('should raise an expection if invalid processor', function() {
-    return pool.retain('foobar').then(
-      function() {
-        throw new Error('Should raise an exception');
-      },
-      function(err) {
-        expect(err).to.be.instanceOf(Error);
-      }
-    );
-  });
-
   it('should return same child if free', function() {
@@ -32,0 +21,0 @@ var processor = __dirname + '/fixtures/fixture_processor_bar.js';

test/test_job.js

@@ -419,18 +419,21 @@ /*eslint-env node */
   describe('.moveToCompleted', function() {
-    it('marks the job as completed', function() {
-      return Job.create(queue, { foo: 'bar' }).then(function(job) {
-        return job
-          .isCompleted()
-          .then(function(isCompleted) {
-            expect(isCompleted).to.be(false);
-          })
-          .then(function() {
-            return job.moveToCompleted('succeeded', true);
-          })
-          .then(function(/*moved*/) {
-            return job.isCompleted().then(function(isCompleted) {
-              expect(isCompleted).to.be(true);
-              expect(job.returnvalue).to.be('succeeded');
+    it('marks the job as completed and returns new job', function() {
+      return Job.create(queue, { foo: 'bar' }).then(function(job1) {
+        return Job.create(queue, { foo: 'bar' }).then(function(job2) {
+          return job2
+            .isCompleted()
+            .then(function(isCompleted) {
+              expect(isCompleted).to.be(false);
+            })
+            .then(function() {
+              return job2.moveToCompleted('succeeded', true);
+            })
+            .then(function(job1Id) {
+              return job2.isCompleted().then(function(isCompleted) {
+                expect(isCompleted).to.be(true);
+                expect(job2.returnvalue).to.be('succeeded');
+                expect(job1Id[1]).to.be(job1.id);
+              });
             });
-          });
+        });
       });
@@ -437,0 +440,0 @@ });

test/test_repeat.js

@@ -337,3 +337,3 @@ /*eslint-env node */
 
-  it('should not re-add a repeatable job after it has been deleted', function() {
+  it('should not re-add a repeatable job after it has been removed', function() {
     var _this = this;
@@ -564,2 +564,26 @@ var date = new Date('2017-02-07 9:24:00');
   });
+
+  it.only('should emit a waiting event when adding a repeatable job to the waiting list', function(done) {
+    var _this = this;
+    var date = new Date('2017-02-07 9:24:00');
+    this.clock.tick(date.getTime());
+    var nextTick = 2 * ONE_SECOND + 500;
+
+    queue.on('waiting', function(jobId) {
+      expect(jobId).to.be.equal(
+        'repeat:93168b0ea97b55fb5a8325e8c66e4300:1486455842000'
+      );
+      done();
+    });
+
+    queue
+      .add('repeat', { foo: 'bar' }, { repeat: { cron: '*/2 * * * * *' } })
+      .then(function() {
+        _this.clock.tick(nextTick);
+      });
+
+    queue.process('repeat', function() {
+      console.error('hiasd');
+    });
+  });
 });

test/test_sandboxed_process.js

@@ -244,3 +244,12 @@ /*eslint-env node */
 
-  it('should process and fail', function(done) {
+  it('should error if processor file is missing', function(done) {
+    try {
+      queue.process(__dirname + '/fixtures/missing_processor.js');
+      done(new Error('did not throw error'));
+    } catch (err) {
+      done();
+    }
+  });
+
+  it('should process and fail using callback', function(done) {
     queue.process(__dirname + '/fixtures/fixture_processor_callback_fail.js');
@@ -247,0 +256,0 @@

No alert changes

Worsened metrics

Total package byte prevSize

1071308

-1.24%

Lines of code

8740

-8.64%

Number of low supply chain risk alerts

4

33.33%

No dependency changes