fh-sync - npm Package Compare versions

Comparing version 1.0.7 to 1.0.8

integration/test_storage.js

@@ -248,3 +248,3 @@ var storageModule = require('../lib/storage');
         function checkSyncUpdatesCreated(callback) {
-          storage.listUpdates(DATASETID, {cuid: TESTCUID}, function(err, updates){
+          storage.listUpdates(DATASETID, {cuid: TESTCUID}, null, function(err, updates){
             assert.ok(!err);
@@ -257,3 +257,3 @@ assert.equal(updates.length, 2);
         function checkSyncUpdatesRemoved(callback) {
-          storage.listUpdates(DATASETID, {cuid: TESTCUID}, function(err, updates){
+          storage.listUpdates(DATASETID, {cuid: TESTCUID}, null, function(err, updates){
             assert.ok(!err);
@@ -260,0 +260,0 @@ assert.equal(updates.length, 1);

lib/api-sync.js

@@ -84,4 +84,21 @@ var _ = require('underscore');
     addAcks: function(callback) {
-      debug('[%s] adding acks to queue. size = %d', datasetId, (params.acknowledgements && params.acknowledgements.length || 0));
-      addToQueue(params.acknowledgements, {datasetId: datasetId, cuid: cuid}, ackQueue, callback);
+      var acknowledgements = params.acknowledgements || [];
+      debug('[%s] found acks in request. size = %d', datasetId, acknowledgements.length);
+      if (syncConfig.syncReqAckLimit && syncConfig.syncReqAckLimit > 0 && acknowledgements.length > syncConfig.syncReqAckLimit) {
+        acknowledgements = acknowledgements.slice(0, syncConfig.syncReqAckLimit);
+        debug('[%s] too many acks in the request. Only process the first %d items', datasetId, acknowledgements.length);
+      }
+      debug('[%s] adding acks to queue. size = %d', datasetId, acknowledgements.length);
+      if (syncConfig.syncReqWaitForAck) {
+        debug('[%s] waiting for ack insert to complete', datasetId);
+        addToQueue(acknowledgements, {datasetId: datasetId, cuid: cuid}, ackQueue, callback);
+      } else {
+        debug('[%s] skip waiting for ack insert', datasetId);
+        addToQueue(acknowledgements, {datasetId: datasetId, cuid: cuid}, ackQueue, function(err){
+          if (err) {
+            debugError('[%s] ack insert error = %s', datasetId, err);
+          }
+        });
+        callback();
+      }
     },
@@ -94,3 +111,8 @@ addPendings: function(callback) {
       debug('[%s] list updates for client cuid = %s', datasetId, cuid);
-      syncStorage.listUpdates(datasetId, {cuid: cuid}, callback);
+      var query = {cuid: cuid};
+      var opts = {};
+      if (syncConfig.syncReqAckLimit && syncConfig.syncReqAckLimit > 0) {
+        opts.limit = syncConfig.syncReqAckLimit;
+      }
+      syncStorage.listUpdates(datasetId, query, opts, callback);
     }
@@ -97,0 +119,0 @@ }, function(err, results) {

lib/api-syncRecords.js

@@ -43,3 +43,3 @@ var async = require('async');
     timestamp: {$gt: lastSyncEndTime}
-  }, cb);
+  }, null, cb);
 }
@@ -46,0 +46,0 @@

lib/storage/sync-updates.js

@@ -61,8 +61,14 @@ var syncUtil = require('../util');
  * @param {Object} criteria
+ * @param {Object} options
  * @param {Function} callback
  */
-function doListUpdates(datasetId, criteria, callback) {
+function doListUpdates(datasetId, criteria, options, callback) {
   debug('[%s] doListUpdates criteria = %j',datasetId,criteria);
   var updatesCollection = mongoClient.collection(getDatasetUpdatesCollectionName(datasetId));
-  updatesCollection.find(criteria).toArray(function(err, updates) {
+  var docLimit = options && options.limit;
+  var cursor = updatesCollection.find(criteria);
+  if (docLimit && docLimit > 0) {
+    cursor = cursor.limit(docLimit);
+  }
+  cursor.toArray(function(err, updates) {
     if (err) {
@@ -103,6 +109,7 @@ debugError('[%s] Failed to doListUpdates due to error %s :: criteria = %j' + criteria,datasetId,err,criteria);
      * @param {Object} criteria the list criteria, a mongodb query object
+     * @param {Object} options options for the find option, like `limit`
      * @param {Function} callback
      */
-    listUpdates: function(datasetId, criteria, callback) {
-      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doListUpdates)(datasetId, criteria, callback);
+    listUpdates: function(datasetId, criteria, options, callback) {
+      return metrics.timeAsyncFunc(metrics.KEYS.MONGODB_OPERATION_TIME, doListUpdates)(datasetId, criteria, options, callback);
     }
@@ -109,0 +116,0 @@ };

lib/sync-server.js

@@ -103,3 +103,8 @@ var metricsModule = require('./sync-metrics');
   /** @type {String} specify the frequency the datasetClient cleaner should run. Default every hour.*/
-  datasetClientCleanerCheckFrequency: '1h'
+  datasetClientCleanerCheckFrequency: '1h',
+
+  /** @type {Boolean} Specify if the server should wait for the ack insert to complete before returning the response for the sync request. Default is true. */
+  syncReqWaitForAck: true,
+  /** @type {Number} Specify the max number of ack items will be processed for a single request. Default is -1 (unlimited).*/
+  syncReqAckLimit: -1
 };
@@ -106,0 +111,0 @@ var syncConfig = _.extend({}, DEFAULT_SYNC_CONF);

package.json

 {
   "name": "fh-sync",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "FeedHenry Data Synchronization Server",
@@ -5,0 +5,0 @@ "main": "index.js",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

225163

decreased by-6.18%

Number of package files

47

decreased by-2.08%

Lines of code

5239

decreased by-7.29%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes