featureservice - npm Package Compare versions

Comparing version 1.4.4 to 1.4.5

CHANGELOG.md

@@ -5,2 +5,7 @@ # Change Log
 
+## [1.4.5] - 2016-01-13
+### Changed
+* Throttle concurrency down on every 2 failed requests
+* Throttle concurrency up on every 10 successful requests
+
 ## [1.4.4] - 2016-01-07
@@ -165,2 +170,3 @@ ### Fixed
 
+[1.4.5]: https://github.com/koopjs/featureservice/compare/v1.4.4...v1.4.5
 [1.4.4]: https://github.com/koopjs/featureservice/compare/v1.4.3...v1.4.4
@@ -167,0 +173,0 @@ [1.4.3]: https://github.com/koopjs/featureservice/compare/v1.4.2...v1.4.3

index.js

@@ -6,2 +6,3 @@ var queue = require('async').queue
 var urlUtils = require('url')
+var utils = require('./lib/utils.js')
 
@@ -23,10 +24,11 @@ /**
   var service = parseUrl(url)
-  this.url = service.url
+  this.server = service.server
   this.options = options || {}
   this.options.size = this.options.size || 5000
   this.options.backoff = this.options.backoff || 1000
-  this.options.concurrency = this.options.concurrency || this.url.split('//')[1].match(/^service/) ? 16 : 4
+  this.options.maxConcurrency = this.options.concurrency || utils.setConcurrency(this.server, this.options.geomType)
   this.options.timeOut = this.options.timeOut || (1.5 * 60 * 1000)
 
   this.layer = service.layer || this.options.layer || 0
+
   this.logger = this.options.logger
@@ -37,3 +39,4 @@
     this._requestFeatures(task, callback)
-  }.bind(this), this.options.concurrency)
+  }.bind(this), this.options.maxConcurrency)
+  this.concurrency = this.options.maxConcurrency
 }
@@ -45,3 +48,3 @@
     layer: layer && layer[1] ? layer[1] : undefined,
-    url: url.match(/.+\/(feature|map)server/i)[0]
+    server: url.match(/.+\/(feature|map)server/i)[0]
   }
@@ -153,3 +156,3 @@ }
 
-  return this.url + '/' + this.layer + '/query?f=json&outFields=&outStatistics=' + JSON.stringify(json)
+  return this.server + '/' + this.layer + '/query?f=json&outFields=&outStatistics=' + JSON.stringify(json)
 }
@@ -183,3 +186,3 @@
 FeatureService.prototype.info = function (callback) {
-  var url = this.url + '?f=json'
+  var url = this.server + '?f=json'
   this.request(url, function (err, json) {
@@ -213,3 +216,3 @@ /**
 FeatureService.prototype.layerInfo = function (callback) {
-  var url = this.url + '/' + this.layer + '?f=json'
+  var url = this.server + '/' + this.layer + '?f=json'
 
@@ -259,3 +262,3 @@ this.request(url, function (err, json) {
 FeatureService.prototype.layerIds = function (callback) {
-  var url = this.url + '/' + this.layer + '/query?where=1=1&returnIdsOnly=true&f=json'
+  var url = this.server + '/' + this.layer + '/query?where=1=1&returnIdsOnly=true&f=json'
   this.request(url, function (err, json) {
@@ -283,3 +286,3 @@ if (err || !json.objectIds) {
 FeatureService.prototype.featureCount = function (callback) {
-  var countUrl = this.url + '/' + (this.layer || 0)
+  var countUrl = this.server + '/' + (this.layer || 0)
   countUrl += '/query?where=1=1&returnCountOnly=true&f=json'
@@ -406,3 +409,3 @@
   var resultOffset
-  var url = this.url
+  var url = this.server
 
@@ -439,3 +442,3 @@ for (var i = 0; i < pages; i++) {
       var where = [oidField, ' >= ', pageMin, ' AND ', oidField, '<=', pageMax].join('')
-      var pageUrl = this.url + '/' + (this.layer) + '/query?outSR=4326&where=' + where + '&f=json&outFields=*'
+      var pageUrl = this.server + '/' + (this.layer) + '/query?outSR=4326&where=' + where + '&f=json&outFields=*'
       pageUrl += '&geometry=&returnGeometry=true&geometryPrecision=10'
@@ -465,3 +468,3 @@ reqs.push({req: pageUrl})
 
-  var url = this.url
+  var url = this.server
   var pages = Math.max((stats.max === size) ? stats.max : Math.ceil((stats.max - stats.min) / size), 1)
@@ -535,2 +538,3 @@
           }
+          self._throttleQueue()
           cb(null, json, task)
@@ -613,3 +617,3 @@ })
 FeatureService.prototype._catchErrors = function (task, error, url, cb) {
-  var self = this
+  this._throttleQueue(error)
   // be defensive in case there was no json payload
@@ -628,5 +632,2 @@ error.body = error.body || {}
 
-  // dial back concurrency if we start hitting errors
-  throttleQueue()
-
   this.log('info', 'Re-requesting page ' + task.req + ' attempt ' + task.retry)
@@ -637,7 +638,9 @@
   }.bind(this), task.retry * this.options.backoff)
+}
 
-  function throttleQueue () {
-    var concurrency = self.pageQueue.concurrency / 2
-    self.pageQueue.concurrency = concurrency > 1 ? Math.ceil(concurrency) : 1
-  }
+FeatureService.prototype._throttleQueue = function (fail) {
+  if (fail) this.concurrency -= 0.5
+  else this.concurrency += 0.1
+  if (this.concurrency > this.options.maxConcurrency) this.concurrency = this.options.maxConcurrency
+  this.pageQueue.concurrency = this.concurrency >= 1 ? Math.floor(this.concurrency) : 1
 }
@@ -644,0 +647,0 @@

package.json

 {
   "name": "featureservice",
   "description": "Get all features from an Esri Feature Service",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "author": "Chris Helm",
@@ -6,0 +6,0 @@ "bugs": {

test/index.js

@@ -20,3 +20,3 @@ var sinon = require('sinon')
   t.equal(serv.layer.toString(), '2')
-  t.equal(serv.url, 'http://koop.whatever.com/FeatureServer')
+  t.equal(serv.server, 'http://koop.whatever.com/FeatureServer')
   t.end()
@@ -28,3 +28,3 @@ })
   t.equal(serv.layer.toString(), '0')
-  t.equal(serv.url, 'http://koop.whatava.com/FeatureServer')
+  t.equal(serv.server, 'http://koop.whatava.com/FeatureServer')
   t.end()
@@ -512,1 +512,25 @@ })
 })
+
+test('setting concurrency for a hosted polygon service', function (t) {
+  t.plan(1)
+  var service = new FeatureService('http://services.arcgis.com/mapserver/3', {geomType: 'Polygon'})
+  t.equal(service.options.maxConcurrency, 4)
+})
+
+test('setting concurrency for an on-premise line service', function (t) {
+  t.plan(1)
+  var service = new FeatureService('http://foo.com/mapserver/2', {geomType: 'Line'})
+  t.equal(service.options.maxConcurrency, 1)
+})
+
+test('setting concurrency for a hosted point service', function (t) {
+  t.plan(1)
+  var service = new FeatureService('http://services.arcgis.com/featureserver/3', {geomType: 'Point'})
+  t.equal(service.options.maxConcurrency, 16)
+})
+
+test('setting concurrency for an on-premise point service', function (t) {
+  t.plan(1)
+  var service = new FeatureService('http://foo.com/featureserver/3', {geomType: 'Point'})
+  t.equal(service.options.maxConcurrency, 4)
+})

No alert changes

Improved metrics

Total package byte prevSize

1371204

0.15%

Number of package files

31

3.33%

Lines of code

1805

2.21%

No dependency changes