featureservice - npm Package Compare versions

Comparing version 1.5.0 to 1.5.1

CHANGELOG.md

@@ -5,2 +5,7 @@ # Change Log
 
+## [1.5.1] - 2016-03-02
+### Changed
+* Don't use resultOffset for services not hosted on ArcGIS Online, it's not reliable
+* Decode streaming server response
+
 ## [1.5.0] - 2016-02-28
@@ -184,2 +189,3 @@ ### Added
 
+[1.5.1]: https://github.com/koopjs/featureservice/compare/v1.5.0...v1.5.1
 [1.5.0]: https://github.com/koopjs/featureservice/compare/v1.4.6...v1.5.0
@@ -186,0 +192,0 @@ [1.4.6]: https://github.com/koopjs/featureservice/compare/v1.4.5...v1.4.6

index.js

@@ -5,2 +5,3 @@ var queue = require('async').queue
 var zlib = require('zlib')
+var stream = require('stream')
 var urlUtils = require('url')
@@ -24,2 +25,3 @@ var Utils = require('./lib/utils.js')
   var service = Utils.parseUrl(url)
+  this.hosted = service.hosted
   this.server = service.server
@@ -96,14 +98,11 @@ this.options = options || {}
   var req = ((uri.protocol === 'https:') ? https : http).request(opts, function (response) {
+    var encoding = response.headers['content-encoding']
     var data = []
-    response.on('data', function (chunk) {
-      data.push(chunk)
-    })
-
-    response.on('error', function (err) {
-      callback(err)
-    })
-
-    response.on('end', function () {
-      self._decode(response, data, callback)
-    })
+    response
+    // TODO standaridize these errors
+    .on('error', function (err) { return callback(err) })
+    .pipe(decode(encoding))
+    .on('error', function (err) { return callback(err) })
+    .on('data', function (chunk) { data.push(chunk) })
+    .on('end', function () { parse(data, callback) })
   })
@@ -337,3 +336,3 @@
     if (err) return callback(err)
-    this.concurrency = this.options.concurrency || Utils.setConcurrency(this.server, meta.layer.geometryType)
+    this.concurrency = this.options.concurrency || Utils.setConcurrency(this.hosted, meta.layer.geometryType)
     this.maxConcurrency = this.concurrency
@@ -350,3 +349,3 @@ this.pageQueue.concurrency = this.concurrency
     var canPage = layer.advancedQueryCapabilities && layer.advancedQueryCapabilities.supportsPagination
-    if (canPage) return callback(null, this._offsetPages(nPages, size))
+    if (canPage && !this.hosted) return callback(null, this._offsetPages(nPages, size))
 
@@ -509,17 +508,15 @@ if (!meta.oid) return callback(new Error('ObjectID type field not found, unable to page'))
     var req = ((url_parts.protocol === 'https:') ? https : http).request(opts, function (response) {
-      var data = []
-
-      response.on('data', function (chunk) {
-        data.push(chunk)
+      var encoding = response.headers['content-encoding']
+      var buffer = []
+      response
+      .on('error', function (err) { self._catchErrors(task, err, uri, cb) })
+      .pipe(decode(encoding))
+      .on('error', function (error) {
+        return self._catchErrors(task, error, uri, cb)
       })
-
-      response.on('error', function (err) {
-        self._catchErrors(task, err, uri, cb)
-      })
-
-      response.on('end', function () {
-        self._decode(response, data, function (err, json) {
-          // the error coming back here is already well formed in _decode
+      .on('data', function (chunk) { buffer.push(chunk) })
+      .on('end', function () {
+        // server responds 200 with error in the payload so we have to inspect
+        parse(buffer, function (err, json) {
           if (err) return self._catchErrors(task, err, uri, cb)
-          // server responds 200 with error in the payload so we have to inspect
           if (!json || json.error) {
@@ -534,3 +531,3 @@ if (!json) json = {error: {}}
           self._throttleQueue()
-          cb(null, json, task)
+          cb(null, json)
         })
@@ -564,24 +561,6 @@ })
 
-/* Decodes a response for features
- * @param {object} res - the response received from the GIS Server
- * @param {array} data - an array of chunks received from the server
- * @param {function} callback - calls back with either an error or the decoded feature json
- */
-FeatureService.prototype._decode = function (res, data, callback) {
-  var encoding = res.headers['content-encoding']
-  if (!data.length > 0) return callback(new Error('Response from the server was empty'))
-  var buffer = Buffer.concat(data)
-  if (encoding === 'gzip') {
-    zlib.gunzip(buffer, function (err, decompressed) {
-      if (err) return callback(err)
-      parse(decompressed, callback)
-    })
-  } else if (encoding === 'deflate') {
-    zlib.inflate(buffer, function (err, decompressed) {
-      if (err) return callback(err)
-      parse(decompressed, callback)
-    })
-  } else {
-    parse(buffer, callback)
-  }
+function decode (encoding) {
+  if (encoding === 'gzip') return zlib.createGunzip()
+  else if (encoding === 'deflate') return zlib.createInflate()
+  else return stream.PassThrough()
 }
@@ -593,5 +572,6 @@
   try {
-    response = buffer.toString()
+    response = Buffer.concat(buffer).toString()
     parsed = JSON.parse(response)
   } catch (e) {
+    console.log(e)
     // sometimes we get html or plain strings back
@@ -598,0 +578,0 @@ var pattern = new RegExp(/[^{\[]/)

lib/utils.js

@@ -5,9 +5,8 @@ module.exports = {
    *
-   * @param {string} service - the feature or map service targeted
+   * @param {boolean} hosted - whether or not the service is hosted on ArcGIS Online
    * @param {string} geomType - the geometry type of the features in the service
    * @return {integer} the suggested concurrency
    */
-  setConcurrency: function (service, geomType) {
-    var isHosted = service.match(/services(\d)?(qa|dev)?.arcgis.com/)
-    var naieve = isHosted ? 16 : 4
+  setConcurrency: function (hosted, geomType) {
+    var naieve = hosted ? 16 : 4
     if (!geomType) return naieve
@@ -17,2 +16,7 @@ var concurrency = geomType.match(/point/i) ? naieve : naieve / 4
   },
+  /**
+   * Parsed the layer and server from a feature service url
+   * @param {string} url - a link to a feature service
+   * @return {object} contains the layer, the server and whether or not the server is hosted
+   */
   parseUrl: function (url) {
@@ -22,5 +26,6 @@ var layer = url.match(/(?:.+\/(?:feature|map)server\/)(\d+)/i)
       layer: layer && layer[1] ? layer[1] : undefined,
-      server: url.match(/.+\/(feature|map)server/i)[0]
+      server: url.match(/.+\/(feature|map)server/i)[0],
+      hosted: /services(\d)?(qa|dev)?.arcgis.com/.test(url)
     }
   }
 }

package.json

 {
   "name": "featureservice",
   "description": "Get all features from an Esri Feature Service",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "author": "Chris Helm",
@@ -6,0 +6,0 @@ "bugs": {

test/index.js

@@ -7,4 +7,4 @@ var sinon = require('sinon')
 var fs = require('fs')
+var _ = require('lodash')
 var zlib = require('zlib')
-var _ = require('lodash')
 
@@ -283,76 +283,2 @@ var service = new FeatureService('http://koop.dc.esri.com/socrata/seattle/2tje-83f6/FeatureServer/1', {objectIdField: 'OBJECTID'})
 
-test('decoding something that is gzipped', function (t) {
-  var json = JSON.stringify(JSON.parse(fs.readFileSync('./test/fixtures/uncompressed.json')))
-  zlib.gzip(json, function (err, gzipped) {
-    t.error(err)
-    var data = [gzipped]
-    var res = {headers: {'content-encoding': 'gzip'}}
-    var service = new FeatureService('http://service.com/mapserver/2')
-
-    service._decode(res, data, function (error, json) {
-      t.equal(error, null)
-      t.equal(json.features.length, 2000)
-      t.end()
-    })
-  })
-})
-
-test('decoding something that is deflated', function (t) {
-  var json = JSON.stringify(JSON.parse((fs.readFileSync('./test/fixtures/uncompressed.json'))))
-  zlib.deflate(json, function (err, deflated) {
-    t.error(err)
-    var data = [deflated]
-    var res = {headers: {'content-encoding': 'deflate'}}
-
-    service._decode(res, data, function (error, json) {
-      t.equal(error, null)
-      t.equal(json.features.length, 2000)
-      t.end()
-    })
-  })
-})
-
-test('decoding something that is not compressed', function (t) {
-  var uncompressed = JSON.stringify(JSON.parse(fs.readFileSync('./test/fixtures/uncompressed.json')))
-  var buffer = new Buffer(uncompressed)
-  var buf1 = buffer.slice(0, -1)
-  var buf2 = buffer.slice(-1)
-  var data = [buf1, buf2]
-  var res = {headers: {}}
-
-  service._decode(res, data, function (err, json) {
-    t.error(err)
-    t.equal(json.features.length, 2000)
-    t.end()
-  })
-})
-
-test('decoding an empty response', function (t) {
-  var empty = []
-  var res = {headers: {'content-encoding': 'gzip'}}
-
-  service._decode(res, empty, function (err, json) {
-    t.notEqual(typeof err, 'undefined')
-    t.end()
-  })
-})
-
-test('decoding an unexpected HMTL response', function (t) {
-  var res = {headers: {'content-encoding': ''}}
-  var data = [new Buffer('</html></html>')]
-  service._decode(res, data, function (err) {
-    t.equal(err.message, 'Received HTML or plain text when expecting JSON')
-    t.end()
-  })
-})
-
-test('decoding an unexpected plain text response', function (t) {
-  var res = {headers: {'content-encoding': ''}}
-  var data = [new Buffer('Bad request')]
-  service._decode(res, data, function (err) {
-    t.equal(err.message, 'Received HTML or plain text when expecting JSON')
-    t.end()
-  })
-})
-
 test('should trigger catchErrors with an error when receiving json with an error in the response', function (t) {
@@ -387,7 +313,7 @@ var data = {
 test('requesting a page of features', function (t) {
-  var page = fs.readFileSync('./test/fixtures/page.json')
+  var page = fs.createReadStream('./test/fixtures/page.json')
   var fixture = nock('http://servicesqa.arcgis.com')
 
   fixture.get('/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision=')
-  .reply(200, page)
+  .reply(200, function () { return page })
 
@@ -404,2 +330,71 @@ var service = new FeatureService('http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0')
 
+test('requesting a page of features that is gzipped', function (t) {
+  var page = fs.createReadStream('./test/fixtures/page.json').pipe(zlib.createGzip())
+  var fixture = nock('http://servicesqa.arcgis.com')
+
+  fixture.get('/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision=')
+  .reply(200, function () { return page }, {'content-encoding': 'gzip'})
+
+  var service = new FeatureService('http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0')
+  var task = {req: 'http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision='}
+
+  service._requestFeatures(task, function (err, json) {
+    t.error(err)
+    t.equal(json.features.length, 1000)
+    t.end()
+  })
+})
+
+test('requesting a page of features that is deflate encoded', function (t) {
+  var page = fs.createReadStream('./test/fixtures/page.json').pipe(zlib.createDeflate())
+  var fixture = nock('http://servicesqa.arcgis.com')
+
+  fixture.get('/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision=')
+  .reply(200, function () { return page }, {'content-encoding': 'deflate'})
+
+  var service = new FeatureService('http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0')
+  var task = {req: 'http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision='}
+
+  service._requestFeatures(task, function (err, json) {
+    t.error(err)
+    t.equal(json.features.length, 1000)
+    t.end()
+  })
+})
+
+test('requesting a page of features and getting an html response', function (t) {
+  var page = new Buffer('</html></html>')
+  var fixture = nock('http://servicesqa.arcgis.com')
+
+  fixture.get('/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision=')
+  .times(4)
+  .reply(200, function () { return page })
+
+  var service = new FeatureService('http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0', {backoff: 1})
+  var task = {req: 'http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision='}
+
+  service._requestFeatures(task, function (err, json) {
+    t.ok(err)
+    t.equal(err.message, 'Paging aborted: Received HTML or plain text when expecting JSON')
+    t.end()
+  })
+})
+
+test('requesting a page of features and getting an empty response', function (t) {
+  var fixture = nock('http://servicesqa.arcgis.com')
+
+  fixture.get('/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision=')
+  .times(4)
+  .reply(200, function () { return undefined })
+
+  var service = new FeatureService('http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0', {backoff: 1})
+  var task = {req: 'http://servicesqa.arcgis.com/97KLIFOSt5CxbiRI/arcgis/rest/services/QA_data_simple_point_5000/FeatureServer/0/query?outSR=4326&f=json&outFields=*&where=1=1&resultOffset=1000&resultRecordCount=1000&geometry=&returnGeometry=true&geometryPrecision='}
+
+  service._requestFeatures(task, function (err, json) {
+    t.ok(err)
+    t.equal(err.message, 'Paging aborted: Failed to parse server response')
+    t.end()
+  })
+})
+
 // paging integration tests
@@ -561,3 +556,3 @@ test('building pages for a service that supports pagination', function (t) {
   t.plan(1)
-  var concurrency = Utils.setConcurrency('http://services.arcgis.com/mapserver/3', 'esriGeometryPolygon')
+  var concurrency = Utils.setConcurrency(true, 'esriGeometryPolygon')
   t.equal(concurrency, 4)
@@ -568,3 +563,3 @@ })
   t.plan(1)
-  var concurrency = Utils.setConcurrency('http://foo.com/mapserver/2', 'esriGeometryLine')
+  var concurrency = Utils.setConcurrency(false, 'esriGeometryLine')
   t.equal(concurrency, 1)
@@ -575,3 +570,3 @@ })
   t.plan(1)
-  var concurrency = Utils.setConcurrency('http://services.arcgis.com/featureserver/3', 'esriGeometryPoint')
+  var concurrency = Utils.setConcurrency(true, 'esriGeometryPoint')
   t.equal(concurrency, 16)
@@ -582,4 +577,4 @@ })
   t.plan(1)
-  var concurrency = Utils.setConcurrency('http://foo.com/featureserver/3', 'esriGeometryPoint')
+  var concurrency = Utils.setConcurrency(false, 'esriGeometryPoint')
   t.equal(concurrency, 4)
 })

No alert changes

Improved metrics

Total package byte prevSize

1274113

increased by0.15%

Worsened metrics

Lines of code

1838

decreased by-0.92%

No dependency changes