osm2json - npm Package Compare versions

Comparing version 0.0.1 to 1.0.0

lib/osm2json.js

@@ -1,2 +0,3 @@
-/* Copyright (c) 2013 Digital Democracy
+/**
+ * Copyright (c) 2016 Digital Democracy
  * MIT +no-false-attribs License
@@ -6,46 +7,180 @@ * <https://github.com/digidem/osm2json/blob/master/LICENSE>
 
+var Transform = require('readable-stream').Transform
+var sax = require('sax')
 var util = require('util')
-  , Transform = require('stream').Transform
-  , expat = require('node-expat');
 
-util.inherits(Osm2Json, Transform);
+var STRICT = true
 
-function Osm2Json (options) {
-  if (!(this instanceof Osm2Json))
-    return new Osm2Json(options);
-  
-  this.currentElement = null;
-  
-  Transform.call(this, options);
-  this.parser = new expat.Parser('UTF-8');
-  
-  this.parser.on('startElement', onStartElement.bind(this));
+// These attributes are "id-like" and will be coerced to Number if opts.coerceIds === true
+var ID_ATTRIBUTES = ['id', 'uid', 'version', 'changeset', 'ref']
+// These attributes are always coerced to Number
+var NUMBER_ATTRIBUTES = ['lat', 'lon', 'comments_count', 'min_lat', 'min_lon', 'max_lat', 'max_lon',
+  'minlon', 'minlat', 'maxlon', 'maxlat']
 
-  this.parser.on('endElement', onEndElement.bind(this));
+// Any nodes in the XML that are not listed below will throw an error
+var VALID_ROOTS = ['osm', 'osmChange']
+var VALID_ACTIONS = ['create', 'modify', 'delete']
+var VALID_NODES = ['node', 'way', 'relation', 'changeset', 'bounds']
+
+// Attributes that are not in these whitelists are ignored.
+// Any node that is a child of a VALID_NODE that is not in `children` will throw an error
+var ELEMENT_ATTRIBUTES = ['id', 'user', 'uid', 'visible', 'version', 'changeset', 'timestamp']
+var WHITELISTS = {
+  node: {
+    attributes: ELEMENT_ATTRIBUTES.concat(['lat', 'lon']),
+    children: ['tag']
+  },
+  way: {
+    attributes: ELEMENT_ATTRIBUTES,
+    children: ['nd', 'tag']
+  },
+  relation: {
+    attributes: ELEMENT_ATTRIBUTES,
+    children: ['member', 'tag']
+  },
+  changeset: {
+    attributes: ['id', 'user', 'uid', 'created_at', 'closed_at', 'open',
+        'min_lat', 'min_lon', 'max_lat', 'max_lon', 'comments_count'],
+    children: ['tag']
+  },
+  bounds: {
+    attributes: ['minlon', 'minlat', 'maxlon', 'maxlat']
+  }
 }
 
-function onStartElement (name, attrs) {
-  if (name.match(/node|way|relation/)) {
-    this.currentElement = { type: name, attrs: attrs };
-  } else if (name == 'tag') {
-    if (!this.currentElement.tags) this.currentElement.tags = []
-    this.currentElement.tags.push(attrs);
-  } else if (name == 'nd') {
-    if (!this.currentElement.nodes) this.currentElement.nodes = []
-    this.currentElement.nodes.push(attrs);
+function is(list, value) {
+  return list.indexOf(value) > -1
+}
+
+function isValidChild(name, childname) {
+  return WHITELISTS[name] &&
+    WHITELISTS[name].children &&
+    WHITELISTS[name].children.indexOf(childname) > -1
+}
+
+function isValidAttribute(name, attr) {
+  return WHITELISTS[name] && WHITELISTS[name].attributes.indexOf(attr) > -1
+}
+
+function parseNumber(str) {
+  if (!isNaN(str) && str.length) {
+    return str % 1 === 0 ? parseInt(str, 10) : parseFloat(str)
   }
+  return str
 }
 
-function onEndElement (name) {
-  if (name.match(/node|way|relation/)) {
-    this.push(JSON.stringify(this.currentElement)+'\n');
-    this.currentElement = null;
+function parseBoolean(str) {
+  if (/^(?:true|false)$/i.test(str)) {
+    return str.toLowerCase() === 'true'
   }
+  return str
 }
 
-Osm2Json.prototype._transform = function (chunk, encoding, done) {
-  this.parser.parse(chunk);
-  done();
+module.exports = Osm2Json
+
+function Osm2Json(opts) {
+  if (!(this instanceof Osm2Json)) return new Osm2Json(opts)
+  this.opts = opts || {}
+  this.opts.coerceIds = this.opts.coerceIds !== false
+  this.parser = sax.parser(STRICT)
+  this.parser.onerror = this.onError.bind(this)
+  this.parser.onopentag = this.onOpenTag.bind(this)
+  this.parser.onclosetag = this.onCloseTag.bind(this)
+  Transform.call(this, { readableObjectMode : true })
 }
 
-module.exports = Osm2Json;
+util.inherits(Osm2Json, Transform)
+
+Osm2Json.prototype._transform = function(chunk, enc, done) {
+  if (this.error) return done(this.error)
+  this.parser.write(chunk.toString())
+  done(this.error)
+}
+
+Osm2Json.prototype.onError = function(err) {
+  err.message = 'Invalid XML at line #' + this.parser.line +
+    ', column #' + this.parser.column + ':\n' +
+    err.message
+  this.error = err
+}
+
+Osm2Json.prototype.onOpenTag = function(node) {
+  if (this.error) return
+  if (!this.root && is(VALID_ROOTS, node.name)) {
+    this.root = node.name
+  } else if (this.root === 'osmChange' && !this.currentAction && is(VALID_ACTIONS, node.name)) {
+    this.currentAction = node.name
+  } else if (!this.currentNode && is(VALID_NODES, node.name)) {
+    this.processNode(node)
+  } else if (this.currentNode && isValidChild(this.currentNode.type, node.name)) {
+    this.processChild(node)
+  } else {
+    this.onError(new Error('invalid tag <' + node.name + '>'))
+  }
+}
+
+Osm2Json.prototype.onCloseTag = function(name) {
+  if (this.root === 'osmChange' && is(VALID_ACTIONS, name)) {
+    this.currentAction = null
+  } else if (is(VALID_NODES, name)) {
+    this.push(this.currentNode)
+    this.currentNode = null
+  }
+}
+
+Osm2Json.prototype.processNode = function(node) {
+  this.currentNode = {}
+  this.currentNode.type = node.name
+  var attr = node.attributes
+  for (var attrName in attr) {
+    if (!attr.hasOwnProperty(attrName)) continue
+    if (!isValidAttribute(node.name, attrName)) continue
+    this.currentNode[attrName] = this.coerce(attrName, attr[attrName])
+  }
+  if (this.currentAction) this.currentNode.action = this.currentAction
+}
+
+Osm2Json.prototype.processChild = function(node) {
+  var currentNode = this.currentNode
+  var attr = node.attributes
+  switch (node.name) {
+    case 'tag':
+      if (!attr.k || !attr.v) {
+        return this.onError(new Error('<tag> missing k or v attribute'))
+      }
+      currentNode.tags = currentNode.tags || {}
+      currentNode.tags[attr.k] = attr.v
+      break
+    case 'nd':
+      if (!attr.ref) {
+        return this.onError(new Error('<nd> missing ref attribute'))
+      }
+      currentNode.nodes = currentNode.nodes || []
+      currentNode.nodes.push(this.coerce('ref', attr.ref))
+      break
+    case 'member':
+      if (!attr.ref || !attr.type) {
+        // NB: we don't throw an error for members with no role attribute
+        return this.onError(new Error('<member> missing ref or type attribute'))
+      }
+      currentNode.members = currentNode.members || []
+      currentNode.members.push({
+        type: attr.type,
+        ref: this.coerce('ref', attr.ref),
+        role: attr.role ? attr.role : ''
+      })
+      break
+  }
+}
+
+Osm2Json.prototype.coerce = function(attrName, value) {
+  var shouldCoerceToNumber = is(NUMBER_ATTRIBUTES, attrName)
+  if (this.opts.coerceIds) {
+    shouldCoerceToNumber = shouldCoerceToNumber || is(ID_ATTRIBUTES, attrName)
+  }
+  if (shouldCoerceToNumber) {
+    return parseNumber(value)
+  } else {
+    return parseBoolean(value)
+  }
+}

package.json

 {
   "name": "osm2json",
-  "version": "0.0.1",
-  "description": "Converts an OSM XML file to JSON objects as a transform stream",
+  "version": "1.0.0",
+  "description": "Converts an OSM XML file to OSM JSON objects as a transform stream",
   "main": "lib/osm2json.js",
@@ -13,11 +13,9 @@ "scripts": {
   },
-  "engines": {
-    "node": ">=0.10.0"
-  },
   "dependencies": {
-    "node-expat": ">=1.4.4"
+    "readable-stream": "^2.1.5",
+    "sax": "^1.2.1"
   },
   "keywords": [
     "xml",
-    "expat",
+    "sax",
     "osm",
@@ -30,3 +28,8 @@ "stream"
     "url": "https://github.com/digidem/osm2json/issues"
+  },
+  "devDependencies": {
+    "concat-stream": "^1.5.1",
+    "standard": "^8.0.0-beta.5",
+    "tape": "^4.6.0"
   }
 }

README.md

@@ -1,63 +0,104 @@
-Implements a [Node Transport Stream](http://nodejs.org/api/stream.html#stream_class_stream_transform). Takes a readable stream of [OSM XML](http://wiki.openstreetmap.org/wiki/OSM_XML) and outputs a stream of JSON in the following format:
+# osm2json
 
-```JSON
+[![npm](https://img.shields.io/npm/v/osm2json.svg?maxAge=2592000)](https://www.npmjs.com/package/osm2json)
+
+> Streaming parser from OSM XML to OSM JSON objects
+
+Implements a [Node Transport Stream](http://nodejs.org/api/stream.html#stream_class_stream_transform). Takes a readable stream of [OSM XML](http://wiki.openstreetmap.org/wiki/OSM_XML) and outputs a stream of objects compatible with Overpass [OSM JSON](http://overpass-api.de/output_formats.html#json). Also reads [OsmChange](http://wiki.openstreetmap.org/wiki/OsmChange) XML and outputs the same format but with an additional property `action` which is one of `create`, `modify`, `delete`. Uses [sax-js](https://github.com/isaacs/sax-js) to work in both node and the browser.
+
+## Table of Contents
+
+- [Install](#install)
+- [Usage](#usage)
+- [API](#api)
+- [Contribute](#contribute)
+- [License](#license)
+
+## Install
+
+```
+npm install osm2json
+```
+
+## Usage
+
+```js
+var fs = require('fs')
+var Osm2Json = require('../lib/osm2json')
+
+var rs = fs.createReadableStream(__dirname + './test.osm')
+
+rs.pipe(new Osm2Json()).pipe(process.stdout)
+```
+
+## Example Output
+
+```js
+// node
 {
-    "attrs": {
-        "changeset": "50", 
-        "id": "29", 
-        "lat": "38.9003573", 
-        "lon": "-77.0232578", 
-        "timestamp": "2013-09-05T19:38:11Z", 
-        "version": "1"
-    }, 
-    "tags": [
-        {
-            "k": "amenity", 
-            "v": "place_of_worship"
-        }
-    ], 
-    "type": "node"
+  type: 'node',
+  id: 1,
+  version: 0,
+  timestamp: '2013-09-05T19:38:11.187Z',
+  uid: 1,
+  user: 'gregor',
+  lat: 0,
+  lon: 0,
+  tags: { null: 'island' }
 }
-```
 
-```JSON
+// way
 {
-  "type": "way",
-  "attrs": {
-    "id": "3",
-    "version": "3",
-    "timestamp": "2013-09-05T19:38:11Z",
-    "changeset": "49"
-  },
-  "nodes": [{
-    "ref": "19"
-  }, {
-    "ref": "20"
-  }, {
-    "ref": "21"
-  }, {
-    "ref": "22"
-  }, {
-    "ref": "26"
-  }, {
-    "ref": "27"
-  }],
-  "tags": [{
-    "k": "name",
-    "v": "York St"
-  }]
+  type: 'way',
+  id: 3,
+  version: 3,
+  timestamp: '2013-09-05T19:38:11Z',
+  changeset: 49,
+  nodes: [ 19, 20, 21, 22, 26, 27 ],
+  tags: { name: 'York St' }
 }
+
+// relation
+{
+  type: 'relation',
+  id: 1,
+  members: [
+    {
+      type: 'relation',
+      ref: 1745069,
+      role: ''
+    },
+    {
+      type: 'relation',
+      ref: 172789,
+      role: ''
+    }
+  ],
+  tags: {
+    from: 'Konrad-Adenauer-Platz',
+    name: 'VRS 636'
+  }
+}
 ```
 
-##Example
+## API
 
-```Javascript
-var fs = require('fs')
-  , Osm2Json = require('../lib/osm2json');
-  
-var rs = fs.createReadStream('./test.osm');
+```js
+var Osm2Json = require('osm2json')
+```
 
-var osm2Json = new Osm2Json();
+### var stream = new Osm2Json(opts)
 
-rs.pipe(osm2Json).pipe(process.stdout);
-```
+Create a transform stream with:
+
+* `opts.coerceIds` - coerce ids to `Number` (defaults to *true*)
+
+The readable side of the stream is in `objectMode`.
+
+## Contribute
+
+PRs welcome. Right now this could do with some tests. If you are feeling ambitious, this could be sped up by using [node-expat](https://github.com/astro/node-expat) on node. The interface is similar to sax-js and it should be possible to wrap this to use sax-js on the browser and node-expat on the server using the [browserify `browser` field](https://github.com/substack/browserify-handbook#browser-field)
+
+## License
+
+MIT (c) 2016, Digital Democracy.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

13223

increased by93.06%

Number of package files

8

increased by14.29%

Lines of code

170

increased by295.35%

Number of low quality alerts

2

decreased by-33.33%

Number of lines in readme file

105

increased by66.67%

Worsened metrics

Dependency count

2

increased by100%

Dev dependency count

3

increased byInfinity%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedreadable-stream@^2.1.5

• + Addedsax@^1.2.1

• + Addedcore-util-is@1.0.3(transitive)

• + Addedinherits@2.0.4(transitive)

• + Addedisarray@1.0.0(transitive)

• + Addedprocess-nextick-args@2.0.1(transitive)

• + Addedreadable-stream@2.3.8(transitive)

• + Addedsafe-buffer@5.1.2(transitive)

• + Addedsax@1.4.1(transitive)

• + Addedstring_decoder@1.1.1(transitive)

• + Addedutil-deprecate@1.0.2(transitive)

• - Removednode-expat@>=1.4.4

• - Removedbindings@1.5.0(transitive)

• - Removedfile-uri-to-path@1.0.0(transitive)

• - Removednan@2.22.0(transitive)

• - Removednode-expat@2.4.1(transitive)