mapnik-omnivore - npm Package Compare versions

Comparing version 1.0.3 to 1.1.0

index.js

 var fs = require('fs'),
     path = require('path'),
     invalid = require('./lib/invalid'),
-    processDatasource = require('./lib/datasourceProcessor');
+    processDatasource = require('./lib/datasourceProcessor'),
+    mapnik = require('mapnik');
+// Register datasource plugins
+mapnik.register_default_input_plugins()
 var _options = {
     encoding: 'utf8'
-}
+};
+/**
+ * Initializes the module
+ * @param file (filepath)
+ * @returns metadata {filesize, projection, filename, center, extent, json, minzoom, maxzoom, layers, dstype, filetype}
+ */
 function digest(file, callback) {
@@ -14,3 +22,7 @@ getMetadata(file, function(err, metadata) {
 };
-
+/**
+ * Validates size of file and processes the file
+ * @param file (filepath)
+ * @returns metadata {filesize, projection, filename, center, extent, json, minzoom, maxzoom, layers, dstype, filetype}
+ */
 function getMetadata(file, callback) {
@@ -25,5 +37,5 @@ var metadata = {};
             if (err) return callback(err);
-            processDatasource.init(file, filesize, filetype, function(err, dsConfigs) {
+            processDatasource.init(file, filesize, filetype, function(err, metadata) {
                 if (err) return callback(err);
-                return callback(null, dsConfigs);
+                return callback(null, metadata);
             });
@@ -33,3 +45,7 @@ });
 };
-
+/**
+ * Validates filetype based on the file's contents
+ * @param file (filepath)
+ * @returns (error, filetype);
+ */
 function getFileType(file, callback) {
@@ -50,2 +66,4 @@ //get file contents
             else if ((head.indexOf('<?xml') !== -1) && (head.indexOf('<gpx') !== -1)) return callback(null, '.gpx');
+            //should detect all geo CSV type files, regardless of file extension (e.g. '.txt' or '.tsv')
+            else if (isCSV(file)) return callback(null, '.csv');
             else return callback(invalid('Incompatible filetype.'));
@@ -59,6 +77,25 @@ //Close file
 };
+/**
+ * Checks if tile is valid geoCSV
+ * @param file (filepath)
+ * @returns boolean;
+ */
+function isCSV(file) {
+    var options = {
+        type: 'csv',
+        file: file
+    };
+    // Using mapnik CSV plugin to validate geocsv files, since mapnik is eventually what 
+    // will be digesting it to obtain fields, extent, and center point
+    try {
+        var ds = new mapnik.Datasource(options);
+        return true;
+    } catch (err) {
+        return false;
+    }
+};
 module.exports = {
-	digest: digest,
-	getFileType: getFileType,
-	getMetadata: getMetadata
+    digest: digest,
+    getFileType: getFileType,
+    getMetadata: getMetadata
 };

lib/datasourceProcessor.js

@@ -9,3 +9,9 @@ var fs = require('fs'),
 mapnik.register_default_input_plugins()
-
+/**
+ * Initializes the module
+ * @param file (filepath)
+ * @param filesize
+ * @param filetype
+ * @returns metadata {filesize, projection, filename, center, extent, json, minzoom, maxzoom, layers, dstype, filetype}
+ */
 function init(file, filesize, filetype, callback) {
@@ -17,3 +23,9 @@ getDatasourceConfigs(file, filesize, filetype, function(err, configs) {
 };
-
+/**
+ * Obtains metadata depending on file type
+ * @param file (filepath)
+ * @param filesize
+ * @param filetype
+ * @returns metadata {filesize, projection, filename, center, extent, json, minzoom, maxzoom, layers, dstype, filetype}
+ */
 function getDatasourceConfigs(file, filesize, filetype, callback) {
@@ -25,4 +37,4 @@ getProjection(file, filetype, function(err, proj) {
         // shapefile datasource
-        if (filetype === '.shp') {
-            processShapeDatasource(file, filesize, name, proj, function(err, info) {
+        if (filetype === '.shp' || filetype === '.csv') {
+            processDatasource(file, filesize, name, proj, filetype, function(err, info) {
                 if (err) return (err);
@@ -43,3 +55,3 @@ return callback(null, {
             });
-            // else OGR datasource
+        // else OGR datasource
         } else {
@@ -65,5 +77,10 @@ processOgrDatasource(file, filesize, name, proj, filetype, function(err, info) {
 };
-
+/**
+ * Obtains projection based on filetype
+ * @param file (filepath)
+ * @param filetype
+ * @returns projection String
+ */
 function getProjection(file, filetype, callback) {
-    if (filetype === '.geo.json') return callback(null, '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs');
+    if (filetype === '.geo.json' || filetype === '.csv') return callback(null, '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs');
     else if (filetype === '.shp') {
@@ -79,3 +96,3 @@ projectionFromShape(file, function(err, proj) {
 /**
- * Obtains center point (lat/lng) from the extent of the file
+ * Obtains extent from the datasource and the center point (lat/lng)
  * @param ds (Mapnik datasource)
@@ -88,3 +105,3 @@ * @param projection
     var extent;
-    if (filetype === '.shp') {
+    if (filetype === '.shp' || filetype === '.csv') {
         // Convert datasource extent to lon/lat when saving
@@ -159,17 +176,39 @@ var fromProj = new mapnik.Projection(projection);
 };
-/**************
- * Shape files
- **************/
-function processShapeDatasource(file, filesize, name, proj, callback) {
-    var info = {};
-    var dstype = 'shape';
-    var options = {
-        type: dstype,
-        file: file,
-        layer: name
+/**********************
+ * Shape and CSV files
+ **********************/
+ /**
+ * Creates a mapnik Datasource and uses that to obtain the file's spatial properties
+ * @param file (filepath)
+ * @param filesize
+ * @param name
+ * @param proj
+ * @param filetype
+ * @returns object that contains spatial properties of the file {extent, center, json, minzoom, maxzoom, layers, dstype}
+ */
+function processDatasource(file, filesize, name, proj, filetype, callback) {
+    var options = {};
+    var dstype;
+    //setup shape datasource options
+    if(filetype === '.shp'){
+        dstype = 'shape';
+        options = {
+            type: dstype,
+            file: file,
+            layer: name
+        };
+    //else setup csv datasource options
+    } else {
+        dstype = 'csv';
+        options = {
+            type: dstype,
+            file: file,
+            filesize_max:10,
+            layer: name
+        };
     }
     var ds = new mapnik.Datasource(options);
     //Center of bounding box/extent
-    var results = getCenterAndExtent(ds, proj, '.shp');
-    //Get shapefile fields and setup vector layers json
+    var results = getCenterAndExtent(ds, proj, filetype);
+    //Get fields and setup vector layers json
     var actual = ds.describe();
@@ -196,7 +235,11 @@ var fields = actual.fields;
             layers: [options.layer],
-            dstype: 'shape'
+            dstype: dstype
         });
     });
 };
-
+ /**
+ * Obtains projection from a shapefile by reading the .prj file in the same directory
+ * @param file (filepath)
+ * @returns result.proj4
+ */
 function projectionFromShape(file, callback) {
@@ -226,5 +269,15 @@ var fileDir = path.dirname(file);
 };
+
 /************
  * OGR files
  ************/
+  /**
+ * Obtains the OGR file's spatial properties
+ * @param file (filepath)
+ * @param filesize
+ * @param name
+ * @param proj
+ * @param filetype
+ * @returns object that contains spatial properties of the file {extent, center, json, minzoom, maxzoom, layers, dstype}
+ */
 function processOgrDatasource(file, filesize, name, proj, filetype, callback) {
@@ -253,3 +306,11 @@ getDatasource(file, name, filetype, function(err, layers, json, ds) {
 };
-
+ /**
+ * Creates a mapnik Datasource and uses that to obtain the file's spatial properties
+ * @param file (filepath)
+ * @param name
+ * @param filetype
+ * @returns layers (file's layer names) 
+ *          json (vector_layers array) 
+ *          ds (mapnik Datasource)
+ */
 function getDatasource(file, name, filetype, callback) {
@@ -256,0 +317,0 @@ var dstype = 'ogr';

package.json

 {
     "name": "mapnik-omnivore",
-    "version": "1.0.3",
+    "version": "1.1.0",
     "description": "Node module that returns metadata of spatial files.",
     "main": "index.js",
-    "keywords": [
-        "mapbox",
-        "mapnik"
-    ],
+    "keywords": ["mapbox", "mapnik"],
     "dependencies": {
@@ -15,2 +12,6 @@ "srs": "0.3.12",
     },
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/mapbox/mapnik-omnivore"
+    },
     "engines": {
@@ -26,2 +27,2 @@ "node": "0.10.x"
     "author": "Carol B. Hansen"
-}
+}

README.md

 # mapnik-omnivore
 Node module that returns metadata of spatial files.
+Version format follows [Semantic Version](http://semver.org/)
 
 [![Build Status](https://travis-ci.org/mapbox/mapnik-omnivore.svg?branch=master)](https://travis-ci.org/mapbox/mapnik-omnivore)
 
-
 Currently supports the following file types:
@@ -11,6 +11,7 @@ - `.kml`
 - `.geo.json`
-- `.shp`  (In order to set the projection, the `.prj` file must be in the same directory and have the same name as the `.shp` file)
+- `.csv` : must be valid geo CSV, and can be in the form of `.csv`, `.txt`, or `.tsv`
+- `.shp` : In order to set the projection, the `.prj` file must be in the same directory and have the same name as the `.shp` file
 
 
-# How to Use
+# Example
 
@@ -32,2 +33,3 @@ ```
 
+
 ### mapnikOmnivore.digest(filepath, callback)
@@ -73,3 +75,9 @@ - filepath `required`
 
+## Install
+With npm:
+```
+npm install mapnik-omnivore
+```
 
+
 Run `node runModule.js` from the root to try it out.
@@ -76,0 +84,0 @@

test/datasourceProcessor.test.js

@@ -10,2 +10,3 @@ var assert = require('assert'),
 var expectedMetadata_DC_polygon = JSON.parse(fs.readFileSync(path.resolve('test/fixtures/metadata_DC_polygon.json')));
+var expectedMetadata_bbl_csv = JSON.parse(fs.readFileSync(path.resolve('test/fixtures/metadata_bbl_current_csv.json')));
 var expectedMetadata_1week_earthquake = JSON.parse(fs.readFileSync(path.resolve('test/fixtures/metadata_1week_earthquake.json')));    
@@ -21,3 +22,4 @@ /**
             type: 'shape',
-            file: shapefile
+            file: shapefile,
+            layer: 'world_merc'
         });
@@ -37,2 +39,26 @@ var type = '.shp';
 });
+describe('[CSV] Getting center of extent', function() {
+    it('should return expected values', function() {
+        var proj = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs';
+        var csvFile = path.resolve('test/data/csv/bbl_current_csv.csv');
+        var filetype = '.csv';
+        var options = {
+            type: 'csv',
+            file: csvFile,
+            filesize_max:10,
+            layer: 'bbl_current_csv'
+        };
+        var ds = new mapnik.Datasource(options);
+        var expectedCenter = [-77.0481382917019, 38.93765872502635];
+        var expectedExtent = [-77.0925920175155,38.9142786070481,-77.0036845658883,38.9610388430046];
+        var result = datasourceProcessor.getCenterAndExtent(ds, proj, filetype);
+        assert.ok(result);
+        assert.ok(result.center);
+        assert.ok(result.extent);
+        assert.ok(typeof result.extent == 'object');
+        assert.ok(typeof result.center == 'object');
+        assert.deepEqual(result.center, expectedCenter);
+        assert.deepEqual(result.extent, expectedExtent);
+    });
+});
 describe('[KML] Getting center of extent', function() {
@@ -63,3 +89,3 @@ it('should return expected values', function() {
     it('should return expected values', function() {
-        var proj = '+init=epsg:4326';
+        var proj = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs';
         var geoJsonFile = path.resolve('test/data/geojson/DC_polygon.geo.json');
@@ -131,2 +157,21 @@ var type = '.geo.json';
 });
+describe('[CSV] Getting datasources', function() {
+    it('should return expected layers and json', function(done) {
+        var csvFile = path.resolve('test/data/csv/bbl_current_csv.csv');
+        var filesize = 1667;
+        var type = '.csv';
+        datasourceProcessor.init(csvFile, filesize, type, function(err, metadata) {
+            if (err) return done(err);
+            assert.ok(err === null);
+            try {
+                assert.deepEqual(metadata, expectedMetadata_bbl_csv);
+            } catch (err) {
+                console.log(err);
+                console.log("Expected mapnik-omnivore metadata has changed. Writing new metadata to file.");
+                fs.writeFileSync(path.resolve('test/fixtures/metadata_bbl_current_csv.json'), JSON.stringify(metadata));
+            }
+            done();
+        });
+    });
+});
 describe('[KML] Getting datasources', function() {
@@ -133,0 +178,0 @@ it('should return expected layers and json', function(done) {

test/index.test.js

@@ -9,3 +9,4 @@ var assert = require('assert'),
 var expectedMetadata_DC_polygon = JSON.parse(fs.readFileSync(path.resolve('test/fixtures/metadata_DC_polygon.json')));
-var expectedMetadata_1week_earthquake = JSON.parse(fs.readFileSync(path.resolve('test/fixtures/metadata_1week_earthquake.json')));    
+var expectedMetadata_bbl_csv = JSON.parse(fs.readFileSync(path.resolve('test/fixtures/metadata_bbl_current_csv.json')));
+var expectedMetadata_1week_earthquake = JSON.parse(fs.readFileSync(path.resolve('test/fixtures/metadata_1week_earthquake.json')));
 /**
@@ -31,2 +32,19 @@ * Testing mapnik-omnivore.digest
 });
+describe('[CSV] Getting datasources', function() {
+    it('should return expected metadata', function(done) {
+        var file = path.resolve('test/data/csv/bbl_current_csv.csv');
+        mapnik_omnivore.digest(file, function(err, metadata) {
+            if (err) return done(err);
+            assert.ok(err === null);
+            try {
+                assert.deepEqual(metadata, expectedMetadata_bbl_csv);
+            } catch (err) {
+                console.log(err);
+                console.log("Expected mapnik-omnivore metadata has changed. Writing new metadata to file.");
+                fs.writeFileSync(path.resolve('test/fixtures/metadata_bbl_current_csv.json'), JSON.stringify(metadata));
+            }
+            done();
+        });
+    });
+});
 describe('[KML] Getting datasources', function() {
@@ -38,3 +56,3 @@ it('should return expected metadata', function(done) {
             assert.ok(err === null);
-  try {
+            try {
                 assert.deepEqual(metadata, expectedMetadata_1week_earthquake);
@@ -88,3 +106,3 @@ } catch (err) {
         var file = path.resolve('test/data/errors/incompatible.txt');
-        mapnik_omnivore.getFileType(file, function(err, type){
+        mapnik_omnivore.getFileType(file, function(err, type) {
             assert.ok(err instanceof Error);
@@ -98,5 +116,16 @@ assert.equal('EINVALID', err.code);
 describe('Getting filetype ', function() {
+    it('should return an error due to non-geo CSV file', function(done) {
+        var file = path.resolve('test/data/errors/nongeo.csv');
+        mapnik_omnivore.getFileType(file, function(err, type) {
+            assert.ok(err instanceof Error);
+            assert.equal('EINVALID', err.code);
+            assert.equal(err.message, 'Incompatible filetype.');
+            done();
+        });
+    });
+});
+describe('Getting filetype ', function() {
     it('should return an error because file does not exist.', function(done) {
         var file = 'doesnt/exist.shp';
-        mapnik_omnivore.getMetadata(file, function(err, configs){
+        mapnik_omnivore.getMetadata(file, function(err, configs) {
             assert.ok(err instanceof Error);
@@ -103,0 +132,0 @@ assert.equal('EINVALID', err.code);

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No repository

Supply chain risk

Package does not have a linked source code repository. Without this field, a package will have no reference to the location of the source code use to generate the package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1666456

increased by2.81%

Number of package files

29

increased by11.54%

Lines of code

1092

increased by19.47%

Number of lines in readme file

84

increased by10.53%

Number of low supply chain risk alerts

4

decreased by-20%

No dependency changes