tilelive - npm Package Compare versions

Comparing version 5.2.3 to 5.3.0

CHANGELOG.md

@@ -0,1 +1,5 @@
+## 5.3.0
+
+* Allows read streams to be split into parallel jobs.
+
 ## 5.2.0
@@ -106,2 +110,1 @@
 * options.mapfile is now options.datasource, and can accept a Carto MML object.
-

lib/stream-deserialize.js

@@ -12,4 +12,7 @@ var stream = require('stream');
 
-function Deserialize() {
+function Deserialize(options) {
     stream.Transform.call(this);
+
+    options = options || {};
+
     this._writableState.objectMode = false;
@@ -19,2 +22,3 @@ this._readableState.objectMode = true;
     this._decoder = new StringDecoder('utf8');
+    this._job = options.job || false;
 }
@@ -27,13 +31,5 @@
 
-    var obj, line;
     for (var i = 0; i < lines.length; i++) {
-        line = lines[i];
-
-        if (line.toString() === serialHeader) continue;
-
-        try { obj = deserialize(line); }
-        catch (err) { return callback(err); }
-        if (obj instanceof Info) this.emit('info', obj);
-        if (obj instanceof Tile) this.emit('tile', obj)
-        this.push(obj);
+        try { this.deserialize(lines[i]); }
+        catch(err) { return callback(err); }
     }
@@ -47,12 +43,22 @@
     if (leftover) {
-        var obj;
+        try { this.deserialize(leftover); }
+        catch(err) { return callback(err); }
+    }
+    callback();
+};
 
-        try { obj = deserialize(leftover); }
-        catch (err) { return callback(err); }
-        if (obj instanceof Info) this.emit('info', obj);
-        if (obj instanceof Tile) this.emit('tile', obj)
-        this.push(obj);
+Deserialize.prototype.deserialize = function(serializedObj) {
+    if (serializedObj.toString() === serialHeader) return;
+
+    if (this._job) {
+        var x = deserialize(serializedObj, 'x');
+        if (x % this._job.total !== this._job.num - 1)
+            return;
     }
 
-    callback();
+    var obj = deserialize(serializedObj);
+
+    if (obj instanceof Info) this.emit('info', obj);
+    if (obj instanceof Tile) this.emit('tile', obj);
+    this.push(obj);
 };

lib/stream-list.js

@@ -20,2 +20,3 @@ var sm = new (require('sphericalmercator'))();
     this.length = 0;
+    this.job = options.job || false;
 
@@ -63,2 +64,6 @@ // Determine when the writable stream is finished so the
         stream.stats.ops++;
+
+        if (stream.job && zxy.x % stream.job.total !== stream.job.num - 1)
+            return skip();
+
         stream.source.getTile(zxy.z, zxy.x, zxy.y, function(err, buffer) {
@@ -68,8 +73,3 @@ if (err && !(/does not exist$/).test(err.message)) {
             } else if (err || isEmpty(buffer)) {
-                stream.stats.skipped++;
-                stream.stats.done++;
-                // Update length
-                stream.length--;
-                stream.emit('length', stream.length);
-                get(push);
+                skip();
             } else {
@@ -81,5 +81,13 @@ stream.stats.done++;
 
+        function skip() {
+            stream.stats.skipped++;
+            stream.stats.done++;
+            // Update length
+            stream.length--;
+            stream.emit('length', stream.length);
+            get(push);
+        }
+
         return true;
     });
 };
-

lib/stream-pyramid.js

@@ -37,2 +37,3 @@ var sm = new (require('sphericalmercator'))();
     this.length = 0;
+    this.job = options.job || false;
 
@@ -53,2 +54,12 @@ stream.Readable.call(this, { objectMode: true });
 
+        if (stream.job) {
+            var dx = (stream.bounds[2] - stream.bounds[0]) / stream.job.total;
+            stream.bounds = [
+                stream.bounds[0] + (stream.job.num - 1) * dx,
+                stream.bounds[1],
+                stream.bounds[0] + stream.job.num * dx,
+                stream.bounds[3]
+            ];
+        }
+
         stream.bboxes = {};
@@ -154,2 +165,1 @@ for (var z = stream.minzoom; z <= stream.maxzoom; z++) {
 }
-

lib/stream-scanline.js

@@ -33,2 +33,3 @@ var sm = new (require('sphericalmercator'))();
     this.length = 0;
+    this.job = options.job || false;
 
@@ -87,2 +88,6 @@ stream.Readable.call(this, { objectMode: true });
         nextDeep(stream);
+
+        if (stream.job && x % stream.job.total !== stream.job.num - 1)
+            return skip();
+
         stream.source.getTile(z, x, y, function(err, buffer) {
@@ -92,8 +97,3 @@ if (err && !(/does not exist$/).test(err.message)) {
             } else if (err || isEmpty(buffer)) {
-                stream.stats.skipped++;
-                stream.stats.done++;
-                // Update length
-                stream.length--;
-                stream.emit('length', stream.length);
-                get(push);
+                skip();
             } else {
@@ -104,2 +104,12 @@ stream.stats.done++;
         });
+
+        function skip() {
+            stream.stats.skipped++;
+            stream.stats.done++;
+            // Update length
+            stream.length--;
+            stream.emit('length', stream.length);
+            get(push);
+        }
+
         return true;
@@ -132,2 +142,1 @@ });
 }
-

lib/stream-util.js

@@ -44,5 +44,7 @@ var concurrency = Math.ceil(require('os').cpus().length * 16);
 
-function deserialize(data) {
+function deserialize(data, property) {
+    if (property) return getSerializedProperty(data, property);
     if (data.indexOf('{"z":') === 0) return deserializeTile(data);
     if (data.indexOf('{') === 0) return deserializeInfo(data);
+
     throw new DeserializationError('Invalid data');
@@ -88,2 +90,8 @@ }
 
+function getSerializedProperty(data, property) {
+    var re = new RegExp('"' + property + '":(.+?)[,}]');
+    var m = re.exec(data);
+    return m ? m[1] : undefined;
+}
+
 function deserializeInfo(data) {
@@ -90,0 +98,0 @@ try { obj = JSON.parse(data); }

lib/tilelive.js

@@ -326,2 +326,2 @@ var tilelive = exports;
 tilelive.serialize = function() { return new Serialize(); };
-tilelive.deserialize = function() { return new Deserialize(); };
+tilelive.deserialize = function(options) { return new Deserialize(options); };

package.json

 {
   "name": "tilelive",
-  "version": "5.2.3",
+  "version": "5.3.0",
   "main": "./lib/tilelive.js",
@@ -39,8 +39,8 @@ "description": "API for various map tile backends",
   "devDependencies": {
+    "coveralls": "~2.11.1",
+    "istanbul": "~0.3.0",
+    "mbtiles": "~0.4.3",
     "tape": "2.13.3",
-    "mbtiles": "~0.4.3",
     "tilejson": "~0.8.0",
-    "tilelive-http": "^0.3.0",
-    "istanbul": "~0.3.0",
-    "coveralls": "~2.11.1"
+    "tilelive-http": "^0.3.0"
   },
@@ -47,0 +47,0 @@ "bin": {

README.md

@@ -58,2 +58,12 @@ # tilelive.js
 
+## Parallel read streams
+
+Tilelive can split a read operation into an arbitrary number of jobs. Pass a `job` parameter to options when using `tilelive.createReadStream` or `tilelive.deserialize`:
+
+```javascript
+var readable = tilelive.createReadStream(src, { type: 'scanline', job: { total: 4, num: 1 } });
+```
+
+This instructs tilelive to only read tiles that would fall into job `1` of `4`. A complete read would mean four calls each with a different `num`.
+
 ## bin/tilelive-copy
@@ -68,2 +78,1 @@
     npm test
-

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

55588

increased by3.04%

Lines of code

953

increased by3.47%

Number of lines in readme file

77

increased by13.24%

Worsened metrics

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes