gtfs2lc - npm Package Compare versions

Comparing version 0.3.0 to 0.4.0

bin/gtfs2lc.js

@@ -6,4 +6,20 @@ #!/usr/bin/env node
     N3 = require('n3'),
-    jsonldstream = require('jsonld-stream');
+    jsonldstream = require('jsonld-stream'),
+    fs = require('fs');
 
+//ty http://www.geedew.com/remove-a-directory-that-is-not-empty-in-nodejs/
+var deleteFolderRecursive = function(path) {
+  if( fs.existsSync(path) ) {
+    fs.readdirSync(path).forEach(function(file,index){
+      var curPath = path + "/" + file;
+      if(fs.lstatSync(curPath).isDirectory()) { // recurse
+        deleteFolderRecursive(curPath);
+      } else { // delete file
+        fs.unlinkSync(curPath);
+      }
+    });
+    fs.rmdirSync(path);
+  }
+};
+
 console.error("GTFS to linked connections converter use --help to discover more functions");
@@ -15,10 +31,18 @@
   .option('-f, --format <format>', 'Format of the output. Possibilities: csv, ntriples, turtle, json or jsonld (default: json)')
+  .option('-s, --startDate <startDate>', 'startDate in YYYYMMDD format')
+  .option('-e, --endDate <endDate>', 'endDate in YYYYMMDD format')
   .parse(process.argv);
 
 if (!program.path) {
-  program.path = './';
+  console.error('Give a path using the -p option');
+  process.exit();
 }
 
-var mapper = new gtfs2lc.Connections();
+var mapper = new gtfs2lc.Connections({
+  startDate : program.startDate,
+  endDate : program.endDate
+});
+var resultStream = null;
 mapper.resultStream(program.path, function (stream) {
+  resultStream = stream;
   if (!program.format || program.format === "json") {
@@ -60,2 +84,22 @@ stream.on('data', function (connection) {
   }
+  stream.on('end', function () {
+    //clean up the leveldb
+    deleteFolderRecursive(program.path + "/.services");
+    deleteFolderRecursive(program.path + "/.trips");
+  });
 });
+
+process.on('SIGINT', function () {
+  console.error("\nCleaning up");
+  if (resultStream) {
+    resultStream.end();
+  } else {
+    deleteFolderRecursive(program.path + "/.services");
+    deleteFolderRecursive(program.path + "/.trips");
+  }
+  process.exit(0);
+});
+
+
+
+                 

lib/ConnectionsBuilder.js

@@ -28,4 +28,5 @@ /**
       if (error) {
-        console.error('Error: Or you didn\'t sort the file, or there\'s an undocumented service_id in the data, or there\'s a bug in our code:', error);
-        process.exit();
+        //This may happen if -s and/or -e option have been set
+        //console.error('Error: Or you didn\'t sort the file, or there\'s an undocumented service_id in the data, or there\'s a bug in our code:', error);
+        //process.exit();
       } else {
@@ -32,0 +33,0 @@ service = JSON.parse(service);

lib/gtfs2connections.js

@@ -5,8 +5,11 @@ var csv = require('fast-csv'),
     Services = require('./services/calendar.js'),
+    DateInterval = require('./DateInterval.js');
     through2 = require('through2'),
-    level = require('level');
+    level = require('level'),
+    moment = require('moment'),
     fs = require('fs');
 
 var Mapper = function (options) {
-    this._options = options;
+  this._options = options;
+  this._options.interval = new DateInterval(options.startDate, options.endDate);
 };
@@ -26,3 +29,3 @@
   var calendarDates = fs.createReadStream(path + '/calendar_dates.txt', {encoding:'utf8', objectMode: true}).pipe(csv({objectMode:true,headers: true}));
-  var services = fs.createReadStream(path + '/calendar.txt', {encoding:'utf8', objectMode: true}).pipe(csv({objectMode:true,headers: true})).pipe(new Services(calendarDates));
+  var services = fs.createReadStream(path + '/calendar.txt', {encoding:'utf8', objectMode: true}).pipe(csv({objectMode:true,headers: true})).pipe(new Services(calendarDates, this._options));
   //Preparations for step 4
@@ -41,3 +44,3 @@ var connectionRules = fs.createReadStream(path + '/stop_times.txt', {encoding:'utf8', objectMode: true}).pipe(csv({objectMode:true,headers: true})).pipe(new ConnectionRules());
       //Step 4 and 5: let's create our connections!
-      done(connectionRules.pipe(new ConnectionsBuilder(tripsdb, servicesdb)));
+      done(connectionRules.pipe(new ConnectionsBuilder(tripsdb, servicesdb, this._options)));
     }
@@ -44,0 +47,0 @@ };

lib/services/calendar.js

@@ -10,5 +10,6 @@ /**
 
-var CalendarToServices = function (calendarDatesStream) {
+var CalendarToServices = function (calendarDatesStream, options) {
   Transform.call(this, {objectMode : true});
-  this._calendarDatesIterator = new StreamIterator(calendarDatesStream);
+  this._calendarDatesIterator = new StreamIterator(calendarDatesStream, options.interval);
+  this._options = options;
 };
@@ -42,10 +43,12 @@
     var d = currentCD['date'];
-    if (currentCD['exception_type'] === '1' && calendar.indexOf(d) === -1) {
-      //This date has been added and doesn't already exist: push it to the back
-      calendar.push(d);
-    } else if (currentCD['exception_type'] === '2') {
-      //Has been removed: remove it from the array
-      var index = calendar.indexOf(d);
-      if (index > -1) {
-        calendar.splice(index, 1);
+    if (this._options.interval.inclusiveBetween(moment(d, 'YYYYMMDD'))) {
+      if (currentCD['exception_type'] === '1' && calendar.indexOf(d) === -1) {
+        //This date has been added and doesn't already exist: push it to the back
+        calendar.push(d);
+      } else if (currentCD['exception_type'] === '2') {
+        //Has been removed: remove it from the array
+        var index = calendar.indexOf(d);
+        if (index > -1) {
+          calendar.splice(index, 1);
+        }
       }
@@ -85,3 +88,3 @@ }
     calendar.push(currentCD['date']);
-    this._calendarDatesIterator.next(function (calendarDate) {
+    self._calendarDatesIterator.next(function (calendarDate) {
       if (calendarDate) {
@@ -102,3 +105,7 @@ self._processCalendarDates(calendar, serviceId, done);
   var expanded = [];
-  while (d.format('YYYYMMDD') !== calendar['end_date']) {
+  if (this._options.startDate && d < this._options.startDate) {
+    d = this._options.startDate;
+  }
+  
+  while (d.format('YYYYMMDD') !== calendar['end_date'] && this._options.interval.beforeEnd(d)) {
     if (calendar[d.format('dddd').toLowerCase()] === '1') {
@@ -105,0 +112,0 @@ expanded.push(d.format('YYYYMMDD'));

lib/StreamIterator.js

 var util = require('util'),
+    moment = require('moment'),
     EventEmitter = require('events').EventEmitter;
 
-var StreamIterator = function (stream) {
+var StreamIterator = function (stream, interval) {
+  this._interval = interval;
   this._stream = stream;
@@ -32,9 +34,12 @@ this._currentObject = null;
     });
-  } else if (object) {
+  } else if (object && this._interval.inclusiveBetween(moment(object['date'], 'YYYYMMDD'))) {
     this._currentObject = object;
     callback(object);
-  } else {
+  } else if (!object) {
     //stream ended
     this._currentObject = null;
     callback(null);
+  } else {
+    //We didn't find a solution this time, let's find it next time
+    this.next(callback);
   }
@@ -41,0 +46,0 @@ };

package.json

 {
   "name": "gtfs2lc",
-  "version": "0.3.0",
-  "description": "Mapping script from gtfs to linked connnections",
+  "version": "0.4.0",
+  "description": "Mapping script from gtfs to linked connections",
   "main": "lib/gtfs-csv2lc.js",
@@ -34,5 +34,4 @@ "bin": {
     "n3": "^0.4.5",
-    "through2": "^2.0.0",
-    "unzip": "~0.1.9"
+    "through2": "^2.0.0"
   }
 }

README.md

@@ -18,3 +18,3 @@ # GTFS to Linked Connections
 
-Now, we need to make sure that a couple of files are ordered in a specific fashion, not required by the GTFS spec. You can do these orderings through bash as follows:
+Now, we need to make sure that a couple of files are ordered in a specific fashion, not required by the GTFS spec:
  * __stop_times.txt__ must be ordered by `trip_id` and `stop_sequence`. Mind that the number of the columns are also not standardized by GTFS and you might need to tweak the sort command in this repo.
@@ -24,8 +24,8 @@ * __calendar.txt__ must be ordered by `service_id`.
  
-We've enclosed a bash script which ensures this for you. It isn't perfect however and may not return the desired result.
+We've enclosed a bash script which ensures this for you. It isn't perfect however and may not return the desired result. You can run this bash script using `gtfs2lc-sort $pathname`.
 
-If you've ensured this, you can install this library using: `npm install -g gtfs2lc` and use it as follows:
+If you've ensured this, you can use this tool on the command line as follows:
 
 ```bash
-gtfs2lc -p /path/to/extracted/gtfs -f csv
+gtfs2lc -p /path/to/extracted/gtfs -f csv --startDate 20151101  -e 20160101
 ```
@@ -32,0 +32,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

31054

increased by10.97%

Dependency count

7

decreased by-12.5%

Number of package files

18

increased by5.88%

Lines of code

670

increased by12.98%

Worsened metrics

Number of low supply chain risk alerts

6

increased by20%

Dependency changes

• - Removedunzip@~0.1.9

• - Removedbalanced-match@1.0.2(transitive)

• - Removedbinary@0.3.0(transitive)

• - Removedbrace-expansion@1.1.11(transitive)

• - Removedbuffers@0.1.1(transitive)

• - Removedchainsaw@0.1.0(transitive)

• - Removedconcat-map@0.0.1(transitive)

• - Removedfs.realpath@1.0.0(transitive)

• - Removedfstream@0.1.31(transitive)

• - Removedglob@7.2.3(transitive)

• - Removedgraceful-fs@3.0.12(transitive)

• - Removedinflight@1.0.6(transitive)

• - Removedmatch-stream@0.0.2(transitive)

• - Removedminimatch@3.1.2(transitive)

• - Removednatives@1.1.6(transitive)

• - Removedover@0.0.5(transitive)

• - Removedpath-is-absolute@1.0.1(transitive)

• - Removedpullstream@0.4.1(transitive)

• - Removedreadable-stream@1.0.34(transitive)

• - Removedrimraf@2.7.1(transitive)

• - Removedsetimmediate@1.0.5(transitive)

• - Removedslice-stream@1.0.0(transitive)

• - Removedtraverse@0.3.9(transitive)

• - Removedunzip@0.1.11(transitive)