gtfs2lc - npm Package Compare versions

Comparing version 0.8.7 to 0.8.8

baseUris-example.json

 {
-  "stop": "http://data.gtfs.org/example/stops/{stop_id}",
-  "route": "http://data.gtfs.org/example/routes/{routes.route_short_name}",
-  "trip": "http://data.gtfs.org/example/trips/{trip_id}/{routes.route_short_name}{trips.startTime}",
-  "connection": "http://example/linkedconnections.org/connections/{trips.startTime}/{connection.departureStop}/{trips.trip_id}",
+  "stop": "http://example.org/stations/{stops.stop_id}",
+  "route": "http://example.org/routes/{routeFrom}/{routeTo}/{routes.route_id}",
+  "trip": "http://example.org/trips/{tripLabel}/{tripStartTime}",
+  "connection": "http://example.org/connections/{tripLabel}/00{connection.departureStop}/{tripStartTime}/",
   "resolve": {
-    "route_short_id": "connection.trip.route.route_id.substring(0,5)",
-    "trip_id": "connection.trip.trip_id",
-    "trip_startTime": "format(connection.trip.startTime, 'YYYYMMDDTHHmm');",
-    "departureStop": "connection.departureStop"
+      "routeFrom": "routes.route_long_name.replace(/\\s/gi, '').split('--')[0];",
+      "routeTo": "routes.route_long_name.replace(/\\s/gi, '').split('--')[1];",
+      "tripLabel": "routes.route_short_name + routes.route_id;",
+      "tripStartTime": "format(trips.startTime, 'YYYYMMDDTHHmm');"
   }
-}
+}

bin/gtfs2lc.js

@@ -53,3 +53,3 @@ #!/usr/bin/env node
 var resultStream = null;
-mapper.resultStream(program.path, function (stream) {
+mapper.resultStream(program.path, function (stream, stopsdb) {
   resultStream = stream;
@@ -82,2 +82,3 @@ if (!program.format || program.format === "json") {
         Connection : 'lc:Connection',
+        CancelledConnection: 'lc:CancelledConnection',
         departureTime : { '@type' : 'xsd:dateTime', '@id' : 'lc:departureTime' },
@@ -90,3 +91,3 @@ departureStop : { '@type' : '@id', '@id' : 'lc:departureStop' },
     //convert triples stream to jsonld stream
-    stream = stream.pipe(new Connections2JSONLD(baseUris, context));
+    stream = stream.pipe(new Connections2JSONLD(baseUris, stopsdb, context));
     //prepare the output
@@ -99,3 +100,3 @@ if (program.format === 'mongold') {
   } else if (['ntriples','turtle'].indexOf(program.format) > -1) {
-    stream = stream.pipe(new gtfs2lc.Connections2Triples(baseUris));
+    stream = stream.pipe(new gtfs2lc.Connections2Triples(baseUris, stopsdb));
     if (program.format === 'ntriples') {
@@ -102,0 +103,0 @@ stream = stream.pipe(new N3.StreamWriter({ format : 'N-Triples'}));

lib/Connections2JSONLD.js

@@ -9,3 +9,3 @@ /**
 
-var Connections2JSONLD = function (baseUris, context) {
+var Connections2JSONLD = function (baseUris, stopsdb, context) {
   Transform.call(this, { objectMode: true });
@@ -31,3 +31,3 @@ this.context = context || {
 
-  this._uris = new URIStrategy(baseUris);
+  this._uris = new URIStrategy(baseUris, stopsdb);
   this._count = 0;
@@ -39,36 +39,41 @@ };
 Connections2JSONLD.prototype._transform = function (connection, encoding, done) {
-  var id = this._uris.getId(connection);
-  const types = ['gtfs:Regular', 'gtfs:NotAvailable', 'gtfs:MustPhone', 'gtfs:MustCoordinateWithDriver'];
+  try {
+    var id = this._uris.getId(connection);
+    const types = ['gtfs:Regular', 'gtfs:NotAvailable', 'gtfs:MustPhone', 'gtfs:MustCoordinateWithDriver'];
 
-  var lc = {
-    "@id": id,
-    "@type": "Connection",
-    "departureStop": this._uris.getStopId(connection.departureStop),
-    "arrivalStop": this._uris.getStopId(connection.arrivalStop),
-    "departureTime": connection.departureTime.toISOString(),
-    "arrivalTime": connection.arrivalTime.toISOString(),
-    "gtfs:trip": this._uris.getTripId(connection),
-    "gtfs:route": this._uris.getRouteId(connection)
-  };
+    var lc = {
+      "@id": id,
+      "@type": "Connection",
+      "departureStop": this._uris.getStopId(connection.departureStop),
+      "arrivalStop": this._uris.getStopId(connection.arrivalStop),
+      "departureTime": connection.departureTime.toISOString(),
+      "arrivalTime": connection.arrivalTime.toISOString(),
+      "gtfs:trip": this._uris.getTripId(connection),
+      "gtfs:route": this._uris.getRouteId(connection)
+    };
 
-  //the headsign is already the result here of earlier checking whether there’s a trip headsign or a route headsign if connection headsign was not set. It can be used reliably
-  if (connection.headsign) {
-    lc["direction"] = connection.headsign;
-  }
+    //the headsign is already the result here of earlier checking whether there’s a trip headsign or a route headsign if connection headsign was not set. It can be used reliably
+    if (connection.headsign) {
+      lc["direction"] = connection.headsign;
+    }
 
-  var pickupType = types[0];
-  if (connection['pickup_type'] && connection['pickup_type'] !== null) {
-    pickupType = types[connection['pickup_type']];
-    lc["gtfs:pickupType"] = pickupType;
-  }
+    var pickupType = types[0];
+    if (connection['pickup_type'] && connection['pickup_type'] !== null) {
+      pickupType = types[connection['pickup_type']];
+      lc["gtfs:pickupType"] = pickupType;
+    }
 
-  var dropOffType = types[0];
-  if (connection['drop_off_type'] && connection['drop_off_type'] !== null) {
-    dropOffType = types[connection['drop_off_type']];
-    lc["gtfs:dropOffType"] = dropOffType;
+    var dropOffType = types[0];
+    if (connection['drop_off_type'] && connection['drop_off_type'] !== null) {
+      dropOffType = types[connection['drop_off_type']];
+      lc["gtfs:dropOffType"] = dropOffType;
+    }
+
+    done(null, lc);
+  } catch (err) {
+    console.error(err);
+    done(null, {});
   }
-
-  done(null, lc);
 };
 
 module.exports = Connections2JSONLD;

lib/Connections2Triples.js

@@ -9,5 +9,5 @@ /**
 
-var Connections2Triples = function (baseUris) {
+var Connections2Triples = function (baseUris, stopsdb) {
   Transform.call(this, {objectMode : true});
-  this._uris = new URIStrategy(baseUris);
+  this._uris = new URIStrategy(baseUris, stopsdb);
   this._count = 0;
@@ -14,0 +14,0 @@ };

lib/gtfs2connections.js

 const csv = require('fast-csv'),
-      ConnectionsBuilder = require('./ConnectionsBuilder.js'),
-      Services = require('./services/calendar.js'),
-      Store = require('./stores/Store.js'),
-      {AsyncIterator} = require('asynciterator'),
-      StreamIterator = require('./StreamIterator.js'),
-      fs = require('fs');
+  ConnectionsBuilder = require('./ConnectionsBuilder.js'),
+  Services = require('./services/calendar.js'),
+  Store = require('./stores/Store.js'),
+  { AsyncIterator } = require('asynciterator'),
+  StreamIterator = require('./StreamIterator.js'),
+  fs = require('fs');
 
@@ -26,2 +26,6 @@ var Mapper = function (options) {
 Mapper.prototype.resultStream = function (path, done) {
+  var stops = fs.createReadStream(path + '/stops.txt', { encoding: 'utf8', objectMode: true }).pipe(csv({ objectMode: true, headers: true })).on('error', function (e) {
+    console.error(e);
+  });
+
   var routes = fs.createReadStream(path + '/routes.txt', { encoding: 'utf8', objectMode: true }).pipe(csv({ objectMode: true, headers: true })).on('error', function (e) {
@@ -34,7 +38,7 @@ console.error(e);
   }));
-  
+
   var calendarDates = fs.createReadStream(path + '/calendar_dates.txt', { encoding: 'utf8', objectMode: true }).pipe(csv({ objectMode: true, headers: true })).on('error', function (e) {
     console.error(e);
   });
-  
+
   var services = fs.createReadStream(path + '/calendar.txt', { encoding: 'utf8', objectMode: true }).pipe(csv({ objectMode: true, headers: true })).pipe(new Services(calendarDates, this._options)).on('error', function (e) {
@@ -45,2 +49,3 @@ console.error(e);
   //Step 2 & 3: store in leveldb in 3 hidden directories, or in memory, depending on the options
+  var stopsdb = new Map();
   var routesdb = Store(path + '/.routes', this._options.store);
@@ -53,4 +58,4 @@ var servicesdb = Store(path + '/.services', this._options.store);
     //wait for the 2 streams to finish (services and routes) to write to the stores
-    if (count === 2) {
-      console.error("Indexing services and routes succesful!");
+    if (count === 3) {
+      console.error("Indexing services and routes successful!");
       //Step 4 and 5: let's create our connections!
@@ -63,3 +68,3 @@ var connectionRules = fs.createReadStream(path + '/connections.txt', { encoding: 'utf8', objectMode: true }).pipe(csv({ objectMode: true, headers: true })).on('error', function (e) {
       let connectionsStream = connectionRules.pipe(connectionsBuilder);
-      done(connectionsStream);
+      done(connectionsStream, stopsdb);
     }
@@ -70,4 +75,5 @@ };
   var routesIterator = AsyncIterator.wrap(routes);
-  
-  servicesIterator.transform((service, doneService) =>{
+  var stopsIterator = AsyncIterator.wrap(stops);
+
+  servicesIterator.transform((service, doneService) => {
     if (service['service_id']) {
@@ -82,3 +88,3 @@ servicesdb.put(service['service_id'], service['dates'], doneService);
   }).on('end', finished);
-  
+
   routesIterator.transform((route, doneRoute) => {
@@ -94,4 +100,14 @@ if (route['route_id']) {
   }).on('end', finished);
+
+  stopsIterator.transform((stop, doneStop) => {
+    if (stop['stop_id']) {
+      stopsdb.set(stop['stop_id'], stop);
+    } 
+    doneStop();
+  }).on('data', () => {
+  }).on('error', function (e) {
+    console.error(e);
+  }).on('end', finished);
 };
 
 module.exports = Mapper;

lib/URIStrategy.js

@@ -9,5 +9,5 @@ /**
 
-var URIStrategy = function(baseUris) {
+var URIStrategy = function (baseUris, stopsdb) {
   var defaultBaseUris = {
-    stop: 'http://example.org/stops/{stop_id}',
+    stop: 'http://example.org/stops/{stops.stop_id}',
     route: 'http://example.org/routes/{routes.route_id}',
@@ -17,3 +17,3 @@ trip:
     connection:
-      'http://example.org/connections/{trips.startTime(YYYYMMDD)}/{connection.departureStop}/{trips.trip_id}',
+      'http://example.org/connections/{trips.startTime(YYYYMMDD)}/{connection.departureStop}/{trips.trip_id}'
   };
@@ -37,2 +37,6 @@ if (!baseUris) {
 
+  // Stops index from GTFS (stops.txt) to allow using other params besides 'stop_id' in URI templates
+  // e.g. 'stop_code' (used by De Lijn!), 'stop_name', stop_lat', 'stop_lon', etc.S
+  this._stopsdb = stopsdb;
+
   this._stopTemplate = uri_templates(baseUris.stop);
@@ -48,19 +52,23 @@ this._routeTemplate = uri_templates(baseUris.route);
  */
-URIStrategy.prototype.getId = function(connection) {
-  return resolveURI(this._connectionTemplate, connection, this._resolve);
+URIStrategy.prototype.getId = function (connection) {
+  return resolveURI(this._connectionTemplate, connection, null, this._resolve);
 };
 
-URIStrategy.prototype.getStopId = function(id) {
-  return this._stopTemplate.fill({ [this._stopTemplate.varNames[0]]: id });
+URIStrategy.prototype.getStopId = function (id) {
+  if(this._stopsdb.has(id)) {
+    return resolveURI(this._stopTemplate, null, this._stopsdb.get(id), this._resolve);
+  } else {
+    throw new Error('Stop ' + id + ' is not defined stops.txt');
+  }
 };
 
-URIStrategy.prototype.getTripId = function(connection) {
-  return resolveURI(this._tripTemplate, connection, this._resolve);
+URIStrategy.prototype.getTripId = function (connection) {
+  return resolveURI(this._tripTemplate, connection, null, this._resolve);
 };
 
-URIStrategy.prototype.getRouteId = function(connection) {
-  return resolveURI(this._routeTemplate, connection, this._resolve);
+URIStrategy.prototype.getRouteId = function (connection) {
+  return resolveURI(this._routeTemplate, connection, null, this._resolve);
 };
 
-function resolveURI(template, connection, resolve) {
+function resolveURI(template, connection, stop, resolve) {
   let varNames = template.varNames;
@@ -70,3 +78,3 @@ let fillerObj = {};
   for (let i in varNames) {
-    fillerObj[varNames[i]] = resolveValue(varNames[i], connection, resolve);
+    fillerObj[varNames[i]] = resolveValue(varNames[i], connection, stop, resolve);
   }
@@ -77,9 +85,16 @@
 
-function resolveValue(param, connection, resolve) {
-  // try first to resolve using keys in 'resolve' object
+function resolveValue(param, connection, stop, resolve) {
+  // Entity objects to be resolved as needed
+  let trips = null;
+  let routes = null;
+
+  // Try first to resolve using keys in 'resolve' object
   if (resolve[param]) {
+    trips = connection ? connection.trip : null;
+    routes = connection ? connection.trip.route : null;
+    let stops = stop;
     return eval(resolve[param]);
   }
 
-  // otherwise, keep behaviour for backward compatibility
+  // Otherwise, keep behaviour for backward compatibility
 
@@ -89,8 +104,2 @@ // GTFS source file and attribute name
   let attr = param.split('.')[1];
-
-  // Entity objects to be resolved as needed
-  let trip = null;
-  let route = null;
-  let caldate = null;
-
   let value = null;
@@ -100,14 +109,17 @@
     case 'trips':
-      trip = connection.trip;
+      trips = connection.trip;
       if (attr.indexOf('startTime') >= 0) {
         let dateformat = attr.match(/\((.*?)\)/)[1];
-        value = format(trip.startTime, dateformat);
+        value = format(trips.startTime, dateformat);
       } else {
-        value = trip[attr];
+        value = trips[attr];
       }
       break;
     case 'routes':
-      route = connection.trip.route;
-      value = route[attr];
+      routes = connection.trip.route;
+      value = routes[attr];
       break;
+    case 'stops':
+      value = stop[attr];
+      break;
     case 'connection':
@@ -114,0 +126,0 @@ if (attr.indexOf('departureTime') >= 0) {

package.json

 {
   "name": "gtfs2lc",
-  "version": "0.8.7",
+  "version": "0.8.8",
   "description": "Mapping script from gtfs to (linked) connections",
@@ -5,0 +5,0 @@ "main": "lib/gtfs2lc.js",

README.md

@@ -9,4 +9,4 @@ # GTFS to Linked Connections
 
-A _connection_ is the combination of a departure and its successive arrival of the same trip. 
-Our goal is to retrieve a list of connections that is sorted by departure time, better known as a Directed Acyclic Graph. This way, routeplanning algorithms can be performed.
+A _connection_ is the combination of a departure and its successive arrival of the same trip.
+Our goal is to retrieve a list of connections that is sorted by departure time, better known as a Directed Acyclic Graph. This way, route planning algorithms can be performed.
 
@@ -27,6 +27,7 @@ More information and live demo at https://linkedconnections.org
 
-If you haven’t yet picked a GTFS file you want to work with, different repositories exist. Our favourite ones:
- * [Transit.land’s feed registry](http://transit.land/feed-registry/)
- * [Transit Feeds](https://transitfeeds.com/)
+If you haven’t yet picked a GTFS file you want to work with, different repositories exist. Our favorite ones:
 
+* [Transit.land’s feed registry](http://transit.land/feed-registry/)
+* [Transit Feeds](https://transitfeeds.com/)
+
 Yet, you may also directly ask your local public transport authority for a copy.
@@ -42,9 +43,10 @@
 
-We’ve enclosed a bash script which ensures this for you. You can run this bash script using `gtfs2lc-sort <path>`. Next to sorting, it also unifies newlines and removed UTF-8 artefacts.
+We’ve enclosed a bash script which ensures this for you. You can run this bash script using `gtfs2lc-sort <path>`. Next to sorting, it also unifies newlines and removes UTF-8 artifacts.
 
 If _step 4_ would not give the desired result, you might want to tweak the script manually. In order for our script to work:
- * __stop_times.txt__ must be ordered by `trip_id` and `stop_sequence`.
- * __calendar.txt__ must be ordered by `service_id`.
- * __calendar_dates.txt__ must be ordered by `service_id`.
 
+* __stop_times.txt__ must be ordered by `trip_id` and `stop_sequence`.
+* __calendar.txt__ must be ordered by `service_id`.
+* __calendar_dates.txt__ must be ordered by `service_id`.
+
 ### Step 4: Generate connections!
@@ -60,9 +62,9 @@
 
-For _big_ GTFS files, your memory may not be sufficient. Luckily, we’ve implemented a way to use your harddisk instead of your RAM. You can enable this with an option: `gtfs2lc /path/to/extracted/gtfs -f json --store LevelStore`.
+For _big_ GTFS files, your memory may not be sufficient. Luckily, we’ve implemented a way to use your hard disk instead of your RAM. You can enable this with an option: `gtfs2lc /path/to/extracted/gtfs -f json --store LevelStore`.
 
 ### Step 5: Generate *Linked* Connections!
 
-When you download a new GTFS file, all identifiers in there will might change and conflict with your previous export. Therefore, we need to think about a way to create global identifiers for the connections, trips, routes and stops in our system. As we are publishing our data on the Web, we will also use Web addresses for these global identifiers.
+When you download a new GTFS file, all identifiers in there might change and conflict with your previous export. Therefore, we need to think about a way to create global identifiers for the connections, trips, routes and stops in our system. As we are publishing our data on the Web, we will also use Web addresses for these global identifiers.
 
-See `baseUris-example.json` for an example or URI templates of what a stable identifier strategy could look like. Copy it and edit it to your likings.
+See `baseUris-example.json` for an example on URI templates of what a stable identifier strategy could look like. Copy it and edit it to your likings. For a more detailed explanation of how to use the URI templates see the description at our [`GTFS-RT2LC`](https://github.com/linkedconnections/gtfsrt2lc#uri-templates) tool, which uses the same strategy.
 
@@ -69,0 +71,0 @@ Now you can generate Linked Data in JSON-LD as follows:

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

55540

increased by2.78%

Lines of code

1030

increased by3.1%

Number of lines in readme file

121

increased by1.68%

No dependency changes