climate-data-crawler - npm Package Compare versions

Comparing version 1.2.1 to 1.3.0

app.js

 "use strict";
 var fs = require('fs');
 var cliArgs = require('command-line-args');
+var readlineSync = require('readline-sync');
 var CdoDataCrawler = require('./cdoDataCrawler');
@@ -35,7 +36,3 @@ var DataProbingBounds = require('./dataProbingBounds');
 
-var crawler = CdoDataCrawler.createInstance(
-  options.dataset, options.datatype, locations, dataProbingBounds,
-  options.offset, options.count);
-
-crawler.run(function(results, locationsNoData){
+var successCallback = function(results, locationsNoData){
   var filenameBase = options.dataset + '-' + options.datatype + '-offset-'
@@ -46,3 +43,18 @@ + options.offset + '-count-' + options.count;
   resultsWriter.write('data/' + filenameBase + '_nodata.json', locationsNoData);
-});
+};
 
+var errorCallback = function(error){
+  console.log("Error: " + error);
+
+  var result = readlineSync.question('Do you want to retry? [y/n] :');
+  if (result.toLowerCase() === 'y'){
+    crawler.continue();
+  }
+};
+
+var crawler = CdoDataCrawler.createInstance(
+  options.dataset, options.datatype, locations, dataProbingBounds,
+  options.offset, options.count);
+
+crawler.run(successCallback, errorCallback);
+

cdoApiClient.js

@@ -8,3 +8,4 @@ "use strict";
   var queryResults;
-  var resultDelegate;
+  var onResults = function(){};
+  var onError = function(){};
 
@@ -35,5 +36,3 @@ var queryPath =
 
-    httpClient.request(options, onRequestCompleted, function(error) {
-      logger.error(error);
-    });
+    httpClient.request(options, onRequestCompleted, onError);
   }
@@ -60,3 +59,3 @@
         eventEmitter.emit('done', queryResults);
-        resultDelegate(queryResults);
+        onResults(queryResults);
       }
@@ -72,3 +71,3 @@ else {
         eventEmitter.emit('done', null);
-        resultDelegate(null);
+        onResults(null);
       }
@@ -79,10 +78,9 @@ }
   // privileged functions
-  this.query = function(resultCallback) {
-    if (resultCallback) {
-      resultDelegate = resultCallback;
-    }
-    else {
-      resultDelegate = function(){};
-    }
+  this.query = function(resultCallback, errorCallback) {
+    if (resultCallback)
+      onResults = resultCallback;
 
+    if (errorCallback)
+      onError = errorCallback;
+
     queryNext();
@@ -89,0 +87,0 @@ };

cdoDataCrawler.js

@@ -13,3 +13,5 @@ "use strict";
 
-  var resultsDelegate = function(){};
+  var onResults = function(){};
+  var onError = function(){};
+
   var results = [];
@@ -33,6 +35,5 @@ var locationsNoData = [];
 
-      dataQuery.run(onQueryComplete);
-      index++;
+      dataQuery.run(onQueryComplete, onError);
     } else {
-      resultsDelegate(results, locationsNoData);
+      onResults(results, locationsNoData);
     }
@@ -48,2 +49,3 @@ };
 
+    index++;
     reportProgress();
@@ -66,7 +68,14 @@
   // privileged functions
-  this.run = function(resultsCallback){
-    if (resultsCallback) {
-      resultsDelegate = resultsCallback;
-    }
+  this.run = function(resultsCallback, errorCallback){
+    if (resultsCallback)
+      onResults = resultsCallback;
+
+    if (errorCallback)
+      onError = errorCallback;
+
     queryNext();
+  };
+
+  this.continue = function(){
+    queryNext();
   }
@@ -73,0 +82,0 @@ }

cdoDataProbingQuery.js

@@ -11,4 +11,6 @@ "use strict";
   var queryYear = startYear;
-  var onQueryComplete;
 
+  var onQueryComplete = function(){};
+  var onError = function(){};
+
   var onApiCallComplete = function(result){
@@ -36,14 +38,13 @@ if (result){
 
-    cdoApiClient.query(onApiCallComplete);
+    cdoApiClient.query(onApiCallComplete, onError);
   };
 
   // privileged functions
-  this.run = function(onQueryCompleteCallback){
-    if (onQueryCompleteCallback){
-      onQueryComplete = onQueryCompleteCallback;
-    }
-    else {
-      onQueryComplete = function(){}
-    }
+  this.run = function(queryCompleteCallback, errorCallback){
+    if (queryCompleteCallback)
+      onQueryComplete = queryCompleteCallback;
 
+    if (errorCallback)
+      onError = errorCallback;
+
     queryNext();
@@ -50,0 +51,0 @@ }

helpers/httpClient.js

@@ -21,2 +21,10 @@ "use strict";
     });
+
+    request.on('socket', function (socket) {
+      socket.setTimeout(60000);
+      socket.on('timeout', function() {
+        request.abort();
+      });
+    });
+
     request.on('error', function(error){
@@ -23,0 +31,0 @@ errorCallback(error.message);

package.json

 {
   "name": "climate-data-crawler",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "Data Crawler for CDO (Climate Data Online) web services",
@@ -30,4 +30,5 @@ "main": "app.js",
     "mocha": "^2.2.1",
+    "readline-sync": "^1.2.15",
     "sinon": "^1.14.1"
   }
 }

tests/unit/cdoApiClientTests.js

@@ -180,18 +180,30 @@ "use strict";
 
-    it('should throw exception when status != 200', function(){
-      // arrange
-      var stubRequest = function(options, successCallback, errorCallback){
-        errorCallback("error");
-      };
-      sinon.stub(httpClient, 'request',stubRequest);
+    describe('error handling', function(){
+      it('should invoke onErrorCallback', function(){
+        // arrange
+        var wasCalled = false;
+        var callArguments = null;
+        var errorCallback = function(){
+          wasCalled = true;
+          callArguments = arguments;
+        };
 
-      var client = getInstance();
+        var expectedError = "this is an error";
+        var httpRequestStub = function(options, successCallback, errorCallback){
+          errorCallback(expectedError);
+        };
+        sinon.stub(httpClient, 'request', httpRequestStub);
 
-      // act
-      client.query();
+        var client = getInstance();
 
-      // assert
-      assert.equal(logger.error.called, true);
+        // act
+        client.query(function(){}, errorCallback);
+
+        // assert
+        assert.equal(wasCalled, true);
+        assert.equal(callArguments[0], expectedError);
+      })
     });
 
+
     it('should use httpClient using correct options', function(){
@@ -198,0 +210,0 @@ // arrange

tests/unit/cdoDataCrawlerTests.js

@@ -11,7 +11,8 @@ "use strict";
 describe('CdoDataCrawler', function(){
-
   var timer;
   var dataQuery;
   var dataQueryFactory;
+
   var simulateQueryCompleted;
+  var simulateError;
 
@@ -79,4 +80,5 @@ var dataset;
     dataQuery = {
-      run: function(queryCompleteCallback){
+      run: function(queryCompleteCallback, errorCallback){
         simulateQueryCompleted = queryCompleteCallback;
+        simulateError = errorCallback;
       }
@@ -402,2 +404,51 @@ };
     });
+
+    describe('error handling', function(){
+      it('should invoke onError callback', function(){
+        // arrange
+        var error = "error 123";
+
+        var wasCalled = false;
+        var callArguments = null;
+        var errorCallback = function(){
+          wasCalled = true;
+          callArguments = arguments;
+        };
+
+        var crawler = getInstance();
+
+        // act
+        crawler.run(function(){}, errorCallback);
+        simulateError(error);
+
+        // assert
+        assert.equal(wasCalled, true);
+        assert.equal(callArguments[0], error);
+      });
+    });
+
+    describe('#continue', function(){
+      it('should run query again', function(){
+        // arrange
+        sinon.spy(dataQuery, 'run');
+        var crawler = getInstance();
+
+        // act + assert
+        crawler.run();
+        assert.equal(dataQuery.run.callCount, 1);
+
+        simulateError('error');
+        assert.equal(dataQuery.run.callCount, 1);
+
+        crawler.continue();
+        assert.equal(dataQuery.run.callCount, 2);
+
+        var locationIdStopCall =
+          dataQueryFactory.createInstance.getCall(0).args[0];
+        var locationIdContinueCall =
+          dataQueryFactory.createInstance.getCall(1).args[0];
+
+        assert.equal(locationIdStopCall, locationIdContinueCall);
+      });
+    })
   });
@@ -404,0 +455,0 @@

tests/unit/cdoDataProbingQueryTests.js

@@ -242,2 +242,32 @@ "use strict";
 
+    describe('error handling', function(){
+      it('should invoke error callback', function(){
+        // arrange
+        var errorTrigger = null;
+        sinon.stub(api, 'query', function(successCallback, errorCallback){
+          errorTrigger = errorCallback;
+        });
+
+        var wasCalled = false;
+        var callArguments = null;
+        var errorHandler = function(){
+          wasCalled = true;
+          callArguments = arguments;
+        };
+
+        var error = "something wrong happened";
+
+        var query = getInstance();
+
+        // act
+        query.run(function(){}, errorHandler);
+        errorTrigger(error);
+
+        // assert
+        assert.equal(wasCalled, true);
+        assert.equal(callArguments[0], error);
+      });
+
+    });
+
   });
@@ -244,0 +274,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

63656

increased by5.31%

Lines of code

1690

increased by5.56%

Worsened metrics

Dependency count

4

increased by33.33%

Dependency changes

• + Addedreadline-sync@^1.2.15

• + Addedreadline-sync@1.4.10(transitive)